[Fortigate] UTM Virus url.path should be url.full

[nicpenning]

Fortios.url contains the full url when the type is utm with the subtybe of virus.

It seems that this is the only subtype of UTM that does this instead of just putting in the path.

With the current way the pipelines are laid out in the project, I don't think we can inject and if, elseif into the copy statements.

I re-engineered the pipelines to use if [field] {mutate {copy ...}} for all of them for more granular control.

This was my solution to ensure that the UTM Virus logs would put the full url into url.full:

	if [subtype] =="virus" {
            mutate { copy =>{ "[fortios][url]"=> "[url][full]" }}
        }
        else if [fortios][url] {
            mutate { copy =>{ "[fortios][url]"=> "[url][path]" }}
        }

The full section would look like this, but not sure if you want a PR to make this large of a change if you have a better way of handling this logic.

    if [type] == "traffic" {
            if [app] {mutate { copy => { "[app]"=> "[network][application]" }}}
            if [collectedemail] {mutate { copy => { "[collectedemail]"=> "[source][user][email]" }}}
            if [comment] {mutate { copy => { "[comment]"=> "[rule][description]" }}}
            if [dstcollectedemail] {mutate { copy => { "[dstcollectedemail]"=> "[destination][user][email]" }}}
            if [dstintf] {mutate { copy => { "[dstintf]"=> "[observer][egress][interface][name]" }}}
            if [dstintfrole] {mutate { copy => { "[dstintfrole]"=> "[observer][egress][interface][role]" }}}
            if [dstip] {mutate { copy => { "[dstip]"=> "[destination][ip]" }}}
            if [dstmac] {mutate { copy => { "[dstmac]"=> "[destination][mac]" }}}
            if [dstname] {mutate { copy => { "[dstname]"=> "[destination][address]" }}}
            if [dstport] {mutate { copy => { "[dstport]"=> "[destination][port]" }}}
            if [duration] {mutate { copy => { "[duration]"=> "[event][duration]" }}}
            if [group] {mutate { copy => { "[group]"=> "[source][user][group][name]" }}}
            if [msg] {mutate { copy => { "[msg]"=> "[message]" }}}
            if [policyid] {mutate { copy => { "[policyid]"=> "[rule][id]" }}}
            if [policyname] {mutate { copy => { "[policyname]"=> "[rule][name]" }}}
            if [policytype] {mutate { copy => { "[policytype]"=> "[rule][ruleset]" }}}
            if [poluuid] {mutate { copy => { "[poluuid]"=> "[rule][uuid]" }}}
            if [proto] {mutate { copy => { "[proto]"=> "[network][iana_number]" }}}
            if [rcvdbyte] {mutate { copy => { "[rcvdbyte]"=> "[destination][bytes]" }}}
            if [rcvdpkt] {mutate { copy => { "[rcvdpkt]"=> "[destination][packets]" }}}
            if [sentbyte] {mutate { copy => { "[sentbyte]"=> "[source][bytes]" }}}
            if [sentpkt] {mutate { copy => { "[sentpkt]"=> "[source][packets]" }}}
            if [fortios][service] {mutate { copy => { "[fortios][service]"=> "[network][protocol]" }}}
            if [sessionid] {mutate { copy => { "[sessionid]"=> "[network][session_id]" }}}
            if [srcdomain] {mutate { copy => { "[srcdomain]"=> "[source][domain]" }}}
            if [srcintf] {mutate { copy => { "[srcintf]"=> "[observer][ingress][interface][name]" }}}
            if [srcintfrole] {mutate { copy => { "[srcintfrole]"=> "[observer][ingress][interface][role]" }}}
            if [srcip] {mutate { copy => { "[srcip]"=> "[source][ip]" }}}
            if [srcmac] {mutate { copy => { "[srcmac]"=> "[source][mac]" }}}
            if [srcport] {mutate { copy => { "[srcport]"=> "[source][port]" }}}
            if [tranip] {mutate { copy => { "[tranip]"=> "[destination][nat][ip]" }}}
            if [tranport] {mutate { copy => { "[tranport]"=> "[destination][nat][port]" }}}
            if [transip] {mutate { copy => { "[transip]"=> "[source][nat][ip]" }}}
            if [transport] {mutate { copy => { "[transport]"=> "[source][nat][port]" }}}
            if [unauthuser] {mutate { copy => { "[unauthuser]"=> "[source][user][name]" }}}
            if [fortios][url] {mutate { copy => { "[fortios][url]"=> "[url][path]" }}}

            if [dstunauthuser] {mutate { copy => { "[dstunauthuser]"=> "[destination][user][name]" }}}
            if [fortios][user] {mutate { copy => { "[fortios][user]"=> "[source][user][name]" }}}

    # ECS categorization fields
        mutate {
            add_field => { "[event][kind]" => "event" }
            add_field => { "[event][category]" => "network" }
            add_field => { "[event][type]" => "connection" }
        }
        if [action] == "deny" or [utmaction] == "block" {
            mutate { add_field => { "[event][type]" => "denied" } }
        }
        else {
            mutate { add_field => { "[event][type]" => "allowed" } }
        }
        if [action] == "start" {
            mutate { add_field => { "[event][type]" => "start" } }
        }
        else {
            mutate { add_field => { "[event][type]" => "end" } }
        }
        if [action] in [ "dns" , "ip-conn" ] {
            mutate { add_field => { "[event][type]" => "error" } }
        }
        if [network][application] {
            mutate { add_field => { "[event][type]" => "protocol" } }
        }
    }

    # type=dns for version 6.0 and below. On 6.2, dns is subtype of utm

    else if [type] == "utm" or [type] == "dns" {
		if [fortios] {mutate { copy =>{ "[fortios][agent]"=> "[user_agent][original]" }}}
		if [app] {mutate { copy =>{ "[app]"=> "[network][application]" }}}
		if [appcat] {mutate { copy =>{ "[appcat]"=> "[rule][category]" }}}
		if [applist] {mutate { copy =>{ "[applist]"=> "[rule][ruleset]" }}}
		
		if [dir] {mutate { copy =>{ "[dir]"=> "[network][direction]" }}}
		
		if [dst_int] {mutate { copy =>{ "[dst_int]"=> "[observer][egress][interface][name]" }}}
		if [dst_port] {mutate { copy =>{ "[dst_port]"=> "[destination][port]" }}}
		
		if [dstintfrole] {mutate { copy =>{ "[dstintfrole]"=> "[observer][egress][interface][role]" }}}
		if [dstip] {mutate { copy =>{ "[dstip]"=> "[destination][ip]" }}}
		
		if [duration] {mutate { copy =>{ "[duration]"=> "[event][duration]" }}}
		if [fortios][error] {mutate { copy =>{ "[fortios][error]"=> "[error][message]" }}}
		if [errorcode] {mutate { copy =>{ "[errorcode]"=> "[error][code]" }}}
		if [event_id] {mutate { copy =>{ "[event_id]"=> "[event][id]" }}}
		
		if [eventtype] {mutate { copy =>{ "[eventtype]"=> "[event][action]" }}}
		if [filehash] {mutate { copy =>{ "[filehash]"=> "[file][hash][crc32]" }}}
		if [filename] {mutate { copy =>{ "[filename]"=> "[file][name]" }}}
		if [filesize] {mutate { copy =>{ "[filesize]"=> "[file][size]" }}}
		if [filetype] {mutate { copy =>{ "[filetype]"=> "[file][extension]" }}}
		if [fortios][group] {mutate { copy =>{ "[fortios][group]"=> "[source][user][group][name]" }}}
		if [ipaddr]{mutate {split => { "ipaddr" => ", " }}}
		if [ipaddr] {mutate { copy =>{ "[ipaddr]"=> "[dns][resolved_ip]" }}}
		if [msg] {mutate { copy =>{ "[msg]"=> "[message]" }}}
		if [policy_id] {mutate { copy =>{ "[policy_id]"=> "[rule][id]" }}}
		
		if [profile] {mutate { copy =>{ "[profile]"=> "[rule][ruleset]" }}}
		if [proto] {mutate { copy =>{ "[proto]"=> "[network][iana_number]" }}}
		if [qclass] {mutate { copy =>{ "[qclass]"=> "[dns][question][class]" }}}
		if [qname] {mutate { copy =>{ "[qname]"=> "[dns][question][name]" }}}
		if [qtype] {mutate { copy =>{ "[qtype]"=> "[dns][question][type]" }}}
		if [rcvdbyte] {mutate { copy =>{ "[rcvdbyte]"=> "[destination][bytes]" }}}
		if [reason] {mutate { copy =>{ "[reason]"=> "[event][reason]" }}}
		
		if [sentbyte] {mutate { copy =>{ "[sentbyte]"=> "[source][bytes]" }}}
		if [fortios][service] {mutate { copy =>{ "[fortios][service]"=> "[network][protocol]" }}}
		if [session_id] {mutate { copy =>{ "[session_id]"=> "[network][session_id]" }}}
		
		if [src_int] {mutate { copy =>{ "[src_int]"=> "[observer][ingress][interface][name]" }}}
		if [src_port] {mutate { copy =>{ "[src_port]"=> "[source][port]" }}}
		if [srcdomain] {mutate { copy =>{ "[srcdomain]"=> "[source][domain]" }}}
		
		if [srcintfrole] {mutate { copy =>{ "[srcintfrole]"=> "[observer][ingress][interface][role]" }}}
		if [srcip] {mutate { copy =>{ "[srcip]"=> "[source][ip]" }}}
		if [srcmac] {mutate { copy =>{ "[srcmac]"=> "[source][mac]" }}}
		
		if [unauthuser] {mutate { copy =>{ "[unauthuser]"=> "[source][user][name]" }}}

#Inconsistencies in the UTM logging forces us to place the UTM virus URL path into url.full since it contains everything, not just the path.
		if [subtype] =="virus" {
            mutate { copy =>{ "[fortios][url]"=> "[url][full]" }}
        }
        else if [fortios][url] {
            mutate { copy =>{ "[fortios][url]"=> "[url][path]" }}
        }
		
		if [vrf] {mutate { copy =>{ "[vrf]"=> "[network][vrf]" }}}
		if [xid] {mutate { copy =>{ "[xid]"=> "[dns][id]" }}}
		if [hostname] {mutate { copy =>{ "[hostname]"=> "[url][domain]" }}}


		if [catdesc] {mutate { copy =>{ "[catdesc]"=> "[rule][category]" }}}
		if [direction] {mutate { copy =>{ "[direction]"=> "[network][direction]" }}}
		if [dstintf] {mutate { copy =>{ "[dstintf]"=> "[observer][egress][interface][name]" }}}
		if [eventid] {mutate { copy =>{ "[eventid]"=> "[event][id]" }}}
		if [locip] {mutate { copy =>{ "[locip]"=> "[source][ip]" }}}
		if [locport] {mutate { copy =>{ "[locport]"=> "[source][port]" }}}
		if [policyid] {mutate { copy =>{ "[policyid]"=> "[rule][id]" }}}
		if [sessionid] {mutate { copy =>{ "[sessionid]"=> "[network][session_id]" }}}
		if [srcintf] {mutate { copy =>{ "[srcintf]"=> "[observer][ingress][interface][name]" }}}
		if [fortios][user] {mutate { copy =>{ "[fortios][user]"=> "[source][user][name]" }}}
		if [remip] {mutate { copy =>{ "[remip]"=> "[destination][ip]" }}}
		if [remport] {mutate { copy =>{ "[remport]"=> "[destination][port]" }}}


		if [dstport] {mutate { copy =>{ "[dstport]" => "[destination][port]" }}}
		if [srcport] {mutate { copy =>{ "[srcport]" => "[source][port]" }}}

    }
    else if [type] == "event" {

		if [fortios][agent] {mutate { copy =>{ "[fortios][agent]"=> "[user_agent][original]" }}}
		if [daddr] {mutate { copy =>{ "[daddr]"=> "[destination][address]" }}}

		if [direction] {mutate { copy =>{ "[direction]"=> "[network][direction]" }}}
		if [dstip] {mutate { copy =>{ "[dstip]"=> "[destination][ip]" }}}
		if [dstport] {mutate { copy =>{ "[dstport]"=> "[destination][port]" }}}
		if [duration] {mutate { copy =>{ "[duration]"=> "[event][duration]" }}}
		if [fortios][error] {mutate { copy =>{ "[fortios][error]"=> "[error][message]" }}}
		if [error_num] {mutate { copy =>{ "[error_num]"=> "[error][code]" }}}

		if [fortios][file] {mutate { copy =>{ "[fortios][file]"=> "[file][name]" }}}
		if [filesize] {mutate { copy =>{ "[filesize]"=> "[file][size]" }}}
		if [fortios][group] {mutate { copy =>{ "[fortios][group]"=> "[user][group][name]" }}}
		if [hostname] {mutate { copy =>{ "[hostname]"=> "[url][domain]" }}}

		if [msg] {mutate { copy =>{ "[msg]"=> "[message]" }}}
		if [policyid] {mutate { copy =>{ "[policyid]"=> "[rule][id]" }}}
		if [proto] {mutate { copy =>{ "[proto]"=> "[network][iana_number]" }}}
		if [rcvdbyte] {mutate { copy =>{ "[rcvdbyte]"=> "[destination][bytes]" }}}
		if [saddr] {mutate { copy =>{ "[saddr]"=> "[source][address]" }}}
		if [sentbyte] {mutate { copy =>{ "[sentbyte]"=> "[source][bytes]" }}}
		if [fortios][service] {mutate { copy =>{ "[fortios][service]"=> "[network][protocol]" }}}
		if [sess_duration] {mutate { copy =>{ "[sess_duration]"=> "[event][duration]" }}}
		if [source_mac] {mutate { copy =>{ "[source_mac]"=> "[source][mac]" }}}

		if [fortios][user] {mutate { copy =>{ "[fortios][user]"=> "[user][name]" }}}
		if [fortios][url] {mutate { copy =>{ "[fortios][url]"=> "[url][path]" }}}

		if [dst_host] {mutate { copy =>{ "[dst_host]"=> "[destination][address]" }}}

		if [srcmac] {mutate { copy =>{ "[srcmac]"=> "[source][mac]" }}}
		if [srcport] {mutate { copy =>{ "[srcport]"=> "[source][port]" }}}

		if [srcip] {mutate { copy =>{ "[srcip]"=> "[source][ip]" }}}

    }

[enotspe]

you are right @nicpenning ! Your logic seems totally OK, I have just updated it on the pipeline. Thanks again for you suggestions and fine-tunning.

[nicpenning]

Sounds great, thank you!

Fixed by:

9ec3b32

[enotspe]

@P1llus you should probably take into account this change for elastic/beats#17890