esmyl911
By:
esmyl911

Logstash filter by tags for different websites

June 2, 2015 4.3k views
DigitalOcean Articles DigitalOcean Security Logging

Hi,

got my ELK Stack up and running.

But i am a bit confused

Got a pool of different websites which I want to see statistics separately for; believe it could be done using [tags]?

but could someone tell me how I can do this?

This is my logstash forwarder config

{
  "network": {
    "servers": [ "logsvr1.logs.local:5000", "logsvr2.logs.local:5000" ],
    "timeout": 15,
    "ssl ca": "logstash-forwarder-new.crt"
  },
  "files": [
    {
      "paths": [
         "logs\\svr08\\ex*",
         "logs\\svr05\\ex*",
         "logs\\svr04\\ex*",
         "logs\\svr03\\ex*"
       ],
      "fields": { "type": "iis" },
      "dead time": "24h" 
        }
   ]
}

This is my IIS config for logstash

filter {
    if [type] == "iis" {
            if [message] =~ "^#" {
                            drop {}
            }
            grok {
                    break_on_match => false
                    match => [
                            "message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:s-sitename} %{IPORHOST:s-ip} %{URIPROTO:cs-method} %{URIPATH:cs-uri-stem} (?:%{NOTSPACE:cs_query}|-) %{NUMBER:src_port} %{NOTSPACE:cs_username} %{IP:clientip} %{NOTSPACE:useragent} %{NUMBER:sc-substatus} %{NUMBER:sc_win32_status} %{NUMBER:sc-bytes} %{NUMBER:cs-bytes} %{NUMBER:timetaken}"
                    ]
            }

            date {

                    locale => "en"
                    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
                    target => "@timestamp"
                    timezone => "Indian/Maldives"
            }
            useragent {
                    source=> "useragent"
                    prefix=> "browser"
            }
            geoip {
                    source => "clientip"
                    target => "geoip"
                    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
            }

            mutate {
                    add_field => [ "src_ip", "%{clientip}" ]
                    convert => [ "[geoip][coordinates]", "float" ]
                    replace => [ "@source_host", "%{clientip}" ]
                    replace => [ "@message", "%{message}" ]
                    rename => [ "cs_method", "method" ]
                    rename => [ "cs_stem", "request" ]
                    rename => [ "useragent", "agent" ]
                    rename => [ "cs_username", "username" ]
                    rename => [ "sc_status", "response" ]
                    rename => [ "timetaken", "time_request" ]
           }
    }
}
filter
  {
    if [type] == "iis" {
            mutate {
                    remove_field => [ "clientip", "host", "hostname", "logtime" ]
            }
    }
}

Suppose I want to send logs different apps

app1.egov.mv
app2.egov.mv

how can i add tags for these different IIS applications? and filter them in the discovery module to make graphs for specific websites using the tag? :|

regards,

Ismail

1 Answer

i haven't done any IIS parsing but you can add a tag with

#EXISTING CODE
grok {
                    break_on_match => false
                    match => [
                            "message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:s-sitename} %{IPORHOST:s-ip} %{URIPROTO:cs-method} %{URIPATH:cs-uri-stem} (?:%{NOTSPACE:cs_query}|-) %{NUMBER:src_port} %{NOTSPACE:cs_username} %{IP:clientip} %{NOTSPACE:useragent} %{NUMBER:sc-substatus} %{NUMBER:sc_win32_status} %{NUMBER:sc-bytes} %{NUMBER:cs-bytes} %{NUMBER:timetaken}"
                    ]
            }
#END EXISTING CODE
#START NEW CODE 
if [s-sitename] == "app1.egov.mv" {
      grok {
        add_tag        => ["app1.egov.mv"]
     }
if [s-sitename] == "app2.egov.mv" {
      grok {
        add_tag        => ["app2.egov.mv"]
     }
#END NEW CODE

this was modified from a working config in my ELK stack

Have another answer? Share your knowledge.