Ingesting logs from legacy Dell Storages

[lukas-lr]

Hi All,

We are already ingesting logs from Dell EMC PowerStore (with a custom parser), but there is also a requirement to ingest logs from Dell Compellent/SC Series and Dell EqualLogic - the predecessors of PowerStore, as far as I know.

Does anyone have experience with these log sources? Are the logs similar enough to use the same data type and parser or do we need separate ones?

Thanks and kind regards
Lukas

[deeshu]

Can you share the parser that you built for Powrestore? I also have to develop one for this.

[lukas-lr]

Hi deeshu,

This should give you a start. For the details I'm still looking for a good method how to handle the different event types, maybe you can figure something out.

filter {

    mutate {

        replace => {

            "event.idm.read_only_udm.metadata.vendor_name" => "Dell"

            "event.idm.read_only_udm.metadata.product_name" => "EMC PowerStore"

            "event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"

        }

    }

    #statedump {label="initial"}

    # Grok out syslog header and place message into :kvdata

    grok {

        match => {

            "message" => [

                "(<\\d+>)?%{SYSLOGTIMESTAMP:syslogTime}\\s+%{IPORHOST:syslogHost}\\s+\\[(?P<syslogPid>\\d+)\\]:\\s+(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3})\\d+\\s+%{HOST:applianceNode}\\s+%{WORD:clusterId}\\s+(?P<msgPid>\\d+)@(?P<serviceTag>\\w+)\\s+(?P<logType>\\w+)\\s+\\[(?P<eventid>\\S+)\\s+(?P<kvdata>[^\\]]+)\\]\\s+%{GREEDYDATA:details}"

            ]

        }

        on_error => "grok_extraction_failure"

    }

    if [grok_extraction_failure] {

        drop {}

    }

    #statedump{label => "postgrok"}

    # Parse Date

    date {

        match => ["timestamp", "yyyy-MM-ddTHH:mm:ss.SSS"]

    }

    # Default values for KV

    mutate {

        replace => {

            "client_ip" => ""

            "status" => ""

        }

    }

    # Split the key/value pairs in kvdata

    kv {

        source => "kvdata"

        value_split => "="

        field_split => " "

        trim_value => "\""

    }

    #statedump{label => "postkv"}

    # Populate some UDM

    mutate {

        rename => {

            #"eventid" => "event.idm.read_only_udm.metadata.product_event_type"

            "id" => "event.idm.read_only_udm.metadata.product_log_id"

            "appliance" => "event.idm.read_only_udm.principal.hostname"

            "applianceNode" => "applianceNodeLabel.value"

            "clusterId" => "clusterIdLabel.value"

            "serviceTag" => "serviceTagLabel.value"

        }

        replace => {

            "applianceNodeLabel.key" => "Appliance Node"

            "clusterIdLabel.key" => "Cluster ID"

            "serviceTagLabel.key" => "Service Tag"

            "event.idm.read_only_udm.metadata.description" => "%{details}"

        }

        merge => {

            "event.idm.read_only_udm.principal.labels" => "applianceNodeLabel"

        }

    }

    mutate {

        merge => {

            "event.idm.read_only_udm.principal.labels" => "clusterIdLabel"

        }

    }

    mutate {

        merge => {

            "event.idm.read_only_udm.principal.labels" => "serviceTagLabel"

        }

        remove_field => ["applianceNodeLabel", "clusterIdLabel", "serviceTagLabel"]

    }

    if [status] == "success" {

        mutate {

            replace => {

                "secAction" => "ALLOW"

            }

            merge => {

                "security_result.action" => "secAction"

            }

        }

        mutate {

            merge => {

                "event.idm.read_only_udm.security_result" => "security_result"

            }

        remove_field => ["secAction", "security_result"]

        }

    }

    if [client_ip] != "" && [client_ip] != "None" && [client_ip] != "not applicable" {

        mutate {

            merge => {

                "event.idm.read_only_udm.src.ip" => "client_ip"

            }

        }

    }

    #statedump{label => "mapping1"}

    # Trigger the UDM merge

    mutate {

        merge => {

            "@output" => "event"

        }

    }

}

Do you have experience with the other Dell storages?