logstash-simple.conf

input { stdin { } }

filter {
  grok {
    match => {
      "message" => [
        "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:level}\s+\[%{DATA:service},%{DATA:traceId},%{DATA:spanId},%{DATA:spanExportable}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread_name}\]\s+%{DATA:logger_name}\s+:\s+(?<keyValuePairs>(?:\[[^\[\]\:]+\:(?:[^\[\]\:]*)\])?(?: \[[^\[\]\:]+\:(?:[^\[\]\:]*)\])*)\s*%{GREEDYDATA:text}",
        "\[%{TIMESTAMP_ISO8601:logdate}\]\s+\[%{DATA:thread}\]\s+\[%{LOGLEVEL:level}\]\s+\[%{DATA:logger}\]\s+\=\>\s+(?<keyValuePairs>(?:\[[^\[\]\:]+\:(?:[^\[\]\:]*|%{TIMESTAMP_ISO8601})\])?(?:\s+\[[^\[\]\:]+\:(?:[^\[\]\:]*|%{TIMESTAMP_ISO8601})\])*)[\s+]*%{GREEDYDATA:text}"
        ]
    }
  }
  kv {
    source => "keyValuePairs"
    field_split => " "
    trim_key => "\["
    trim_value =>  "\]"
    value_split => ":"
    remove_field => [ "keyValuePairs" ]
  }
  date {
    match => ["logdate", "yyyy-MM-dd HH:mm:ss,SSS", "ISO8601"]
    timezone => "Australia/Sydney"
    target => "logdate"
  }
}

output {
  elasticsearch { hosts => ["localhost:9200"] }
  stdout { codec => rubydebug }
}