Ruby过滤器可以满足您的要求。
input {
stdin {
}
}
filter {
csv {
columns => ["num","date","time","orig","type","action","alert","i/f_name","i/f_dir","product","Internal_CA:","serial_num:","dn:","sys_message:","inzone","outzone","rule","rule_uid","rule_name","service_id","src","dst","proto","service","s_port","dynamic object","change type","message_info","StormAgentName","StormAgentAction","TCP packet out of state","tcp_flags","xlatesrc","xlatedst","NAT_rulenum","NAT_addtnl_rulenum","xlatedport","xlatesport","fw_message","ICMP","ICMP Type","ICMP Code","DCE-RPC Interface UUID","rpc_prog","log_sys_message","scheme:","Validation log:","Reason:","Serial num:","Instruction:","fw_subproduct","vpn_feature_name","srckeyid","dstkeyid","user","methods:","peer gateway","IKE:","CookieI","CookieR","msgid","IKE notification:","Certificate DN:","IKE IDs:","partner","community","Session:","L2TP:","PPP:","MAC:","OM:","om_method:","assigned_IP:","machine:","reject_category","message:","VPN internal source IP","start_time","connection_uid","encryption failure:","vpn_user","Log ID","message","old IP","old port","new IP","new port","elapsed","connectivity_state","ctrl_category","description","description ","severity","auth_status","identity_src","snid","src_user_name","endpoint_ip","src_machine_name","src_user_group","src_machine_group","auth_method","identity_type","Authentication trial","roles","dst_user_name","dst_machine_name","spi","encryption fail reason:","information","error_description","domain_name","termination_reason","duration"]
separator => "|"
}
ruby {
code => "
hash = event.to_hash
hash.each do |k,v|
if v == nil
event.remove(k)
end
end
"
}
}
output {
stdout { codec => rubydebug }
}
您可以使用 ruby 插件来过滤所有字段nil值（Ruby 中为 null）
Updated:
这是我的环境：Windows server 2008 和 Logstash 1.4.1。
你的日志样本对我有用！我已经更新了配置、输入和输出。
Input
2|8Jun2012|16:52:39|10.0.0.1|log|keyinst||daemon|inbound|VPN-1 & FireWall-1|Certificate initialized|86232|CN=fw-KO,O=sc-KO.KO.dc.obn8cx|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
Output:
{
"@version" => "1",
"@timestamp" => "2015-03-12T00:30:34.123Z",
"host" => "BENLIM",
"num" => "2",
"date" => "8Jun2012",
"time" => "16:52:39",
"orig" => "10.0.0.1",
"type" => "log",
"action" => "keyinst",
"i/f_name" => "daemon",
"i/f_dir" => "inbound",
"product" => "VPN-1 & FireWall-1",
"Internal_CA:" => "Certificate initialized",
"serial_num:" => "86232",
"dn:" => "CN=fw-KO,O=sc-KO.KO.dc.obn8cx"
}
