说 明：
1. vi home/shsnc/snc_product/zabbix_server/etc/zabbix_server.conf
ExportDir=/home/shsnc/snc_product/zabbix_server/data
#确认此目录存在且有读写权限
ExportFileSize=1G
注：以root权限执行以下命令, 否则curl命令会报错访问证书错误：
cat /home/shsnc/snc_product/elasticsearch/config/root-
ca.pem >> etc/pki/tls/certs/ca-bundle.crt
1. 创建pipeline
{
"description": "zabbix exportdir history",
"processors": [{
"grok": {
"field": "message",
"patterns": [
"{\"\\S+itemid\":(?<itemid>\\d+),\".*clock\":(?<clock>\\d+),\"ns\":(?<ns>\\d+).*value\":(?<value>[\\s\\S]*)}"
]
}
},{
"date" : {
"field" : "clock",
"formats" : ["UNIX"],
"timezone" : "Asia/Shanghai",
"locale" : "zh-CN"
}
},{
"date_index_name" : {
"field" : "clock",
"date_formats" : ["UNIX"],
"timezone" : "Asia/Shanghai",
"index_name_prefix" : "history_",
"index_name_format" : "yyyyMMdd",
"date_rounding" : "d"
}
},{
"remove": {
"field": "message"
}
}]
}
{
"description": "zabbix exportdir trend.ndjson",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"{\"\\S+itemid\":(?<itemid>\\d+),\".*clock\":(?<clock>\\d+),\"count\":(?<count>\\d+),\"min\":(?<min>.*),\"avg\":(?<avg>.*),\"max\":(?<max>.*)}"
]
}
},{
"date" : {
"field" : "clock",
"formats" : ["UNIX"],
"timezone" : "Asia/Shanghai",
"locale" : "zh-CN"
}
},{
"date_index_name" : {
"field" : "clock",
"date_formats" : ["UNIX"],
"timezone" : "Asia/Shanghai",
"index_name_prefix" : "trends_",
"index_name_format" : "yyyyMM",
"date_rounding" : "M"
}
},{
"remove": {
"field": "message"
}
}
]
}
curl -k --header "Content-Type:
application/json;charset=UTF-8" --user admin:admin -XPUT
"https://xx.xx.50.195:9200/_ingest/pipeline/history_
pipeline" -d@history.pipeline
curl -k --header "Content-Type:
application/json;charset=UTF-8" --user admin:admin -XPUT
"https://xx.xx.50.195:9200/_ingest/pipeline/trends_
pipeline" -d@trends.pipeline
{
"template": "history_*",
"order": 0,
"settings": {
"index": {
"number_of_replicas": 1,
"number_of_shards": 5
}
},
"mappings" : {
"doc" : {
"dynamic" : true,
"properties": {
"itemid": {
"type": "long"
},
"clock": {
"format": "epoch_second",
"type": "date"
},
"value": {
"fields": {
"analyzed": {
"index": true,
"type": "text",
"analyzer": "standard"
}
},
"index": false,
"type": "keyword"
},
"ns": {
"type": "long"
},
"id": {
"type": "keyword"
}
}
}
}
}
{
"template": "trends_*",
"order": 0,
"settings": {
"index": {
"number_of_replicas": 1,
"number_of_shards": 5
}
},
"mappings" : {
"doc" : {
"dynamic" : true,
"properties": {
"itemid": {
"type": "long"
},
"clock": {
"format": "epoch_second",
"type": "date"
},
"num": {
"type": "integer"
},
"valueMax": {
"type": "double"
},
"valueAvg": {
"type": "double"
},
"valueMin": {
"type": "double"
},
"id": {
"type": "keyword"
}
}
}
}
}
curl -k --header "Content-Type:
application/json;charset=UTF-8" --user admin:admin -XPUT
https://xx.xx.50.195:9200/_template/history_template
-d@history_template.json
curl -k --header "Content-Type:
application/json;charset=UTF-8" --user admin:admin -XPUT
https://xx.xx.50.195:9200/_template/trends_template
-d@trends_template.json
1. 安装filebeat 7.6版本
cd /home/shsnc/snc_product/filebeat-7.6.0-linux-x86_64
mkdir -p config data/{history,trends} logs/{history,trends}
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/shsnc/snc_product/zabbix_server/data/history-history-syncer-*.ndjson
filetype: zabb_history
fields_under_root: true
setup.template.name: "history_template"
setup.template.pattern: "history_*"
setup.ilm.enabled: false
output.elasticsearch:
hosts: ["xx.xx.50.195:9200"]
pipeline: history_pipeline
index: "history_%{+yyyyMMdd}"
protocol: "https"
username: "admin"
password: "admin"
ssl.verification_mode: "none"
processors:
- script:
lang: javascript
id: my_filter
file: ${path.config}/config/history_filter.js
- timestamp:
field: clock
timezone: Asia/Shanghai
layouts:
- '1579651200'
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/shsnc/snc_product/zabbix_server/data/trends-history-syncer-*.ndjson
filetype: zabb_trends
fields_under_root: true
setup.template.name: "trends_template"
setup.template.pattern: "trends_*"
setup.ilm.enabled: false
output.elasticsearch:
hosts: ["xx.xx.50.195:9200"]
pipeline: trends_pipeline
index: "trends_%{+yyyyMM}"
protocol: "https"
username: "admin"
password: "admin"
ssl.verification_mode: "none"
processors:
- script:
lang: javascript
id: my_filter
file: ${path.config}/config/trends_filter.js
function process(event) {
event.Delete("log");
event.Delete("agent");
event.Delete("ecs");
event.Delete("input");
event.Delete("host");
var message = JSON.parse(event.fields.message);
var result = {itemid:message.itemid,clock:message.clock,ns:message.ns,value:message.value};
var _value = message.value;
var isNum = isRealNum(_value);
result.numberFlag=isNum;
if(isNum){
result.numericalValue=parseFloat(_value);
}
event.Put("numberFlag",result.numberFlag);
event.Put("numericalValue",result.numericalValue);
var id = message.itemid+"_"+message.clock+"_"+message.ns;
event.Put("id",id);
// event.Put("message2",JSON.stringify(result));
}
function isRealNum(val) {
if (val === "" || val == null) {
return false;
}
if (!isNaN(val)) {
return true;
} else {
return false;
}
}
function process(event) {
event.Delete("log");
event.Delete("agent");
event.Delete("ecs");
event.Delete("input");
event.Delete("host");
var message = JSON.parse(event.fields.message);
var id = message.itemid+"_"+message.clock;
event.Put("id",id);
}
function isRealNum(val) {
if (val === "" || val == null) {
return false;
}
if (!isNaN(val)) {
return true;
} else {
return false;
}
}
#!/bin/bash
filebeat_home=/home/shsnc/snc_product/filebeat-7.6.0-linux-x86_64
conf_home=${filebeat_home}/config
#check path
if [ ! -e "${filebeat_home}" ];then
echo "${filebeat_home} not exists, script exit ..."
exit 1
fi
start() {
pid=`ps -ef |grep ${filebeat_home} |grep -v grep |wc -l`
if [ ${pid} -eq 0 ];then
if [ ! -e ${conf_home}/history.yml -o ! -e ${conf_home}/trends.yml ];then
echo "${conf_home}/history.yml not exits, script exit ..."
exit 1
fi
if [ ! -e ${filebeat_home}/data/history ];then
mkdir -p ${filebeat_home}/data/history
fi
if [ ! -e ${filebeat_home}/data/trends ];then
mkdir -p ${filebeat_home}/data/trends
fi
if [ ! -e ${filebeat_home}/logs/history ];then
mkdir -p ${filebeat_home}/logs/history
fi
if [ ! -e ${filebeat_home}/logs/trends ];then
mkdir -p ${filebeat_home}/logs/trends
fi
${filebeat_home}/filebeat -c ${conf_home}/history.yml --path.data ${filebeat_home}/data/history --path.logs ${filebeat_home}/logs/history &
${filebeat_home}/filebeat -c ${conf_home}/trends.yml --path.data ${filebeat_home}/data/trends --path.logs ${filebeat_home}/logs/trends &
if [ $? == '0' ];then
echo "Starting filebeat ok"
else
echo "Starting filebeat failed"
fi
else
echo "filebeat is running!"
exit
fi
}
stop() {
echo -n $"Stopping filebeat: "
pid=`ps -ef |grep ${filebeat_home} |grep -v grep |wc -l`
pidd=`ps -ef |grep ${filebeat_home} |grep -v grep |awk '{print $2}' |xargs`
if [ ${pid} -eq 0 ];then
echo "filebeat is not running"
else
kill $pidd
echo "stop filebeat ok"
fi
}
restart() {
stop
start
}
status(){
pid=`ps -ef |grep ${filebeat_home} |grep -v grep |wc -l`
if [ ${pid} -eq 0 ];then
echo "filebeat is not running"
else
echo "filebeat is running"
fi
}
case "${1}" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
status)
status
;;
*)
echo $"Usage: $0 {start|stop|restart|status}"
exit 1
esac
chmod 444 config/*
chmod o+x script.sh
sh script.sh start
本文作者：事业二部(上海新炬中北团队）
本文来源：“IT那活儿”公众号
文章转载自IT那活儿，如果涉嫌侵权，请发送邮件至：contact@modb.pro进行举报，并提供相关证据，一经查实，墨天轮将立刻删除相关内容。
