今日闲来无事希望将ELK的Nginx访问dashboard美化一下，却发现kibana Visualize中的Metrics需要是number字段才可以使用除count外的统计字段，由于之前并未注意此方面的东西，导致部分需要统计的字段是string类型，需要修改字段类型，但是ELK并不能直接修改字段类型，需要重建索引（相关命令在kibana的Dev Tools =>consle中执行），于是写此篇记录一下过程。
一、环境信息
1、ELK版本
6.8.2
2、nignx日志格式
log_format json '{"@timestamp":"$time_iso8601",'
'"@version":"1",'
'"client":"$remote_addr",'
'"realip":"$http_x_forwarded_for",'
'"url":"$request_uri",'
'"status":"$status",'
'"domain":"$host",'
'"host":"$server_addr",'
'"size":"$body_bytes_sent",'
'"responsetime":"$request_time",'
'"referer":"$http_referer",'
'"ua":"$http_user_agent"'
'}'
3、logstash配置文件内容
input {
file {
path => ["/var/log/nginx/access.log"]
codec => json
}
}
filter {
geoip {
source => "[realip]"
target => "geoip"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
# 此处添加convert，将string转化成float
convert => [ "responsetime", "float"]
}
}
output {
elasticsearch {
hosts => ["1.2.3.4"]
manage_template => true
index => "logstash-lb01-%{+YYYY.MM.dd}"
}
}
二、重建索引流程
1、查找源索引的mapping
![图片[1]|ELK修改字段后重建索引|leon的博客](https://blog.leonshadow.cn/wp-content/uploads/2020/03/elk重建索引01.png)
2、创建字段修改后的中间索引
PUT logstash-lb01-2020.03.14_1
# 注意此处新的mapping需要将源索引mapping中冗余的内容去掉后方可创建成功
PUT logstash-lb01-2020.03.14_1/doc/_mapping
{
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "keyword"
},
"client": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"city_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"coordinates": {
"type": "float"
},
"country_code2": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
},
"region_code": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"host": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"path": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"realip": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"referer": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"responsetime": {
"type": "float" #修改此处的string为float
},
"size": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ua": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
3、停止服务，将旧索引的数据导入到中间索引中
POST _reindex
{
"source": {
"index": "logstash-lb01-2020.03.14"
},
"dest": {
"index": "logstash-lb01-2020.03.14_1"
}
}
4、检查索引是否导入成功
GET /logstash-lb01-2020.03.14/doc/_search GET /logstash-lb01-2020.03.14_1/doc/_search
5、删除旧索引
DELETE logstash-lb01-2020.03.14
6、创建新索引
PUT logstash-lb01-2020.03.14
PUT logstash-lb01-2020.03.14/doc/_mapping
{
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "keyword"
},
"client": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"city_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"coordinates": {
"type": "float"
},
"country_code2": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
},
"region_code": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"host": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"path": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"realip": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"referer": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"responsetime": {
"type": "float"
},
"size": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ua": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
7、从中间索引将数据导入到新索引中
POST _reindex
{
"source": {
"index": "logstash-lb01-2020.03.14_1"
},
"dest": {
"index": "logstash-lb01-2020.03.14"
}
}
8、检查索引导入情况
GET /logstash-lb01-2020.03.14/doc/_search
9、删除中间索引
DELETE logstash-lb01-2020.03.14_1
三、重建索引后的工作
1、重新刷新索引
![图片[2]|ELK修改字段后重建索引|leon的博客](https://blog.leonshadow.cn/wp-content/uploads/2020/03/elk重建索引02.png)
2、制作Visualize
![图片[3]|ELK修改字段后重建索引|leon的博客](https://blog.leonshadow.cn/wp-content/uploads/2020/03/elk重建索引03.png)
3、后记
由于索引是按照每日新建的，所以仅修改当日的索引后刷新索引会导致修改的字段出现冲突，需要将所有的索引都重建后方可完成索引重建，但是由于我这边个人使用，数据不多，为了方便直接将之前的索引全部删掉了，同时生产环境中需要不停服务重建索引可以使用alias的方式重建索引，同时如果数据量很大也需要滚动重建索引，此处个人使用要求不高，所以直接停掉服务重建。
四、参考资料
https://www.cnblogs.com/huangxiufen/p/12461191.html
https://blog.csdn.net/qq_36762677/article/details/97244174
https://www.cnblogs.com/bigben0123/p/10059289.html
https://www.cnblogs.com/Creator/p/3722408.html
https://www.cnblogs.com/minseo/p/10949802.html
https://blog.csdn.net/u010603691/article/details/79310495
http://www.eryajf.net/2367.html
https://blog.csdn.net/u012107143/article/details/93870999
温馨提示：本文最后更新于
转载请注明本文链接：https://blog.leonshadow.cn/763482/1961.html
2022-12-20 20:57:41，已超过486天没有更新。某些文章具有时效性，若文章内容或图片资源有错误或已失效，请联系站长。谢谢！转载请注明本文链接：https://blog.leonshadow.cn/763482/1961.html
© 版权声明
THE END
