9.4. elk入门
ELK由ElasticSearch、Logstash和Kiabana三个开源工具组成。官方网站：https://www.elastic.co/products ,Logstash非常重量级，且效率很低 ，吃内存厉害，Beats是一个轻量化版本的logStatsh。
Elasticsearch是基于 JSON 的分布式搜索和分析引擎，专为实现水平扩展、高可用和管理便捷性而设计。
Logstash 是动态数据收集管道，拥有可扩展的插件生态系统，能够与 Elasticsearch 产生强大的协同作用。
Kibana Kibana 能够以图表的形式呈现数据，并且具有可扩展的用户界面，供您全方位配置和管理 Elastic Stack。
Beats 是轻量型采集器的平台，从边缘机器向 Logstash 和 Elasticsearch 发送数据。
组件:
索引组件： Lucene
搜索组件： Solr,ElasticSearch
关于Lucene的核心术语:
索引（index）相对于数据库中的数据库
类型(type)相当于数据库中的表
文档(document)相当于数据库中的行
9.4.1. 准备工作
hosts文件解析配置
1[root@centos-151 ~]# vim /etc/hosts
2[root@centos-151 ~]# cat /etc/hosts
3127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
4::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
5
6192.168.46.151 centos-151.linuxpanda.tech centos-151 c151
7192.168.46.152 centos-152.linuxpanda.tech centos-152 c152
8192.168.46.153 centos-153.linuxpanda.tech centos-153 c153
9192.168.46.154 centos-154.linuxpanda.tech centos-154 c154
10
11[root@centos-151 ~]# scp /etc/hosts c152:/etc/
12[root@centos-151 ~]# scp /etc/hosts c153:/etc/
13[root@centos-151 ~]# scp /etc/hosts c154:/etc/
关闭防火墙和selinux
安装jdk环境
1[root@centos-151 ~]# yum install java-1.8.0-openjdk-devel
2[root@centos-152 ~]# yum install java-1.8.0-openjdk-devel
3[root@centos-153 ~]# yum install java-1.8.0-openjdk-devel
4[root@centos-154 ~]# yum install java-1.8.0-openjdk-devel
9.4.2. 安装elk
官方下载的实在是太慢了。 好在清华大学镜像站点提供了镜像，美滋滋。
1[root@centos-151 ~]# yum-config-manager --add-repo=https://mirrors.tuna.tsinghua.edu.cn/elasticstack/5.x/yum/
2[root@centos-152 ~]# yum-config-manager --add-repo=https://mirrors.tuna.tsinghua.edu.cn/elasticstack/5.x/yum/
3[root@centos-153 ~]# yum-config-manager --add-repo=https://mirrors.tuna.tsinghua.edu.cn/elasticstack/5.x/yum/
4[root@centos-154 ~]# yum-config-manager --add-repo=https://mirrors.tuna.tsinghua.edu.cn/elasticstack/5.x/yum/
5
6[root@centos-151 ~]# yum install elasticsearch
9.4.3. 修改配置文件
1[root@centos-151 ~]# cd /etc/elasticsearch/
2[root@centos-151 elasticsearch]# ls
3elasticsearch.yml jvm.options log4j2.properties scripts
4[root@centos-151 elasticsearch]# vim elasticsearch.yml
5# 编辑如下行
6cluster.name: my-application
7node.name: c151
8node.attr.rack: r1
9path.data: /elk/data
10path.logs: /elk/logs
11network.host: 192.168.46.151
12discovery.zen.ping.unicast.hosts: ["c151", "c152", "c153"]
13discovery.zen.minimum_master_nodes: 2
14
15[root@centos-151 elasticsearch]# vim jvm.options
16# 修改如下2行
17-Xms1g
18-Xmx1g
19[root@centos-151 elasticsearch]# scp elasticsearch.yml jvm.options c153:/etc/elasticsearch/
20[root@centos-151 elasticsearch]# mkdir /elk/{data,logs} -pv
21[root@centos-151 elasticsearch]# chown elasticsearch.elasticsearch /elk -R
22[root@centos-151 elasticsearch]# scp -r -p /elk c152:/
23[root@centos-151 elasticsearch]# scp -r -p /elk c153:/
24[root@centos-152 ~]# vim /etc/elasticsearch/elasticsearch.yml
25node.name: c152
26network.host: 192.168.46.152
27[root@centos-153 ~]# vim /etc/elasticsearch/elasticsearch.yml
28node.name: c153
29network.host: 192.168.46.13
30
31# 启动服务
32[root@centos-151 elasticsearch]# systemctl daemon-reload && systemctl start elasticsearch
33[root@centos-152 elasticsearch]# systemctl daemon-reload && systemctl start elasticsearch
34[root@centos-153 elasticsearch]# systemctl daemon-reload && systemctl start elasticsearch
35[root@centos-151 elasticsearch]# ss -tunl |grep 9.00
36tcp LISTEN 0 128 ::ffff:192.168.46.151:9200 :::*
37tcp LISTEN 0 128 ::ffff:192.168.46.151:9300 :::*
9.4.4. 测试elasticsearch
1[root@centos-151 ~]# curl http://c151:9200/
2{
3"name" : "c151",
4"cluster_name" : "my-application",
5"cluster_uuid" : "2pSvD0vGTHaxZny1EFvmFQ",
6"version" : {
7 "number" : "5.6.8",
8 "build_hash" : "688ecce",
9 "build_date" : "2018-02-16T16:46:30.010Z",
10 "build_snapshot" : false,
11 "lucene_version" : "6.6.1"
12},
13"tagline" : "You Know, for Search"
14}
15[root@centos-151 ~]# curl http://c152:9200/
16{
17"name" : "c152",
18"cluster_name" : "my-application",
19"cluster_uuid" : "2pSvD0vGTHaxZny1EFvmFQ",
20"version" : {
21 "number" : "5.6.8",
22 "build_hash" : "688ecce",
23 "build_date" : "2018-02-16T16:46:30.010Z",
24 "build_snapshot" : false,
25 "lucene_version" : "6.6.1"
26},
27"tagline" : "You Know, for Search"
28}
29[root@centos-151 ~]# curl http://c153:9200/
30{
31"name" : "c153",
32"cluster_name" : "my-application",
33"cluster_uuid" : "2pSvD0vGTHaxZny1EFvmFQ",
34"version" : {
35 "number" : "5.6.8",
36 "build_hash" : "688ecce",
37 "build_date" : "2018-02-16T16:46:30.010Z",
38 "build_snapshot" : false,
39 "lucene_version" : "6.6.1"
40},
41"tagline" : "You Know, for Search"
42}
43
44# 集群健康
45[root@centos-151 ~]# curl http://c151:9200/_cluster/health?pretty
46# 集群健康
47[root@centos-151 ~]# curl http://c151:9200/_cluster/health | python -m json.tool
48# 集群统计信息
49[root@centos-151 ~]# curl http://c151:9200/_cluster/stats?pretty
50# 查看节点信息
51[root@centos-151 ~]# curl http://c151:9200/_cat/nodes
52# 查看健康信息
53[root@centos-151 ~]# curl http://c151:9200/_cat/health
9.4.5. 插入数据
1curl -XPUT 'http://c151:9200/book/it/1' -d '{"name": "zhaojiedi", "publisher": "zhao"}'
2curl -XPUT 'http://c151:9200/book/it/2' -d '{"name": "xiaojia", "publisher": "zhao"}'
3curl -XPUT 'http://c151:9200/book/it/3' -d '{"name": "zhao", "publisher": "zhaojiedi"}'
9.4.6. 查询数据
1[root@centos-151 ~]# curl 'http://c151:9200/_search?q=zhao&pretty=true'
2{
3"took" : 7,
4"timed_out" : false,
5"_shards" : {
6 "total" : 5,
7 "successful" : 5,
8 "skipped" : 0,
9 "failed" : 0
10},
11"hits" : {
12 "total" : 3,
13 "max_score" : 0.25811607,
14 "hits" : [
15 {
16 "_index" : "book",
17 "_type" : "it",
18 "_id" : "2",
19 "_score" : 0.25811607,
20 "_source" : {
21 "name" : "xiaojia",
22 "publisher" : "zhao"
23 }
24 },
25 {
26 "_index" : "book",
27 "_type" : "it",
28 "_id" : "1",
29 "_score" : 0.25811607,
30 "_source" : {
31 "name" : "zhaojiedi",
32 "publisher" : "zhao"
33 }
34 },
35 {
36 "_index" : "book",
37 "_type" : "it",
38 "_id" : "3",
39 "_score" : 0.25811607,
40 "_source" : {
41 "name" : "zhao",
42 "publisher" : "zhaojiedi"
43 }
44 }
45 ]
46}
47}
手工去写这个查询语句是很麻烦的，官方提供了web查询方式的。
9.4.7. elasticsearch-head插件安装
1[root@centos-151 ~]# vim /etc/elasticsearch/elasticsearch.yml
2# 添加如下2行
3http.cors.enabled: true
4http.cors.allow-origin: "*"
5
6[root@centos-151 ~]# yum install git npm
7
8[root@centos-151 elasticsearch]# cd /usr/local/
9[root@centos-151 local]# ls
10bin etc games include lib lib64 libexec sbin share src
11[root@centos-151 local]# git clone https://github.com/mobz/elasticsearch-head
12[root@centos-151 local]# ls
13bin elasticsearch-head etc games include lib lib64 libexec sbin share src
14[root@centos-151 local]# cd elasticsearch-head/
15[root@centos-151 phantomjs]# yum install lbzip2 bzip2
16[root@centos-151 elasticsearch-head]# npm install -g cnpm --registry=https://registry.npm.taobao.org
17[root@centos-151 elasticsearch-head]# cnpm install
18[root@centos-151 elasticsearch-head]# npm run start
9.4.8. elasticsearch-head插件配置测试
9.4.9. logstash配置
1[root@centos-154 ~]# yum install logstash
2[root@centos-154 ~]# rpm -ql logstash |wc -l
311293
4[root@centos-154 ~]# rpm -ql logstash |grep logstash$
5/usr/share/logstash/bin/logstash
6/var/lib/logstash
7/var/log/logstash
9.4.9.1. 配置从标准输入到标准输出
1[root@centos-154 conf.d]# cd /etc/logstash/conf.d/
2[root@centos-154 conf.d]# vim sample.conf
3[root@centos-154 conf.d]# cat sample.conf
4input {
5 stdin {}
6}
7
8output {
9 stdout {
10 codec => rubydebug
11 }
12}
13
14# 测试下
15[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./sample.conf -t
16# 运行
17[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./sample.conf
18OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
19WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
20Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
21The stdin plugin is now waiting for input:
22# 输入样例
23hello world
24{
25 "@version" => "1",
26 "host" => "centos-154.linuxpanda.tech",
27 "@timestamp" => 2018-04-03T13:25:49.970Z,
28 "message" => "hello world"
29}
30zhaojiedi
31{
32 "@version" => "1",
33 "host" => "centos-154.linuxpanda.tech",
34 "@timestamp" => 2018-04-03T13:25:52.776Z,
35 "message" => "zhaojiedi "
36}
9.4.9.2. 配置从文件输入到标准输出
1[root@centos-154 conf.d]# yum install httpd
2[root@centos-154 conf.d]# systemctl start httpd
3[root@centos-154 conf.d]# curl 192.168.46.154
4sorry page
5[root@centos-154 conf.d]# cat /var/log/httpd/access_log
6192.168.46.154 - - [03/Apr/2018:21:29:00 +0800] "GET / HTTP/1.1" 200 11 "-" "curl/7.29.0"
7[root@centos-154 conf.d]# vim sample.conf
8[root@centos-154 conf.d]# cat sample.conf
9input {
10 file {
11 path => ["/var/log/httpd/access_log"]
12 start_position => "beginning"
13 }
14}
15
16output {
17 stdout {
18 codec => rubydebug
19 }
20}
21
22[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./sample.conf
23OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
24WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
25Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
26{
27 "@version" => "1",
28 "host" => "centos-154.linuxpanda.tech",
29 "path" => "/var/log/httpd/access_log",
30 "@timestamp" => 2018-04-03T13:33:54.464Z,
31 "message" => "192.168.46.154 - - [03/Apr/2018:21:29:00 +0800] \"GET / HTTP/1.1\" 200 11 \"-\" \"curl/7.29.0\""
32}
这个是提取到messsage了， 但是太笼统了，我们需要让它切开为各个属性的。
9.4.9.3. filter的grok使用
1[root@centos-154 conf.d]# !vim
2vim sample.conf
3[root@centos-154 conf.d]# cat sample.conf
4input {
5 file {
6 path => ["/var/log/httpd/access_log"]
7 start_position => "beginning"
8 }
9}
10
11filter {
12 grok {
13 match => {
14
15 "message" => "%{COMBINEDAPACHELOG}"
16 }
17 }
18
19
20}
21
22output {
23 stdout {
24 codec => rubydebug
25 }
26}
27
28[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./sample.conf
29OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
30WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
31Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
32
33# 这个时候在另一个终端执行下下面语句
34[root@centos-154 ~]# echo '192.168.46.154 - - [03/Apr/2018:21:29:00 +0800] "GET / HTTP/1.1" 200 11 "-" "curl/7.29.0"' >> /var/log/httpd/access_log
35# 在终端就可以看到如下输出了。
36{
37 "request" => "/",
38 "agent" => "\"curl/7.29.0\"",
39 "auth" => "-",
40 "ident" => "-",
41 "verb" => "GET",
42 "message" => "192.168.46.154 - - [03/Apr/2018:21:29:00 +0800] \"GET / HTTP/1.1\" 200 11 \"-\" \"curl/7.29.0\"",
43 "path" => "/var/log/httpd/access_log",
44 "referrer" => "\"-\"",
45 "@timestamp" => 2018-04-03T13:59:31.356Z,
46 "response" => "200",
47 "bytes" => "11",
48 "clientip" => "192.168.46.154",
49 "@version" => "1",
50 "host" => "centos-154.linuxpanda.tech",
51 "httpversion" => "1.1",
52 "timestamp" => "03/Apr/2018:21:29:00 +0800"
53}
9.4.9.4. filter的date使用
默认你的日期字符串是不能被识别出来是日期的， 你需要指定你的字符串是日期就需要date过滤器。
1[root@centos-154 conf.d]# !vim
2vim sample.conf
3[root@centos-154 conf.d]# cat sample.conf
4input {
5 file {
6 path => ["/var/log/httpd/access_log"]
7 start_position => "beginning"
8 }
9}
10
11filter {
12 grok {
13 match => {
14
15 "message" => "%{COMBINEDAPACHELOG}"
16 }
17 }
18
19 date {
20 match => ["timestamp" ,"dd/MMM/YYYY:H:m:s Z"]
21 }
22
23
24}
25
26output {
27 stdout {
28 codec => rubydebug
29 }
30}
9.4.9.5. geoip使用
这个很厉害的，根据ip可以获取到地理位置的，城市信息和经纬度信息。
geoip下载
1[root@centos-154 conf.d]# cd /etc/logstash/
2[root@centos-154 logstash]# wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
3[root@centos-154 logstash]# tar xf GeoLite2-City.tar.gz
4[root@centos-154 logstash]# ls
5conf.d GeoLite2-City_20180327 GeoLite2-City.tar.gz jvm.options log4j2.properties logstash.yml startup.options
6
7[root@centos-154 logstash]# ll GeoLite2-City_20180327/
8total 59936
9-rw-r--r-- 1 2000 2000 55 Mar 27 20:52 COPYRIGHT.txt
10-rw-r--r-- 1 2000 2000 61361390 Mar 27 20:52 GeoLite2-City.mmdb
11-rw-r--r-- 1 2000 2000 433 Mar 27 20:52 LICENSE.txt
12-rw-r--r-- 1 2000 2000 116 Mar 27 20:52 README.txt
13
14[root@centos-154 logstash]# mv GeoLite2-City_20180327/ maxmind
15
16[root@centos-154 logstash]# cd conf.d/
17[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./sample.conf
18
19# 另一个终端
20[root@centos-154 ~]# echo '223.5.5.5 - - [03/Apr/2018:21:29:00 +0800] "GET / HTTP/1.1" 200 11 "-" "curl/7.29.0"' >> /var/log/httpd/access_log
21
22# 可以看到
23{
24 "request" => "/",
25 "agent" => "\"curl/7.29.0\"",
26 "geoip" => {
27 "city_name" => "Hangzhou",
28 "timezone" => "Asia/Shanghai",
29 "ip" => "223.5.5.5",
30 "latitude" => 30.2936,
31 "country_name" => "China",
32 "country_code2" => "CN",
33 "continent_code" => "AS",
34 "country_code3" => "CN",
35 "region_name" => "Zhejiang",
36 "location" => {
37 "lon" => 120.1614,
38 "lat" => 30.2936
39 },
40 "region_code" => "ZJ",
41 "longitude" => 120.1614
42 },
43 "auth" => "-",
44 "ident" => "-",
45 "verb" => "GET",
46 "message" => "223.5.5.5 - - [03/Apr/2018:21:29:00 +0800] \"GET / HTTP/1.1\" 200 11 \"-\" \"curl/7.29.0\"",
47 "path" => "/var/log/httpd/access_log",
48 "referrer" => "\"-\"",
49 "@timestamp" => 2018-04-03T13:29:00.000Z,
50 "response" => "200",
51 "bytes" => "11",
52 "clientip" => "223.5.5.5",
53 "@version" => "1",
54 "host" => "centos-154.linuxpanda.tech",
55 "httpversion" => "1.1",
56 "timestamp" => "03/Apr/2018:21:29:00 +0800"
57}
我们可以看到它识别出来223.5.5.5这个阿里云的dns为杭州的啦。
9.4.9.6. redis配合logstash
redis作为输入
1input {
2 redis {
3 batch_count => 1
4 data_type => "list"
5 key => "logstash-list"
6 host=> "192.168.46.154"
7 port => 6379
8 threads => 6
9 }
10}
redis作为输出
1[root@centos-154 conf.d]# yum install redis
2[root@centos-154 conf.d]# vim /etc/redis.conf
3# 修改如下2行
4requirepass redis
5bind 0.0.0.0
6[root@centos-154 conf.d]# systemctl restart redis
7[root@centos-154 conf.d]# pwd
8/etc/logstash/conf.d
9[root@centos-154 conf.d]# vim redis.conf
10[root@centos-154 conf.d]# cat redis.conf
11input {
12 file {
13 path => ["/var/log/httpd/access_log"]
14 start_position => "beginning"
15 }
16}
17
18filter {
19 grok {
20 match => {
21
22 "message" => "%{COMBINEDAPACHELOG}"
23 }
24 }
25
26 date {
27 match => ["timestamp" ,"dd/MMM/YYYY:H:m:s Z"]
28 }
29
30 geoip {
31 source => "clientip"
32 target => "geoip"
33 database => "/etc/logstash/maxmind/GeoLite2-City.mmdb"
34 }
35
36}
37
38output {
39 redis {
40 host => [ "192.168.46.154" ]
41 port => 6379
42 db => 8
43 data_type => "list"
44 key => "logstash-%{+yyyy.MM.dd}"
45 }
46}
47
48
49[root@centos-154 conf.d]# /usr/share/logstash/bin/logstash -f ./redis.conf
50OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
51
52# 一个终端模拟访问
53[root@centos-154 ~]# curl localhost
54sorry page
55# 另一个终端查看redis情况
56[root@centos-154 ~]# redis-cli -h 192.168.46.154
57192.168.46.154:6379> select *
58(error) ERR invalid DB index
59192.168.46.154:6379> select *
60(error) ERR invalid DB index
61192.168.46.154:6379> select 8
62OK
63192.168.46.154:6379[8]> keys *
641) "logstash-2018.04.04"
65192.168.46.154:6379[8]> lpop logstash-2018.04.04
66"{\"request\":\"/\",\"agent\":\"\\\"curl/7.29.0\\\"\",\"geoip\":{},\"auth\":\"-\",\"ident\":\"-\",\"verb\":\"GET\",\"message\":\"::1 - - [04/Apr/2018:19:21:32 +0800] \\\"GET / HTTP/1.1\\\" 200 11 \\\"-\\\" \\\"curl/7.29.0\\\"\",\"tags\":[\"_geoip_lookup_failure\"],\"path\":\"/var/log/httpd/access_log\",\"referrer\":\"\\\"-\\\"\",\"@timestamp\":\"2018-04-04T11:21:32.000Z\",\"response\":\"200\",\"bytes\":\"11\",\"clientip\":\"::1\",\"@version\":\"1\",\"host\":\"centos-154.linuxpanda.tech\",\"httpversion\":\"1.1\",\"timestamp\":\"04/Apr/2018:19:21:32 +0800\"}"
9.4.9.7. logstash写数据到els
1output {
2 elasticsearch {
3 hosts => ["http://192.168.46.151:9200/","http://192.168.46.152:9200/","http://192.168.46.153:9200/"]
4 index => "logstash-%{+YYYY.MM.dd}"
5 document_type => "apache_logs"
6 }
7}
9.4.9.8. 使用filebeat收集数据
1[root@centos-154 conf.d]# yum install filebeat
2[root@centos-154 filebeat]# vim filebeat.yml
3# 修改如下几行内容
4hosts: ["192.168.46.151:9200","192.168.46.152:9200","192.168.46.153:9200"]
5paths:
6 - /var/log/httpd/access_log
7[root@centos-154 filebeat]# systemctl restart filebeat
8# 查看
9[root@centos-154 filebeat]# curl 192.168.46.151:9200/_cat/indices
10green open book jRS4oLXvQDusRzkZqTvo4Q 5 1 3 0 23.9kb 11.9kb
11green open filebeat-2018.04.04 HXRGZ6sKRGaM4GDFGwf2Zg 5 1 3 0 38.6kb 19.3kb
9.4.9.9. kibana使用
1[root@centos-154 filebeat]# vim /etc/kibana/kibana.yml
2# 修改如下2行
3elasticsearch.url: "http://192.168.46.151:9200"
4server.host: "192.168.46.154"
5[root@centos-154 filebeat]# yum install elasticsearch
6
7在浏览器输入192.168.46.154:5601就可以访问了
8
9.. image:: /images/elk/filebeat-1.png