elastic中文官网
注释:在inputs中配置了两个目录的.log文件,在output中也配置了两个会在es中产生的index
[root@VM-0-17-centos ~]# cd /elk/ [root@VM-0-17-centos elk]# tar -xf filebeat-7.9.0-linux-x86_64.tar.gz -C /usr/local/ [root@VM-0-17-centos elk]# cd /usr/local/filebeat-7.9.0-linux-x86_64/ [root@VM-0-17-centos filebeat-7.9.0-linux-x86_64]# vim filebeat.yml 15 filebeat.inputs: 16 - type: log 17 enabled: true 18 paths: 19 - /usr/local/nginx/logs/*.log 146 # ---------------------------- Elasticsearch Output ---------------------------- 147 #output.elasticsearch: 148 # Array of hosts to connect to. 149 # hosts: ["localhost:9200"] 159 # ------------------------------ Logstash Output ------------------------------- 160 output.logstash: 161 # The Logstash hosts 162 hosts: ["localhost:5044"] [root@VM-0-17-centos filebeat-7.9.0-linux-x86_64]# nohup ./filebeat -e -c filebeat.yml & [1] 11733 [root@VM-0-17-centos filebeat-7.9.0-linux-x86_64]# nohup: ignoring input and appending output to ‘nohup.out’ [root@VM-0-17-centos filebeat-7.9.0-linux-x86_64]# ps -elf | grep filebeat 0 S root 11733 7222 0 80 0 - 228233 futex_ 09:06 pts/0 00:00:00 ./filebeat -e -c filebeat.yml 0 S root 12434 7222 0 80 0 - 28203 pipe_w 09:09 pts/0 00:00:00 grep --color=auto filebeat 13 # ============================== Filebeat inputs =============================== 14 15 filebeat.inputs: 16 - type: log 17 enabled: true 18 paths: 19 - /var/log/test.log 20 multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Exception|^Caused by' 21 multiline.negate: false 22 max_lines: 20 23 multiline.match: after 24 document_type: "osquery" 25 tags: ["osquery"] 26 fields: 27 type: 'osquery' 28 29 - type: log 30 enabled: true 31 paths: 32 - /var/log/ida/ida-restful-api/*.log 33 multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Exception|^Caused by' 34 multiline.negate: false 35 max_lines: 20 36 multiline.match: after 37 document_type: "restful" 38 tags: ["restful"] 39 fields: 40 type: 'restful' 123 # ---------------------------- Elasticsearch Output ---------------------------- 124 output.elasticsearch: 125 hosts: ["localhost:9200"] 126 indices: 127 - index: "osquery-%{+yyyy.MM.dd}" 128 when.equals: 129 fields.type: "osquery" 130 - index: "restful-%{+yyyy.MM.dd}" 131 when.equals: 132 fields.type: "restful"# 集群健康值为yellow状态的解决办法 正常情况下,Elasticsearch 集群健康状态分为三种: green 最健康得状态,说明所有的分片包括备份都可用; 这种情况Elasticsearch集群所有的主分片和副本分片都已分配, Elasticsearch集群是 100% 可用的。 yellow 基本的分片可用,但是备份不可用(或者是没有备份); 这种情况Elasticsearch集群所有的主分片已经分片了,但至少还有一个副本是缺失的。不会有数据丢失,所以搜索结果依然是完整的。不过,你的高可用性在某种程度上被弱化。如果 更多的 分片消失,你就会丢数据了。把 yellow 想象成一个需要及时调查的警告。 red 部分的分片可用,表明分片有一部分损坏。此时执行查询部分数据仍然可以查到,遇到这种情况,还是赶快解决比较好; 这种情况Elasticsearch集群至少一个主分片(以及它的全部副本)都在缺失中。这意味着你在缺少数据:搜索只能返回部分数据,而分配到这个分片上的写入请求会返回一个异常。 Elasticsearch 集群不健康时的排查思路 -> 首先确保 es 主节点最先启动,随后启动数据节点; -> 允许 selinux(非必要),关闭 iptables; -> 确保数据节点的elasticsearch配置文件正确; -> 系统最大打开文件描述符数是否够用; -> elasticsearch设置的内存是否够用 ("ES_HEAP_SIZE"内存设置 和 "indices.fielddata.cache.size"上限设置); -> elasticsearch的索引数量暴增 , 删除一部分索引(尤其是不需要的索引); [root@VM-0-17-centos ~]# curl http://localhost:9200/_cluster/health?pretty { "cluster_name" : "elasticsearch", # 集群名 "status" : "yellow", # 集群健康状态,正常的话是“green”,缺少副本分片为“yellow”,缺少主分片为“red” "timed_out" : false, "number_of_nodes" : 1, # 集群节点数 "number_of_data_nodes" : 1, # 数据节点数 "active_primary_shards" : 1, # 主分片数 "active_shards" : 1, # 可用的分片数 "relocating_shards" : 0, # 正在迁移的分片数 "initializing_shards" : 0, # 正在初始化的分片数 "unassigned_shards" : 1, # 未分配的分片,但在集群中存在 "delayed_unassigned_shards" : 0, # 延时待分配到具体节点上的分片数 "number_of_pending_tasks" : 0, # 待处理的任务数,指主节点创建索引并分配 "number_of_in_flight_fetch" : 0, "task_max_waiting_in_queue_millis" : 0, "active_shards_percent_as_number" : 50.0 # 可用分片数占总分片的比例 } [root@VM-0-17-centos ~]# curl -XGET http://localhost:9200/_cat/indices\?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size yellow open .kibana ixYbdO9ARHmTtCx6FgaP2Q 1 1 2 0 8.2kb 8.2kb 解决办法: 添加请求头 [root@VM-0-17-centos ~]# curl -H "Content-Type: application/json" -XPUT localhost:9200/_settings -d '{ "index" : { "number_of_replicas" : 0 } }' {"acknowledged":true} [root@VM-0-17-centos ~]# curl -XGET http://localhost:9200/_cat/indices\?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open .kibana ixYbdO9ARHmTtCx6FgaP2Q 1 0 2 0 8.2kb 8.2kb [root@VM-0-17-centos ~]# curl http://localhost:9200/_cluster/health?pretty { "cluster_name" : "elasticsearch", "status" : "green", "timed_out" : false, "number_of_nodes" : 1, "number_of_data_nodes" : 1, "active_primary_shards" : 1, "active_shards" : 1, "relocating_shards" : 0, "initializing_shards" : 0, "unassigned_shards" : 0, "delayed_unassigned_shards" : 0, "number_of_pending_tasks" : 0, "number_of_in_flight_fetch" : 0, "task_max_waiting_in_queue_millis" : 0, "active_shards_percent_as_number" : 100.0 } # 如果curl访问9200端口可以访问但9100端口无法访问,可以进行如下操作: [root@VM-0-17-centos ~]# curl -get localhost:9200 { "name" : "node-1", "cluster_name" : "elasticsearch", "cluster_uuid" : "yaYvP4NyQiSX-jIBPCLvaA", "version" : { "number" : "7.9.0", "build_flavor" : "default", "build_type" : "tar", "build_hash" : "a479a2a7fce0389512d6a9361301708b92dff667", "build_date" : "2020-08-11T21:36:48.204330Z", "build_snapshot" : false, "lucene_version" : "8.6.0", "minimum_wire_compatibility_version" : "6.8.0", "minimum_index_compatibility_version" : "6.0.0-beta1" }, "tagline" : "You Know, for Search" } [root@VM-0-17-centos ~]# curl http://localhost:9200/_cluster/health?pretty { "cluster_name" : "elasticsearch", "status" : "green", "timed_out" : false, "number_of_nodes" : 1, "number_of_data_nodes" : 1, "active_primary_shards" : 9, "active_shards" : 9, "relocating_shards" : 0, "initializing_shards" : 0, "unassigned_shards" : 0, "delayed_unassigned_shards" : 0, "number_of_pending_tasks" : 0, "number_of_in_flight_fetch" : 0, "task_max_waiting_in_queue_millis" : 0, "active_shards_percent_as_number" : 100.0 } 但在浏览器访问ip:9100被拒绝,解决: [root@VM-0-17-centos ~]# vim /usr/local/elasticsearch-7.9.0/config/elasticsearch.yml network.host: 172.17.0.17 # 将此行改为本机的ip,如果是云服务器,建议改成云服务器的内网ip,否则可能会出现Cannot assign requested address [es@VM-0-17-centos ~]$ /usr/local/elasticsearch-7.9.0/bin/elasticsearch -d # 重启 [root@VM-0-17-centos ~]# ss -nutlp | grep 9200 tcp LISTEN 0 128 [::ffff:172.17.0.17]:9200 [::]:* users:(("java",pid=9558,fd=251)) 也可能会出现如下问题: [root@VM-0-17-centos elasticsearch-head]# nohup ./node_modules/grunt/bin/grunt server & [4] 14409 [root@VM-0-17-centos elasticsearch-head]# nohup: ignoring input and appending output to ‘nohup.out’ [Exit]....... /root/node_modules/chalk/source/index.js:106 ...styles, ^^^ SyntaxError: Unexpected token ... at createScript (vm.js:53:10) at Object.runInThisContext (vm.js:95:10) at Module._compile (module.js:543:28) at Object.Module._extensions..js (module.js:580:10) at Module.load (module.js:488:32) at tryModuleLoad (module.js:447:12) at Function.Module._load (module.js:439:3) at Module.require (module.js:498:17) at require (internal/module.js:20:19) at Object.<anonymous> (/root/node_modules/grunt-legacy-log-utils/index.js:12:13) 解决办法: 其实这是因为npm和node的版本比较低的原因 进行如下的操作: [root@VM-0-17-centos ~]# npm uninstall npm -g # 卸载npm [root@VM-0-17-centos ~]# yum install gcc gcc-c++ # 安装gcc [root@VM-0-17-centos ~]# wget https://npm.taobao.org/mirrors/node/v10.14.1/node-v10.14.1-linux-x64.tar.gz # 下载高版本的node [root@VM-0-17-centos ~]# tar -xvf node-v10.14.1-linux-x64.tar.gz # 解压 [root@VM-0-17-centos ~]# mv node-v10.14.1-linux-x64 /usr/local/node [root@VM-0-17-centos ~]# vim /etc/profile export NODE_HOME=/usr/local/node export PATH=$PATH:$NODE_HOME/bin:$PATH export NODE_PATH=$NODE_HOME/lib/node_modules:$PATH [root@VM-0-17-centos ~]# source /etc/profile [root@VM-0-17-centos ~]# ls /usr/local/node bin CHANGELOG.md include lib LICENSE README.md share [root@VM-0-17-centos ~]# source /etc/profile [root@VM-0-17-centos ~]# node -v v10.14.1 [root@VM-0-17-centos ~]# npm -v 6.4.1
