Docker环境 ELK 快速部署

Docker环境 ELK快速部署

环境 Centos 7.4 , Docker version 17.12
Docker至少3GB内存；

内核配置

echo '
vm.max_map_count = 262144
net.core.somaxconn=65535
'>>/etc/sysctl.conf
sysctl -p

#创建elk

 #下载elk镜像  
docker pull sebp/elk

 #创建volume(推荐)  
docker volume create elk-es
docker volume ls

 #创建elk容器  
sudo docker run -dit --name elk 
  -p 5601:5601 -p 9200:9200 -p 5044:5044 
  -v elk-es:/var/lib/elasticsearch 
  -v /etc/localtime:/etc/localtime 
  sebp/elk 

保持时区一致-v /etc/localtime:/etc/localtime
内存限制 -e ES_MIN_MEM=1G -e ES_MAX_MEM=3G

查看

docker ps -l  
 #访问测试  
curl localhost:9200
curl localhost:5601
 #浏览器访问kabana  ip:5601  

#logstash 客户端访问配置

 #logstash配置文件目录 /etc/logstash/conf.d/
 #关闭logstash的ssl验证(生产环境建议使用自签证书)
docker exec -it elk sed -i 's/ssl/#ssl/' /etc/logstash/conf.d/02-beats-input.conf
 #重启ELK容器
docker restart elk

##############################

#客户端使用centos+nginx+filebeat测试

下载centos镜像

docker pull centos

创建Dockerfile文档，自定义镜像，安装filebeat、nginx

echo '
FROM centos

MAINTAINER Elven <elven89@qq.com> && 
ENV TZ "Asia/Shanghai" && 
ENV TERM xterm

 #use aliyun source,and install#
RUN curl -s http://mirrors.aliyun.com/repo/Centos-7.repo>/etc/yum.repos.d/CentOS-Base.repo && 
 curl -s http://mirrors.aliyun.com/repo/epel-7.repo>/etc/yum.repos.d/epel.repo && 
 sed -i "/aliyuncs.com/d" /etc/yum.repos.d/*.repo && 
 yum install -y net-tools tar && 
 rm -rf  /var/cache/yum/* /tmp/* /var/tmp/* /root/*.cfg

 #install filebeat
ENV FILEBEAT_VERSION=6.2.3
RUN rpm -Uvh https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-x86_64.rpm && 
 systemctl enable filebeat.service 
ADD filebeat.yml /etc/filebeat/filebeat.yml

 #install nginx
RUN rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm && 
 yum install nginx -y && 
 systemctl enable nginx.service && 
 yum clean all

EXPOSE 80

ENTRYPOINT ["/usr/sbin/init"]
'>Dockerfile

创建filebeat配置文件

echo '#filebeat#
filebeat.prospectors:
#nginx
- input_type: log
  enable: yes
  #tags: nginx-access
  paths:
    - /var/log/nginx/access.log
  exclude_lines: ["^$"]
  fields:
    type: "nginx-access"
  fields_under_root: true
#logstash
output.logstash:
  hosts: ["elk:5044"]
'>filebeat.yml

创建镜像filebeat-nginx

docker build -t filebeat-nginx .

查看镜像

docker images

创建容器filebeat-nginx

sudo docker run --privileged -dit --name filebeat-nginx 
  --link elk -p 82:80  filebeat-nginx

挖坑 centos镜像使用--privileged参数，启动/usr/sbin/init ，才可使用systemctl管理服务

查看

docker ps -l
netstat -lntp |grep 82
docker exec -it filebeat-nginx netstat -lntp
curl localhost:82

浏览器访问ip:82 能访问nginx页面

首次打开，需要添加索引
Management管理-->Index Patterns索引模式-->Create index pattern创建索引模式
填写filebeat-* (索引名)-->Next step-->选择如@timestamp-->Create index pattern ,完成

elk镜像自带nginx日志切割实例文件
/opt/logstash/patterns/nginx
/etc/logstash/conf.d/11-nginx.conf

#调试

 #进入elk容器
docker exec -it elk /bin/bash

 #安装网络工具net-tools
apt install net-tools -y
 #查看启动端口
netstat -lntp

 #logstash检测配置
/opt/logstash/bin/logstash -t -f /opt/logstash/config/logstash.yml
 #终端启动
service logstash stop
/opt/logstash/bin/logstash -f /opt/logstash/config/logstash.yml

 #进入filebeat-nginx容器  
docker exec -it filebeat-nginx /bin/bash

 #filebeat调试
systemctl stop filebeat
/usr/share/filebeat/bin/filebeat -configtest -c /etc/filebeat/filebeat.yml
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -d "publish"

#汉化kibana (可选)

查看ELK Dockerfile文档得知系统基于ubuntu:16

 #进入elk容器
docker exec -it elk /bin/bash

 #配置国内源
echo 'deb http://mirrors.aliyun.com/ubuntu/ xenial xenial-updates main universe restricted multiverse'>>/etc/apt/sources.list
echo 'deb-src http://mirrors.aliyun.com/ubuntu/ xenial xenial-updates main restricted multiverse universe'>>/etc/apt/sources.list
apt-get update 

 #安装git
apt install git -y
git --version
 #安装python
apt install python -y
python -V

 #汉化kibana
cd /opt
git clone https://github.com/anbai-inc/Kibana_Hanization.git
cd Kibana_Hanization
python main.py /opt/kibana

 #重启kibana
service kibana restart

 #Ctrl+D快捷键退出容器

##############################

使用curl命令操作Elasticsearch索引

 #查询索引
curl 'localhost:9200/_cat/indices?v'

 #创建索引test-index
curl -XPUT 'localhost:9200/test-index?pretty'

 #删除索引
curl -XDELETE 'localhost:9200/test-index'

##############################
官方文档

Docker ELK文档
http://elk-docker.readthedocs.io/

Docker Hub官网 ELK
https://hub.docker.com/r/sebp/elk/

github elk
https://github.com/spujadas/elk-docker
https://github.com/spujadas/elk-docker/blob/master/nginx-filebeat/Dockerfile