logstash同步nginx日志到数据库

系统运维

本文参考：https://www.cnblogs.com/yanshicheng/articles/9436373.html

? ? ? ? ? ? ? ?https://www.cnblogs.com/fawaikuangtu123/articles/10360264.html

1.logstash安装（jdk提前安装1.8）

rpm -ivh? logstash-6.6.2.rpm

2.上传数据库驱动jar包

wget? https://dev.mysql.com/get/Downloads/Connector-J/Mysql-connector-java-5.1.48.tar.gz

mkdir?-p?/usr/share/logstash/vendor/jar/jdbc
cd?/usr/share/logstash/vendor/jar/jdbc
[root@localhost?soft]#?cd?/usr/share/logstash/vendor/jar/jdbc
[root@localhost?jdbc]#?ll
total?984
-rw-r--r--.?1?logstash?logstash?1006959?Jul?11?19:43?MYSQL-connector-java-5.1.48-bin.jar
[root@localhost?jdbc]#

3.数据库创建库 并授权用户

create?database?nginxlog;
use?nginxlog;
CREATE?TABLE?`consumerlog`?(
`client_ip`?varchar(128)?DEFAULT?NULL,
`log_time`?timestamp?NOT?NULL?DEFAULT?CURRENT_TIMESTAMP,
`status`?int(6)?DEFAULT?NULL,
`http_referer`?text,
`AgentVersion`?varchar(512)?DEFAULT?NULL,
)?
grant?allonnginxlog.*?tonginxlog@\\\'%\\\'identifiedby\\\'123456\\\';

4.安装?logstash-output-jdbc插件

vim?/usr/share/logstash/Gemfile
#?source?https://rubygems.org将国外的源注释,换成国内的
source?https://gems.ruby-china.com/

/usr/share/logstash/bin/logstash-plugin?install?logstash-output-jdbc
Validating?logstash-output-jdbc
Installing?logstash-output-jdbc
Installation?successful
?
/usr/share/logstash/bin/logstash-plugin?list?|?grep?jdbc
logstash-input-jdbc
logstash-output-jdbc

5.nginx日志格式设置

log_format?access_log_json?\\\'{client_ip:$remote_addr,log_time:$time_local,request:$request,status:$status,body_bytes_sent:$body_bytes_sent,http_referer:$http_referer,AgentVersion:$http_user_agent,upstream_addr:$upstream_addr,request_time:$request_time,upstream_response_time:$upstream_response_time}\\\';

6.

[root@localhost conf.d]# cat consumer_log.conf?

/etc/logstash/conf.d

input{
file{
path?=>?/usr/local/tengine-2.1.2/logs/sxt-consumer.log
start_position?=>?beginning
stat_interval?=>?2
codec?=>?json
}?
}
?
filter?{
if?[status]?!=?200?{?
drop{}
}
}
output{
jdbc{
connection_string?=>?jdbc:mysql://192.168.14.61/nginxlog?user=nginxlog&password=123456&useUnicode=true&characterEncoding=UTF8
statement?=>?[insert?into?consumerlog(client_ip,status,http_referer,AgentVersion)?VALUES(?,?,?,?),?client_ip,status,http_referer,AgentVersion]
?}
}

7.启动logstash

systemctl start logstash

8.验证数据

更多关于云服务器，域名注册，虚拟主机的问题，请访问西部数码官网：www.west.cn