rsyslog 和 logrotate

Posted on Posted in linux

rsyslog 日志服务器(cs架构)

版本

rsyslogd 8.16.0

client

/etc/rsyslog.conf

#################
#### MODULES ####
#################

#提供本地系统日志记录,比如使用logger模拟发送日志
module(load="imuxsock") # provides support for local system logging
#提供内核级别的日志记录
module(load="imklog")   # provides kernel logging support
#提供标记message的能力
module(load="immark")  # provides --MARK-- message capability

#提供UDP的514端口来接收UDP协议发送过来的数据
module(load="imudp")
input(type="imudp" port="514")

#提供TCP的514端口来接收TCP协议发送过来的数据
module(load="imtcp")
input(type="imtcp" port="514")

# Enable non-kernel facility klog messages
$KLogPermitNonKernelFacility on

###########################
#### GLOBAL DIRECTIVES ####
###########################

global(net.enableDNS = "on")
global(net.ipprotocol = "ipv4-only")
global(debug.onShutdown = "on")
maxMessageSize      256K

# 设置默认的timestamp格式
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Filter duplicated messages
$RepeatedMsgReduction on

#
# Set the default permissions for all log files.
#
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf
# 所有日志发送到远程rsyslog服务器,@表示UDP协议,@@表示TCP协议
*.*     @@192.168.213.130:514

/etc/rsyslog.d/50-default.conf

日志类型
  • auth –pam产生的日志
  • authpriv –ssh,ftp等登录信息的验证信息
  • cron –时间任务相关
  • kern –内核
  • lpr –打印
  • mail –邮件
  • mark(syslog)–rsyslog服务内部的信息,时间标识
  • news –新闻组
  • user –用户程序产生的相关信息
  • uucp –unix to unix copy, unix主机之间相关的通讯
  • local 1~7 –自定义的日志设备
连接符号
  • .xxx: 表示大于等于xxx级别的信息
  • .=xxx:表示等于xxx级别的信息
  • .!xxx:表示在xxx之外的等级的信息
日志级别:
  • 级别从低到高,记录的信息越来越少
  • debug –有调式信息的,日志信息最多
  • info –一般信息的日志,最常用
  • notice –最具有重要性的普通条件的信息
  • warning –警告级别
  • err –错误级别,阻止某个功能或者模块不能正常工作的信息
  • crit –严重级别,阻止整个系统或者整个软件不能正常工作的信息
  • alert –需要立刻修改的信息
  • emerg –内核崩溃等严重信息
  • none –什么都不记录
处理方式:
  • /var/log/file 发送到日志文件
  • @@192.168.0.1 发送到TCP server
  • @192.168.0.1 发送到UDP server
  • user1,user2 发送到在线用户user1,user2
  • ~ 丢弃该日志
  • ^/path/script 执行的脚本,^后面跟可以执行的脚本,日志内容可以作为脚本的第一个参数,可以用来触发告警
#  Default rules for rsyslog.

# authpriv相关的日志
auth,authpriv.*         /var/log/auth.log
*.*;auth,authpriv.none      -/var/log/syslog
# 定时任务的日志
cron.*              /var/log/cron.log
#daemon.*           -/var/log/daemon.log
# 内核日志
kern.*              /var/log/kern.log
#lpr.*              -/var/log/lpr.log
# 邮件相关的日志
mail.*              /var/log/mail.log
#user.*             -/var/log/user.log
# 在一个特殊的文件里面保存crit或者级别更高的uucp日志
uucp,news.crit      /var/log/spooler
# 记录启动信息到/var/log/boot.log
local7.*            /var/log/boot.log

#
# Logging for the mail system.  Split it up so that
# it is easy to write scripts to parse these files.
#
#mail.info          -/var/log/mail.info
#mail.warn          -/var/log/mail.warn
mail.err            /var/log/mail.err

#
# Logging for INN news system.
#
news.crit           /var/log/news/news.crit
news.err            /var/log/news/news.err
news.notice         /var/log/news/news.notice

#
# Some "catch-all" log files.
#
#*.=debug;\
#   auth,authpriv.none;\
#   news.none;mail.none -/var/log/debug
#*.=info;*.=notice;*.=warn;\
#   auth,authpriv.none;\
#   cron,daemon.none;\
#   mail,news.none      -/var/log/messages
# 记录所有事件日志级别大于info的日志到/var/log/message,但是mail、news的日志除外.忽略掉自定义的local5,local6日志类型,避免消息跑到messages文件,导致本地磁盘爆掉
*.info;mail.none,news.none,local5.none,local6.none      /var/log/messages

#
# Emergencies are sent to everybody logged in.
# 所有级别大于emerg的信息,每个人都会看到
*.emerg                                :omusrmsg:*

#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
#   news.=crit;news.=err;news.=notice;\
#   *.=debug;*.=info;\
#   *.=notice;*.=warn   /dev/tty8

# The named pipe /dev/xconsole is for the `xconsole' utility.  To use it,
# you must invoke `xconsole' with the `-file' option:
#
#    $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
#      busy site..
# 日志发送到登录端
daemon.*;mail.*;\
    news.err;\
    *.=debug;*.=info;\
    *.=notice;*.=warn   |/dev/xconsole

module(load="imfile")
$template CleanMsgFormat,"%msg%\n"

$InputFileName /root/tmp.log
# 文件唯一标识tag,最好保持唯一,用于接收端区分原始log文件,可以包含特殊字符,如":"、","等
$InputFileTag proname
# 【重要】需要保证发送端唯一,记录读取到哪儿,状态文件保存在$WorkDirectory,默认为 /var/lib/rsyslog
#  如果某个要监控的文件名变化了,一定要重新设置该值
$InputFileStateFile stat_proname
# log级别:info,warning,默认notice
$InputFileSeverity info
# log类型,默认local0, local开头的表示自定义类型
$InputFileFacility local1
# 全局设置,默认轮询是10s
$InputFilePollInterval 10
# 每多少行更新state文件状态
$InputFilePersistStateInterval 20000
$RepeatedMsgReduction off
# 启动监控当前的文件,如果忘记这行,则啥事也不会发生
$InputRunFileMonitor

$InputFileName /root/tmp01.log
$InputFileTag proname
$InputFileStateFile stat_proname
$InputFileSeverity info
$InputFilePersistStateInterval 20000
$InputFileFacility local1
$InputFilePollInterval 10
$RepeatedMsgReduction off
$InputRunFileMonitor

$InputFileName /root/tmp02.log
$InputFileTag proname2
$InputFileStateFile stat_proname2
$InputFileSeverity info
$InputFilePersistStateInterval 20000
$InputFileFacility local1
$InputFilePollInterval 10
$RepeatedMsgReduction off
$InputRunFileMonitor

server

server 端只增加一个 template 即可

template(name="remote_syslog" type="string" string="/var/log/%FROMHOST-IP%/%$YEAR%-%$MONTH%-%$DAY%/%PROGRAMNAME%.log")

模版变量

  • msg 匹配message中的msg部分

  • rawmsg 从socket收到的信息,一般用来debug

  • rawmsg-after-pri 和rawmsg类似,但是syslog PRI被移除了

  • hostname message的主机名

  • source HOSTNAME的别名

  • fromhost message来源的主机名,一般是用在relay chain中

  • fromhost-ip 同fromhost,不过获取的是ip

  • syslogtag message的tag

  • programname 是tag的静态部分,例如tag是named[123456],则programname是named

  • pri message的PRI,undecoded格式

  • pri-text text格式的PRI

  • syslogfacility the facility from the message - in numerical form

  • syslogfacility-text the facility from the message - in text form

  • syslogseverity severity from the message - in numerical form

  • syslogseverity-text severity from the message - in text form

  • timegenerated timestamp when the message was RECEIVED. message被本地syslog接收到的时间

  • timereported timestamp from the message,包含message被创建的时间

  • timestamp alias for timereported

  • bom The UTF-8 encoded Unicode byte-order mask (BOM)

  • myhostname The name of the current host as it knows itself

  • now 当前日期,格式YYYY-MM-DD,now是指当前message被处理的时间

  • year 当前年份(4-digit)

  • month 当前月份(2-digit)

  • day 当前日期(2-digit)

  • hour 当前小时(24 hour) time (2-digit)

  • hhour From minute 0 to 29, this is always 0 while from 30 to 59 it is always 1.

  • minute 当前分钟(2-digit)

特殊配置

client

$InputFileName /root/tmp03.log
# 这里有逗号,
$InputFileTag proname,file03
$InputFileStateFile stat_proname_file03
$InputFileSeverity info
$InputFilePersistStateInterval 20000
$InputFileFacility local1
$InputFilePollInterval 10
$RepeatedMsgReduction off
$InputRunFileMonitor

server

  • F, 代表自定义一个分隔符
  • 44 是逗号 , 的 ASCII 码值,如需要别的分隔符,需要查对应 ASCII 值
  • 2 取分隔后的第二个字段
template(name="remote_syslog" type="string" string="/var/log/%FROMHOST-IP%/%syslogtag:F,44:2%/%$YEAR%-%$MONTH%-%$DAY%/%PROGRAMNAME%.log")
if $syslogtag startswith 'proname' then ?remote_syslog;CleanMsgFormat
stop

template(name="remote_syslog02" type="string" string="/var/log/%FROMHOST-IP%/%syslogtag:F,44:2%/%$YEAR%-%$MONTH%-%$DAY%/%PROGRAMNAME%.log")
if $syslogtag startswith 'othername' then ?remote_syslog02;CleanMsgFormat
stop

logrotate

/etc/logrotate.conf

# see "man logrotate" for details
# rotate log files weekly
weekly

# use the syslog group by default, since this is the owning group
# of /var/log/syslog.
su root syslog

# keep 4 weeks worth of backlogs
rotate 4

# create new (empty) log files after rotating old ones
create

# uncomment this if you want your log files compressed
#compress

# packages drop log rotation information into this directory
include /etc/logrotate.d

# no packages own wtmp, or btmp -- we'll rotate them here
/var/log/wtmp {
    missingok
    monthly
    create 0664 root utmp
    rotate 1
}

/var/log/btmp {
    missingok
    monthly
    create 0660 root utmp
    rotate 1
}

# system-specific logs may be configured here

运行

crontab

*/30 * * * * /usr/sbin/logrotate /etc/logrotate.d/rsyslog > /dev/null 2>&1 &

手动运行

  • logrotate -d 并不会真正进行 rotate 或者 compress 操作,但是会打印出整个执行的流程,和调用的脚本等详细信息。
  • logrotate -v 会真正执行操作,打印出详细信息(debug模式,默认是开启verbose)

系统自带 cron

cat /etc/cron.daily/logrotate 
#!/bin/sh

# Clean non existent log file entries from status file
cd /var/lib/logrotate
test -e status || touch status
head -1 status > status.clean
sed 's/"//g' status | while read logfile date
do
    [ -e "$logfile" ] && echo "\"$logfile\" $date"
done >> status.clean
mv status.clean status

test -x /usr/sbin/logrotate || exit 0
/usr/sbin/logrotate /etc/logrotate.conf

例子

# 匹配的文件pattern,可以是通配符,注意:如果对应的log不存在会报错,中断处理,可以自行用 debug 模式测试。(可以添加 missingok 缓解)
/Data/logs/production/*/*/*.log
/Data/logs/erp/*/*/*.log
/Data/logs/erp/mq_order/*/*/*.log
{ # { ... } 花括号里面的就是具体的指令参数了, logrotate 支持一些hook预处理,可以在rotate执行之前或者之后调用命令或者自己的脚本。
   prerotate
        # ....
   endscript
   #daily
   rotate 10 # 保留个数
   size 5M # 限制大小
   create 0644 liuhonghe liuhonghe # 权限
   dateformat  -%Y%m%d-%s #  rotate的文件后缀格式
   compress # 压缩: compress, 默认gzip,后缀为gz。 也可以指定其他压缩程序,如bzip2,后缀名也可以修改
   missingok
   postrotate # 这个是最常用的,用来 reopen 被rotate后的文件
     /bin/kill -HUP $(/bin/cat /var/run/syslogd.pid 2>/dev/null) &>/dev/null
   endscript
}
#其余hook:
#prerotate/endscript
#firstaction/endscript
#lastaction/endscript
#preremove/endscript
#sharedscripts
» 转载请注明来源:若我若鱼 » rsyslog 和 logrotate

Leave a Reply

Your email address will not be published. Required fields are marked *

five × 2 =