rsyslog

Install rsyslog

https://www.rsyslog.com/doc/

https://access.redhat.com/documentation/de-de/red_hat_enterprise_linux/8/html/configuring_basic_system_settings/assembly_configuring-a-remote-logging-solution_configuring-basic-system-settings

# verify packages installed
yum list installed
dnf list installed | grep rsyslog
repoquery -a --installed | grep "rsyslog"
rpm -qa rsyslog*

dnf -y install rsyslog

# check what SELinux is set to permit on port 514
# Ensure TCP and UDP 514 is listed as allowed port
semanage port -l | grep 514

#** If change port for rsyslog, then run command
    semanage port -a -t syslogd_port_t -p tcp <port-number>
semanage port -a -t syslogd_port_t -p tcp 514

# Restart rsyslog to take effect
service rsyslog restart

# Verify what port that rsyslog is listening
netstat -tnlp | grep rsyslog

Configure rsyslog

# Configure /etc/rsyslog.conf <--------- keep this file as default

# Create a new file in the /etc/rsyslog.d/ directory named, for example, remotelog.conf
##********** Rsyslog Server only **********
# Define templates before the rules that use them
# Per-Host templates for remote systems
# Save log to /data/syslog/remote directory
template(name="TmplAuthpriv" type="list") {
    constant(value="/data/syslog/remote/auth/")
    property(name="hostname")
    constant(value="/")
    property(name="programname" SecurePath="replace")
    constant(value=".log")
    }

template(name="TmplMsg" type="list") {
    constant(value="/data/syslog/remote/msg/")
    property(name="hostname")
    constant(value="/")
    property(name="programname" SecurePath="replace")
    constant(value=".log")
    }

# Provides UDP syslog reception
# for parameters see http://www.rsyslog.com/doc/imudp.html
module(load="imudp") # needs to be done just once
input(type="imudp" port="514" ruleset="remote1")


# Adding this ruleset to process remote messages
ruleset(name="remote1"){
    authpriv.* action(type="omfile" DynaFile="TmplAuthpriv")
    *.info;mail.none;authpriv.none;cron.none action(type="omfile" DynaFile="TmplMsg") 
    }
module(load="imtcp")
input(type="imtcp" port="514" ruleset="remote1")

Verify and troubleshooting

# Restart rsyslog service
systemctl restart rsylog
systemctl status rsyslog

# verify rsyslogd service
# It shows that rsyslog is listening on TCP 514
netstat -tnlp
netstat -tnlp | grep rsyslog*


#***************** Troubleshooting
# Test connection from other system to new syslog server
nc -z -v 192.168.8.17 514     # Test TCP
nc -z -v -u 192.168.8.17 514  # Test UDP

#*********** Run the following command at the rsylog server
# verify rsyslog service
ps -A | grep rsyslog

# Check rsyslog configuration
rsyslogd -N1

### Verify
netstat -taupn | grep syslog
tcpdump -A dst 192.168.8.17 # where 192.168.8.17 is the rsyslog server
tcpdump -A src 192.168.8.16 # where 192.168.8.16 is the rsyslog client

tcpdump -nni ens192 dst 192.168.9.17
tcpdump -nni ens192  -l port 514
tcpdump -i ens192 -A tcp and port 514


# On rsyslog server:
# tcpdump -i lo -A tcp and port 514    # different network adapater
tcpdump -i ens192 -A tcp and port 514   # VMware VM network adapter

# Troubleshooting if failed only
a. On rsyslog server
systemctl disable rsyslog.service   # systemctl enable rsyslog.service
systemctl stop rsyslog.service  # stop rsyslog service
nc -k -l 192.168.8.17 514       # listening on tcp 514

b. On client
echo “test message 1” | nc 192.168.8.17 514

Configure SNMP

#***** configure snmp *********
Please save normal configuration tokens for snmpd in /etc/snmp/snmpd.conf

[root@testsys01 ~]# grep ^[^#] /etc/snmp/snmpd.conf
view    systemview    included   .1.3.6.1.2.1.1
view    systemview    included   .1.3.6.1.2.1.25.1.1
syslocation TEST site01
syscontact TEST admin
dontLogTCPWrappersConnects yes
rouser ObsUser


####
# Third, create a view for us to let the group have rights to:

# Make at least  snmpwalk -v 1 localhost -c public system fast again.
#       name           incl/excl     subtree         mask(optional)
view    systemview    included   .1.3.6.1.2.1.1
view    systemview    included   .1.3.6.1.2.1.25.1.1

###############################################################################
# System contact information
#

# It is also possible to set the sysContact and sysLocation system
# variables through the snmpd.conf file:

syslocation TEST site01
syscontact TEST admin



###############################################################################
# Logging
#

# We do not want annoying "Connection from UDP: " messages in syslog.
# If the following option is commented out, snmpd will print each incoming
# connection, which can be useful for debugging.

dontLogTCPWrappersConnects yes

Create syslog auth and msg directory in rsyslog server

# Create syslog auth and msg directories
mkdir -p /data/syslog/remote/{auth,msg}

Sample rsyslog configuration

### using secure TLS 
# SAMPLE RSYSLOG CONFIGURATIONS
# rsyslog configuration file
# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
#### MODULES ####
# Provides UDP syslog reception
# for parameters see http://www.rsyslog.com/doc/imudp.html
module(load="imudp") # needs to be done just once
input(
 type="imudp"
 name="udp"
 port="20514"
 ruleset="udp_20514"
)
# Provides plain TCP syslog reception
# for parameters see http://www.rsyslog.com/doc/imptcp.html
module(load="imptcp")
input(
 type="imptcp"
 name="tcp"
 port="20514"
 ruleset="tcp_20514"
)
# Provides TLS TCP syslog reception
# for parameters see http://www.rsyslog.com/doc/imtcp.html
module(
 load="imtcp"
 StreamDriver.Name="gtls"
 StreamDriver.Mode="1"
 StreamDriver.AuthMode="x509/name"
 PermittedPeer=["127.0.0.1", "tlsforwarder1.lan"]
)
input(
 type="imtcp"
 name="tls"
 port="10514"
 ruleset="tls_10514"
)
#### GLOBAL DIRECTIVES ####
# Where to place auxiliary files
global(
 workDirectory="/var/lib/rsyslog"
 defaultNetSTreamDriver="ptcp"
 defaultNetstreamDriverCAFile="/var/lib/rsyslog/log-ca-root.crt"
 defaultNetstreamDriverCertFile="/var/lib/rsyslog/rsyslog.lan.crt"
 defaultNetstreamDriverKeyFile="/var/lib/rsyslog/rsyslog.lan.key"
)

# Use default timestamp format
module(load="builtin:omfile" Template="RSYSLOG_FileFormat")
RuleSet (name="udp_20514")
{ action (
 name="persist_udp_20514"
 type="omfile"
 file="/var/log/messages.udp"
 )
}
RuleSet (name="tcp_20514")
{ action (
 name="persist_tcp_20514"
 type="omfile"
 file="/var/log/messages.tcp"
 )
}
RuleSet (name="tls_10514")
{ action (
 name="persist_tls_10514"
 type="omfile"
 file="/var/log/messages.tls"
 )
}


#************* Another sample rsyslog configuration file /etc/rsyslog.conf ******************
# rsyslog configuration file

# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html 
# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html

#### MODULES ####

module(load="imuxsock"    # provides support for local system logging (e.g. via logger command)
       SysSock.Use="off") # Turn off message reception via local log socket; 
              # local messages are retrieved through imjournal now.
module(load="imjournal"         # provides access to the systemd journal
       StateFile="imjournal.state") # File to store the position in the journal
#module(load="imklog") # reads kernel messages (the same are read from journald)
#module(load="immark") # provides --MARK-- message capability

template(name="TmplAuthpriv" type="list") {
    constant(value="/data/syslog/remote/auth/")
    property(name="hostname")
    constant(value="/")
    property(name="programname" SecurePath="replace")
    constant(value=".log")
    }

template(name="TmplMsg" type="list") {
    constant(value="/data/syslog/remote/msg/")
    property(name="hostname")
    constant(value="/")
    property(name="programname" SecurePath="replace")
    constant(value=".log")
    }

ruleset(name="remote1"){
    authpriv.* action(type="omfile" DynaFile="TmplAuthpriv")
    *.info;mail.none;authpriv.none;cron.none action(type="omfile" DynaFile="TmplMsg") 
    }

# Provides UDP syslog reception
# for parameters see http://www.rsyslog.com/doc/imudp.html
module(load="imudp") # needs to be done just once
input(type="imudp" port="514" ruleset="remote1")

# Provides TCP syslog reception
# for parameters see http://www.rsyslog.com/doc/imtcp.html

module(load="imtcp") # needs to be done just once

# Adding this ruleset to process remote messages
input(type="imtcp" port="514" ruleset="remote1")

#### GLOBAL DIRECTIVES ####

# Where to place auxiliary files
global(workDirectory="/var/lib/rsyslog")

# Use default timestamp format
module(load="builtin:omfile" Template="RSYSLOG_TraditionalFileFormat")

# Discard F5 UDP health check message
:rawmsg, isequal, "default send string" stop

# Include all config files in /etc/rsyslog.d/
include(file="/etc/rsyslog.d/*.conf" mode="optional")

#### RULES ####

# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.*                                                 /dev/console

# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
*.info;mail.none;authpriv.none;cron.none                /var/log/messages

# The authpriv file has restricted access.
authpriv.*                                              /var/log/secure

# Log all the mail messages in one place.
mail.*                                                  -/var/log/maillog


# Log cron stuff
cron.*                                                  /var/log/cron

# Everybody gets emergency messages
*.emerg                                                 :omusrmsg:*

# Save news errors of level crit and higher in a special file.
uucp,news.crit                                          /var/log/spooler

# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log


# ### sample forwarding rule ###
#action(type="omfwd"  
# An on-disk queue is created for this action. If the remote host is
# down, messages are spooled to disk and sent when it is up again.
#queue.filename="fwdRule1"       # unique name prefix for spool files
#queue.maxdiskspace="1g"         # 1gb space limit (use as much as possible)
#queue.saveonshutdown="on"       # save messages to disk on shutdown
#queue.type="LinkedList"         # run asynchronously
#action.resumeRetryCount="-1"    # infinite retries if host is down
# Remote Logging (we use TCP for reliable delivery)
# remote_host is: name/ip, e.g. 192.168.0.1, port optional e.g. 10514
#Target="remote_host" Port="XXX" Protocol="tcp")

# Send syslog to other syslog server
*.* @@1.2.3.4:514

Verify rsyslog

# Verify the rsyslog, example search
tail -f /var/log/rsyslog/test.yyyy-mm-dd.log | grep -v 10.10.10.11 | less
tail -f /var/log/rsyslog/test.yyyy-mm-dd.log | egrep "10.10.10.11|10.10.10.12"
tail -f /var/log/rsyslog/test.yyyy-mm-dd.log | grep -v ICMP | grep 10.10.10.11
tail -f /var/log/rsyslog/test.yyyy-mm-dd.log | grep 10.10.10.11 | grep denied
grep '/<port-number> /var/log/rsyslog/test.yyyy-mm-dd.log | egrep -v '<phrase1> | <phrase2>' | less