Syslog Server (rsyslog, Elasticsearch, Logstash, Kibana)

log server:
* 4 cores
* 4 GB RAM
* Debian 9
* 8 GB system SSD
log.mcp.lcl

> /etc/motd
apt-get --no-install-recommends install aptitude
aptitude install screen rsync psmisc file patch ethtool strace tcpdump vim less net-tools man-db bzip2 xz-utils
aptitude --without-recommends install dnsutils
aptitude install ssh # but deselect xauth (pulls in dbus)
aptitude full-upgrade # pulls in irqbalance
adduser tdobes adm
adduser tdobes systemd-journal

mkdir -p /etc/systemd/system/ssh.socket.d
echo '[Socket]' > /etc/systemd/system/ssh.socket.d/port-2222.conf
echo 'ListenStream=2222' >> /etc/systemd/system/ssh.socket.d/port-2222.conf
systemctl disable ssh.service && systemctl enable ssh.socket


mkdir -p /etc/systemd/system/getty\@tty1.service.d
echo '[Service]' > /etc/systemd/system/getty\@tty1.service.d/noclear.conf
echo 'TTYVTDisallocate=no' >> /etc/systemd/system/getty\@tty1.service.d/noclear.conf

aptitude install unattended-upgrades
dpkg-reconfigure -plow unattended-upgrades # select "Yes", and select default at origin screen

aptitude install nfs-common
echo >> /etc/fstab
echo '10.2.25.244:/tank/logs   /mnt/logs   nfs     udp,intr,nfsvers=3,_netdev     0 0' >> /etc/fstab
mkdir /mnt/logs
mount /mnt/logs

aptitude install ca-certificates default-jre-headless
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
aptitude install apt-transport-https
echo 'deb https://artifacts.elastic.co/packages/5.x/apt stable main' > /etc/apt/sources.list.d/elastic-5.x.list
aptitude update && aptitude forget-new && aptitude install elasticsearch logstash kibana

mv /etc/rsyslog.conf /etc/rsyslog.conf.orig
echo '$FileOwner root' > /etc/rsyslog.conf
echo '$FileGroup adm' >> /etc/rsyslog.conf
echo '$FileCreateMode 0640' >> /etc/rsyslog.conf
echo '$DirCreateMode 0755' >> /etc/rsyslog.conf
echo '$Umask 0022' >> /etc/rsyslog.conf
echo >> /etc/rsyslog.conf
echo '$WorkDirectory /var/spool/rsyslog' >> /etc/rsyslog.conf
echo '$IncludeConfig /etc/rsyslog.d/*.conf' >> /etc/rsyslog.conf
echo '# provides UDP syslog reception' > /etc/rsyslog.d/01-remote.conf
echo 'module(load="imudp")' >> /etc/rsyslog.d/01-remote.conf
echo 'input(type="imudp" port="514")' >> /etc/rsyslog.d/01-remote.conf
echo >> /etc/rsyslog.d/01-remote.conf
echo '# provides TCP syslog reception' >> /etc/rsyslog.d/01-remote.conf
echo 'module(load="imtcp")' >> /etc/rsyslog.d/01-remote.conf
echo 'input(type="imtcp" port="514")' >> /etc/rsyslog.d/01-remote.conf

echo 'template(name="json-template"' > /etc/rsyslog.d/01-json-template.conf
echo '  type="list") {' >> /etc/rsyslog.d/01-json-template.conf
echo '    constant(value="{")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\"@timestamp\":\"")     property(name="timereported" dateFormat="rfc3339")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"@version\":\"1")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"message\":\"")     property(name="msg" format="json")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"sysloghost\":\"")  property(name="hostname")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"programname\":\"") property(name="programname")' >> /etc/rsyslog.d/01-json-template.conf
echo '      constant(value="\",\"procid\":\"")      property(name="procid")' >> /etc/rsyslog.d/01-json-template.conf
echo '    constant(value="\"}\n")' >> /etc/rsyslog.d/01-json-template.conf
echo '}' >> /etc/rsyslog.d/01-json-template.conf

echo '*.* @127.0.0.1:10514;json-template' > /etc/rsyslog.d/60-output_to_logstash.conf

echo 'input {' > /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '  udp {' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    host => "127.0.0.1"' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    port => 10514' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    codec => "json"' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    type => "rsyslog"' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '  }' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '}' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo 'filter { }' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo 'output {' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '  if [type] == "rsyslog" {' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    elasticsearch {' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '      hosts => [ "127.0.0.1:9200" ]' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '    }' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '  }' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf
echo '}' >> /etc/logstash/conf.d/rsyslog_to_elasticsearch.conf

# echo >> /etc/elasticsearch/elasticsearch.yml
# echo 'path.data: /mnt/logs' >> /etc/elasticsearch/elasticsearch.yml
echo >> /etc/elasticsearch/elasticsearch.yml
echo 'network.bind_host: [ _site_, _local_ ]' >> /etc/elasticsearch/elasticsearch.yml
sed -i -e 's|#DATA_DIR=/var/lib/elasticsearch|DATA_DIR=/mnt/logs|g' /etc/default/elasticsearch
chown elasticsearch:elasticsearch /mnt/logs

echo >> /etc/kibana/kibana.yml
echo 'server.host: "0.0.0.0"' >> /etc/kibana/kibana.yml

systemctl enable elasticsearch.service
systemctl start elasticsearch.service

systemctl enable logstash.service
systemctl start logstash.service

systemctl enable kibana.service
systemctl start kibana.service


Elasticsearch @ http://log:9200/
Recent messages @ http://log:9200/_all/_search?q=*&pretty
Kibana @ http://log:5601/





*** on devices sending in syslog messages ***

mv /etc/rsyslog.conf /etc/rsyslog.conf.orig
echo '$FileOwner root' > /etc/rsyslog.conf
echo '$FileGroup adm' >> /etc/rsyslog.conf
echo '$FileCreateMode 0640' >> /etc/rsyslog.conf
echo '$DirCreateMode 0755' >> /etc/rsyslog.conf
echo '$Umask 0022' >> /etc/rsyslog.conf
echo >> /etc/rsyslog.conf
echo '$WorkDirectory /var/spool/rsyslog' >> /etc/rsyslog.conf
echo '$IncludeConfig /etc/rsyslog.d/*.conf' >> /etc/rsyslog.conf
echo '$ModLoad imuxsock # provides support for local system logging' > /etc/rsyslog.d/01-local_logs.conf
echo '$ModLoad imklog   # provides kernel logging support' >> /etc/rsyslog.d/01-local_logs.conf
# single @ == UDP; double @ == TCP
echo '*.* @@log:514' > /etc/rsyslog.d/02-send_to_server.conf

References:

computer/log_server.txt · Last modified: 2017/11/04 18:07 by tdobes
Recent changes RSS feed Driven by DokuWiki Valid XHTML 1.0 Valid CSS