Logfile Monitoring

From EWIKI
Jump to navigation Jump to search

logstash is a wonderful framework to collect different logs, put them into a searchable data store and view them online via a web browser.

How it works:

  1. download logstash [1]
  2. create a config file
  3. start logstash: java -jar logstash-1.2.2-flatjar.jar agent -f logstash.conf -- web
  4. some new logs must come into the configured file
  5. open URL http://localhost:9292/ in browser

logstash also supports collecting logs in a server farm, ref [2]

Web GUI

Logstash.png

logstash configurations

Advice: To test grok patterns, use grok debugger

Apache HTTPD Logs

Apache HTTPD is supported by default, so it is easy to configure the logs:

input {
 	file {
		path => "/var/log/apache2/access.log"
		type => "apache"
	}
 	file {
		path => "/var/log/apache2/error.log"
		type => "apache"
	}
}

output { 
	elasticsearch { embedded => true }
}

WSO2 Carbon logs

WSO2 products have a special log format, but since this is in a unique format, it all works as follows

 input {
	stdin {
		type => "stdin-type"
	}

	file {
		# Wildcards work, here :)
		path => [ "/var/log/wso2as/repository/logs/wso2carbon.log", "/var/log/wso2esb/wso2carbon.log" ]
	}
 }

 filter {
	grok {
		pattern => "TID\: \[%{WORD:pid}\] \[%{WORD:product}\] \[%{TIMESTAMP_ISO8601:logdate}\]  %{LOGLEVEL:level} \{%{DATA:classname}\} -  %{GREEDYDATA:wso2_message}"
		add_tag => [ "level_%{level}" ]
	}
 }

 output {
	elasticsearch { embedded => true }
 }

Cassandra Logs

input {
	stdin {
		type => "stdin-type"
	}

	file {
		path => [ "/var/log/cassandra/system.log" ]
	}
}

filter {
	grok {
		pattern => " %{LOGLEVEL:level} \[%{DATA:source}\] %{TIMESTAMP_ISO8601:logdate} %{DATA:classname} \(line %{WORD:line_no}\) %{GREEDYDATA:cassandra_message}"
		add_tag => [ "level_%{level}" ]
	}
}

output {
	stdout { codec => rubydebug }
	elasticsearch { embedded => true }
}

Shipper Installation

Install script (expects jar, init-script and conf in the current folder):

# binary
mkdir /opt/logstash
mkdir /var/logstash
cp logstash-1.2.2-flatjar.jar /opt/logstash
(cd /opt/logstash;  ln -s logstash-1.2.2-flatjar.jar logstash.jar)
# config
mkdir /etc/logstash
cp shipper.conf /etc/logstash
# copy init scripts
cp logstash-shipper /etc/init.d/
chmod +x /etc/init.d/logstash-shipper
chkconfig --add logstash-shipper
chkconfig --level 345 logstash-shipper on
service logstash-shipper start

CentOS init scripts

Source: logstash on CentOS/Ubuntu/RHEL

Config: /etc/logstash/send.conf Binary: /opt/logstash/logstash.jar

#! /bin/sh
#
#    /etc/rc.d/init.d/logstash-shipper
#
#    Starts Logstash-shipper as a daemon
#
# description: Starts Logstash-shipper as a daemon
 
### BEGIN INIT INFO
# Provides: logstash-shipper
# Required-Start: $local_fs $remote_fs
# Required-Stop: $local_fs $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: S 0 1 6
# Short-Description: Logstash-shipper
# Description: Starts Logstash-shipper as a daemon.
# Author: MH
 
### END INIT INFO
 
# The name of the config file
SUFFIX=shipper
 
# Amount of memory for Java
JAVAMEM=256M
 
LOCATION=/opt/logstash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
DESC="Logstash Daemon"
NAME=java
DAEMON=`which java`
CONFIG=/etc/logstash/${SUFFIX}.conf
LOGFILE="/var/log/logstash-${SUFFIX}.log"
PATTERNSPATH="/opt/logstash/patterns"
JARNAME=logstash.jar
ARGS="-Xmx$JAVAMEM -Xms$JAVAMEM -jar ${JARNAME} agent -f ${CONFIG} --log ${LOGFILE}"
SCRIPTNAME=/etc/init.d/logstash-${SUFFIX}
base=logstash-${SUFFIX}
 
# Exit if the package is not installed
if [ ! -x "$DAEMON" ]; then
{
  echo "Couldn't find $DAEMON"
  exit 99
}
fi
 
. /etc/init.d/functions
 
#
# Function that starts the daemon/service
#
do_start()
{
  cd $LOCATION && \
  ($DAEMON $ARGS &) \
  && success || failure
}
 
get_pid()
{
  pid=`ps auxww | grep 'logstash.jar' | grep java | grep ${SUFFIX}.conf | awk '{print $2}'`
}
 
#
# Function that stops the daemon/service
#
do_stop()
{
    get_pid
		   if checkpid $pid 2>&1; then
			   # TERM first, then KILL if not dead
			   kill -TERM $pid >/dev/null 2>&1
			   usleep 100000
			   if checkpid $pid && sleep 1 &&
				  checkpid $pid && sleep $delay &&
				  checkpid $pid ; then
					kill -KILL $pid >/dev/null 2>&1
					usleep 100000
			   fi
			fi
			checkpid $pid
			RC=$?
			[ "$RC" -eq 0 ] && failure $"$base shutdown" || success $"$base shutdown"

}
 
case "$1" in
  start)
    if [[ -f /var/lock/subsys/$JARNAME.${SUFFIX} ]] ; then
        get_pid
        echo -n "$DESC already running ($pid)" && success
    else
        echo -n "Starting $DESC: "
        do_start
        touch /var/lock/subsys/$JARNAME.${SUFFIX}
    fi
    ;;
  stop)
    echo -n "Stopping $DESC: "
    do_stop
    rm /var/lock/subsys/$JARNAME.${SUFFIX}
    ;;
  restart|reload)
    echo -n "Restarting $DESC: "
    do_stop
    do_start
    ;;
  status)
    get_pid
    if [[ $pid == "" ]] ; then
        echo -n "$DESC stopped"
    else
        get_pid
        echo -n "$DESC running ($pid)"
    fi
    ;;
  *)
    echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2
    exit 3
    ;;
esac
 
echo
exit 0