Installation and configuration of the ELK Stack (Elasticsearch, Logstash, Kibana)

# Overview and download homepage
http://www.elasticsearch.org/overview/elkdownloads/

#
# Prerequirements (Elasticsearch and Logstash are Java packages so please install Java JRE first)
#
# Install Java JRE package on Debian
apt-get install -y openjre-7-jre

#
# Elasticsearch (distributed restful search and analytics)
#
# Install Elasticsearch package on Debian
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.2.deb -P /tmp
dpkg -i /tmp/elasticsearch-1.3.2.deb

# Enable Elasticsearch daemon
update-rc.d elasticsearch defaults 95 10

# Start Elasticsearch manually
/etc/init.d/elasticsearch start

#
# Logstash (manage events and logs)
#
# Install Logstash package on Debian
wget https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash_1.4.2-1-2c0f5a1_all.deb -P /tmp
dpkg -i /tmp/logstash_1.4.2-1-2c0f5a1_all.deb

# Optional: Install Logstash contrib package (plug-ins contributed by the community and not supported by Elasticsearch)
wget https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash-contrib_1.4.2-1-efd53ef_all.deb -P /tmp
dpkg -i /tmp/logstash-contrib_1.4.2-1-efd53ef_all.deb

# Enable Logstash daemon by default
update-rc.d logstash defaults 96 10

# Start Logstash manually
/etc/init.d/elasticsearch start

#
# Kibana (webinterface to visualize ElasticSearch data)
#
# Kibana is already included in the Logstash Debian package.
# URL: http://:9292
#
# Optinal: There is also a stand-alone archive avaiable with can by installed on a different webserver:
https://download.elasticsearch.org/kibana/kibana/kibana-3.1.0.tar.gz

# Enable Kibana webservie by default
update-rc.d logstash-web defaults 97 10

# Start Kibana manually
/etc/init.d/logstash-web start

# Optional: configure the Elasticsearch server FQHN
Open config.js and edit the "elasticsearch" parameter to the fully qualified hostname of your Elasticsearch server

# Logstash config for apache.log

cat < /etc/logstash/conf.d/logstash.conf
/etc/logstash/conf.d/apache_access_log.conf
input {
file {
path => "/var/log/apache2/access.log"
start_position => "beginning"
# sincedb_path => "/dev/null" # dont track the position of monitored log files
}
}

filter {
grok {
pattern => "%{IP:remote_ip} - - \[%{HTTPDATE:time}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes:int}|-) %{HOST:host} \"(?[^\"]*)\" \"(?[^\"]*)\" \"(?[^\"]*)\" %{NUMBER:duration:int} microsec"
}
}

output {
# DEBUG: output to console
# stdout {
# codec => rubydebug
# }

elasticsearch {
host => localhost
}
}
EOF

# get total index size
http://YOUR_ELASTCSEARCH_SERVER:9200/_stats/?pretty