您是否有任何有用的awk和grep脚本来解析Apache日志？ [关闭]

#1 楼

awk -F'[ "]+' ' == "/" { ipcount[]++ }
    END { for (i in ipcount) {
        printf "%15s - %d\n", i, ipcount[i] } }' logfile.log

awk -F'[ "]+' ' == "1.2.3.4" { pagecount[]++ }
    END { for (i in pagecount) {
        printf "%15s - %d\n", i, pagecount[i] } }' logfile.log

awk -F'[ "]+' ' == "/" { ipcount[]++ }
    END { for (i in ipcount) {
        printf "%15s - %d\n", i, ipcount[i] | sort } }' logfile.log

#2 楼

# We don't want to log bots, they're our friends
BrowserMatch Pingdom.com robot

# Custom log format, for testing
#
#         date          proto   ipaddr  status  time    req     referer         user-agent
LogFormat "%{%F %T}t    %p      %a      %>s     %D      %r      %{Referer}i     %{User-agent}i" standard
CustomLog /var/log/apache2/access.log standard env=!robot

for line in file("access.log"):
  line = line.split("\t")
  if line[3] != "200":
    print line

if line[6] in ("","-") and "/images" in line[5]:

grep -o "[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" logfile | sort -n | uniq -c | sort -n

cut -f 3 log | uniq -c | sort -n

cut -f 8 log | uniq -c | sort -n

#3 楼

asql v0.6 - type 'help' for help.
asql> load /home/skx/hg/engaging/logs/access.log
Loading: /home/skx/hg/engaging/logs/access.log
sasql> select COUNT(id) FROM logs
46
asql> alias hits SELECT COUNT(id) FROM logs
ALIAS hits SELECT COUNT(id) FROM logs
asql> alias ips SELECT DISTINCT(source) FROM logs;
ALIAS ips SELECT DISTINCT(source) FROM logs;
asql> hits
46
asql> alias
ALIAS hits SELECT COUNT(id) FROM logs
ALIAS ips SELECT DISTINCT(source) FROM logs;

#4 楼

#!/bin/bash
# Usage
# ls-httpd type count
# Eg: 
# ls-httpd url 1000
# will find top URLs in the last 1000 access log entries
# ls-httpd ip 1000
# will find top IPs in the last 1000 access log entries
# ls-httpd agent 1000
# will find top user agents in the last 1000 access log entries

type=
length=

if [ "" == "" ]; then
  log_file="/var/log/httpd/example.com-access_log"
else
  log_file=""
fi

if [ "$type" = "ip" ]; then
  tail -n $length $log_file | grep -o "[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | sort -n | uniq -c | sort -n
elif [ "$type" = "agent" ]; then
  tail -n $length $log_file | awk -F\" '{print }'| sort -n | uniq -c | sort -n
elif [ "$type" = "url" ]; then
  tail -n $length $log_file | awk -F\" '{print }'| sort -n | uniq -c | sort -n
fi

#5 楼

cat log | grep -o "[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | sort -n | uniq -c | sort -n

netstat -an | awk '{print }' | grep -o "[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | egrep -v "(`for i in \`ip addr | grep inet |grep eth0 | cut -d/ -f1 | awk '{print }'\`;do echo -n "$i|"| sed 's/\./\\./g;';done`127\.|0\.0\.0)" | sort -n | uniq -c | sort -n

#6 楼

LogFormat "%h %l %u %t \"%r\" %>s %b 
    \"%{Referer}i\" \"%{User-Agent}i\" %T" custom

tail -9000 access_log | grep -v '" 200 ' | cut -d: -f2,3 | uniq -c

#!/usr/bin/perl
# script to show changes in hitrates for any regex pattern
# results displayed with arbitrary intervals
# and ascii indication of frequency
# gaps are also displayed properly
use Date::Manip;
use POSIX qw(strftime);
$pattern=shift || ".";
$ival=shift || 60;
$tick=shift || 10;
$minb=undef;
while (<>){
    next unless /$pattern/;
    $stamp=" " if m[(../.../....):(..:..:..)];
    $epoch = UnixDate(ParseDate($stamp),"%s");
    $bucket= int($epoch/$ival)*$ival;
    $minb=$bucket if $bucket<$minb || !defined($minb);
    $maxb=$bucket if $bucket>$maxb;
    $count{$bucket}++;
}
# loop thru the min/max range to expose any gaps
for($t=$minb;$t<=$maxb;$t+=$ival){
    printf "%s %s %4d %s\n",
            $t,
            strftime("%m/%d/%Y %H:%M:%S",localtime($t)),
            $count{$t}+0,
            substr("x"x100,0,$count{$t}/$tick
    );
}

#7 楼

cat access.log | \ 
  sed 's/^\(.*\) \(.*\) \(.*\) \[\(.*\)\] \"\(.*\)\" \(.*\) \(.*\) \"\(.*\)\" \"\(.*\)\"$/########/g'

#8 楼

#  is the status code in my log file

tail -f ${APACHE_LOG} |  awk  ' ~ /(404|500)/ {print }'

echo ""
#echo  "Hits by source IP:"
echo "======================================================================"

awk '{print }' "" | grep -ivE "(127.0.0.1|192.168.100.)" | sort | uniq -c | sort -rn | head -25

echo ""
echo ""
#echo "The 25 most popular pages:"
echo "======================================================================"

awk '{print }' "" | grep -ivE '(mod_status|favico|crossdomain|alive.txt)' | grep -ivE '(.gif|.jpg|.png)' | \
 sed 's/\/$//g' | sort | \
 uniq -c | sort -rn | head -25

echo ""    
echo ""
echo "The 25 most popular pages (no js or css):"
echo "======================================================================"

awk '{print }' "" | grep -ivE '(mod_status|favico|crossdomain|alive.txt)' | grep -ivE '(.gif|.jpg|.png|.js|.css)' | \
 sed 's/\/$//g' | sort | \
   uniq -c | sort -rn | head -25

   echo ""


#echo "The 25 most common referrer URLs:"
echo "======================================================================"

awk '{print }' "" | \
 grep -vE "(^"-"$|/www.$host|/$host)" | \
 sort | uniq -c | sort -rn | head -25

echo ""

#echo "Longest running requests"
echo "======================================================================"

awk  '{print ,}' "" | grep -ivE '(.gif|.jpg|.png|.css|.js)'  | awk '{secs=0.000001*;req=;printf("%.2f minutes req time for %s\n", secs / 60,req )}' | sort -rn | head -50

exit 0

#9 楼

awk -F\" '( ~ /\.(jpg|gif)/ &&  !~ /^http:\/\/www\.mydomain\.com/){print }' access_log | sort | uniq -c | sort

#10 楼

#!/bin/bash
#This script should return a set of lines between 2 values, the main purpose is for searching a log file between 2 times
#Script usage: logship.sh "start" "stop" file

#If the file contains any "/" in the date range the following 2 lines add the escape character so that the search can be performed for those characters
start=$(echo "" | sed 's/\//\\//g')
stop=$(echo "" | sed 's/\//\\//g')

zipped=$(echo "" | grep -c "gz$")     #figures out if the file is zipped or not

if [ "$zipped" ==  "1" ]; then          #If the file is zipped then pass it through zcat before sed
        zcat  | sed -n "/$start/,/$stop/p";
else
        sed -n "/$start/,/$stop/p" ;  #if it's not zipped just run sed
fi

#11 楼

cat access.log | awk '{ sum +=  } END { print sum }'

cat access.log | awk '{ sum +=  } END { print sum }'

#12 楼