Searching Logs
From Psygen Wiki
Tips and one-liners to help search logs.
1.) Domlog Diving
1.1) Get stuff from domlogs on cPanel:
echo -e "\e[93m\e[1mChecking Apache Domlogs:\e[0m";if [ -f /etc/cpanel/ea4/is_ea4 ]; then DOMLOGDIR='/var/log/apache2/domlogs/'; else DOMLOGDIR='/usr/local/apache/domlogs/'; fi;_tdominfo=$(grep -s `date +%d/%b/%Y` "$DOMLOGDIR"*);_tdiget=$(echo "$_tdominfo" | grep GET);_tdipost=$(echo "$_tdominfo" | grep POST);_tga1=$(echo "$_tdiget" | awk '{print $1}');_tga7=$(echo "$_tdiget" | awk '{print $7}');_tpa1=$(echo "$_tdipost" | awk '{print $1}');_tpa7=$(echo "$_tdipost" | awk '{print $7}');echo -e "\e[93m \e[1mTop hits per site:\e[0m";echo "$_tdominfo.*" | awk '{print $1}' | sort | uniq -c | sort -rnk1 | head;echo "";echo -e "\e[93m \e[1mTop POST Today:\e[0m";echo "$_tpa1" | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop GET Today:\e[0m";echo "$_tga1" | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mBots:\e[0m";echo "$_tdominfo" | egrep -i '(crawl|bot|spider|yahoo|bing|Googlebot)'| awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop IPs:\e[0m";echo "$_tpa1" | cut -d: -f2 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop URIs POSTed to:\e[0m";echo "$_tpa7" | cut -d: -f2 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop URIs Requested with GET:\e[0m";echo "$_tga7" | cut -d: -f2 | sort | uniq -c | sort -rn | head;
OLD:
if [ -f /etc/cpanel/ea4/is_ea4 ]; then DOMLOGDIR='/var/log/apache2/domlogs/*'; else DOMLOGDIR='/usr/local/apache/domlogs/*'; fi;echo "";echo -e "\e[93m \e[1mTop hits per site:\e[0m";grep `date +%d/%b/%Y` $DOMLOGDIR.* | awk '{print $1}' | sort | uniq -c | sort -rnk1 | head;echo "";echo -e "\e[93m \e[1mTop POST Today:\e[0m";grep -s `date +%d/%b/%Y` $DOMLOGDIR | grep POST | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop GET Today:\e[0m";grep -s `date +%d/%b/%Y` $DOMLOGDIR | grep GET | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mBots:\e[0m";grep -s `date +%d/%b/%Y` $DOMLOGDIR | egrep -i '(crawl|bot|spider|yahoo|bing|Googlebot)'| awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop IPs:\e[0m";grep -s `date +%d/%b/%Y` $DOMLOGDIR | grep POST | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head;echo "";echo -e "\e[93m \e[1mTop URIs Requested:\e[0m";grep -s `date +%d/%b/%Y` $DOMLOGDIR | grep POST | awk '{print $7}' | cut -d: -f2 | sort | uniq -c | sort -rn | head;echo "";
Number of hits per site:
grep `date +%d/%b/%Y` /var/log/apache2/domlogs/*.* | awk '{print $1}' | sort | uniq -c | sort -rnk1 | head
top 10 POST today:
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/*.* | grep POST | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
top 10 GET today:
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/*.* | grep GET | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
Bots (from wiki):
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/*.* | egrep -i '(crawl|bot|spider|yahoo|bing|Googlebot)'| awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
Top 10 IP's:
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/*.* | grep POST | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Top URI's POSTed to:
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/* | grep POST | awk '{print $7}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Most visited pages/links:
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/* | grep GET | awk '{print $7}' | cut -d: -f1 | sort | uniq -c | sort -rn | head -n25
Top IP's asking for wp-login.php
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/* | grep wp-login.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Top IP's asking for xmlrpc.php
grep -s `date +%d/%b/%Y` /var/log/apache2/domlogs/* | grep xmlrpc.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
1.2) Get stuff from domlogs on Plesk:
top 10 POST today:
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep POST | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
top 10 GET today:
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep GET | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
Bots (from wiki):
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | egrep -i '(crawl|bot|spider|yahoo|bing|Googlebot)'| awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
Top 10 IP's:
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep POST | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Top URI's POSTed to:
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep POST | awk '{print $7}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Most visited pages/links:
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep GET | awk '{print $7}' | cut -d: -f1 | sort | uniq -c | sort -rn | head -n25
Top IP's asking for wp-login.php
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep wp-login.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Top IP's asking for xmlrpc.php
grep -s `date +%d/%b/%Y` /var/www/vhosts/*/logs/access_* | grep xmlrpc.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
1.3) Get stuff from domlogs on Interworx:
Number of hits per site:
grep `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | awk '{print $1}' | sort | uniq -c | sort -rnk1 | head
top 10 POST today:
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep POST | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
top 10 GET today:
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep GET | awk '{print $1}' | cut -d: -f1 | sort | uniq -c | sort -rn | head
Top URI's POSTed to:
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep POST | awk '{print $7}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Most visited pages/links:
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep GET | awk '{print $7}' | cut -d: -f1 | sort | uniq -c | sort -rn | head -n25
Top IP's asking for wp-login.php
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep wp-login.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head
Top IP's asking for xmlrpc.php
grep -s `date +%d/%b/%Y` /home/*/var/*/logs/transfer-ssl.log | grep xmlrpc.php | awk '{print $1}' | cut -d: -f2 | sort | uniq -c | sort -rn | head