all_time_stats.sh

#!/bin/bash
echo "=============================================="
echo "    ALL-TIME WEB INTERFACE STATISTICS"
echo "=============================================="
echo "Generated: $(date)"
echo ""

# First, let's combine all logs
echo "📁 Analyzing all available logs..."
ALL_LOGS=$(sudo zcat /var/log/nginx/access.log*.gz 2>/dev/null | cat - /var/log/nginx/access.log)

# Core metrics
echo ""
echo "🎯 CORE METRICS"
echo "---------------"
TOTAL_VISITS=$(echo "$ALL_LOGS" | grep -c "GET / HTTP.*200")
TOTAL_UNIQUE=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | awk '{print $1}' | sort -u | wc -l)
TOTAL_REQUESTS=$(echo "$ALL_LOGS" | wc -l)
FIRST_VISIT=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | head -1 | awk '{print $4}' | cut -d'[' -f2)
LAST_VISIT=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | tail -1 | awk '{print $4}' | cut -d'[' -f2)

echo "Total homepage visits: $TOTAL_VISITS"
echo "Unique visitors: $TOTAL_UNIQUE"
echo "Total HTTP requests: $TOTAL_REQUESTS"
echo "Average visits per user: $(echo "scale=2; $TOTAL_VISITS / $TOTAL_UNIQUE" | bc)"
echo "First recorded visit: $FIRST_VISIT"
echo "Latest visit: $LAST_VISIT"

echo ""
echo "📈 GROWTH ANALYSIS"
echo "------------------"
# Monthly breakdown
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $4}' | cut -d'[' -f2 | cut -d':' -f1 | \
    awk -F'/' '{print $2 "/" $3}' | sort | uniq -c | \
    awk '{print $2 ": " $1 " visits"}'

echo ""
echo "🌍 TOP 30 VISITORS (by frequency)"
echo "---------------------------------"
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $1}' | sort | uniq -c | sort -rn | head -30 | \
    awk '{printf "%3d visits - %s\n", $1, $2}'

echo ""
echo "🕐 PEAK ACTIVITY HOURS (All-Time)"
echo "----------------------------------"
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk -F'[: ]' '{print $6":00"}' | sort | uniq -c | sort -rn | \
    awk '{printf "%s - %4d visits\n", $2, $1}'

echo ""
echo "📅 BUSIEST DAYS EVER"
echo "--------------------"
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $4}' | cut -d'[' -f2 | cut -d':' -f1 | \
    sort | uniq -c | sort -rn | head -10 | \
    awk '{printf "%4d visits - %s\n", $1, $2}'

echo ""
echo "🖥️  DEVICE/BROWSER BREAKDOWN"
echo "----------------------------"
# Browser stats
echo "=== Browsers ==="
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk -F'"' '{print $6}' | grep -v "^-$" | \
    grep -Eo "(Chrome|Firefox|Safari|Edge|Opera)" | \
    sort | uniq -c | sort -rn

echo -e "\n=== Operating Systems ==="
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk -F'"' '{print $6}' | grep -v "^-$" | \
    grep -Eo "(Windows|Mac|Linux|Android|iPhone|iPad)" | \
    sort | uniq -c | sort -rn

echo -e "\n=== Mobile vs Desktop ==="
MOBILE=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk -F'"' '{print $6}' | grep -cE "(Mobile|Android|iPhone|iPad)")
DESKTOP=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk -F'"' '{print $6}' | grep -cvE "(Mobile|Android|iPhone|iPad|^-$)")
echo "Mobile: $MOBILE"
echo "Desktop: $DESKTOP"

echo ""
echo "🤖 BOT/CRAWLER ACTIVITY"
echo "-----------------------"
BOT_VISITS=$(echo "$ALL_LOGS" | grep -cE "(bot|Bot|crawler|spider|scraper)")
echo "Bot/Crawler visits: $BOT_VISITS"
echo "Top bots:"
echo "$ALL_LOGS" | grep -E "(bot|Bot|crawler|spider)" | \
    awk -F'"' '{print $6}' | \
    sed -E 's/.*(bot|Bot|crawler|spider|Googlebot|bingbot|facebookexternalhit|Twitterbot|LinkedInBot|WhatsApp|Slackbot)[^ ]*/\1/' | \
    sort | uniq -c | sort -rn | head -10

echo ""
echo "🌐 REFERRER ANALYSIS"
echo "--------------------"
echo "=== Traffic Sources ==="
DIRECT=$(echo "$ALL_LOGS" | grep 'GET / HTTP.*200' | grep -c '"-"')
REFERRED=$(echo "$ALL_LOGS" | grep 'GET / HTTP.*200' | grep -cv '"-"')
echo "Direct traffic: $DIRECT ($(echo "scale=1; $DIRECT * 100 / ($DIRECT + $REFERRED)" | bc)%)"
echo "Referred traffic: $REFERRED ($(echo "scale=1; $REFERRED * 100 / ($DIRECT + $REFERRED)" | bc)%)"

echo -e "\n=== Top Referrers ==="
echo "$ALL_LOGS" | grep 'GET / HTTP.*200' | \
    awk -F'"' '{print $4}' | grep -v "^-$" | \
    sort | uniq -c | sort -rn | head -15

echo ""
echo "📊 VISITOR LOYALTY METRICS"
echo "--------------------------"
# Count visit frequency
echo "=== Visit Frequency Distribution ==="
echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $1}' | sort | uniq -c | \
    awk '{visits[$1]++} END {for(v in visits) print v " visits: " visits[v] " users"}' | \
    sort -n | tail -10

echo ""
echo "🔥 INTERESTING PATTERNS"
echo "-----------------------"
# Weekend vs Weekday
WEEKEND=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $4}' | cut -d'[' -f2 | \
    xargs -I {} date -d "{}" "+%u" 2>/dev/null | \
    grep -cE "^[67]$")
WEEKDAY=$(echo "$ALL_LOGS" | grep "GET / HTTP.*200" | \
    awk '{print $4}' | cut -d'[' -f2 | \
    xargs -I {} date -d "{}" "+%u" 2>/dev/null | \
    grep -cE "^[1-5]$")
echo "Weekend visits: $WEEKEND"
echo "Weekday visits: $WEEKDAY"

echo ""
echo "💾 LOG STATISTICS"
echo "-----------------"
echo "Total log entries: $(echo "$ALL_LOGS" | wc -l)"
echo "Date range: $FIRST_VISIT to $LAST_VISIT"
echo "404 errors: $(echo "$ALL_LOGS" | grep -c " 404 ")"
echo "500 errors: $(echo "$ALL_LOGS" | grep -c " 500 ")"
echo "Security scan attempts: $(echo "$ALL_LOGS" | grep -cE "(\.php|\.env|wp-admin|phpmyadmin)")"