Notes

#!/bin/bash

###############################################################################
# Network Connectivity Monitoring Script
# Tests connectivity between downstream (US) and upstream (Bangalore) clusters
#
# Usage:
# ./network_monitor.sh # Run in foreground
# ./network_monitor.sh --background # Run in background
# ./network_monitor.sh --help # Show help
###############################################################################

# Configuration
UPSTREAM_HOST="<upstream-server-ip-or-hostname>" # Change this to your upstream server
UPSTREAM_PORT="443"
TEST_DURATION_MINUTES=1 # Change to 240 for 4 hours
PING_COUNT=10
MTR_COUNT=20
MTR_MAX_HOPS=30
CHECK_INTERVAL_SECONDS=30

# Parse command line arguments
BACKGROUND_MODE=false
SHOW_HELP=false

for arg in "$@"; do
case $arg in
--background|-b)
BACKGROUND_MODE=true
shift
;;
--help|-h)
SHOW_HELP=true
shift
;;
*)
# Unknown option
;;
esac
done

# Show help if requested
if [ "$SHOW_HELP" = true ]; then
cat << EOF
Network Connectivity Monitoring Script

Usage: $0 [OPTIONS]

Options:
--background, -b Run in background mode (creates background.log)
--help, -h Show this help message

Configuration (edit script to change):
UPSTREAM_HOST Target server to monitor
TEST_DURATION_MINUTES How long to run (1 for testing, 240 for 4 hours)
CHECK_INTERVAL_SECONDS Time between tests (default: 30)
MTR_MAX_HOPS Maximum hops for MTR (default: 30)

Examples:
$0 # Run in foreground (see output in terminal)
$0 --background # Run in background (can close terminal)

For 4-hour monitoring, edit the script and change:
TEST_DURATION_MINUTES=240

Output files will be in: network_test_TIMESTAMP/
Compressed archive: network_test_TIMESTAMP.tar.gz

EOF
exit 0
fi

# Calculate total iterations
TOTAL_ITERATIONS=$((TEST_DURATION_MINUTES * 60 / CHECK_INTERVAL_SECONDS))

# Generate timestamp for filenames
START_TIME=$(date +%Y%m%d_%H%M%S)
OUTPUT_DIR="network_test_${START_TIME}"
mkdir -p "$OUTPUT_DIR"

LOG_FILE="$OUTPUT_DIR/connectivity_test.log"
MTR_FILE="$OUTPUT_DIR/mtr_results.log"
PING_FILE="$OUTPUT_DIR/ping_detailed.log"
SUMMARY_FILE="$OUTPUT_DIR/summary.txt"

# Color codes for terminal output
RED='33[0;31m'
GREEN='33[0;32m'
YELLOW='33[1;33m'
NC='33[0m' # No Color

###############################################################################
# Helper Functions
###############################################################################

log_message() {
echo -e "$1" | tee -a "$LOG_FILE"
}

check_dependencies() {
local missing_deps=()

for cmd in ping curl mtr ss; do
if ! command -v $cmd &> /dev/null; then
missing_deps+=($cmd)
fi
done

if [ ${#missing_deps[@]} -ne 0 ]; then
echo -e "${RED}Error: Missing required commands: ${missing_deps[*]}${NC}"
echo "Please install missing dependencies:"
echo " Ubuntu/Debian: sudo apt-get install iputils-ping curl mtr-tiny iproute2"
echo " RHEL/CentOS: sudo yum install iputils curl mtr iproute"
exit 1
fi
}

###############################################################################
# Test Functions
###############################################################################

run_ping_test() {
local timestamp=$1
local iteration=$2

echo "" >> "$PING_FILE"
echo "=== PING Test at $timestamp (Iteration $iteration) ===" >> "$PING_FILE"

PING_OUTPUT=$(ping -c $PING_COUNT -W 2 $UPSTREAM_HOST 2>&1)
PING_STATUS=$?

# Save full ping output to file
echo "$PING_OUTPUT" >> "$PING_FILE"
echo "" >> "$PING_FILE"

if [ $PING_STATUS -eq 0 ]; then
PING_STATS=$(echo "$PING_OUTPUT" | tail -1)
PACKET_LOSS=$(echo "$PING_OUTPUT" | grep -oP 'd+(?=% packet loss)')
RTT_AVG=$(echo "$PING_OUTPUT" | tail -1 | awk -F'/' '{print $5}')
echo "${timestamp} | PING: SUCCESS | Loss: ${PACKET_LOSS}% | Avg RTT: ${RTT_AVG}ms | $PING_STATS"
return 0
else
echo "${timestamp} | PING: ${RED}FAILED${NC} | Check ping_detailed.log for details"
return 1
fi
}

run_tcp_test() {
local timestamp=$1
TCP_START=$(date +%s.%N)

if timeout 5 bash -c "cat < /dev/null > /dev/tcp/$UPSTREAM_HOST/$UPSTREAM_PORT" 2>/dev/null; then
TCP_END=$(date +%s.%N)
TCP_TIME=$(echo "$TCP_END - $TCP_START" | bc)
echo "${timestamp} | TCP: ${GREEN}SUCCESS${NC} | Connect time: ${TCP_TIME}s"
return 0
else
echo "${timestamp} | TCP: ${RED}FAILED${NC} | Port $UPSTREAM_PORT unreachable"
return 1
fi
}

run_http_test() {
local timestamp=$1
CURL_OUTPUT=$(curl -k -s -o /dev/null -w "Status:%{http_code}|DNS:%{time_namelookup}s|Connect:%{time_connect}s|Total:%{time_total}s"
--connect-timeout 5 --max-time 10 https://$UPSTREAM_HOST 2>&1)
CURL_STATUS=$?

if [ $CURL_STATUS -eq 0 ]; then
echo "${timestamp} | HTTP: ${GREEN}SUCCESS${NC} | $CURL_OUTPUT"
return 0
else
echo "${timestamp} | HTTP: ${RED}FAILED${NC} | Error: $CURL_OUTPUT"
return 1
fi
}

run_mtr_test() {
local timestamp=$1
local iteration=$2

echo "" >> "$MTR_FILE"
echo "=== MTR Test at $timestamp (Iteration $iteration) ===" >> "$MTR_FILE"

MTR_OUTPUT=$(mtr -r -c $MTR_COUNT -m $MTR_MAX_HOPS -n $UPSTREAM_HOST 2>&1)
MTR_STATUS=$?

if [ $MTR_STATUS -eq 0 ]; then
echo "$MTR_OUTPUT" >> "$MTR_FILE"
echo "" >> "$MTR_FILE"

# Extract summary (last hop statistics)
LAST_HOP=$(echo "$MTR_OUTPUT" | grep -v "^HOST:" | tail -1)
AVG_LATENCY=$(echo "$LAST_HOP" | awk '{print $6}')
LOSS=$(echo "$LAST_HOP" | awk '{print $3}' | tr -d '%')

echo "${timestamp} | MTR: ${GREEN}SUCCESS${NC} | Avg latency: ${AVG_LATENCY}ms | Loss: ${LOSS}% | Max hops: $MTR_MAX_HOPS"
else
echo "$MTR_OUTPUT" >> "$MTR_FILE"
echo "" >> "$MTR_FILE"
echo "${timestamp} | MTR: ${YELLOW}WARNING${NC} | Check $MTR_FILE for details"
fi
}

check_system_stats() {
local timestamp=$1

# Connection tracking
if [ -f /proc/sys/net/netfilter/nf_conntrack_count ]; then
CONNTRACK_COUNT=$(cat /proc/sys/net/netfilter/nf_conntrack_count)
CONNTRACK_MAX=$(cat /proc/sys/net/netfilter/nf_conntrack_max)
CONNTRACK_PCT=$((CONNTRACK_COUNT * 100 / CONNTRACK_MAX))
else
CONNTRACK_PCT="N/A"
fi

# Established connections
ESTABLISHED=$(ss -tan | grep -c ESTABLISHED)
TIME_WAIT=$(ss -tan state time-wait | wc -l)

echo "${timestamp} | SYSTEM: Established: $ESTABLISHED | TimeWait: $TIME_WAIT | ConnTrack: ${CONNTRACK_PCT}%"
}

###############################################################################
# Main Monitoring Loop
###############################################################################

# Function to run the actual monitoring
run_monitoring() {
# Print configuration
log_message "Configuration:"
log_message " Upstream Host: $UPSTREAM_HOST"
log_message " Upstream Port: $UPSTREAM_PORT"
log_message " Test Duration: $TEST_DURATION_MINUTES minutes"
log_message " Check Interval: $CHECK_INTERVAL_SECONDS seconds"
log_message " Total Iterations: $TOTAL_ITERATIONS"
log_message " MTR Max Hops: $MTR_MAX_HOPS"
log_message " Output Directory: $OUTPUT_DIR"
log_message " Background Mode: $BACKGROUND_MODE"
log_message ""
log_message "Starting monitoring at $(date '+%Y-%m-%d %H:%M:%S')"
log_message "=========================================="
log_message ""

# Initialize counters
ITERATION=0
PING_FAILURES=0
TCP_FAILURES=0
HTTP_FAILURES=0

# Main loop
while [ $ITERATION -lt $TOTAL_ITERATIONS ]; do
ITERATION=$((ITERATION + 1))
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

log_message "${YELLOW}>>> Iteration $ITERATION/$TOTAL_ITERATIONS at $TIMESTAMP${NC}"

# Run tests
run_ping_test "$TIMESTAMP" "$ITERATION" | tee -a "$LOG_FILE"
PING_RESULT=$?

run_tcp_test "$TIMESTAMP" | tee -a "$LOG_FILE"
TCP_RESULT=$?

run_http_test "$TIMESTAMP" | tee -a "$LOG_FILE"
HTTP_RESULT=$?

run_mtr_test "$TIMESTAMP" "$ITERATION"
check_system_stats "$TIMESTAMP" | tee -a "$LOG_FILE"

log_message ""

# Track failures using return codes
[ $PING_RESULT -ne 0 ] && PING_FAILURES=$((PING_FAILURES + 1))
[ $TCP_RESULT -ne 0 ] && TCP_FAILURES=$((TCP_FAILURES + 1))
[ $HTTP_RESULT -ne 0 ] && HTTP_FAILURES=$((HTTP_FAILURES + 1))

# Sleep until next iteration (unless it's the last one)
if [ $ITERATION -lt $TOTAL_ITERATIONS ]; then
sleep $CHECK_INTERVAL_SECONDS
fi
done

# Generate summary
END_TIME=$(date '+%Y-%m-%d %H:%M:%S')

log_message "=========================================="
log_message "Monitoring completed at $END_TIME"
log_message ""

# Create summary file
cat > "$SUMMARY_FILE" << EOF
Network Connectivity Test Summary
==================================
Upstream Host: $UPSTREAM_HOST
Upstream Port: $UPSTREAM_PORT
Test Duration: $TEST_DURATION_MINUTES minutes
Check Interval: $CHECK_INTERVAL_SECONDS seconds
MTR Max Hops: $MTR_MAX_HOPS

Start Time: $START_TIME
End Time: $END_TIME
Total Iterations: $TOTAL_ITERATIONS

Results:
--------
Ping Failures: $PING_FAILURES / $TOTAL_ITERATIONS ($((PING_FAILURES * 100 / TOTAL_ITERATIONS))%)
TCP Failures: $TCP_FAILURES / $TOTAL_ITERATIONS ($((TCP_FAILURES * 100 / TOTAL_ITERATIONS))%)
HTTP Failures: $HTTP_FAILURES / $TOTAL_ITERATIONS ($((HTTP_FAILURES * 100 / TOTAL_ITERATIONS))%)

Files Generated:
----------------
- Detailed Log: $LOG_FILE
- Ping Details: $PING_FILE
- MTR Results: $MTR_FILE
- Summary: $SUMMARY_FILE

EOF

cat "$SUMMARY_FILE" | tee -a "$LOG_FILE"

# Create compressed archive
ARCHIVE_NAME="network_test_${START_TIME}.tar.gz"
tar -czf "$ARCHIVE_NAME" "$OUTPUT_DIR"

log_message ""
log_message "${GREEN}Archive created: $ARCHIVE_NAME${NC}"
log_message "${GREEN}You can share this file for analysis${NC}"
log_message ""

# Show quick analysis
echo -e "${YELLOW}Quick Analysis:${NC}"
echo "Checking for patterns in failures..."

if [ $PING_FAILURES -gt 0 ] || [ $TCP_FAILURES -gt 0 ] || [ $HTTP_FAILURES -gt 0 ]; then
echo -e "${RED}⚠ Network instability detected!${NC}"
echo "Check the following in the logs:"
echo " - Pattern of failures (intermittent vs continuous)"
echo " - MTR results for packet loss and high latency hops"
echo " - Ping details for packet loss patterns"
echo " - System stats during failure periods"
else
echo -e "${GREEN}✓ No failures detected during test period${NC}"
echo "Network appears stable, but review MTR and ping results for latency patterns"
fi
}

# Trap Ctrl+C to create archive before exit
trap 'echo ""; echo "Interrupted! Creating archive..."; tar -czf "network_test_${START_TIME}_partial.tar.gz" "$OUTPUT_DIR"; exit 1' INT

# Main function
main() {
# Check dependencies first
check_dependencies

if [ "$BACKGROUND_MODE" = true ]; then
# Background mode
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Network Connectivity Monitoring Script ║${NC}"
echo -e "${GREEN}║ BACKGROUND MODE ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
echo ""
echo "Starting background monitoring..."
echo "Output directory: $OUTPUT_DIR"
echo "Background log: ${OUTPUT_DIR}/background.log"
echo ""
echo "To check progress:"
echo " tail -f ${OUTPUT_DIR}/background.log"
echo " tail -f ${OUTPUT_DIR}/connectivity_test.log"
echo ""
echo "To stop monitoring:"
echo " kill $(cat ${OUTPUT_DIR}/monitor.pid)"
echo ""

# Save PID
echo $ > "${OUTPUT_DIR}/monitor.pid"

# Redirect all output to background log
exec > "${OUTPUT_DIR}/background.log" 2>&1

echo "Background monitoring started at $(date '+%Y-%m-%d %H:%M:%S')"
echo "PID: $"
echo ""

# Run monitoring
run_monitoring

# Cleanup PID file
rm -f "${OUTPUT_DIR}/monitor.pid"

else
# Foreground mode
clear
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Network Connectivity Monitoring Script ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
echo ""

# Run monitoring
run_monitoring

echo ""
echo "For 4-hour monitoring, change TEST_DURATION_MINUTES=240 in the script"
echo "To run in background: $0 --background"
fi
}

# Run main function
main