Linux Automation | Advanced Linux Administration

# Field positions: # .--------- minute (0-59) # | .------- hour (0-23) # | | .---- day of month (1-31) # | | | .- month (1-12 or jan-dec) # | | | | . day of week (0-7 or sun-sat; 0 and 7 = Sunday) # | | | | | # * * * * * command # Every minute * * * * * /usr/local/bin/heartbeat.sh # Every day at 02:30 30 2 * * * /usr/local/bin/backup.sh # Every Monday at 04:00 0 4 * * 1 /usr/local/bin/weekly-report.sh # Every 15 minutes */15 * * * * /usr/local/bin/check-disk.sh # First day of every month at 00:00 0 0 1 * * /usr/local/bin/monthly-cleanup.sh # Weekdays (Mon-Fri) at 08:00 0 8 * * 1-5 /usr/local/bin/workday-start.sh # Multiple specific hours: 06:00, 12:00, 18:00 0 6,12,18 * * * /usr/local/bin/sync.sh

# /etc/cron.d/sector-backups — system cron job with username field # Format: minute hour dom month dow USERNAME command 30 2 * * * root /usr/local/bin/sector-backup.sh >> /var/log/backup.log 2>&1 # Check when run-parts will execute scripts in cron.daily grep -r 'run-parts' /etc/crontab /etc/cron.d/ # View all cron jobs for a user crontab -u deploy -l # Validate cron syntax before saving (crontab -e opens $EDITOR) # Use https://crontab.guru/ or: man 5 crontab

# Top of crontab: set environment variables explicitly SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin MAILTO=ops@sector.local # email job output here; set to "" to silence HOME=/root # Always redirect output in the crontab entry itself 0 3 * * * /usr/local/bin/backup.sh >> /var/log/backup.log 2>&1 # Suppress all output (use only when you know it works) */5 * * * * /usr/local/bin/heartbeat.sh >/dev/null 2>&1 # Debug a failing cron job: run it with cron's environment manually env -i HOME=/root PATH=/usr/bin:/bin SHELL=/bin/bash /usr/local/bin/backup.sh

# List all active timers and their next trigger time systemctl list-timers --all # Enable and start a timer systemctl enable --now backup.timer # Check timer status and last run time systemctl status backup.timer # View output from the last run of the associated service journalctl -u backup.service -n 50

# /etc/systemd/system/sector-backup.service [Unit] Description=Sector Node Backup After=network-online.target Wants=network-online.target [Service] Type=oneshot User=backup ExecStart=/usr/local/bin/sector-backup.sh StandardOutput=journal StandardError=journal # SyslogIdentifier makes filtering in journalctl easy SyslogIdentifier=sector-backup

# /etc/systemd/system/sector-backup.timer [Unit] Description=Daily sector backup at 02:30 [Timer] # Calendar schedule (same expressive power as cron, clearer syntax) OnCalendar=*-*-* 02:30:00 # Catch up on missed runs (e.g., server was off at 02:30) Persistent=true # Randomize within 15 minutes to avoid thundering herd on multiple nodes RandomizedDelaySec=15min [Install] WantedBy=timers.target

# Deploy and activate systemctl daemon-reload systemctl enable --now sector-backup.timer systemctl list-timers sector-backup.timer

# Validate any calendar expression (prints next trigger times) systemd-analyze calendar '*-*-* 02:30:00' systemd-analyze calendar 'Mon *-*-* 08:00:00' systemd-analyze calendar 'weekly' # Common calendar expressions OnCalendar=daily # *-*-* 00:00:00 OnCalendar=weekly # Mon *-*-* 00:00:00 OnCalendar=monthly # *-*-01 00:00:00 OnCalendar=hourly # *-*-* *:00:00 OnCalendar='*:0/15' # every 15 minutes OnCalendar='Mon..Fri *-*-* 09:00' # weekdays at 09:00 OnCalendar='*-*-1,15 02:00' # 1st and 15th of every month # Monotonic (relative) triggers OnBootSec=2min # 2 minutes after boot OnUnitActiveSec=1h # 1 hour after the last activation OnStartupSec=30s # 30 seconds after systemd started

# /etc/logrotate.d/sector-app /var/log/sector/app.log /var/log/sector/worker.log { daily # rotate frequency rotate 14 # keep 14 rotated files compress # compress rotated files with gzip delaycompress # compress the previous rotation, not the current missingok # do not error if log file does not exist notifempty # skip rotation if the file is empty create 0640 sectorapp adm # permissions and owner for new log file dateext # add date to rotated filename: app.log-20260409 sharedscripts # run postrotate once for all logs in this block postrotate systemctl reload sector-app 2>/dev/null || true endscript }

# Size-based rotation (not time-based) /var/log/nginx/access.log { size 100M # rotate when file reaches 100MB rotate 5 compress copytruncate # copy then truncate -- no daemon reload needed # WARNING: copytruncate has a brief race window -- use postrotate when possible } # Hourly rotation for high-volume logs /var/log/sector/audit.log { hourly rotate 168 # 168 hours = 7 days of hourly files compress delaycompress missingok notifempty postrotate kill -USR1 $(cat /var/run/sector-audit.pid 2>/dev/null) 2>/dev/null || true endscript }

# Test logrotate config without actually rotating logrotate --debug /etc/logrotate.d/sector-app # Force rotation immediately (for testing postrotate scripts) logrotate --force /etc/logrotate.d/sector-app # Check logrotate status (last run times per log) cat /var/lib/logrotate/status | grep sector

# Core flags you will use on every rsync call # -a archive mode: -rlptgoD (recursive, preserve links/perms/times/owner/group/devices) # -v verbose (add to see what was transferred) # -z compress in transit (useful on slow links) # --delete remove files from destination not in source (true sync) # -n / --dry-run show what would change without changing anything # Local sync rsync -av /opt/sector/ /mnt/backup/sector/ # Remote sync over SSH rsync -avz --delete /opt/sector/ backup-node:/mnt/backup/sector/ # Dry run first -- always rsync -avn --delete /opt/sector/ backup-node:/mnt/backup/sector/ # Show progress and transfer statistics rsync -av --progress --stats /data/ /mnt/backup/data/ # Exclude directories rsync -av --exclude='*.tmp' --exclude='.git/' /opt/sector/ /mnt/backup/sector/ # Bandwidth limit (useful for background sync, units: KB/s) rsync -avz --bwlimit=5000 /data/ backup-node:/backup/data/

#!/usr/bin/env bash # incremental-backup.sh -- daily snapshot backup with hard-link deduplication set -euo pipefail SRC="/opt/sector" DEST="/mnt/backup" TODAY="$(date +%Y-%m-%d)" SNAPSHOT="${DEST}/snapshots/${TODAY}" LATEST="${DEST}/latest" mkdir -p "$SNAPSHOT" # --link-dest: files unchanged since LATEST are hard-linked, not copied rsync -av --delete \ --link-dest="${LATEST}" \ "${SRC}/" "${SNAPSHOT}/" # Update the 'latest' symlink to point to today's snapshot ln -sfn "${SNAPSHOT}" "${LATEST}" # Remove snapshots older than 30 days find "${DEST}/snapshots" -maxdepth 1 -type d -mtime +30 -exec rm -rf {} + echo "Backup complete: ${SNAPSHOT}"

# Generate a dedicated key for backup automation (no passphrase) ssh-keygen -t ed25519 -C "backup-auto@sector" -f /root/.ssh/id_backup -N "" # On the backup destination server: restrict the key to rsync only # ~/.ssh/authorized_keys entry: # command="rsync --server --daemon .",from="10.0.0.0/8" ssh-ed25519 AAAA... backup-auto@sector # Custom SSH options for the rsync call rsync -av -e "ssh -i /root/.ssh/id_backup -o StrictHostKeyChecking=yes -p 2222" \ /opt/sector/ backup-node:/backup/sector/ # rsync with SSH config alias (/root/.ssh/config) # Host backup-node # HostName 10.0.1.50 # User backup # IdentityFile /root/.ssh/id_backup # Port 2222 rsync -av /opt/sector/ backup-node:/backup/sector/

# Schedule a command at a specific time echo '/usr/local/bin/maintenance.sh >> /var/log/maint.log 2>&1' | at 02:30 # Human-readable time expressions at 4pm tomorrow at noon next friday at now + 2 hours at 2026-04-10 08:00 # Interactive mode (type commands, Ctrl+D to submit) at 03:00 <<'EOF' apt-get update -qq apt-get upgrade -y systemctl reboot EOF # View pending at jobs atq # View the commands in a job (job number from atq) at -c 3 # Remove a pending job atrm 3 # batch: defer heavy work to low-load time echo '/usr/local/bin/reindex-db.sh' | batch

#!/usr/bin/env bash # disk-monitor.sh — alert when any filesystem exceeds threshold set -euo pipefail WARN_THRESHOLD=80 CRIT_THRESHOLD=90 ALERT_EMAIL="ops@sector.local" HOSTNAME="$(hostname -f)" df -h --output=source,pcent,target | tail -n +2 | while IFS= read -r line; do PCT="$(echo "$line" | awk '{gsub(/%/,"",$2); print $2}')" MOUNT="$(echo "$line" | awk '{print $3}')" DEV="$(echo "$line" | awk '{print $1}')" if (( PCT >= CRIT_THRESHOLD )); then BODY="CRITICAL: ${MOUNT} on ${HOSTNAME} is at ${PCT}% (${DEV})\n" BODY+="$(du -sh ${MOUNT}/* 2>/dev/null | sort -rh | head -10)" echo -e "$BODY" | mail -s "[CRITICAL] Disk ${PCT}% on ${HOSTNAME}:${MOUNT}" "$ALERT_EMAIL" elif (( PCT >= WARN_THRESHOLD )); then logger -t disk-monitor -p local0.warning \ "WARN: ${MOUNT} at ${PCT}% on ${DEV}" fi done

#!/usr/bin/env bash # watchdog.sh — monitor critical processes, restart if down set -euo pipefail # Array of: "process-name:start-command" declare -A PROCESSES PROCESSES["nginx"]="systemctl start nginx" PROCESSES["sector-worker"]="/opt/sector/bin/worker --daemonize" PROCESSES["redis-server"]="systemctl start redis" MAX_RESTARTS=3 STATE_DIR="/var/run/watchdog" mkdir -p "$STATE_DIR" for proc in "${!PROCESSES[@]}"; do STATE_FILE="${STATE_DIR}/${proc}.restarts" RESTARTS="$(cat "$STATE_FILE" 2>/dev/null || echo 0)" if ! pgrep -x "$proc" >/dev/null 2>&1; then if (( RESTARTS < MAX_RESTARTS )); then echo "[WARN] $proc down -- restarting (attempt $((RESTARTS+1)))" eval "${PROCESSES[$proc]}" echo "$((RESTARTS+1))" > "$STATE_FILE" else echo "[CRIT] $proc down -- max restarts reached, alerting" logger -t watchdog -p local0.crit "$proc failed after $RESTARTS restarts" fi else echo 0 > "$STATE_FILE" # reset counter when process is healthy fi done

#!/usr/bin/env bash # net-check.sh — verify reachability and service response set -euo pipefail declare -A ENDPOINTS ENDPOINTS["gateway"]="10.0.0.1:icmp" ENDPOINTS["web-lb"]="10.0.1.10:443" ENDPOINTS["db-primary"]="10.0.1.20:5432" ENDPOINTS["api-health"]="https://api.sector.local/health:http" FAIL=0 check_tcp() { local host="$1" port="$2" timeout 3 bash -c "echo >/dev/tcp/${host}/${port}" 2>/dev/null } check_http() { local url="$1" curl -sf --max-time 5 "$url" >/dev/null } for name in "${!ENDPOINTS[@]}"; do spec="${ENDPOINTS[$name]}" case "$spec" in *:icmp) ping -c1 -W2 "${spec%:icmp}" >/dev/null 2>&1 || { echo "FAIL: $name"; FAIL=1; } ;; https://*:http) check_http "${spec%:http}" || { echo "FAIL: $name"; FAIL=1; } ;; *:*) check_tcp "${spec%:*}" "${spec##*:}" || { echo "FAIL: $name"; FAIL=1; } ;; esac done (( FAIL > 0 )) && { logger -t net-check -p local0.error "$FAIL endpoint(s) unreachable"; exit 1; } echo "All endpoints OK"

#!/usr/bin/env bash # log-anomaly.sh — scan recent logs for error patterns set -euo pipefail SINCE="1 hour ago" PATTERNS=( "CRITICAL" "OOM killer" "segfault" "Connection refused" "disk quota exceeded" "authentication failure" "permission denied" ) REPORT="" for pattern in "${PATTERNS[@]}"; do COUNT="$(journalctl --since="$SINCE" -q 2>/dev/null \ | grep -ci "$pattern" || echo 0)" if (( COUNT > 0 )); then REPORT+="[${COUNT}x] ${pattern}\n" # Show the most recent 3 occurrences for context REPORT+="$(journalctl --since="$SINCE" -q 2>/dev/null \ | grep -i "$pattern" | tail -3 | sed 's/^/ /')\n\n" fi done if [[ -n "$REPORT" ]]; then echo -e "[ANOMALY REPORT] $(hostname) @ $(date)\n$REPORT" | \ mail -s "[Log Anomaly] $(hostname)" ops@sector.local fi

# flock approach 1: wrap in crontab */30 * * * * flock -n /var/lock/backup.lock /usr/local/bin/backup.sh # -n = non-blocking (exit 1 immediately if lock not available) # Default is blocking (wait for lock to be released) # flock approach 2: inside the script (preferred for complex scripts) #!/usr/bin/env bash set -euo pipefail LOCK_FILE="/var/lock/backup.lock" # Open file descriptor 200 and acquire exclusive lock exec 200>"$LOCK_FILE" flock -n 200 || { echo "Another instance is running. Exiting."; exit 0; } # Lock is held for the remainder of the script # Automatically released when script exits (fd 200 closes) echo "$BASHPID" >&200 # write PID into lock file for debugging # ... rest of script

# /etc/anacrontab format: # period(days) delay(minutes) job-id command 1 5 daily-backup /usr/local/bin/backup.sh 7 10 weekly-report /usr/local/bin/weekly-report.sh 30 15 monthly-cleanup /usr/local/bin/cleanup.sh # View last run times ls -la /var/spool/anacron/ cat /var/spool/anacron/daily-backup # contains date of last run # Force anacron to run all due jobs immediately (testing) anacron -f -d # -f force, -d debug output # Check if anacron is installed and active on this system systemctl status cron | grep -i anacron

#!/usr/bin/env bash # verify-backup.sh -- confirm backup integrity and restore readiness set -euo pipefail BACKUP_DIR="/mnt/backup/snapshots/latest" VERIFY_LOG="/var/log/backup-verify.log" FAIL=0 log() { echo "$(date +%T) $*" | tee -a "$VERIFY_LOG"; } # Check 1: backup directory exists and is recent [[ -d "$BACKUP_DIR" ]] || { log "FAIL: backup dir missing"; FAIL=1; } AGE=$(( ($(date +%s) - $(stat -c %Y "$BACKUP_DIR")) / 3600 )) (( AGE > 26 )) && { log "FAIL: backup is ${AGE}h old -- expected <26h"; FAIL=1; } # Check 2: critical files are present for f in /etc/passwd /etc/nginx/nginx.conf /opt/sector/config.json; do [[ -f "${BACKUP_DIR}${f}" ]] || { log "FAIL: missing ${f} in backup"; FAIL=1; } done # Check 3: compare checksums of key configs sha256sum /etc/nginx/nginx.conf > /tmp/live.sha256 sha256sum "${BACKUP_DIR}/etc/nginx/nginx.conf" >> /tmp/live.sha256 awk 'NR==1{ck=$1} NR==2{if($1!=ck) print "CHECKSUM MISMATCH: nginx.conf"}' /tmp/live.sha256 (( FAIL > 0 )) && logger -t backup-verify -p local0.crit "Backup verification FAILED" log "Verification complete: FAIL=$FAIL" exit $FAIL

# Create a dedicated backup user with no shell login useradd -r -s /usr/sbin/nologin -m backup # Grant backup user read access to required directories setfacl -R -m u:backup:rX /opt/sector setfacl -m u:backup:rX /etc/nginx # Allow backup user to run rsync as root without password (sudoers) # /etc/sudoers.d/backup: # backup ALL=(root) NOPASSWD: /usr/bin/rsync --server * # /etc/cron.d/sector-backups (using explicit user field) 30 2 * * * backup /usr/local/bin/backup.sh >> /var/log/backup.log 2>&1 # Restrict crontab access: /etc/cron.allow lists permitted users # /etc/cron.deny lists blocked users

# Resource-limited service unit # /etc/systemd/system/sector-cleanup.service [Service] Type=oneshot User=cleanup-svc ExecStart=/usr/local/bin/cleanup.sh # CPU: maximum 20% of one core CPUQuota=20% # Memory: kill if it tries to use more than 512MB MemoryMax=512M # Nice level: low priority, yield to other processes Nice=15 # I/O: low priority scheduling class IOSchedulingClass=idle # Prevent writing outside /tmp and /var/log/sector ReadWritePaths=/var/log/sector /tmp ProtectSystem=strict

# Transient timer: run once at a specific time, then gone (no unit file needed) systemd-run --on-calendar='2026-04-10 03:00' \ --unit=one-shot-reboot \ /sbin/reboot # Transient relative timer: 30 minutes from now systemd-run --on-active=30m \ --description="Deferred maintenance" \ /usr/local/bin/maintenance.sh # View transient jobs systemctl list-timers

# Basic usage: write to syslog with a custom tag logger -t backup "Backup started for /opt/sector" # Specify facility and priority (facility.priority) logger -p local0.info -t backup "Backup completed successfully" logger -p local0.warning -t backup "Backup completed with 3 skipped files" logger -p local0.err -t backup "Backup FAILED: rsync exited 23" logger -p local0.crit -t backup "Backup destination unreachable" # Structured key=value logging (machine-parseable) logger -t sector-monitor \ "event=disk_warn host=$(hostname) mount=/var/log usage_pct=87" # Log stdin (pipe output of a command directly to syslog) rsync -av /data/ /backup/ 2>&1 | logger -t rsync-backup # See messages written by logger journalctl -t backup -n 20 journalctl -t sector-monitor --since "1 hour ago" # Filter by facility in rsyslog (routes local0.* to a dedicated log) # /etc/rsyslog.d/sector.conf: local0.* /var/log/sector/automation.log

# Run a cron job manually with cron's environment (replicate the context) sudo -u backup env -i HOME=/home/backup \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ SHELL=/bin/bash \ /usr/local/bin/backup.sh # Force a systemd timer to fire now (without waiting for schedule) systemctl start sector-backup.service # runs the service immediately journalctl -u sector-backup.service -f # watch its output

# sector-backup.sh: the actual work, called by the service flock -n /var/lock/sector-backup.lock || { logger -t sector-backup "already running"; exit 0; } rsync -av --delete --link-dest=/mnt/backup/latest \ /opt/sector/ /mnt/backup/snapshots/"$(date +%F)"/ ln -sfn /mnt/backup/snapshots/"$(date +%F)" /mnt/backup/latest find /mnt/backup/snapshots -maxdepth 1 -type d -mtime +30 -exec rm -rf {} + # sector-backup-verify.timer fires 30 minutes after backup completes # OnUnitActiveSec=30m # fires 30 minutes after sector-backup.service ran # Monitor: check last run time from systemd systemctl show sector-backup.service --property=ExecMainExitTimestamp

# /etc/rsyncd.conf on the destination/backup server uid = backup gid = backup use chroot = yes max connections = 4 log file = /var/log/rsyncd.log pid file = /var/run/rsyncd.pid [sector-data] path = /mnt/backup/sector comment = Sector node data backup read only = no hosts allow = 10.0.1.0/24 auth users = sector-backup secrets file = /etc/rsyncd.secrets # /etc/rsyncd.secrets: sector-backup:s3cr3tpassword (chmod 600)

# Client: rsync to daemon (use :: double colon for daemon protocol) rsync -av --password-file=/etc/sector/rsync.pass \ /opt/sector/ backup-node::sector-data/ # Start rsyncd as a systemd service systemctl enable --now rsync # Or: launch from xinetd for on-demand connections (legacy environments) # Or: run with --daemon flag for standalone operation rsync --daemon --config=/etc/rsyncd.conf

# Install: apt install inotify-tools # Wait for a single event (blocking) inotifywait -e close_write /etc/nginx/nginx.conf # Blocks until the file is written and closed systemctl reload nginx # Watch a directory recursively for any file modification inotifywait -m -r -e modify,create,delete /etc/sector/ # Event-driven deployment: sync whenever configs change inotifywait -m -r -e close_write \ --format '%T %w%f' --timefmt '%F %T' \ /etc/sector/configs/ | while read -r ts path; do echo "[$ts] Changed: $path -- redeploying" /usr/local/bin/deploy-config.sh "$path" done # Common events to watch for # modify — file content changed # create — new file created # delete — file removed # moved_to — file moved into watched dir # close_write — file closed after writing (safer than modify for deployments)

#!/usr/bin/env bash # adaptive-tuner.sh -- adjust kernel parameters based on observed load set -euo pipefail LOAD="$(awk '{print int($1)}' /proc/loadavg)" NCPU="$(nproc)" LOAD_PER_CPU=$(( LOAD * 100 / NCPU )) # load as % of capacity if (( LOAD_PER_CPU > 150 )); then # High load: increase I/O scheduler time slice for throughput echo 256 > /sys/block/sda/queue/nr_requests echo "mq-deadline" > /sys/block/sda/queue/scheduler logger -t adaptive-tuner "High load (${LOAD_PER_CPU}%) -- throughput mode" elif (( LOAD_PER_CPU < 30 )); then # Low load: reduce latency for interactive workloads echo 64 > /sys/block/sda/queue/nr_requests echo "bfq" > /sys/block/sda/queue/scheduler logger -t adaptive-tuner "Low load (${LOAD_PER_CPU}%) -- latency mode" fi # Schedule this every 5 minutes via cron or systemd timer # */5 * * * * root flock -n /var/lock/tuner.lock /usr/local/bin/adaptive-tuner.sh

# Email via mail command (requires postfix/sendmail configured) mail -s "[ALERT] Backup failed on $(hostname)" ops@sector.local <<'EOF' Backup script failed at 02:31. Last successful backup: check /var/log/backup.log Action required: verify /mnt/backup is mounted and accessible EOF # Webhook alert (Slack, PagerDuty, Mattermost) via curl send_alert() { local msg="$1" curl -sf -X POST "${WEBHOOK_URL}" \ -H 'Content-Type: application/json' \ -d "{\"text\": \"[${HOSTNAME}] ${msg}\"}" >/dev/null } # Integrate alert into any script run_backup || send_alert "Backup FAILED -- exit code $?" # Throttle alerts: write a timestamp file, alert only if not alerted recently ALERT_FILE="/var/run/backup-alert.ts" LAST="$(cat "$ALERT_FILE" 2>/dev/null || echo 0)" NOW="$(date +%s)" if (( NOW - LAST > 3600 )); then # alert at most once per hour send_alert "Backup FAILED" echo "$NOW" > "$ALERT_FILE" fi

# Comprehensive cron audit: all sources on the system echo "=== /etc/crontab ==="; cat /etc/crontab echo "=== /etc/cron.d/ ==="; ls -la /etc/cron.d/ && cat /etc/cron.d/* echo "=== User crontabs ===" for user in "$(cut -d: -f1 /etc/passwd)"; do CTAB="$(crontab -u "$user" -l 2>/dev/null)" [[ -n "$CTAB" ]] && { echo "-- $user --"; echo "$CTAB"; } done echo "=== /etc/cron.{hourly,daily,weekly,monthly} ===" for dir in hourly daily weekly monthly; do echo "-- cron.$dir --" ls -la /etc/cron."$dir"/ 2>/dev/null done echo "=== systemd timers ==="; systemctl list-timers --all