#!/bin/bash # Shared ban list updater — pulls URLs from sources.json and refreshes static/bans.txt. # Runs nightly via systemd timer on the master (DNS3). set -uo pipefail REPO_DIR="/var/lib/shared-bans" SOURCES_JSON="$REPO_DIR/sources.json" STATIC_OUT="$REPO_DIR/static/bans.txt" LOG="/var/log/shared-bans-update.log" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG"; } if [ ! -d "$REPO_DIR/.git" ]; then log "ERROR: $REPO_DIR not a git repo, run shared-bans-apply.sh first" exit 1 fi cd "$REPO_DIR" || exit 1 # Refresh git index (in case dynamic bans were pushed from elsewhere) git pull --ff-only >>"$LOG" 2>&1 || log "WARN: git pull failed, using local" # Parse sources.json with python (jq may not be installed everywhere) python3 << PYEOF > "$WORK/sources.txt" import json, sys try: with open("$SOURCES_JSON") as f: data = json.load(f) for src in data.get("sources", []): print(f"# source: {src['name']} ({src.get('comment','')})") print(f"# url: {src['url']}") print(f"URL\t{src['url']}") except Exception as e: print(f"# ERROR parsing sources.json: {e}", file=sys.stderr) sys.exit(1) PYEOF if [ ! -s "$WORK/sources.txt" ]; then log "ERROR: sources.txt empty (python parse failed?)" exit 1 fi # Fetch each URL, dedupe > "$WORK/all.txt" while IFS=$'\t' read -r tag url; do [ "$tag" != "URL" ] && continue [ -z "$url" ] && continue echo "# fetching $url ..." >> "$WORK/all.txt" if curl -fsS --max-time 60 "$url" 2>>"$LOG" >> "$WORK/all.txt"; then log " fetched $url" else log " WARN: failed to fetch $url" fi done < "$WORK/sources.txt" # Clean: strip comments/blank/whitespace, keep only IP/CIDR lines grep -E '^[0-9]' "$WORK/all.txt" | tr -d ' \t' | sort -u > "$WORK/clean.txt" COUNT=$(wc -l < "$WORK/clean.txt") log "After dedupe: $COUNT unique CIDRs/IPs" if [ "$COUNT" -lt 50 ]; then log "ERROR: suspiciously few entries ($COUNT), refusing to commit" exit 1 fi # Write new static file { echo "# Auto-generated from sources.json — do not edit by hand." echo "# To add new block sources, edit sources.json instead." echo "# Last regenerated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "# Total unique entries: $COUNT" echo cat "$WORK/clean.txt" } > "$STATIC_OUT.new" # Diff against current if diff -q "$STATIC_OUT" "$STATIC_OUT.new" >/dev/null 2>&1; then log "No changes since last update" rm -f "$STATIC_OUT.new" exit 0 fi mv "$STATIC_OUT.new" "$STATIC_OUT" log "Wrote $STATIC_OUT" # Commit + push git add "$STATIC_OUT" if git diff --cached --quiet; then log "Nothing to commit" exit 0 fi git commit -m "Update static ban list ($(date -u +%Y-%m-%d)) [auto]" >>"$LOG" 2>&1 || { log "ERROR: git commit failed" exit 1 } if git push origin main >>"$LOG" 2>&1; then log "Pushed to origin" else log "ERROR: git push failed" exit 1 fi # Log rotation if [ "$(wc -l < "$LOG")" -gt 5000 ]; then tail -n 3000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG" fi exit 0