sami
114 lines
3.0 KiB
Bash
114 lines
3.0 KiB
Bash
#!/bin/bash
|
|
# Shared ban list updater — pulls URLs from sources.json and refreshes static/bans.txt.
|
|
# Runs nightly via systemd timer on the master (DNS3).
|
|
set -uo pipefail
|
|
|
|
REPO_DIR="/var/lib/shared-bans"
|
|
SOURCES_JSON="$REPO_DIR/sources.json"
|
|
STATIC_OUT="$REPO_DIR/static/bans.txt"
|
|
LOG="/var/log/shared-bans-update.log"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG"; }
|
|
|
|
if [ ! -d "$REPO_DIR/.git" ]; then
|
|
log "ERROR: $REPO_DIR not a git repo, run shared-bans-apply.sh first"
|
|
exit 1
|
|
fi
|
|
|
|
cd "$REPO_DIR" || exit 1
|
|
|
|
# Refresh git index (in case dynamic bans were pushed from elsewhere)
|
|
git pull --ff-only >>"$LOG" 2>&1 || log "WARN: git pull failed, using local"
|
|
|
|
# Parse sources.json with python (jq may not be installed everywhere)
|
|
python3 << PYEOF > "$WORK/sources.txt"
|
|
import json, sys
|
|
try:
|
|
with open("$SOURCES_JSON") as f:
|
|
data = json.load(f)
|
|
for src in data.get("sources", []):
|
|
print(f"# source: {src['name']} ({src.get('comment','')})")
|
|
print(f"# url: {src['url']}")
|
|
print(f"URL\t{src['url']}")
|
|
except Exception as e:
|
|
print(f"# ERROR parsing sources.json: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
PYEOF
|
|
|
|
if [ ! -s "$WORK/sources.txt" ]; then
|
|
log "ERROR: sources.txt empty (python parse failed?)"
|
|
exit 1
|
|
fi
|
|
|
|
# Fetch each URL, dedupe
|
|
> "$WORK/all.txt"
|
|
|
|
while IFS=$'\t' read -r tag url; do
|
|
[ "$tag" != "URL" ] && continue
|
|
[ -z "$url" ] && continue
|
|
echo "# fetching $url ..." >> "$WORK/all.txt"
|
|
if curl -fsS --max-time 60 "$url" 2>>"$LOG" >> "$WORK/all.txt"; then
|
|
log " fetched $url"
|
|
else
|
|
log " WARN: failed to fetch $url"
|
|
fi
|
|
done < "$WORK/sources.txt"
|
|
|
|
# Clean: strip comments/blank/whitespace, keep only IP/CIDR lines
|
|
grep -E '^[0-9]' "$WORK/all.txt" | tr -d ' \t' | sort -u > "$WORK/clean.txt"
|
|
|
|
COUNT=$(wc -l < "$WORK/clean.txt")
|
|
log "After dedupe: $COUNT unique CIDRs/IPs"
|
|
|
|
if [ "$COUNT" -lt 50 ]; then
|
|
log "ERROR: suspiciously few entries ($COUNT), refusing to commit"
|
|
exit 1
|
|
fi
|
|
|
|
# Write new static file
|
|
{
|
|
echo "# Auto-generated from sources.json — do not edit by hand."
|
|
echo "# To add new block sources, edit sources.json instead."
|
|
echo "# Last regenerated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "# Total unique entries: $COUNT"
|
|
echo
|
|
cat "$WORK/clean.txt"
|
|
} > "$STATIC_OUT.new"
|
|
|
|
# Diff against current
|
|
if diff -q "$STATIC_OUT" "$STATIC_OUT.new" >/dev/null 2>&1; then
|
|
log "No changes since last update"
|
|
rm -f "$STATIC_OUT.new"
|
|
exit 0
|
|
fi
|
|
|
|
mv "$STATIC_OUT.new" "$STATIC_OUT"
|
|
log "Wrote $STATIC_OUT"
|
|
|
|
# Commit + push
|
|
git add "$STATIC_OUT"
|
|
if git diff --cached --quiet; then
|
|
log "Nothing to commit"
|
|
exit 0
|
|
fi
|
|
|
|
git commit -m "Update static ban list ($(date -u +%Y-%m-%d)) [auto]" >>"$LOG" 2>&1 || {
|
|
log "ERROR: git commit failed"
|
|
exit 1
|
|
}
|
|
|
|
if git push origin main >>"$LOG" 2>&1; then
|
|
log "Pushed to origin"
|
|
else
|
|
log "ERROR: git push failed"
|
|
exit 1
|
|
fi
|
|
|
|
# Log rotation
|
|
if [ "$(wc -l < "$LOG")" -gt 5000 ]; then
|
|
tail -n 3000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
|
|
fi
|
|
|
|
exit 0 |