Download spamhaus drop list with curl

Here is yet another utility to deploy Spamhaus DROP (Don't Route Or Peer) list by using curl, ipset and iptables.

The following script assumes you have created an IP set of type nethash named spamhaus. You should do this in the script that sets up the firewall, like so:

ipset create spamhaus nethash

Note the syntax change, from ipset -N to ipset create, in version 6.

Then, among other iptables rules, you add something like:

iptables -A my_chain  -m set --match-set spamhaus src,dst -j DROP

There is another tactic, that Mike uses for TCP packets directed against his mail server:
First mangle

-A PREROUTING -m conntrack --ctstate NEW -m set --match-set spamhaus src -j CONNMARK --set-xmark 0x19

and then filter

-A OUTPUT -p tcp -m connmark --mark 0x19 -j REJECT --reject-with tcp-reset

001: #! /bin/bash
002: 
003: log=/var/log/spamhaus-drop.log
004: lock=drop.lock
005: wdir=/var/lib/spamhaus-drop
006: spamhaus='http://www.spamhaus.org/drop'
007: 
008: # log $1
009: function log_this() {
010:    printf '%s: %s\n' "$(date --rfc-3339=seconds)" "$1" >> $log
011: }
012: 
013: 
014: cd $wdir || exit 1
015: lockfile-create --use-pid --retry 1 --lock-name $lock || exit 1
016: 
017: # if the ipset is missing, it's useless to download anything
018: ipset list spamhaus > /dev/null 2>&1
019: if [ $? -ne 0 ]; then
020:    log_this 'error in ipset list spamhaus'
021:    lockfile-remove --lock-name $lock
022:    exit 1
023: fi
024: 
025: curl_flags='--silent --show-error --remote-name --remote-time --max-time 150
026:    --write-out %{http_code}'
027: 
028: if [ "$1" = "-v" ]; then
029:    curl_flags="--verbose $curl_flags"
030:    curl_keep_stderr=true
031: fi
032: 
033: new_files=""
034: old_files=""
035: # download file, name must not be null, and must have no spaces (urlencode).
036: # existing files are downloaded only if newer.
037: # new_ and old_ files are updated according to the result.
038: # stderr displays normally when -v (verbose) is given on the command line
039: function run_curl() {
040:    local name=$1 existing="" time_cond="" rtc=""
041:    if [ -f $name ]; then
042:       time_cond="--time-cond $name"
043:       existing=$name
044:    fi
045:    if [ -n "$curl_keep_stderr" ]; then
046:       rtc=$(curl $curl_flags $time_cond $spamhaus/$name)
047:    else
048:       rtc=$(curl $curl_flags $time_cond $spamhaus/$name 2>&1)
049:    fi
050:    if [ $? -eq 0 ]; then
051:       if [ "$rtc" = "200" -a -f $name ]; then
052:          new_files+="${new_files:+ }$name"
053:          existing=''
054:       elif [ "$rtc" != "304" ]; then
055:          log_this "http $rtc for $name"
056:       fi
057:    else
058:       log_this "${rtc:-curl: $?} for $spamhaus/$name"
059:    fi
060:    test -n "$existing" && old_files+="${old_files:+ }$existing"
061: }
062: 
063: # define what files to process
064: do_ipset=""
065: if [ -f no-download ]; then
066:    do_ipset=$(<no-download)
067:    log_this "no-download${do_ipset:+: $do_ipset}"
068: else
069:    run_curl drop.txt
070:    run_curl edrop.txt
071: 
072:    log_this "download: ${new_files:-none}${old_files:+, existing: $old_files}"
073:    do_ipset=${new_files:+$new_files $old_files}  # do both if any is new
074: fi
075: 
076: # process blocks, if any
077: if [ -n "$do_ipset" ]; then
078:    ipset destroy temp  > /dev/null 2>&1
079:    ipset create temp hash:net
080: 
081:    perl -e 'while(<>){s/;.*//; s/\s//g; print $_, "\n" unless /^$/;}' \
082:       $do_ipset |
083:       (
084:          blockcount=0;
085:          errcount=0
086:          while read block; do
087:             err=$(ipset add temp $block 2>&1)
088:             if [ $? -ne 0 ]; then
089:                (( ++errcount < 6 )) && log_this "$block $err"
090:             else
091:                (( ++blockcount ))
092:             fi
093:          done
094:          test $errcount -gt 6 &&
095:             log_this "$errcount errors in $do_ipset"
096:          log_this "$blockcount blocks set from $do_ipset"
097:       )
098: 
099:    ipset swap temp spamhaus
100:    ipset destroy temp
101: fi
102: 
103: lockfile-remove --lock-name $lock
104: 
105:

#! /bin/bash

log=/var/log/spamhaus-drop.log
lock=drop.lock
wdir=/var/lib/spamhaus-drop
spamhaus='http://www.spamhaus.org/drop'

# log $1
function log_this() {
	printf '%s: %s\n' "$(date --rfc-3339=seconds)" "$1" >> $log
}

cd $wdir || exit 1
lockfile-create --use-pid --retry 1 --lock-name $lock || exit 1

# if the ipset is missing, it's useless to download anything
ipset list spamhaus > /dev/null 2>&1
if [ $? -ne 0 ]; then
	log_this 'error in ipset list spamhaus'
	lockfile-remove --lock-name $lock
	exit 1
fi

curl_flags='--silent --show-error --remote-name --remote-time --max-time 150
	--write-out %{http_code}'

if [ "$1" = "-v" ]; then
	curl_flags="--verbose $curl_flags"
	curl_keep_stderr=true
fi

new_files=""
old_files=""
# download file, name must not be null, and must have no spaces (urlencode).
# existing files are downloaded only if newer.
# new_ and old_ files are updated according to the result.
# stderr displays normally when -v (verbose) is given on the command line
function run_curl() {
	local name=$1 existing="" time_cond="" rtc=""
	if [ -f $name ]; then
		time_cond="--time-cond $name"
		existing=$name
	fi
	if [ -n "$curl_keep_stderr" ]; then
		rtc=$(curl $curl_flags $time_cond $spamhaus/$name)
	else
		rtc=$(curl $curl_flags $time_cond $spamhaus/$name 2>&1)
	fi
	if [ $? -eq 0 ]; then
		if [ "$rtc" = "200" -a -f $name ]; then
			new_files+="${new_files:+ }$name"
			existing=''
		elif [ "$rtc" != "304" ]; then
			log_this "http $rtc for $name"
		fi
	else
		log_this "${rtc:-curl: $?} for $spamhaus/$name"
	fi
	test -n "$existing" && old_files+="${old_files:+ }$existing"
}

# define what files to process
do_ipset=""
if [ -f no-download ]; then
	do_ipset=$(<no-download)
	log_this "no-download${do_ipset:+: $do_ipset}"
else
	run_curl drop.txt
	run_curl edrop.txt

log_this "download: ${new_files:-none}${old_files:+, existing: $old_files}"
	do_ipset=${new_files:+$new_files $old_files}  # do both if any is new
fi

# process blocks, if any
if [ -n "$do_ipset" ]; then
	ipset destroy temp  > /dev/null 2>&1
	ipset create temp hash:net

perl -e 'while(<>){s/;.*//; s/\s//g; print $_, "\n" unless /^$/;}' \
		$do_ipset |
		(
			blockcount=0;
			errcount=0
			while read block; do
				err=$(ipset add temp $block 2>&1)
				if [ $? -ne 0 ]; then
					(( ++errcount < 6 )) && log_this "$block $err"
				else
					(( ++blockcount ))
				fi
			done
			test $errcount -gt 6 &&
				log_this "$errcount errors in $do_ipset"
			log_this "$blockcount blocks set from $do_ipset"
		)

ipset swap temp spamhaus
	ipset destroy temp
fi

lockfile-remove --lock-name $lock

The author of this script has dedicated it to the public domain by waiving all of his rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. You can copy, modify, distribute and perform this script, even for commercial purposes, all without asking permission.

Copy and paste the script to your editor of choice. Note that it uses perl (line 81). You may want to use sed and egrep -v instead. Regular expressions are similar, but sed wants [[:space:]] instead of \s. Just save it in /etc/cron.daily/ or equivalent when done.

You can pass -v to the script to see more of the download. Otherwise, you can prevent unwanted downloads like so:

echo drop.txt edrop.txt > no-download

That way you won't annoy Spamhaus while testing. Recall they don't want people to download more than once per hour. In my limited experience, once per day seems to be enough...

Download spamhaus drop list with curl
Maintain an ipset to be used within iptables on your firewall
[Intro] [Bash script] [Alternatives]

Intro

Bash script

Alternatives

Download spamhaus drop list with curl Maintain an ipset to be used within iptables on your firewall [Intro] [Bash script] [Alternatives]

Intro

Bash script

Alternatives

Download spamhaus drop list with curl
Maintain an ipset to be used within iptables on your firewall
[Intro] [Bash script] [Alternatives]