Initial version

config.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+const defaultTemplate = `$TTL 1H
+
+@ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h)
+  NS  LOCALHOST.
+
+; Blacklist entries
+{{ range .blacklist -}}
+{{ to_punycode .Domain }} CNAME . ; {{ .Comment }}
+{{ end }}`
+
+type providerType string
+
+type providerAction string
+
+const (
+	providerActionBlacklist providerAction = "blacklist"
+	providerActionWhitelist providerAction = "whitelist"
+)
+
+type configfile struct {
+	Providers []providerDefinition `yaml:"providers"`
+
+	Template string `yaml:"template"`
+	tpl      *template.Template
+}
+
+type providerDefinition struct {
+	Action  providerAction `yaml:"action"`
+	Content string         `yaml:"content"`
+	File    string         `yaml:"file"`
+	Name    string         `yaml:"name"`
+	Type    providerType   `yaml:"type"`
+	URL     string         `yaml:"url"`
+}
+
+func (p providerDefinition) GetContent() (io.ReadCloser, error) {
+	switch {
+
+	case p.Content != "":
+		return ioutil.NopCloser(strings.NewReader(p.Content)), nil
+
+	case p.File != "":
+		return os.Open(p.File)
+
+	case p.URL != "":
+		resp, err := http.Get(p.URL)
+		return resp.Body, err
+
+	default:
+		return nil, errors.New("Neither file nor URL specified")
+
+	}
+}
+
+func loadConfigFile(filename string) (*configfile, error) {
+	if _, err := os.Stat(filename); err != nil {
+		return nil, errors.Wrap(err, "Unable to access given file")
+	}
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to open given file")
+	}
+	defer f.Close()
+
+	out := &configfile{Template: defaultTemplate}
+	if err = yaml.NewDecoder(f).Decode(out); err != nil {
+		return nil, errors.Wrap(err, "Unable to parse given file")
+	}
+
+	if out.tpl, err = template.New("configTemplate").Funcs(template.FuncMap{
+		"to_punycode": domainToPunycode,
+	}).Parse(out.Template); err != nil {
+		return nil, errors.Wrap(err, "Unable to parse given template")
+	}
+
+	return out, nil
+}

config.sample.yaml

@@ -0,0 +1,120 @@
+---
+
+# List of third-party lists to download and include into generated
+# blacklist zone file (entries are just examples and copied from the
+# source of https://github.com/StevenBlack/hosts#sources-of-hosts-data-unified-in-this-variant)
+# Please verify the list matches your interest or compile your own!
+# 
+# Whitelists are applied AFTER all blacklists are compiled together
+# which means an entry in the whitelist will finally remove the domain
+# from the whole blacklist. Provider order does not matter in this case.
+providers:
+
+  #- name: Local blacklist
+  #  file: blacklist.local
+  #  action: blacklist
+  #  type: domain-list
+
+  #- name: Local whitelist
+  #  file: whitelist.local
+  #  action: whitelist
+  #  type: domain-list
+  
+  #- name: Local whitelist
+  #  content: |
+  #    my.domain.com
+  #  action: whitelist
+  #  type: domain-list
+
+  - name: Steven Black's ad-hoc list # License: MIT, URL: https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts
+    url: https://raw.githubusercontent.com/StevenBlack/hosts/master/data/StevenBlack/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: Malware Domain List # License: can be used for free by anyone, URL: https://www.malwaredomainlist.com/
+    url: https://www.malwaredomainlist.com/hostslist/hosts.txt
+    action: blacklist
+    type: hosts-file
+
+  - name: add.Dead # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras
+    url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: hostsVN # License: MIT, URL: https://github.com/bigdargon/hostsVN
+    url: https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/hosts-VN
+    action: blacklist
+    type: hosts-file
+
+  - name: add.Spam # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras
+    url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: Dan Pollock - someonewhocares # License: non-commercial with attribution, URL:https://someonewhocares.org/hosts/
+    url: https://someonewhocares.org/hosts/zero/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: MVPS hosts file # License: CC BY-NC-SA 4.0, URL: http://winhelp2002.mvps.org/
+    url: http://winhelp2002.mvps.org/hosts.txt
+    action: blacklist
+    type: hosts-file
+
+  - name: yoyo.org # URL: https://pgl.yoyo.org/adservers/
+    url: https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0
+    action: blacklist
+    type: hosts-file
+
+  - name: Mitchell Krog's - Badd Boyz Hosts # License: non-commercial with attribution, URL: https://github.com/mitchellkrogza/Badd-Boyz-Hosts
+    url: https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: CoinBlocker # License: GPLv3, URL: https://gitlab.com/ZeroDot1/CoinBlockerLists
+    url: https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser
+    action: blacklist
+    type: hosts-file
+
+  - name: UncheckyAds # URL: https://github.com/FadeMind/hosts.extras
+    url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: add.2o7Net # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras
+    url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: KADhosts # License: GPLv3, URL: https://github.com/azet12/KADhosts
+    url: https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt
+    action: blacklist
+    type: hosts-file
+
+  - name: AdAway # License: CC BY 3.0, URL: https://adaway.org/
+    url: https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt
+    action: blacklist
+    type: hosts-file
+
+  - name: add.Risk # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras
+    url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts
+    action: blacklist
+    type: hosts-file
+
+  - name: Tiuxo hostlist - ads # License: CC BY 4.0, URL: https://github.com/tiuxo/hosts
+    url: https://raw.githubusercontent.com/tiuxo/hosts/master/ads
+    action: blacklist
+    type: hosts-file
+
+template: |
+  $TTL 1H
+
+  @ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h)
+    NS  LOCALHOST.
+
+  ; Blacklist entries
+  {{ range .blacklist -}}
+  {{ to_punycode .Domain }} CNAME . ; {{ .Comment }}
+  {{ end }}
+
+...

go.mod

@@ -0,0 +1,12 @@
+module github.com/Luzifer/named-blacklist
+
+go 1.12
+
+require (
+	github.com/Luzifer/go_helpers/v2 v2.9.1
+	github.com/Luzifer/rconfig/v2 v2.2.1
+	github.com/pkg/errors v0.8.1
+	github.com/sirupsen/logrus v1.4.2
+	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
+	gopkg.in/yaml.v2 v2.2.2
+)

go.sum

@@ -0,0 +1,30 @@
+github.com/Luzifer/go_helpers/v2 v2.9.1 h1:MVUOlD6tJ2m/iTF0hllnI/QVZH5kI+TikUm1WRGg/c4=
+github.com/Luzifer/go_helpers/v2 v2.9.1/go.mod h1:ZnWxPjyCdQ4rZP3kNiMSUW/7FigU1X9Rz8XopdJ5ZCU=
+github.com/Luzifer/rconfig/v2 v2.2.1 h1:zcDdLQlnlzwcBJ8E0WFzOkQE1pCMn3EbX0dFYkeTczg=
+github.com/Luzifer/rconfig/v2 v2.2.1/go.mod h1:OKIX0/JRZrPJ/ZXXWklQEFXA6tBfWaljZbW37w+sqBw=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/leekchan/gtf v0.0.0-20190214083521-5fba33c5b00b/go.mod h1:thNruaSwydMhkQ8dXzapABF9Sc1Tz08ZBcDdgott9RA=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 h1:WB265cn5OpO+hK3pikC9hpP1zI/KTwmyMFKloW9eOVc=
+gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

helpers.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"strings"
+
+	"golang.org/x/net/idna"
+
+	"github.com/Luzifer/go_helpers/v2/str"
+)
+
+var genericBlacklist = []string{
+	"broadcasthost",
+	"ip6-allhosts",
+	"ip6-allnodes",
+	"ip6-allrouters",
+	"ip6-localnet",
+	"ip6-mcastprefix",
+	"local",
+	"localhost",
+	"localhost.localdomain",
+}
+
+func lineIsComment(line string) bool {
+	if len(strings.TrimSpace(line)) == 0 {
+		return true
+	}
+
+	return line[0] == '#' || line[0] == ';' || line[0] == '!'
+}
+
+func isBlacklisted(domain string) bool {
+	return str.StringInSlice(domain, genericBlacklist)
+}
+
+func domainToPunycode(name string, v ...string) (string, error) {
+	return idna.ToASCII(name)
+}

main.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/Luzifer/rconfig/v2"
+)
+
+var (
+	cfg = struct {
+		Config         string `flag:"config" default:"config.yaml" description:"Config file to use for generating the file"`
+		LogLevel       string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
+		VersionAndExit bool   `flag:"version" default:"false" description:"Prints current version and exits"`
+	}{}
+
+	config *configfile
+
+	version = "dev"
+)
+
+func init() {
+	rconfig.AutoEnv(true)
+	if err := rconfig.ParseAndValidate(&cfg); err != nil {
+		log.Fatalf("Unable to parse commandline options: %s", err)
+	}
+
+	if cfg.VersionAndExit {
+		fmt.Printf("named-blacklist %s\n", version)
+		os.Exit(0)
+	}
+
+	if l, err := log.ParseLevel(cfg.LogLevel); err != nil {
+		log.WithError(err).Fatal("Unable to parse log level")
+	} else {
+		log.SetLevel(l)
+	}
+}
+
+func main() {
+	var (
+		blacklist []entry
+		whitelist []entry
+		write     = new(sync.Mutex)
+		wg        sync.WaitGroup
+		err       error
+	)
+
+	if config, err = loadConfigFile(cfg.Config); err != nil {
+		log.WithError(err).Fatal("Unable to read config file")
+	}
+
+	wg.Add(len(config.Providers))
+	for _, p := range config.Providers {
+
+		go func(p providerDefinition) {
+			defer wg.Done()
+
+			entries, err := getDomainList(p)
+			if err != nil {
+				log.WithField("provider", p.Name).
+					WithError(err).
+					Fatal("Unable to get domain list")
+			}
+
+			write.Lock()
+			defer write.Unlock()
+
+			for _, e := range entries {
+				switch p.Action {
+
+				case providerActionBlacklist:
+					blacklist = addIfNotExists(blacklist, e)
+
+				case providerActionWhitelist:
+					whitelist = addIfNotExists(whitelist, e)
+
+				default:
+					log.WithField("provider", p.Name).Fatalf("Inavlid action %q", p.Action)
+
+				}
+			}
+		}(p)
+
+	}
+
+	wg.Wait()
+
+	blacklist = cleanFromList(blacklist, whitelist)
+
+	sort.Slice(blacklist, func(i, j int) bool { return blacklist[i].Domain < blacklist[j].Domain })
+
+	config.tpl.Execute(os.Stdout, map[string]interface{}{
+		"blacklist": blacklist,
+	})
+}
+
+func addIfNotExists(entries []entry, e entry) []entry {
+	for _, pe := range entries {
+		if pe.Domain == e.Domain {
+			// Entry already exists, skip
+			return entries
+		}
+	}
+
+	return append(entries, e)
+}
+
+func cleanFromList(blacklist, whitelist []entry) []entry {
+	var tmp []entry
+
+	for _, be := range blacklist {
+		var found bool
+
+		for _, we := range whitelist {
+			if we.Domain == be.Domain {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			tmp = append(tmp, be)
+		}
+	}
+
+	return tmp
+}

provider.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	providerRegistry     = map[providerType]provider{}
+	providerRegistryLock sync.Mutex
+)
+
+type entry struct {
+	Domain  string
+	Comment string
+}
+
+type provider interface {
+	GetDomainList(providerDefinition) ([]entry, error)
+}
+
+func registerProvider(t providerType, p provider) {
+	providerRegistryLock.Lock()
+	defer providerRegistryLock.Unlock()
+
+	providerRegistry[t] = p
+}
+
+func getDomainList(p providerDefinition) ([]entry, error) {
+	pro, ok := providerRegistry[p.Type]
+	if !ok {
+		return nil, errors.Errorf("Unknown provider type %q", p.Type)
+	}
+
+	return pro.GetDomainList(p)
+}

provider_domainList.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bufio"
+	"strings"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	registerProvider("domain-list", providerdomainList{})
+}
+
+type providerdomainList struct{}
+
+func (p providerdomainList) GetDomainList(d providerDefinition) ([]entry, error) {
+	r, err := d.GetContent()
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to get source content")
+	}
+	defer r.Close()
+
+	logger := log.WithField("provider", d.Name)
+
+	var entries []entry
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		if lineIsComment(scanner.Text()) {
+			continue
+		}
+
+		domain := strings.TrimSpace(scanner.Text())
+
+		if isBlacklisted(domain) {
+			logger.WithField("domain", domain).Debug("Skipping because of blacklist")
+			continue
+		}
+
+		entries = append(entries, entry{
+			Domain:  domain,
+			Comment: d.Name,
+		})
+	}
+
+	return entries, nil
+}

provider_hostFile.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	registerProvider("hosts-file", providerHostFile{})
+}
+
+type providerHostFile struct{}
+
+func (p providerHostFile) GetDomainList(d providerDefinition) ([]entry, error) {
+	r, err := d.GetContent()
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to get source content")
+	}
+	defer r.Close()
+
+	logger := log.WithField("provider", d.Name)
+
+	var (
+		entries []entry
+		matcher = regexp.MustCompile(`^(?:[0-9.]+|[a-z0-9:]+)\s+([^\s]+)(?:\s+#(.+))?$`)
+	)
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+
+		if lineIsComment(line) {
+			continue
+		}
+
+		if !matcher.MatchString(line) {
+			logger.WithField("line", line).Warn("Invalid line found (format)")
+			continue
+		}
+
+		groups := matcher.FindStringSubmatch(line)
+		if len(groups) < 2 {
+			logger.WithField("line", line).Warn("Invalid line found (groups)")
+			continue
+		}
+
+		if isBlacklisted(groups[1]) {
+			logger.WithField("domain", groups[1]).Debug("Skipping because of blacklist")
+			continue
+		}
+
+		comment := fmt.Sprintf("From: %q", d.Name)
+		if len(groups) == 3 && strings.Trim(groups[2], "#") != "" {
+			comment = fmt.Sprintf("%s, Comment: %q",
+				comment,
+				strings.TrimSpace(strings.Trim(groups[2], "#")),
+			)
+		}
+
+		entries = append(entries, entry{
+			Domain:  groups[1],
+			Comment: comment,
+		})
+	}
+
+	return entries, nil
+}