From 0ef16ac8f14d1c64266fa315a833a1acf8cc008e Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sun, 25 Aug 2019 20:38:08 +0200 Subject: [PATCH] Initial version --- config.go | 92 ++++++++++++++++++++++++++++ config.sample.yaml | 120 +++++++++++++++++++++++++++++++++++++ go.mod | 12 ++++ go.sum | 30 ++++++++++ helpers.go | 37 ++++++++++++ main.go | 132 +++++++++++++++++++++++++++++++++++++++++ provider.go | 37 ++++++++++++ provider_domainList.go | 48 +++++++++++++++ provider_hostFile.go | 72 ++++++++++++++++++++++ 9 files changed, 580 insertions(+) create mode 100644 config.go create mode 100644 config.sample.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 helpers.go create mode 100644 main.go create mode 100644 provider.go create mode 100644 provider_domainList.go create mode 100644 provider_hostFile.go diff --git a/config.go b/config.go new file mode 100644 index 0000000..296251d --- /dev/null +++ b/config.go @@ -0,0 +1,92 @@ +package main + +import ( + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "text/template" + + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +const defaultTemplate = `$TTL 1H + +@ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h) + NS LOCALHOST. + +; Blacklist entries +{{ range .blacklist -}} +{{ to_punycode .Domain }} CNAME . ; {{ .Comment }} +{{ end }}` + +type providerType string + +type providerAction string + +const ( + providerActionBlacklist providerAction = "blacklist" + providerActionWhitelist providerAction = "whitelist" +) + +type configfile struct { + Providers []providerDefinition `yaml:"providers"` + + Template string `yaml:"template"` + tpl *template.Template +} + +type providerDefinition struct { + Action providerAction `yaml:"action"` + Content string `yaml:"content"` + File string `yaml:"file"` + Name string `yaml:"name"` + Type providerType `yaml:"type"` + URL string `yaml:"url"` +} + +func (p providerDefinition) GetContent() (io.ReadCloser, error) { + switch { + + case p.Content != "": + return ioutil.NopCloser(strings.NewReader(p.Content)), nil + + case p.File != "": + return os.Open(p.File) + + case p.URL != "": + resp, err := http.Get(p.URL) + return resp.Body, err + + default: + return nil, errors.New("Neither file nor URL specified") + + } +} + +func loadConfigFile(filename string) (*configfile, error) { + if _, err := os.Stat(filename); err != nil { + return nil, errors.Wrap(err, "Unable to access given file") + } + + f, err := os.Open(filename) + if err != nil { + return nil, errors.Wrap(err, "Unable to open given file") + } + defer f.Close() + + out := &configfile{Template: defaultTemplate} + if err = yaml.NewDecoder(f).Decode(out); err != nil { + return nil, errors.Wrap(err, "Unable to parse given file") + } + + if out.tpl, err = template.New("configTemplate").Funcs(template.FuncMap{ + "to_punycode": domainToPunycode, + }).Parse(out.Template); err != nil { + return nil, errors.Wrap(err, "Unable to parse given template") + } + + return out, nil +} diff --git a/config.sample.yaml b/config.sample.yaml new file mode 100644 index 0000000..f62bcd9 --- /dev/null +++ b/config.sample.yaml @@ -0,0 +1,120 @@ +--- + +# List of third-party lists to download and include into generated +# blacklist zone file (entries are just examples and copied from the +# source of https://github.com/StevenBlack/hosts#sources-of-hosts-data-unified-in-this-variant) +# Please verify the list matches your interest or compile your own! +# +# Whitelists are applied AFTER all blacklists are compiled together +# which means an entry in the whitelist will finally remove the domain +# from the whole blacklist. Provider order does not matter in this case. +providers: + + #- name: Local blacklist + # file: blacklist.local + # action: blacklist + # type: domain-list + + #- name: Local whitelist + # file: whitelist.local + # action: whitelist + # type: domain-list + + #- name: Local whitelist + # content: | + # my.domain.com + # action: whitelist + # type: domain-list + + - name: Steven Black's ad-hoc list # License: MIT, URL: https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts + url: https://raw.githubusercontent.com/StevenBlack/hosts/master/data/StevenBlack/hosts + action: blacklist + type: hosts-file + + - name: Malware Domain List # License: can be used for free by anyone, URL: https://www.malwaredomainlist.com/ + url: https://www.malwaredomainlist.com/hostslist/hosts.txt + action: blacklist + type: hosts-file + + - name: add.Dead # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts + action: blacklist + type: hosts-file + + - name: hostsVN # License: MIT, URL: https://github.com/bigdargon/hostsVN + url: https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/hosts-VN + action: blacklist + type: hosts-file + + - name: add.Spam # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts + action: blacklist + type: hosts-file + + - name: Dan Pollock - someonewhocares # License: non-commercial with attribution, URL:https://someonewhocares.org/hosts/ + url: https://someonewhocares.org/hosts/zero/hosts + action: blacklist + type: hosts-file + + - name: MVPS hosts file # License: CC BY-NC-SA 4.0, URL: http://winhelp2002.mvps.org/ + url: http://winhelp2002.mvps.org/hosts.txt + action: blacklist + type: hosts-file + + - name: yoyo.org # URL: https://pgl.yoyo.org/adservers/ + url: https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0 + action: blacklist + type: hosts-file + + - name: Mitchell Krog's - Badd Boyz Hosts # License: non-commercial with attribution, URL: https://github.com/mitchellkrogza/Badd-Boyz-Hosts + url: https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts + action: blacklist + type: hosts-file + + - name: CoinBlocker # License: GPLv3, URL: https://gitlab.com/ZeroDot1/CoinBlockerLists + url: https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser + action: blacklist + type: hosts-file + + - name: UncheckyAds # URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts + action: blacklist + type: hosts-file + + - name: add.2o7Net # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts + action: blacklist + type: hosts-file + + - name: KADhosts # License: GPLv3, URL: https://github.com/azet12/KADhosts + url: https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt + action: blacklist + type: hosts-file + + - name: AdAway # License: CC BY 3.0, URL: https://adaway.org/ + url: https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt + action: blacklist + type: hosts-file + + - name: add.Risk # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts + action: blacklist + type: hosts-file + + - name: Tiuxo hostlist - ads # License: CC BY 4.0, URL: https://github.com/tiuxo/hosts + url: https://raw.githubusercontent.com/tiuxo/hosts/master/ads + action: blacklist + type: hosts-file + +template: | + $TTL 1H + + @ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h) + NS LOCALHOST. + + ; Blacklist entries + {{ range .blacklist -}} + {{ to_punycode .Domain }} CNAME . ; {{ .Comment }} + {{ end }} + +... diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..21a161d --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/Luzifer/named-blacklist + +go 1.12 + +require ( + github.com/Luzifer/go_helpers/v2 v2.9.1 + github.com/Luzifer/rconfig/v2 v2.2.1 + github.com/pkg/errors v0.8.1 + github.com/sirupsen/logrus v1.4.2 + golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..96264bc --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/Luzifer/go_helpers/v2 v2.9.1 h1:MVUOlD6tJ2m/iTF0hllnI/QVZH5kI+TikUm1WRGg/c4= +github.com/Luzifer/go_helpers/v2 v2.9.1/go.mod h1:ZnWxPjyCdQ4rZP3kNiMSUW/7FigU1X9Rz8XopdJ5ZCU= +github.com/Luzifer/rconfig/v2 v2.2.1 h1:zcDdLQlnlzwcBJ8E0WFzOkQE1pCMn3EbX0dFYkeTczg= +github.com/Luzifer/rconfig/v2 v2.2.1/go.mod h1:OKIX0/JRZrPJ/ZXXWklQEFXA6tBfWaljZbW37w+sqBw= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/leekchan/gtf v0.0.0-20190214083521-5fba33c5b00b/go.mod h1:thNruaSwydMhkQ8dXzapABF9Sc1Tz08ZBcDdgott9RA= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 h1:WB265cn5OpO+hK3pikC9hpP1zI/KTwmyMFKloW9eOVc= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..3355b45 --- /dev/null +++ b/helpers.go @@ -0,0 +1,37 @@ +package main + +import ( + "strings" + + "golang.org/x/net/idna" + + "github.com/Luzifer/go_helpers/v2/str" +) + +var genericBlacklist = []string{ + "broadcasthost", + "ip6-allhosts", + "ip6-allnodes", + "ip6-allrouters", + "ip6-localnet", + "ip6-mcastprefix", + "local", + "localhost", + "localhost.localdomain", +} + +func lineIsComment(line string) bool { + if len(strings.TrimSpace(line)) == 0 { + return true + } + + return line[0] == '#' || line[0] == ';' || line[0] == '!' +} + +func isBlacklisted(domain string) bool { + return str.StringInSlice(domain, genericBlacklist) +} + +func domainToPunycode(name string, v ...string) (string, error) { + return idna.ToASCII(name) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..0749df7 --- /dev/null +++ b/main.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "os" + "sort" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig/v2" +) + +var ( + cfg = struct { + Config string `flag:"config" default:"config.yaml" description:"Config file to use for generating the file"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + config *configfile + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("named-blacklist %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + var ( + blacklist []entry + whitelist []entry + write = new(sync.Mutex) + wg sync.WaitGroup + err error + ) + + if config, err = loadConfigFile(cfg.Config); err != nil { + log.WithError(err).Fatal("Unable to read config file") + } + + wg.Add(len(config.Providers)) + for _, p := range config.Providers { + + go func(p providerDefinition) { + defer wg.Done() + + entries, err := getDomainList(p) + if err != nil { + log.WithField("provider", p.Name). + WithError(err). + Fatal("Unable to get domain list") + } + + write.Lock() + defer write.Unlock() + + for _, e := range entries { + switch p.Action { + + case providerActionBlacklist: + blacklist = addIfNotExists(blacklist, e) + + case providerActionWhitelist: + whitelist = addIfNotExists(whitelist, e) + + default: + log.WithField("provider", p.Name).Fatalf("Inavlid action %q", p.Action) + + } + } + }(p) + + } + + wg.Wait() + + blacklist = cleanFromList(blacklist, whitelist) + + sort.Slice(blacklist, func(i, j int) bool { return blacklist[i].Domain < blacklist[j].Domain }) + + config.tpl.Execute(os.Stdout, map[string]interface{}{ + "blacklist": blacklist, + }) +} + +func addIfNotExists(entries []entry, e entry) []entry { + for _, pe := range entries { + if pe.Domain == e.Domain { + // Entry already exists, skip + return entries + } + } + + return append(entries, e) +} + +func cleanFromList(blacklist, whitelist []entry) []entry { + var tmp []entry + + for _, be := range blacklist { + var found bool + + for _, we := range whitelist { + if we.Domain == be.Domain { + found = true + break + } + } + + if !found { + tmp = append(tmp, be) + } + } + + return tmp +} diff --git a/provider.go b/provider.go new file mode 100644 index 0000000..938328e --- /dev/null +++ b/provider.go @@ -0,0 +1,37 @@ +package main + +import ( + "sync" + + "github.com/pkg/errors" +) + +var ( + providerRegistry = map[providerType]provider{} + providerRegistryLock sync.Mutex +) + +type entry struct { + Domain string + Comment string +} + +type provider interface { + GetDomainList(providerDefinition) ([]entry, error) +} + +func registerProvider(t providerType, p provider) { + providerRegistryLock.Lock() + defer providerRegistryLock.Unlock() + + providerRegistry[t] = p +} + +func getDomainList(p providerDefinition) ([]entry, error) { + pro, ok := providerRegistry[p.Type] + if !ok { + return nil, errors.Errorf("Unknown provider type %q", p.Type) + } + + return pro.GetDomainList(p) +} diff --git a/provider_domainList.go b/provider_domainList.go new file mode 100644 index 0000000..1f10a3c --- /dev/null +++ b/provider_domainList.go @@ -0,0 +1,48 @@ +package main + +import ( + "bufio" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func init() { + registerProvider("domain-list", providerdomainList{}) +} + +type providerdomainList struct{} + +func (p providerdomainList) GetDomainList(d providerDefinition) ([]entry, error) { + r, err := d.GetContent() + if err != nil { + return nil, errors.Wrap(err, "Unable to get source content") + } + defer r.Close() + + logger := log.WithField("provider", d.Name) + + var entries []entry + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if lineIsComment(scanner.Text()) { + continue + } + + domain := strings.TrimSpace(scanner.Text()) + + if isBlacklisted(domain) { + logger.WithField("domain", domain).Debug("Skipping because of blacklist") + continue + } + + entries = append(entries, entry{ + Domain: domain, + Comment: d.Name, + }) + } + + return entries, nil +} diff --git a/provider_hostFile.go b/provider_hostFile.go new file mode 100644 index 0000000..2bcbe46 --- /dev/null +++ b/provider_hostFile.go @@ -0,0 +1,72 @@ +package main + +import ( + "bufio" + "fmt" + "regexp" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func init() { + registerProvider("hosts-file", providerHostFile{}) +} + +type providerHostFile struct{} + +func (p providerHostFile) GetDomainList(d providerDefinition) ([]entry, error) { + r, err := d.GetContent() + if err != nil { + return nil, errors.Wrap(err, "Unable to get source content") + } + defer r.Close() + + logger := log.WithField("provider", d.Name) + + var ( + entries []entry + matcher = regexp.MustCompile(`^(?:[0-9.]+|[a-z0-9:]+)\s+([^\s]+)(?:\s+#(.+))?$`) + ) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if lineIsComment(line) { + continue + } + + if !matcher.MatchString(line) { + logger.WithField("line", line).Warn("Invalid line found (format)") + continue + } + + groups := matcher.FindStringSubmatch(line) + if len(groups) < 2 { + logger.WithField("line", line).Warn("Invalid line found (groups)") + continue + } + + if isBlacklisted(groups[1]) { + logger.WithField("domain", groups[1]).Debug("Skipping because of blacklist") + continue + } + + comment := fmt.Sprintf("From: %q", d.Name) + if len(groups) == 3 && strings.Trim(groups[2], "#") != "" { + comment = fmt.Sprintf("%s, Comment: %q", + comment, + strings.TrimSpace(strings.Trim(groups[2], "#")), + ) + } + + entries = append(entries, entry{ + Domain: groups[1], + Comment: comment, + }) + } + + return entries, nil +}