commit 0ef16ac8f14d1c64266fa315a833a1acf8cc008e Author: Knut Ahlers Date: Sun Aug 25 20:38:08 2019 +0200 Initial version diff --git a/config.go b/config.go new file mode 100644 index 0000000..296251d --- /dev/null +++ b/config.go @@ -0,0 +1,92 @@ +package main + +import ( + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "text/template" + + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +const defaultTemplate = `$TTL 1H + +@ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h) + NS LOCALHOST. + +; Blacklist entries +{{ range .blacklist -}} +{{ to_punycode .Domain }} CNAME . ; {{ .Comment }} +{{ end }}` + +type providerType string + +type providerAction string + +const ( + providerActionBlacklist providerAction = "blacklist" + providerActionWhitelist providerAction = "whitelist" +) + +type configfile struct { + Providers []providerDefinition `yaml:"providers"` + + Template string `yaml:"template"` + tpl *template.Template +} + +type providerDefinition struct { + Action providerAction `yaml:"action"` + Content string `yaml:"content"` + File string `yaml:"file"` + Name string `yaml:"name"` + Type providerType `yaml:"type"` + URL string `yaml:"url"` +} + +func (p providerDefinition) GetContent() (io.ReadCloser, error) { + switch { + + case p.Content != "": + return ioutil.NopCloser(strings.NewReader(p.Content)), nil + + case p.File != "": + return os.Open(p.File) + + case p.URL != "": + resp, err := http.Get(p.URL) + return resp.Body, err + + default: + return nil, errors.New("Neither file nor URL specified") + + } +} + +func loadConfigFile(filename string) (*configfile, error) { + if _, err := os.Stat(filename); err != nil { + return nil, errors.Wrap(err, "Unable to access given file") + } + + f, err := os.Open(filename) + if err != nil { + return nil, errors.Wrap(err, "Unable to open given file") + } + defer f.Close() + + out := &configfile{Template: defaultTemplate} + if err = yaml.NewDecoder(f).Decode(out); err != nil { + return nil, errors.Wrap(err, "Unable to parse given file") + } + + if out.tpl, err = template.New("configTemplate").Funcs(template.FuncMap{ + "to_punycode": domainToPunycode, + }).Parse(out.Template); err != nil { + return nil, errors.Wrap(err, "Unable to parse given template") + } + + return out, nil +} diff --git a/config.sample.yaml b/config.sample.yaml new file mode 100644 index 0000000..f62bcd9 --- /dev/null +++ b/config.sample.yaml @@ -0,0 +1,120 @@ +--- + +# List of third-party lists to download and include into generated +# blacklist zone file (entries are just examples and copied from the +# source of https://github.com/StevenBlack/hosts#sources-of-hosts-data-unified-in-this-variant) +# Please verify the list matches your interest or compile your own! +# +# Whitelists are applied AFTER all blacklists are compiled together +# which means an entry in the whitelist will finally remove the domain +# from the whole blacklist. Provider order does not matter in this case. +providers: + + #- name: Local blacklist + # file: blacklist.local + # action: blacklist + # type: domain-list + + #- name: Local whitelist + # file: whitelist.local + # action: whitelist + # type: domain-list + + #- name: Local whitelist + # content: | + # my.domain.com + # action: whitelist + # type: domain-list + + - name: Steven Black's ad-hoc list # License: MIT, URL: https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts + url: https://raw.githubusercontent.com/StevenBlack/hosts/master/data/StevenBlack/hosts + action: blacklist + type: hosts-file + + - name: Malware Domain List # License: can be used for free by anyone, URL: https://www.malwaredomainlist.com/ + url: https://www.malwaredomainlist.com/hostslist/hosts.txt + action: blacklist + type: hosts-file + + - name: add.Dead # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts + action: blacklist + type: hosts-file + + - name: hostsVN # License: MIT, URL: https://github.com/bigdargon/hostsVN + url: https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/hosts-VN + action: blacklist + type: hosts-file + + - name: add.Spam # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts + action: blacklist + type: hosts-file + + - name: Dan Pollock - someonewhocares # License: non-commercial with attribution, URL:https://someonewhocares.org/hosts/ + url: https://someonewhocares.org/hosts/zero/hosts + action: blacklist + type: hosts-file + + - name: MVPS hosts file # License: CC BY-NC-SA 4.0, URL: http://winhelp2002.mvps.org/ + url: http://winhelp2002.mvps.org/hosts.txt + action: blacklist + type: hosts-file + + - name: yoyo.org # URL: https://pgl.yoyo.org/adservers/ + url: https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0 + action: blacklist + type: hosts-file + + - name: Mitchell Krog's - Badd Boyz Hosts # License: non-commercial with attribution, URL: https://github.com/mitchellkrogza/Badd-Boyz-Hosts + url: https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts + action: blacklist + type: hosts-file + + - name: CoinBlocker # License: GPLv3, URL: https://gitlab.com/ZeroDot1/CoinBlockerLists + url: https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser + action: blacklist + type: hosts-file + + - name: UncheckyAds # URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts + action: blacklist + type: hosts-file + + - name: add.2o7Net # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts + action: blacklist + type: hosts-file + + - name: KADhosts # License: GPLv3, URL: https://github.com/azet12/KADhosts + url: https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt + action: blacklist + type: hosts-file + + - name: AdAway # License: CC BY 3.0, URL: https://adaway.org/ + url: https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt + action: blacklist + type: hosts-file + + - name: add.Risk # License: GPLv3+, URL: https://github.com/FadeMind/hosts.extras + url: https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts + action: blacklist + type: hosts-file + + - name: Tiuxo hostlist - ads # License: CC BY 4.0, URL: https://github.com/tiuxo/hosts + url: https://raw.githubusercontent.com/tiuxo/hosts/master/ads + action: blacklist + type: hosts-file + +template: | + $TTL 1H + + @ SOA LOCALHOST. dns-master.localhost. (1 1h 15m 30d 2h) + NS LOCALHOST. + + ; Blacklist entries + {{ range .blacklist -}} + {{ to_punycode .Domain }} CNAME . ; {{ .Comment }} + {{ end }} + +... diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..21a161d --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/Luzifer/named-blacklist + +go 1.12 + +require ( + github.com/Luzifer/go_helpers/v2 v2.9.1 + github.com/Luzifer/rconfig/v2 v2.2.1 + github.com/pkg/errors v0.8.1 + github.com/sirupsen/logrus v1.4.2 + golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..96264bc --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/Luzifer/go_helpers/v2 v2.9.1 h1:MVUOlD6tJ2m/iTF0hllnI/QVZH5kI+TikUm1WRGg/c4= +github.com/Luzifer/go_helpers/v2 v2.9.1/go.mod h1:ZnWxPjyCdQ4rZP3kNiMSUW/7FigU1X9Rz8XopdJ5ZCU= +github.com/Luzifer/rconfig/v2 v2.2.1 h1:zcDdLQlnlzwcBJ8E0WFzOkQE1pCMn3EbX0dFYkeTczg= +github.com/Luzifer/rconfig/v2 v2.2.1/go.mod h1:OKIX0/JRZrPJ/ZXXWklQEFXA6tBfWaljZbW37w+sqBw= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/leekchan/gtf v0.0.0-20190214083521-5fba33c5b00b/go.mod h1:thNruaSwydMhkQ8dXzapABF9Sc1Tz08ZBcDdgott9RA= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 h1:WB265cn5OpO+hK3pikC9hpP1zI/KTwmyMFKloW9eOVc= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..3355b45 --- /dev/null +++ b/helpers.go @@ -0,0 +1,37 @@ +package main + +import ( + "strings" + + "golang.org/x/net/idna" + + "github.com/Luzifer/go_helpers/v2/str" +) + +var genericBlacklist = []string{ + "broadcasthost", + "ip6-allhosts", + "ip6-allnodes", + "ip6-allrouters", + "ip6-localnet", + "ip6-mcastprefix", + "local", + "localhost", + "localhost.localdomain", +} + +func lineIsComment(line string) bool { + if len(strings.TrimSpace(line)) == 0 { + return true + } + + return line[0] == '#' || line[0] == ';' || line[0] == '!' +} + +func isBlacklisted(domain string) bool { + return str.StringInSlice(domain, genericBlacklist) +} + +func domainToPunycode(name string, v ...string) (string, error) { + return idna.ToASCII(name) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..0749df7 --- /dev/null +++ b/main.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "os" + "sort" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig/v2" +) + +var ( + cfg = struct { + Config string `flag:"config" default:"config.yaml" description:"Config file to use for generating the file"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + config *configfile + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("named-blacklist %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + var ( + blacklist []entry + whitelist []entry + write = new(sync.Mutex) + wg sync.WaitGroup + err error + ) + + if config, err = loadConfigFile(cfg.Config); err != nil { + log.WithError(err).Fatal("Unable to read config file") + } + + wg.Add(len(config.Providers)) + for _, p := range config.Providers { + + go func(p providerDefinition) { + defer wg.Done() + + entries, err := getDomainList(p) + if err != nil { + log.WithField("provider", p.Name). + WithError(err). + Fatal("Unable to get domain list") + } + + write.Lock() + defer write.Unlock() + + for _, e := range entries { + switch p.Action { + + case providerActionBlacklist: + blacklist = addIfNotExists(blacklist, e) + + case providerActionWhitelist: + whitelist = addIfNotExists(whitelist, e) + + default: + log.WithField("provider", p.Name).Fatalf("Inavlid action %q", p.Action) + + } + } + }(p) + + } + + wg.Wait() + + blacklist = cleanFromList(blacklist, whitelist) + + sort.Slice(blacklist, func(i, j int) bool { return blacklist[i].Domain < blacklist[j].Domain }) + + config.tpl.Execute(os.Stdout, map[string]interface{}{ + "blacklist": blacklist, + }) +} + +func addIfNotExists(entries []entry, e entry) []entry { + for _, pe := range entries { + if pe.Domain == e.Domain { + // Entry already exists, skip + return entries + } + } + + return append(entries, e) +} + +func cleanFromList(blacklist, whitelist []entry) []entry { + var tmp []entry + + for _, be := range blacklist { + var found bool + + for _, we := range whitelist { + if we.Domain == be.Domain { + found = true + break + } + } + + if !found { + tmp = append(tmp, be) + } + } + + return tmp +} diff --git a/provider.go b/provider.go new file mode 100644 index 0000000..938328e --- /dev/null +++ b/provider.go @@ -0,0 +1,37 @@ +package main + +import ( + "sync" + + "github.com/pkg/errors" +) + +var ( + providerRegistry = map[providerType]provider{} + providerRegistryLock sync.Mutex +) + +type entry struct { + Domain string + Comment string +} + +type provider interface { + GetDomainList(providerDefinition) ([]entry, error) +} + +func registerProvider(t providerType, p provider) { + providerRegistryLock.Lock() + defer providerRegistryLock.Unlock() + + providerRegistry[t] = p +} + +func getDomainList(p providerDefinition) ([]entry, error) { + pro, ok := providerRegistry[p.Type] + if !ok { + return nil, errors.Errorf("Unknown provider type %q", p.Type) + } + + return pro.GetDomainList(p) +} diff --git a/provider_domainList.go b/provider_domainList.go new file mode 100644 index 0000000..1f10a3c --- /dev/null +++ b/provider_domainList.go @@ -0,0 +1,48 @@ +package main + +import ( + "bufio" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func init() { + registerProvider("domain-list", providerdomainList{}) +} + +type providerdomainList struct{} + +func (p providerdomainList) GetDomainList(d providerDefinition) ([]entry, error) { + r, err := d.GetContent() + if err != nil { + return nil, errors.Wrap(err, "Unable to get source content") + } + defer r.Close() + + logger := log.WithField("provider", d.Name) + + var entries []entry + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if lineIsComment(scanner.Text()) { + continue + } + + domain := strings.TrimSpace(scanner.Text()) + + if isBlacklisted(domain) { + logger.WithField("domain", domain).Debug("Skipping because of blacklist") + continue + } + + entries = append(entries, entry{ + Domain: domain, + Comment: d.Name, + }) + } + + return entries, nil +} diff --git a/provider_hostFile.go b/provider_hostFile.go new file mode 100644 index 0000000..2bcbe46 --- /dev/null +++ b/provider_hostFile.go @@ -0,0 +1,72 @@ +package main + +import ( + "bufio" + "fmt" + "regexp" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func init() { + registerProvider("hosts-file", providerHostFile{}) +} + +type providerHostFile struct{} + +func (p providerHostFile) GetDomainList(d providerDefinition) ([]entry, error) { + r, err := d.GetContent() + if err != nil { + return nil, errors.Wrap(err, "Unable to get source content") + } + defer r.Close() + + logger := log.WithField("provider", d.Name) + + var ( + entries []entry + matcher = regexp.MustCompile(`^(?:[0-9.]+|[a-z0-9:]+)\s+([^\s]+)(?:\s+#(.+))?$`) + ) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if lineIsComment(line) { + continue + } + + if !matcher.MatchString(line) { + logger.WithField("line", line).Warn("Invalid line found (format)") + continue + } + + groups := matcher.FindStringSubmatch(line) + if len(groups) < 2 { + logger.WithField("line", line).Warn("Invalid line found (groups)") + continue + } + + if isBlacklisted(groups[1]) { + logger.WithField("domain", groups[1]).Debug("Skipping because of blacklist") + continue + } + + comment := fmt.Sprintf("From: %q", d.Name) + if len(groups) == 3 && strings.Trim(groups[2], "#") != "" { + comment = fmt.Sprintf("%s, Comment: %q", + comment, + strings.TrimSpace(strings.Trim(groups[2], "#")), + ) + } + + entries = append(entries, entry{ + Domain: groups[1], + Comment: comment, + }) + } + + return entries, nil +}