Move to zone replication

Signed-off-by: Knut Ahlers <knut@ahlers.me>

.gitignore

@@ -1,2 +1,3 @@
-zones/*
+.env
 *.pyc
+zones/*

Dockerfile

@@ -16,8 +16,5 @@ EXPOSE 53/udp 53
 
 VOLUME ["/src/zones"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-  CMD dig +short @localhost health.server.test TXT || exit 1
-
 ENTRYPOINT ["/src/docker-entrypoint.sh"]
 CMD ["named", "-g"]

Makefile

@@ -4,9 +4,6 @@ container:
 	docker build --no-cache --pull -t luzifer/dns .
 	bash -eo pipefail -c '[ "$(REF)" == "refs/heads/master" ] && docker push luzifer/dns || true'
 
-check_zones: .venv
-	./.venv/bin/python3 checkZonefile.py
-
 .venv:
 	virtualenv --python=python3 .venv
 	./.venv/bin/pip3 install -r requirements.txt
@@ -15,4 +12,4 @@ alpine-prereq:
 	apk --no-cache add make python3
 	pip3 install virtualenv
 
-.PHONY: check_zones container
+.PHONY: container

checkZonefile.py

@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import generateZonefiles as generator
-
-import re
-import sys
-import yaml
-
-MATCH_FQDN = '(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}.$)'
-
-result_code = 0
-
-
-class bcolors:
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-
-def error(msg):
-    global result_code
-    print(bcolors.FAIL + "[ERR] {}".format(msg) + bcolors.ENDC)
-    result_code = 1
-
-
-def fatal(msg):
-    print(bcolors.FAIL + '[FAT] {}'.format(msg) + bcolors.ENDC)
-    sys.exit(1)
-
-
-def warn(msg):
-    print(bcolors.WARNING + "[WRN] {}".format(msg) + bcolors.ENDC)
-
-
-def check_entry(zone_name, entry):
-    optional_entry = ['name', 'type', 'ttl', 'class', 'records', 'alias']
-
-    if 'name' not in entry:
-        error('Zone "{}" contains entry without name'.format(zone_name))
-        return
-
-    if 'alias' not in entry and 'records' not in entry:
-        error('Zone "{}" - Entry "{}" - Neither alias nor records specified'.format(
-              zone_name, entry['name']))
-        return
-
-    for k in entry.keys():
-        if k not in optional_entry:
-            warn('Zone "{}" - Entry "{}" - Unexpected key in entry found: {}'.format(
-                zone_name, entry['name'], k))
-
-    try:
-        parsed_entries = generator.sanitize(entry)
-    except:
-        error('Zone "{}" - Entry "{}" - Alias is not resolvable'.format(
-              zone_name, entry['name']))
-        return
-
-    if len(parsed_entries) == 0:
-        warn('Zone "{}" - Entry "{}" - Resolved to 0 resource records!'.format(
-            zone_name, entry['name']))
-
-    for entry in parsed_entries:
-        if entry['type'] == 'CNAME' and entry['data'][-1] != '.':
-            warn('Zone "{}" - Entry "{}" - CNAME data has no ending dot, pointing to same domain.'.format(
-                zone_name, entry['name']))
-
-    # TODO(kahlers): Add more checks:
-    # - Type is valid
-    # - Class is valid
-
-
-def check_soa(soa, nameservers):
-    # SOA check
-    expected_soa = ['auth_ns', 'contact', 'refresh', 'retry', 'expire', 'ttl']
-    for e in expected_soa:
-        if e not in soa:
-            fatal('Missing required attribute {} in soa'.format(e))
-
-    for k in soa.keys():
-        if k not in expected_soa:
-            warn('Unexpected entry in soa found: {}'.format(k))
-
-    if soa['auth_ns'] not in nameservers:
-        error('SOA auth nameserver not in nameserver list')
-
-    if not re.match("^[a-z.]+.$", soa['contact']):
-        error('SOA contact does not match specificiation')
-
-    if not 1200 <= soa['refresh'] <= 43200:
-        warn('SOA refresh out of recommended bounds 1200 - 43200')
-
-    if not 180 <= soa['retry'] <= 900:
-        warn('SOA retry out of suggested bounds 180 - 900')
-
-    if not 1209600 <= soa['expire'] <= 2419200:
-        warn('SOA expire out of recommended bounds 1209600 - 2419200')
-
-    if not 300 <= soa['ttl'] <= 86400:
-        warn('SOA minimum ttl out of recommended bounds 300 - 86400')
-
-
-def check_mailserver_sets(mailserver_sets):
-    for name, set in mailserver_sets.items():
-        for mx, weight in set.items():
-            if not str(weight).isdigit():
-                error('Mailserver Set {} - entry {} has non-digit weight {}'.format(
-                    name, mx, weight))
-
-            if not re.match(MATCH_FQDN, mx):
-                warn('Mailserver Set {} - MX {} does not match fqdn regexp'.format(
-                    name, mx))
-
-
-def check_nameserver(nameservers):
-    if len(nameservers) < 2:
-        error('Number of nameservers below required 2 servers')
-
-    if not 3 <= len(nameservers) <= 7:
-        warn('Number of nameservers out of recommended bounds 3 - 7 (RFC2182)')
-
-    for n in nameservers:
-        if not re.match(MATCH_FQDN, n):
-            warn('Nameserver {} does not match fqdn regexp'.format(n))
-
-    # TODO(kahlers): Add more checks:
-    # - Nameservers do not have same IP
-    # - Nameservers are reachable
-    # - Nameservers are not in same AS
-
-
-def check_zone(name, config):
-    expected_zone = ['mailserver', 'mailserver_set',
-                     'entries', 'default_ttl', 'from_consul']
-
-    for k in config.keys():
-        if k not in expected_zone:
-            warn('Unexpected entry in zone {} found: {}'.format(name, k))
-
-    if 'mailserver' not in config and 'mailserver_set' not in config and ('entries' not in config or len(config['entries']) == 0) and 'from_consul' not in config:
-        warn('Zone {} has no mailservers and no entries'.format(name))
-
-    if 'mailserver' in config and 'mailserver_set' in config:
-        error('Zone {} contains mailserver and mailserver_set'.format(name))
-
-    if 'from_consul' in config and config['from_consul'] and 'entries' in config:
-        error('Zone {} contains entries and from_consul flag'.format(name))
-
-    if 'from_consul' in config and config['from_consul'] and 'mailserver' in config:
-        warn('Zone {} contains mailserver and from_consul flag'.format(name))
-
-    if 'mailserver' in config:
-        for mx, weight in config['mailserver'].items():
-            if not str(weight).isdigit():
-                error('Zone {} - MX entry {} has non-digit weight {}'.format(
-                    name, mx, weight))
-
-            if not re.match(MATCH_FQDN, mx):
-                warn('Zone {} - MX {} does not match fqdn regexp'.format(name, mx))
-
-    if 'entries' in config:
-        for e in config['entries']:
-            check_entry(name, e)
-
-
-def main():
-    zone_data = yaml.load(open("zones.yml"))
-
-    if 'soa' not in zone_data:
-        fatal("SOA configuration not found")
-
-    if 'nameserver' not in zone_data:
-        fatal("Nameserver list not found")
-
-    if 'zones' not in zone_data:
-        error('No zones configuration found')
-
-    check_soa(zone_data['soa'], zone_data['nameserver'])
-    check_nameserver(zone_data['nameserver'])
-
-    for name, config in zone_data['zones'].items():
-        check_zone(name, config)
-
-    return result_code
-
-
-if __name__ == "__main__":
-    sys.exit(main())

consul.py

@@ -21,54 +21,10 @@ def get_zones():
     if resp.status_code == 404:
         return []
 
-    zones = {}
+    zones = []
     for key in resp.json():
         zone = key.split('/')[1]
         if zone not in zones:
-            zones[zone] = {'from_consul': True}
+            zones.append(zone)
 
     return zones
-
-
-def query_zone_entries(zone):
-    if os.getenv('CONSUL_HTTP_ADDR') == '' or os.getenv('CONSUL_HTTP_TOKEN') == '':
-        raise Exception(
-            'Consul query does not work with CONSUL_HTTP_ADDR or CONSUL_HTTP_TOKEN unset')
-
-    return parse_raw_consul(zone)
-
-
-def read_raw_from_consul(zone):
-    resp = requests.get(
-        '{}/v1/kv/dns/{}?recurse=true'.format(
-            os.getenv('CONSUL_HTTP_ADDR'),
-            zone.rstrip('.'),
-        ),
-        headers={
-            'X-Consul-Token': os.getenv('CONSUL_HTTP_TOKEN'),
-        }
-    )
-
-    if resp.status_code == 404:
-        return []
-
-    return resp.json()
-
-
-def parse_raw_consul(zone):
-    entries = []
-
-    for raw_entry in read_raw_from_consul(zone):
-        sub_entries = json.loads(base64.b64decode(raw_entry['Value']))
-
-        # Key consists of at least 2 elements: dns/ahlers.me/subdomain OR dns/ahlers.me
-        key = raw_entry['Key'].split('/')[2:]
-        name = ''
-        if key not in [[], ['@']]:
-            name = '.'.join(reversed(key))
-
-        for entry in sub_entries:
-            entry['name'] = name
-            entries.append(entry)
-
-    return entries

generateZonefiles.py

@@ -13,13 +13,10 @@ import time
 import consul
 
 # Third-party imports
-import dns.resolver
-import dns.rdatatype
 import jinja2
 import requests
 import yaml
 
-DEFAULT_TTL = 3600
 
 rndc_queue = []
 
@@ -34,12 +31,6 @@ def call_rndc(params):
         subprocess.check_call(command, stdout=sys.stdout, stderr=sys.stderr)
 
 
-def default(d, key, default=None):
-    if key in d:
-        return d[key]
-    return default
-
-
 def diff_files(file1, file2):
     fromlines = []
     tolines = []
@@ -71,8 +62,6 @@ def hash_file(filename):
     with open(filename, 'r') as afile:
         lines = afile.readlines()
 
-    lines = map(replace_soa_line, lines)
-
     hasher.update(''.join(lines).encode('utf-8'))
     return hasher.hexdigest()
 
@@ -83,59 +72,6 @@ def queue_rndc_call(params):
     rndc_queue.append(params)
 
 
-def replace_soa_line(line):
-    if 'SOA' in line:
-        return '; SOA line replaced'
-    return line
-
-
-def resolve_alias(entry):
-    result = []
-
-    answers = []
-    answers.extend(dns.resolver.query(
-        entry['alias'], 'A', raise_on_no_answer=False))
-    answers.extend(dns.resolver.query(
-        entry['alias'], 'AAAA', raise_on_no_answer=False))
-
-    if len(answers) == 0:
-        raise Exception(
-            "Alias {} was not resolvable: No answers!".format(entry['alias']))
-
-    for rdata in answers:
-        new_entry = entry.copy()
-        del new_entry['alias']
-        new_entry['type'] = dns.rdatatype.to_text(rdata.rdtype)
-        new_entry['data'] = rdata.address
-        result.append(new_entry)
-
-    return sorted(result, key=lambda k: k['data'])
-
-
-def sanitize(entry):
-    result = []
-
-    if entry['name'] == '':
-        entry['name'] = '@'
-
-    if 'alias' in entry:
-        return resolve_alias(entry)
-
-    for rr in entry['records']:
-        new_entry = entry.copy()
-        del new_entry['records']
-        new_entry['data'] = rr
-
-        if new_entry['type'] == 'TXT' and new_entry['data'][0] != '"':
-            new_entry['data'] = '"{}"'.format(new_entry['data'])
-
-        result.append(new_entry)
-
-    result.sort(key=lambda k: k['data'])
-
-    return result
-
-
 def write_named_conf(zones):
     tpl = jinja2.Template(open("named.conf").read())
     zone_content = tpl.render({
@@ -155,62 +91,14 @@ def write_named_conf(zones):
         os.unlink("zones/named.conf.new")
 
 
-def write_zone(zone, ttl, soa, nameserver, mailserver, entries):
-    soa['serial'] = int(time.time()) - 946681200  # 2000-01-01
-
-    tpl = jinja2.Template(open("zone_template.j2").read())
-    zone_content = tpl.render({
-        "zone": zone,
-        "ttl": ttl,
-        "soa": soa,
-        "nameserver": nameserver,
-        "mailserver": mailserver,
-        "entries": entries,
-    })
-
-    with open("zones/tmp.{}".format(zone), 'w') as zf:
-        zf.write(zone_content)
-
-    if hash_file("zones/tmp.{}".format(zone)) != hash_file("zones/db.{}".format(zone)):
-        print("Generated and replaced zone file for {}".format(zone))
-        diff_files("zones/db.{}".format(zone), "zones/tmp.{}".format(zone))
-        os.rename("zones/tmp.{}".format(zone), "zones/db.{}".format(zone))
-
-        queue_rndc_call(['reload', zone])
-    else:
-        os.unlink("zones/tmp.{}".format(zone))
-
-
 def healthcheck():
     if os.getenv('HC_PING') is not None:
         requests.get(os.getenv('HC_PING'))
 
 
 def main():
-    zone_data = yaml.load(open("zones.yml"))
-
     consul_zones = consul.get_zones()
-    zone_data['zones'] = {**consul_zones, **zone_data['zones']}
-
-    write_named_conf(zone_data['zones'].keys())
-
-    for zone, config in zone_data['zones'].items():
-        ttl = default(config, "default_ttl", DEFAULT_TTL)
-
-        entries = []
-        for entry in default(config, 'entries', []):
-            entries.extend(sanitize(entry))
-
-        if default(config, 'from_consul', False):
-            for entry in consul.query_zone_entries(zone):
-                entries.extend(sanitize(entry))
-
-        mailserver = default(config, 'mailserver', {})
-        if 'mailserver_set' in config and config['mailserver_set'] in zone_data['mailserver_sets']:
-            mailserver = zone_data['mailserver_sets'][config['mailserver_set']]
-
-        write_zone(zone, ttl, zone_data['soa'],
-                   zone_data['nameserver'], mailserver, entries)
+    write_named_conf(consul_zones)
 
     exec_rndc_queue()
 

named.conf

@@ -33,8 +33,10 @@ options {
 };
 {% for zone in zones %}
 zone "{{ zone }}" IN {
-      type master;
-      file "/src/zones/db.{{ zone }}";
+  type slave;
+  file "/src/zones/db.{{ zone }}.repl";
+  masters port 1053 { 10.231.0.34; };
+  allow-query { any; };
 };
 {% endfor %}
 

requirements.txt

@@ -1,4 +1,3 @@
 jinja2
 PyYAML
-dnspython
 requests

zone_template.j2

@@ -1,18 +0,0 @@
-; Auto-generated using generateZonefiles.py
-
-$ORIGIN {{ zone }}.
-$TTL {{ ttl }}
-
-{{ zone }}. {{ ttl }} IN SOA {{ soa.auth_ns }} {{ soa.contact }} {{ soa.serial }} {{ soa.refresh }} {{ soa.retry }} {{ soa.expire }} {{ soa.ttl }}
-
-{% for ns in nameserver -%}
-{{ zone }}. {{ ttl }} IN NS {{ ns }}
-{% endfor %}
-{%- if mailserver | length > 0 %}
-{% for mailserver, weight in mailserver | dictsort -%}
-{{ zone }}. {{ ttl }} IN MX {{ weight }} {{ mailserver }}
-{% endfor -%}
-{%- endif %}
-{% for entry in entries -%}
-{{ entry.name }} {{ entry.ttl | default(ttl) }} {{ entry.class | default('IN') }} {{ entry.type }} {% if entry.weight %}{{ entry.weight }} {% endif %}{{ entry.data }}
-{% endfor -%}

zones.yml

@@ -1,25 +0,0 @@
----
-
-soa:
-  auth_ns: ns1.kserver.biz.
-  contact: dns.ahlers.me.
-  refresh: 7200
-  retry: 900
-  expire: 1209600
-  ttl: 86400
-
-mailserver_sets:
-
-nameserver:
-  - ns1.kserver.biz.
-  - ns2.kserver.biz.
-  - ns3.kserver.biz.
-
-zones:
-  server.test:
-    entries:
-      - name: health
-        type: TXT
-        records:
-          - "OK"
-...