From faa0db7d8aaf3509db453deceb0d3b8ed0e5381b Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Wed, 30 Jul 2014 19:07:41 -0400 Subject: [PATCH 01/51] adding version to docker client creation --- docker_dns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 6466b1a..af68f2c 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -214,9 +214,9 @@ def main(): # Create docker if CONFIG['docker_url']: - docker_client = docker.Client(CONFIG['docker_url']) + docker_client = docker.Client(CONFIG['docker_url'], CONFIG['version']) else: - docker_client = docker.Client() + docker_client = docker.Client(CONFIG['version']) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) @@ -258,6 +258,7 @@ def main(): # Merge user config over defaults DEFAULT_CONFIG = { 'docker_url': None, + 'version': '1.11', 'bind_interface': '', 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], From abfe4163d68c0ac01a8b2a9560a562cd7d1bebc7 Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Wed, 30 Jul 2014 19:17:11 -0400 Subject: [PATCH 02/51] adding version= to client --- docker_dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index af68f2c..26c1057 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -214,9 +214,9 @@ def main(): # Create docker if CONFIG['docker_url']: - docker_client = docker.Client(CONFIG['docker_url'], CONFIG['version']) + docker_client = docker.Client(CONFIG['docker_url'], 'version'=CONFIG['version']) else: - docker_client = docker.Client(CONFIG['version']) + docker_client = docker.Client('version'=CONFIG['version']) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) From bed9d614258396bbf9b9b4c41ace00b436dea80e Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Thu, 31 Jul 2014 10:36:14 -0400 Subject: [PATCH 03/51] oops --- docker_dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 26c1057..d7d018e 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -214,9 +214,9 @@ def main(): # Create docker if CONFIG['docker_url']: - docker_client = docker.Client(CONFIG['docker_url'], 'version'=CONFIG['version']) + docker_client = docker.Client(CONFIG['docker_url'], version=CONFIG['version']) else: - docker_client = docker.Client('version'=CONFIG['version']) + docker_client = docker.Client(version=CONFIG['version']) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) From 2a4eb870502f42bc73b7b0d4b6e4d276a1480f1f Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 18:03:10 +0000 Subject: [PATCH 04/51] heavy modification for troubleshooting --- docker_dns.py | 128 +++++++++++++++----------------------------------- 1 file changed, 37 insertions(+), 91 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index d7d018e..2f294e5 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -26,37 +26,12 @@ from twisted.python import failure from warnings import warn - -# FIXME replace with a more generic solution like operator.attrgetter -def dict_lookup(dic, key_path, default=None): - """ - Look up value in a nested dict - - Args: - dic: The dictionary to search - key_path: An iterable containing an ordered list of dict keys to - traverse - default: Value to return in case nothing is found - - Returns: - Value of the dict at the nested location given, or default if no value - was found - """ - - for k in key_path: - if k in dic: - dic = dic[k] - else: - return default - return dic - - class DockerMapping(object): """ Look up docker container data """ - id_re = re.compile(r'([a-z0-9]+)\.docker') + id_re = re.compile(r'([a-z0-9]+)\.d') def __init__(self, api): """ @@ -65,29 +40,11 @@ def __init__(self, api): """ self.api = api - - def _ids_from_prop(self, key_path, value): - """ - Get IDs of containers where their config matches a value - - Args: - key_path: An iterable containing an ordered list of container - config keys to traverse - value: What the value at key_path must match to qualify - - Returns: - Generator with a list of containers that match the config value - """ - - return ( - c['ID'] - for c in ( - self.api.inspect_container(c_lite['Id']) - for c_lite in self.api.containers(all=True) - ) - if dict_lookup(c, key_path, None) == value - if 'ID' in c - ) + try: + print('connected to docker instance running api version %s' % \ + self.api.version()['ApiVersion']) + except docker.client.APIError as ex: + raise Exception(ex) def lookup_container(self, name): """ @@ -100,31 +57,27 @@ def lookup_container(self, name): Returns: Container config dict for the first matching container """ + key_path = 'Name' + warn(name) + + cid_all = [ c['Id'] for c in self.api.containers(all=True) ] + print(cid_all) + + for cid in cid_all: + warn(cid) + cdic = self.api.inspect_container(cid_all['Id']) + warn(cdic[key_path]) + if cdic[key_path] == name: + container_id = cdic[key_path] + else: + container_id = None + warn(container_id) - match = self.id_re.match(name) - if match: - container_id = match.group(1) - else: - ids = self._ids_from_prop(('Config', 'Hostname'), unicode(name)) - # FIXME Should be able to support multiple - try: - container_id = ids.next() - except StopIteration: - return None - - try: - return self.api.inspect_container(container_id) - - except docker.client.APIError as ex: - # 404 is valid, others aren't - if ex.response.status_code != 404: - warn(ex) - - return None - - except RequestException as ex: - warn(ex) - return None +# try: + return self.api.inspect_container(container_id) +# except RequestException as ex: +# warn(ex) +# return None def get_a(self, name): """ @@ -138,6 +91,8 @@ def get_a(self, name): """ container = self.lookup_container(name) + print 'container:' + print(container) if container is None: return None @@ -181,6 +136,7 @@ def _a_records(self, name): """ addr = self.mapping.get_a(name) + print addr if not addr: raise DomainError(name) @@ -191,21 +147,11 @@ def _a_records(self, name): ]) def lookupAddress(self, name, timeout=None): - try: - records = self._a_records(name) - return defer.succeed((records, (), ())) - - # We need to catch everything. Uncaught exceptian will make the server - # stop responding - except: # pylint:disable=bare-except - if CONFIG['no_nxdomain']: - # FIXME surely there's a better way to give SERVFAIL - exception = DNSQueryTimeoutError(name) - else: - exception = DomainError(name) - - return defer.fail(failure.Failure(exception)) - + print('attempting lookup') + records = self._a_records(name) + print('records') + print(records) + return defer.succeed((records, (), ())) def main(): """ @@ -214,7 +160,7 @@ def main(): # Create docker if CONFIG['docker_url']: - docker_client = docker.Client(CONFIG['docker_url'], version=CONFIG['version']) + docker_client = docker.Client(base_url=CONFIG['docker_url'], version=CONFIG['version']) else: docker_client = docker.Client(version=CONFIG['version']) @@ -257,8 +203,8 @@ def main(): # Merge user config over defaults DEFAULT_CONFIG = { - 'docker_url': None, - 'version': '1.11', + 'docker_url': 'unix://var/run/docker.sock', + 'version': '1.13', 'bind_interface': '', 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], From 5f46b4ce96ffecbad756bd1c022c57166dac0686 Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 18:21:09 +0000 Subject: [PATCH 05/51] . --- docker_dns.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 2f294e5..5fe93b2 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -61,17 +61,14 @@ def lookup_container(self, name): warn(name) cid_all = [ c['Id'] for c in self.api.containers(all=True) ] - print(cid_all) for cid in cid_all: warn(cid) - cdic = self.api.inspect_container(cid_all['Id']) - warn(cdic[key_path]) + cdic = self.api.inspect_container(cid) if cdic[key_path] == name: - container_id = cdic[key_path] + container_id = cid else: container_id = None - warn(container_id) # try: return self.api.inspect_container(container_id) From 051da4646f9ba47b382f8ae6876cb5e9e7a4e96a Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 18:59:28 +0000 Subject: [PATCH 06/51] fixed name lookup, string comp --- docker_dns.py | 52 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 5fe93b2..946ba9d 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -31,8 +31,6 @@ class DockerMapping(object): Look up docker container data """ - id_re = re.compile(r'([a-z0-9]+)\.d') - def __init__(self, api): """ Args: @@ -58,23 +56,34 @@ def lookup_container(self, name): Container config dict for the first matching container """ key_path = 'Name' - warn(name) + name = name.strip('.d') cid_all = [ c['Id'] for c in self.api.containers(all=True) ] for cid in cid_all: - warn(cid) cdic = self.api.inspect_container(cid) - if cdic[key_path] == name: + cname = str(cdic[key_path].strip('/')) + if cname == name: container_id = cid + break else: container_id = None -# try: - return self.api.inspect_container(container_id) -# except RequestException as ex: -# warn(ex) -# return None + print('container matching %s: %s' % (name, container_id)) + + try: + return self.api.inspect_container(container_id) + + except docker.errors.APIError as ex: + # 404 is valid, others aren't + if ex.response.status_code != 404: + warn(ex) + return None + + except RequestException as ex: + warn(ex) + return None + def get_a(self, name): """ @@ -88,8 +97,6 @@ def get_a(self, name): """ container = self.lookup_container(name) - print 'container:' - print(container) if container is None: return None @@ -133,7 +140,6 @@ def _a_records(self, name): """ addr = self.mapping.get_a(name) - print addr if not addr: raise DomainError(name) @@ -144,11 +150,21 @@ def _a_records(self, name): ]) def lookupAddress(self, name, timeout=None): - print('attempting lookup') - records = self._a_records(name) - print('records') - print(records) - return defer.succeed((records, (), ())) + try: + records = self._a_records(name) + return defer.succeed((records, (), ())) + + # We need to catch everything. Uncaught exceptian will make the server + # stop responding + except: # pylint:disable=bare-except + if CONFIG['no_nxdomain']: + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + def main(): """ From b1aed7c3db367c9b01b7e25f836217995c854dee Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 19:02:00 +0000 Subject: [PATCH 07/51] change descript --- docker_dns.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 946ba9d..eb3ecc5 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -5,18 +5,16 @@ resolution. To look up a container: - - 'A' record query container's hostname with no TLD. Must be an exact match - - 'A' record query an ID that will match a container with a docker inspect - command with '.docker' as the TLD. eg: 0949efde23b.docker + - 'A' record query a container NAME that will match a container with a docker inspect + command with '.d' as the TLD. eg: mysql_server1.d -Code heavily modified from -http://stackoverflow.com/a/4401671/509043 +Code modified from +https://github.com/infoxchange/docker_dns -Author: Ricky Cook +Author: Bradley Cicenas """ import docker -import re from requests.exceptions import RequestException from twisted.application import internet, service From 29a7ba38813d2f69e38514ee929d2f528beb004b Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 20:34:55 +0000 Subject: [PATCH 08/51] update requirements to newer dockerpy --- README.md | 5 ++--- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index df7f605..b405f74 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,8 @@ A simple Twisted DNS server using custom TLD and Docker as the back end for IP resolution. To look up a container: - - 'A' record query container's hostname with no TLD. Must be an exact match - - 'A' record query an ID that will match a container with a docker inspect - command with '.docker' as the TLD. eg: 0949efde23b.docker + - 'A' record query a container NAME that will match a container with a docker inspect + command with '.d' as the TLD. eg: mysql_server1.d Install/Run ----------- diff --git a/requirements.txt b/requirements.txt index b6b285a..087550d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ twisted==12.0.0 --e git+git://github.com/dotcloud/docker-py.git@3754edc2673996e8598b617f7d32d4ce035f81c5#egg=docker \ No newline at end of file +docker-py=0.4.0 From 0bf4031892f9057bf4a8e62b0c2ffbf0e545893d Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 5 Aug 2014 20:36:39 +0000 Subject: [PATCH 09/51] . --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 087550d..b849f5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ twisted==12.0.0 -docker-py=0.4.0 +docker-py==0.4.0 From b4c691d382e4add776fbd678387048b6933c2aae Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Wed, 6 Aug 2014 12:33:44 -0400 Subject: [PATCH 10/51] change apierror location --- docker_dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_dns.py b/docker_dns.py index eb3ecc5..061cc4b 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -39,7 +39,7 @@ def __init__(self, api): try: print('connected to docker instance running api version %s' % \ self.api.version()['ApiVersion']) - except docker.client.APIError as ex: + except docker.errors.APIError as ex: raise Exception(ex) def lookup_container(self, name): From bde11f48353936838e51f9d7cd7348ff49009ab1 Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 12 Aug 2014 17:14:51 -0400 Subject: [PATCH 11/51] changing default bind port to 53000 --- docker_dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_dns.py b/docker_dns.py index 061cc4b..5680930 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -217,7 +217,7 @@ def main(): 'docker_url': 'unix://var/run/docker.sock', 'version': '1.13', 'bind_interface': '', - 'bind_port': 53, + 'bind_port': 53000, 'bind_protocols': ['tcp', 'udp'], 'no_nxdomain': True, 'authoritive': True, From a63de162d6aea1e6eed95554f5344760a6bd1be2 Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 12 Aug 2014 17:32:41 -0400 Subject: [PATCH 12/51] adding default timeout of 30 to client obj --- docker_dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 5680930..0a6bfab 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -171,9 +171,9 @@ def main(): # Create docker if CONFIG['docker_url']: - docker_client = docker.Client(base_url=CONFIG['docker_url'], version=CONFIG['version']) + docker_client = docker.Client(base_url=CONFIG['docker_url'], version=CONFIG['version'], timeout=30) else: - docker_client = docker.Client(version=CONFIG['version']) + docker_client = docker.Client(version=CONFIG['version'], timeout=30) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) From 05fe5ebcc15be2887fc2e1db6324d6f99c8d7eb2 Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Tue, 12 Aug 2014 20:26:35 -0400 Subject: [PATCH 13/51] Revert "changing default bind port to 53000" This reverts commit bde11f48353936838e51f9d7cd7348ff49009ab1. --- docker_dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_dns.py b/docker_dns.py index 0a6bfab..84b121b 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -217,7 +217,7 @@ def main(): 'docker_url': 'unix://var/run/docker.sock', 'version': '1.13', 'bind_interface': '', - 'bind_port': 53000, + 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], 'no_nxdomain': True, 'authoritive': True, From 6f77542af954f0b204008297adb881ba3f660ab4 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 31 Oct 2014 19:09:38 +0100 Subject: [PATCH 14/51] support for nat discovery via SRV --- README.md | 16 +++- docker_dns.py | 216 ++++++++++++++++++++++++++++++++++++++------- docker_dns_test.py | 3 +- test_srv.py | 180 +++++++++++++++++++++++++++++++++++++ 4 files changed, 380 insertions(+), 35 deletions(-) create mode 100644 test_srv.py diff --git a/README.md b/README.md index b405f74..dbbdc07 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,14 @@ Just install from requirements (in a virtualenv if you'd like) pip install -r requirements.txt --use-mirrors -That's it! To run, just +That's it! To run, remember that you may need to set user/group ids on +the process - twistd -y docker_dns.py + + sudo twistd -gdocker -y docker_dns.py This will start a DNS server on port 53 (default DNS port). To make this -useful, you probably want to combine it with your regular DNS in something like -Dnsmasq. +useful, you probably want to combine it with your regular DNS in something like Dnsmasq. Examples -------- @@ -110,6 +111,13 @@ container: ;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 12687 ;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0 + +Nat discovery: you can discover natted ports with queries like this one + + dig _8080._tcp.my-thing.docker srv + ;; ANSWER SECTION: + _8080._tcp.jboss631.docker. 10 IN SRV 100 100 18080 192.168.204.17. + Configuration ------------- Config is done in the `config.py` file. There's a skeleton in diff --git a/docker_dns.py b/docker_dns.py index 84b121b..4d90d55 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -1,6 +1,6 @@ #!/usr/bin/python -""" +""" A simple TwistD DNS server using custom TLD and Docker as the back end for IP resolution. @@ -15,33 +15,93 @@ """ import docker - +from warnings import warn +from socket import getfqdn from requests.exceptions import RequestException + from twisted.application import internet, service from twisted.internet import defer from twisted.names import common, dns, server from twisted.names.error import DNSQueryTimeoutError, DomainError -from twisted.python import failure -from warnings import warn +from twisted.python import failure, log + +from functools import partial + + +def get_preferred_ip(): + """Return the ip associated to the default gw""" + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + # connecting to a UDP address doesn't send packets + s.connect(('8.8.8.8', 0)) + return s.getsockname()[0] + + +class memoize(object): + + """cache the return value of a method + + This class is meant to be used as a decorator of methods. The return value + from a given method invocation will be cached on the instance whose method + was invoked. All arguments passed to a method decorated with memoize must + be hashable. + + If a memoized method is invoked directly on its class the result will not + be cached. Instead the method will be invoked like a static method: + class Obj(object): + @memoize + def add_to(self, arg): + return self + arg + Obj.add_to(1) # not enough arguments + Obj.add_to(1, 2) # returns 3, result is not cached + """ + + def __init__(self, func): + self.func = func + + def __get__(self, obj, objtype=None): + if obj is None: + return self.func + return partial(self, obj) + + def __call__(self, *args, **kw): + obj = args[0] + try: + cache = obj.__cache + except AttributeError: + cache = obj.__cache = {} + key = (self.func, args[1:], frozenset(kw.items())) + try: + res = cache[key] + except KeyError: + res = cache[key] = self.func(*args, **kw) + return res + class DockerMapping(object): + """ - Look up docker container data + Look up docker container data via docker.api. + + XXX Should it be dns-agnostic, and just a wrapper around docker.api """ - def __init__(self, api): + def __init__(self, api=None): """ Args: api: Docker Client instance used to do API communication """ - self.api = api + self.api = api if api else docker.Client() + log.msg("DockerMapping pointing to %r" % self.api.base_url) try: - print('connected to docker instance running api version %s' % \ - self.api.version()['ApiVersion']) + print('connected to docker instance running api version %s' % + self.api.version()['ApiVersion']) except docker.errors.APIError as ex: - raise Exception(ex) + log.err("Cannot instantiate docker api") + raise ex + #@memoize def lookup_container(self, name): """ Gets the container config from a DNS lookup name, or returns None if @@ -53,23 +113,27 @@ def lookup_container(self, name): Returns: Container config dict for the first matching container """ + assert self.api + key_path = 'Name' name = name.strip('.d') + log.msg('lookup container: %r' % name) - cid_all = [ c['Id'] for c in self.api.containers(all=True) ] + try: + cid_all = (c['Id'] for c in self.api.containers(all=True) if c) + log.msg('found containers: %r ' % cid_all) - for cid in cid_all: - cdic = self.api.inspect_container(cid) - cname = str(cdic[key_path].strip('/')) - if cname == name: - container_id = cid - break - else: - container_id = None + for cid in cid_all: + cdic = self.api.inspect_container(cid) + cname = str(cdic[key_path].strip('/')) + if cname == name: + container_id = cid + break + else: + container_id = None - print('container matching %s: %s' % (name, container_id)) + print('container matching %s: %s' % (name, container_id)) - try: return self.api.inspect_container(container_id) except docker.errors.APIError as ex: @@ -79,10 +143,10 @@ def lookup_container(self, name): return None except RequestException as ex: - warn(ex) + log.err() + # warn(ex) return None - def get_a(self, name): """ Get an IPv4 address from a query name to be used in A record lookups @@ -97,20 +161,56 @@ def get_a(self, name): container = self.lookup_container(name) if container is None: + print("No container found") return None addr = container['NetworkSettings']['IPAddress'] - if addr is '': + if not addr: return None return addr + def get_nat(self, container_name, sport=0, sproto=None): + """ @return - a list of natted maps (local, nat, ip) + + @param sport: the port to search + @param sproto: the protocol to search + + eg. [ (8080, 'tcp', 18080, '0.0.0.0'), + (8787, 'tcp', 8787, '0.0.0.0'), + ] + """ + sport = int(sport) + container = self.lookup_container(container_name) + try: + for local, remote in container['NetworkSettings']['Ports'].items(): + port, proto = local.split("/") + port = int(port) + if sport and sport != port: + continue + if sproto and sproto != proto: + continue + if not remote: + continue + + for r in remote: + try: + yield (port, proto, int(r['HostPort']), r['HostIp']) + except (ValueError, KeyError) as e: + log.err() + continue + except KeyError as e: + log.err("Bad network information from docker") + # pylint:disable=too-many-public-methods class DockerResolver(common.ResolverBase): + """ DNS resolver to resolve queries with a DockerMapping instance. + + Twisted Names just uses the lookupXXX method """ def __init__(self, mapping): @@ -147,6 +247,14 @@ def _a_records(self, name): CONFIG['authoritive']) ]) + def _srv_records(self, name): + print("getting srv: %r" + name) + return tuple([ + dns.RRHeader(name, dns.SRV, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + CONFIG['authoritive']) + ]) + def lookupAddress(self, name, timeout=None): try: records = self._a_records(name) @@ -154,8 +262,10 @@ def lookupAddress(self, name, timeout=None): # We need to catch everything. Uncaught exceptian will make the server # stop responding - except: # pylint:disable=bare-except + except Exception as e: # pylint:disable=bare-except if CONFIG['no_nxdomain']: + print("E stampala sta eccezione imbecille %r" % e) + log.err() # FIXME surely there's a better way to give SERVFAIL exception = DNSQueryTimeoutError(name) else: @@ -163,17 +273,63 @@ def lookupAddress(self, name, timeout=None): return defer.fail(failure.Failure(exception)) + def lookupService(self, name, timeout=None): + """ Lookup a docker natted service of + the form: NATTEDPORT._tcp.CONTAINERNAME.docker. + and returns a srv record of the for: + _service._proto.name. TTL class SRV priority weight port target. + + @returns - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) + """ + if not name.endswith(".docker"): + log.err("Domain not ending with .docker: %r" % name) + return defer.fail(failure.Failure(DomainError("not ending with docker"))) + try: + port, proto, container, _ = name.split(".") + port = int(port.strip("_")) + except (IndexError, TypeError, ValueError) as e: + log.err("Domain not of the right form: %r" % name) + return defer.fail(failure.Failure(DomainError("not of the right form"))) + + my_preferred_ip = get_preferred_ip() + mock_records = [dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) + ] + + records = [dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + dns.Record_SRV( + priority=100, weight=100, port=c_nat_port, target=my_preferred_ip, ttl=None), + auth=True) + for c_port, protocol, c_nat_port, target + in self.mapping.get_nat(container) + if c_port == port # eventually filter + ] + return defer.succeed((records, (), ())) + def main(): """ Set everything up """ - # Create docker - if CONFIG['docker_url']: - docker_client = docker.Client(base_url=CONFIG['docker_url'], version=CONFIG['version'], timeout=30) - else: - docker_client = docker.Client(version=CONFIG['version'], timeout=30) + docker_client = docker.Client() + + # Test docker connectivity before starting + print(docker_client.info()) + print(docker_client.base_url) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) diff --git a/docker_dns_test.py b/docker_dns_test.py index 2f54b72..a7f76ee 100755 --- a/docker_dns_test.py +++ b/docker_dns_test.py @@ -17,7 +17,7 @@ import unittest from docker_dns import (DEFAULT_CONFIG, - dict_lookup, + # dict_lookup, DockerMapping, DockerResolver) from twisted.names import dns @@ -60,6 +60,7 @@ def x_back(result): status, result = completed[0] if status != success: + raise AssertionError("Expected: %r, got %r" % (success, result)) return False return result diff --git a/test_srv.py b/test_srv.py new file mode 100644 index 0000000..662600b --- /dev/null +++ b/test_srv.py @@ -0,0 +1,180 @@ +""" + Creating srv records + +""" +from docker_dns import DockerResolver, DockerMapping +from twisted.names import dns +from docker_dns_test import check_deferred +import docker + +mock_list_containers = lambda self, name: [ + {u'Command': u'/bin/bash', + u'Created': 1414430175, + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'eap63_tracer:v6.3.1', + u'Names': [u'/jboss631'], + u'Ports': [{u'IP': u'0.0.0.0', + u'PrivatePort': 8080, + u'PublicPort': 18080, + u'Type': u'tcp'}, + {u'IP': u'0.0.0.0', + u'PrivatePort': 8787, + u'PublicPort': 8787, + u'Type': u'tcp'}, + {u'IP': u'0.0.0.0', + u'PrivatePort': 9999, + u'PublicPort': 19999, + u'Type': u'tcp'}, + {u'PrivatePort': 8443, u'Type': u'tcp'}, + {u'PrivatePort': 9990, u'Type': u'tcp'}], + u'Status': u'Up 2 days'} +] + + +mock_lookup_container = lambda name: { + u'Args': [], + u'Config': {u'AttachStderr': True, + u'AttachStdin': True, + u'AttachStdout': True, + u'Cmd': [u'/bin/bash'], + u'CpuShares': 0, + u'Cpuset': u'', + u'Domainname': u'', + u'Entrypoint': None, + u'Env': [u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], + u'ExposedPorts': {u'8080/tcp': {}, + u'8443/tcp': {}, + u'8787/tcp': {}, + u'9990/tcp': {}, + u'9999/tcp': {}}, + u'Hostname': u'7d564ceb891b', + u'Image': u'eap63_tracer:v6.3.1', + u'Memory': 0, + u'MemorySwap': 0, + u'NetworkDisabled': False, + u'OnBuild': None, + u'OpenStdin': True, + u'PortSpecs': None, + u'StdinOnce': True, + u'Tty': True, + u'User': u'', + u'Volumes': {}, + u'WorkingDir': u''}, + u'Created': u'2014-10-27T17:16:15.261857884Z', + u'Driver': u'devicemapper', + u'ExecDriver': u'native-0.2', + u'HostConfig': {u'Binds': [u'/home/rpolli/Downloads/:/mnt/tmp'], + u'CapAdd': None, + u'CapDrop': None, + u'ContainerIDFile': u'', + u'Devices': [], + u'Dns': None, + u'DnsSearch': None, + u'Links': None, + u'LxcConf': [], + u'NetworkMode': u'bridge', + u'PortBindings': {u'8080/tcp': [{u'HostIp': u'', u'HostPort': u'18080'}], + u'8787/tcp': [{u'HostIp': u'', u'HostPort': u'8787'}], + u'9999/tcp': [{u'HostIp': u'', u'HostPort': u'19999'}]}, + u'Privileged': False, + u'PublishAllPorts': False, + u'RestartPolicy': {u'MaximumRetryCount': 0, u'Name': u''}, + u'VolumesFrom': None}, + u'HostnamePath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hostname', + u'HostsPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hosts', + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', + u'MountLabel': u'', + u'Name': u'/jboss631', + u'NetworkSettings': {u'Bridge': u'docker0', + u'Gateway': u'172.17.42.1', + u'IPAddress': u'172.17.0.10', + u'IPPrefixLen': 16, + u'PortMapping': None, + u'Ports': {u'8080/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'18080'}], + u'8443/tcp': None, + u'8787/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'8787'}], + u'9990/tcp': None, + u'9999/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'19999'}]}}, + u'Path': u'/bin/bash', + u'ProcessLabel': u'', + u'ResolvConfPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/resolv.conf', + u'State': {u'ExitCode': 0, + u'FinishedAt': u'0001-01-01T00:00:00Z', + u'Paused': False, + u'Pid': 8662, + u'Restarting': False, + u'Running': True, + u'StartedAt': u'2014-10-27T17:16:15.756653452Z'}, + u'Volumes': {u'/mnt/tmp': u'/home/rpolli/Downloads'}, + u'VolumesRW': {u'/mnt/tmp': True} +} + +SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" + + +class Test(object): + + def setup(self): + docker_client = docker.Client() + self.mapping = DockerMapping(api=docker_client) + self.mapping.lookup_container = mock_lookup_container + self.resolver = DockerResolver(self.mapping) + + def test_srv(self): + dns.Record_SRV(priority=100, weight=100, port=123, target='', ttl=None) + + mock_mapping = {} + resolver = DockerResolver(mock_mapping) + res = resolver.lookupService("_8888._tcp.jboss63.docker") + return res + + def test_nat_all(self): + host, port = "foo.docker", 8080 + ret = self.mapping.get_nat(host, port) + assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret + + def test_nat_ports(self): + expected = [(8080, 18080), + (9999, 19999), (8787, 8787)] + for pin, pout in expected: + ret = self.mapping.get_nat("foo", pin) + _, _, port, _ = next(ret) + assert port == pout, "unexpected value in %r" % ret + + def test_lookupService_ko(self): + expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( + ) + for n in expect_fail: + ret = self.resolver.lookupService(n) + check_deferred(ret, False) + + def test_lookupService_ok(self): + ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_mapping(self): + container = self.mapping.lookup_container("foo") + + for local, remote in container['NetworkSettings']['Ports'].items(): + port, proto = local.split("/") + if not remote: + continue + try: + remote = remote[0] + except IndexError: + continue + + print(SRV_FMT.format( + svc=port, + proto=proto, + container=container['Name'][1:], + cclass="IN", + priority=100, + weight=100, + port=remote['HostPort'], + target=remote['HostIp'] if remote[ + 'HostIp'] != '0.0.0.0' else "localhost" + ) + ) From e67cb57b801d11bf10cead1526f6684d29bc9135 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 31 Oct 2014 21:14:29 +0100 Subject: [PATCH 15/51] little tweaks --- docker_dns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 4d90d55..469795e 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -1,4 +1,6 @@ #!/usr/bin/python +#from __future__ import print_function, unicode_literals + """ A simple TwistD DNS server using custom TLD and Docker as the back end for IP @@ -16,7 +18,6 @@ import docker from warnings import warn -from socket import getfqdn from requests.exceptions import RequestException from twisted.application import internet, service @@ -25,7 +26,6 @@ from twisted.names.error import DNSQueryTimeoutError, DomainError from twisted.python import failure, log -from functools import partial def get_preferred_ip(): @@ -37,6 +37,7 @@ def get_preferred_ip(): return s.getsockname()[0] +from functools import partial class memoize(object): """cache the return value of a method @@ -79,7 +80,6 @@ def __call__(self, *args, **kw): class DockerMapping(object): - """ Look up docker container data via docker.api. From 4d2e69302590802655a1aeaf3e93ff2eaa705f0a Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 31 Oct 2014 22:19:38 +0100 Subject: [PATCH 16/51] merge error logging from miracle2k --- docker_dns.py | 129 +++++++++++++++++++++++++++++---------------- docker_dns_test.py | 2 +- 2 files changed, 86 insertions(+), 45 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 469795e..50d9759 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -1,19 +1,22 @@ #!/usr/bin/python #from __future__ import print_function, unicode_literals - - -""" +""" A simple TwistD DNS server using custom TLD and Docker as the back end for IP resolution. To look up a container: - 'A' record query a container NAME that will match a container with a docker inspect command with '.d' as the TLD. eg: mysql_server1.d + - 'SRV' record query to _port._srv.container.docker will return the natted address. + eg. _3306._tcp.mysql_server1.docker returns + _18080._tcp.compassionate_poincare.docker. 10 IN SRV 100 100 8080 192.168.42.126. -Code modified from + +Code modified from https://github.com/infoxchange/docker_dns Author: Bradley Cicenas +Author: Roberto Polli """ import docker @@ -27,6 +30,40 @@ from twisted.python import failure, log +# Merge user config over defaults +CONFIG = { + 'docker_url': 'unix://var/run/docker.sock', + 'version': '1.13', + 'bind_interface': '', + 'bind_port': 53, + 'bind_protocols': ['tcp', 'udp'], + 'no_nxdomain': True, + 'authoritive': True, +} + +# FIXME replace with a more generic solution like operator.attrgetter + + +def dict_lookup(dic, key_path, default=None): + """ + Look up value in a nested dict + + Args: + dic: The dictionary to search + key_path: An iterable containing an ordered list of dict keys to + traverse + default: Value to return in case nothing is found + Returns: + Value of the dict at the nested location given, or default if no value + was found + """ + for k in key_path: + if k in dic: + dic = dic[k] + else: + return default + return dic + def get_preferred_ip(): """Return the ip associated to the default gw""" @@ -41,12 +78,12 @@ def get_preferred_ip(): class memoize(object): """cache the return value of a method - + This class is meant to be used as a decorator of methods. The return value from a given method invocation will be cached on the instance whose method was invoked. All arguments passed to a method decorated with memoize must be hashable. - + If a memoized method is invoked directly on its class the result will not be cached. Instead the method will be invoked like a static method: class Obj(object): @@ -82,7 +119,7 @@ def __call__(self, *args, **kw): class DockerMapping(object): """ Look up docker container data via docker.api. - + XXX Should it be dns-agnostic, and just a wrapper around docker.api """ @@ -139,12 +176,12 @@ def lookup_container(self, name): except docker.errors.APIError as ex: # 404 is valid, others aren't if ex.response.status_code != 404: - warn(ex) + warn(str(ex)) return None except RequestException as ex: log.err() - # warn(ex) + warn(str(ex)) return None def get_a(self, name): @@ -178,7 +215,7 @@ def get_nat(self, container_name, sport=0, sproto=None): @param sproto: the protocol to search eg. [ (8080, 'tcp', 18080, '0.0.0.0'), - (8787, 'tcp', 8787, '0.0.0.0'), + (8787, 'tcp', 8787, '0.0.0.0'), ] """ sport = int(sport) @@ -209,7 +246,7 @@ class DockerResolver(common.ResolverBase): """ DNS resolver to resolve queries with a DockerMapping instance. - + Twisted Names just uses the lookupXXX method """ @@ -262,9 +299,14 @@ def lookupAddress(self, name, timeout=None): # We need to catch everything. Uncaught exceptian will make the server # stop responding + except DomainError as e0: + log.err() + return defer.fail(failure.Failure(e)) except Exception as e: # pylint:disable=bare-except + import traceback + traceback.print_exc() + if CONFIG['no_nxdomain']: - print("E stampala sta eccezione imbecille %r" % e) log.err() # FIXME surely there's a better way to give SERVFAIL exception = DNSQueryTimeoutError(name) @@ -277,13 +319,13 @@ def lookupService(self, name, timeout=None): """ Lookup a docker natted service of the form: NATTEDPORT._tcp.CONTAINERNAME.docker. and returns a srv record of the for: - _service._proto.name. TTL class SRV priority weight port target. - + _service._proto.name. TTL class SRV priority weight port target. + @returns - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. - The first element of the tuple gives answers. - The second element of the tuple gives authorities. - The third element of the tuple gives additional information. - The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) """ if not name.endswith(".docker"): log.err("Domain not ending with .docker: %r" % name) @@ -302,7 +344,7 @@ def lookupService(self, name, timeout=None): priority=100, weight=100, port=19999, target='name', ttl=None), auth=True), dns.RRHeader( - name, dns.SRV, dns.IN, self.ttl, + name, dns.SRV, dns.IN, self.ttl, dns.Record_SRV( priority=100, weight=100, port=18080, target='name', ttl=None), auth=True) @@ -310,9 +352,9 @@ def lookupService(self, name, timeout=None): records = [dns.RRHeader( name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=c_nat_port, target=my_preferred_ip, ttl=None), - auth=True) + dns.Record_SRV( + priority=100, weight=100, port=c_nat_port, target=my_preferred_ip, ttl=None), + auth=True) for c_port, protocol, c_nat_port, target in self.mapping.get_nat(container) if c_port == port # eventually filter @@ -325,11 +367,11 @@ def main(): Set everything up """ - docker_client = docker.Client() + # Create docker: by default dict.get returns None on missing keys + docker_client = docker.Client(CONFIG.get('docker_url')) # Test docker connectivity before starting - print(docker_client.info()) - print(docker_client.base_url) + log.msg("Connecting to docker instance: %r" % docker_client.info()) # Create our custom mapping and resolver mapping = DockerMapping(docker_client) @@ -342,17 +384,19 @@ def main(): # Protocols to bind bind_list = [] if 'tcp' in CONFIG['bind_protocols']: - bind_list.append((internet.TCPServer, factory)) # noqa pylint:disable=no-member + bind_list.append( + (internet.TCPServer, factory)) # noqa pylint:disable=no-member if 'udp' in CONFIG['bind_protocols']: proto = dns.DNSDatagramProtocol(factory) proto.noisy = False - bind_list.append((internet.UDPServer, proto)) # noqa pylint:disable=no-member + bind_list.append( + (internet.UDPServer, proto)) # noqa pylint:disable=no-member # Register the service ret = service.MultiService() - for (klass, arg) in bind_list: - svc = klass( + for (InternetServerKlass, arg) in bind_list: + svc = InternetServerKlass( CONFIG['bind_port'], arg, interface=CONFIG['bind_interface'] @@ -362,25 +406,22 @@ def main(): # DO IT NOW ret.setServiceParent(service.IServiceCollection(application)) +# +# This is the effective twisted application +# + # Load the config try: - from config import CONFIG # pylint:disable=no-name-in-module,import-error + from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error + CONFIG.update(appcfg) except ImportError: - CONFIG = {} - -# Merge user config over defaults -DEFAULT_CONFIG = { - 'docker_url': 'unix://var/run/docker.sock', - 'version': '1.13', - 'bind_interface': '', - 'bind_port': 53, - 'bind_protocols': ['tcp', 'udp'], - 'no_nxdomain': True, - 'authoritive': True, -} -CONFIG = dict(DEFAULT_CONFIG.items() + CONFIG.items()) + appcfg = {} -application = service.Application('dnsserver', 1, 1) # noqa pylint:disable=invalid-name +# +# Run it with twisted... it should be named .tac, not .py +# +application = service.Application( + 'dnsserver', 1, 1) # noqa pylint:disable=invalid-name main() diff --git a/docker_dns_test.py b/docker_dns_test.py index a7f76ee..f3dd9e1 100755 --- a/docker_dns_test.py +++ b/docker_dns_test.py @@ -17,7 +17,7 @@ import unittest from docker_dns import (DEFAULT_CONFIG, - # dict_lookup, + dict_lookup, DockerMapping, DockerResolver) from twisted.names import dns From 77c1d876fbdc863e3b70f1c8d5a270da03b78328 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 31 Oct 2014 23:27:37 +0100 Subject: [PATCH 17/51] moved helpers functions in utils.py --- docker_dns.py | 132 ++++++++++----------------------------------- docker_dns_test.py | 18 +++---- test_srv.py | 8 --- utils.py | 76 ++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 120 deletions(-) create mode 100644 utils.py diff --git a/docker_dns.py b/docker_dns.py index 50d9759..14c0549 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -29,9 +29,10 @@ from twisted.names.error import DNSQueryTimeoutError, DomainError from twisted.python import failure, log +from utils import get_preferred_ip, memoize # Merge user config over defaults -CONFIG = { +CONFIG = DEFAULT_CONFIG = { 'docker_url': 'unix://var/run/docker.sock', 'version': '1.13', 'bind_interface': '', @@ -41,79 +42,12 @@ 'authoritive': True, } -# FIXME replace with a more generic solution like operator.attrgetter - - -def dict_lookup(dic, key_path, default=None): - """ - Look up value in a nested dict - - Args: - dic: The dictionary to search - key_path: An iterable containing an ordered list of dict keys to - traverse - default: Value to return in case nothing is found - Returns: - Value of the dict at the nested location given, or default if no value - was found - """ - for k in key_path: - if k in dic: - dic = dic[k] - else: - return default - return dic - - -def get_preferred_ip(): - """Return the ip associated to the default gw""" - import socket - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - # connecting to a UDP address doesn't send packets - s.connect(('8.8.8.8', 0)) - return s.getsockname()[0] - - -from functools import partial -class memoize(object): - - """cache the return value of a method - - This class is meant to be used as a decorator of methods. The return value - from a given method invocation will be cached on the instance whose method - was invoked. All arguments passed to a method decorated with memoize must - be hashable. - - If a memoized method is invoked directly on its class the result will not - be cached. Instead the method will be invoked like a static method: - class Obj(object): - @memoize - def add_to(self, arg): - return self + arg - Obj.add_to(1) # not enough arguments - Obj.add_to(1, 2) # returns 3, result is not cached - """ - - def __init__(self, func): - self.func = func - - def __get__(self, obj, objtype=None): - if obj is None: - return self.func - return partial(self, obj) - - def __call__(self, *args, **kw): - obj = args[0] - try: - cache = obj.__cache - except AttributeError: - cache = obj.__cache = {} - key = (self.func, args[1:], frozenset(kw.items())) - try: - res = cache[key] - except KeyError: - res = cache[key] = self.func(*args, **kw) - return res +# Load the config +try: + from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error + CONFIG.update(appcfg) +except ImportError: + appcfg = {} class DockerMapping(object): @@ -138,7 +72,7 @@ def __init__(self, api=None): log.err("Cannot instantiate docker api") raise ex - #@memoize + @memoize def lookup_container(self, name): """ Gets the container config from a DNS lookup name, or returns None if @@ -150,8 +84,6 @@ def lookup_container(self, name): Returns: Container config dict for the first matching container """ - assert self.api - key_path = 'Name' name = name.strip('.d') log.msg('lookup container: %r' % name) @@ -162,14 +94,15 @@ def lookup_container(self, name): for cid in cid_all: cdic = self.api.inspect_container(cid) - cname = str(cdic[key_path].strip('/')) + # as container names starts with "/" we should strip it + cname = str(cdic[key_path].strip('/')) if cname == name: container_id = cid break else: container_id = None - print('container matching %s: %s' % (name, container_id)) + log.msg('container matching %s: %s' % (name, container_id)) return self.api.inspect_container(container_id) @@ -208,8 +141,9 @@ def get_a(self, name): return addr + #@memoize def get_nat(self, container_name, sport=0, sproto=None): - """ @return - a list of natted maps (local, nat, ip) + """ @return - a generator of natted maps (local, nat, ip) @param sport: the port to search @param sproto: the protocol to search @@ -243,13 +177,22 @@ def get_nat(self, container_name, sport=0, sproto=None): # pylint:disable=too-many-public-methods class DockerResolver(common.ResolverBase): - """ DNS resolver to resolve queries with a DockerMapping instance. Twisted Names just uses the lookupXXX method """ - + mock_records = [dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) + ] def __init__(self, mapping): """ Args: @@ -262,6 +205,7 @@ def __init__(self, mapping): # super(DockerResolver, self).__init__() common.ResolverBase.__init__(self) self.ttl = 10 + self.my_preferred_ip = get_preferred_ip() def _a_records(self, name): """ @@ -299,7 +243,7 @@ def lookupAddress(self, name, timeout=None): # We need to catch everything. Uncaught exceptian will make the server # stop responding - except DomainError as e0: + except DomainError as e: log.err() return defer.fail(failure.Failure(e)) except Exception as e: # pylint:disable=bare-except @@ -327,6 +271,7 @@ def lookupService(self, name, timeout=None): The third element of the tuple gives additional information. The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) """ + #return defer.succeed((self.mock_records, (), ())) if not name.endswith(".docker"): log.err("Domain not ending with .docker: %r" % name) return defer.fail(failure.Failure(DomainError("not ending with docker"))) @@ -337,23 +282,10 @@ def lookupService(self, name, timeout=None): log.err("Domain not of the right form: %r" % name) return defer.fail(failure.Failure(DomainError("not of the right form"))) - my_preferred_ip = get_preferred_ip() - mock_records = [dns.RRHeader( - name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=19999, target='name', ttl=None), - auth=True), - dns.RRHeader( - name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=18080, target='name', ttl=None), - auth=True) - ] - records = [dns.RRHeader( name, dns.SRV, dns.IN, self.ttl, dns.Record_SRV( - priority=100, weight=100, port=c_nat_port, target=my_preferred_ip, ttl=None), + priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip, ttl=None), auth=True) for c_port, protocol, c_nat_port, target in self.mapping.get_nat(container) @@ -410,12 +342,6 @@ def main(): # This is the effective twisted application # -# Load the config -try: - from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error - CONFIG.update(appcfg) -except ImportError: - appcfg = {} # # Run it with twisted... it should be named .tac, not .py diff --git a/docker_dns_test.py b/docker_dns_test.py index f3dd9e1..60ca980 100755 --- a/docker_dns_test.py +++ b/docker_dns_test.py @@ -16,8 +16,8 @@ import itertools import unittest +from utils import traverse_tree from docker_dns import (DEFAULT_CONFIG, - dict_lookup, DockerMapping, DockerResolver) from twisted.names import dns @@ -145,7 +145,7 @@ class DictLookupTest(unittest.TestCase): def test_basic_one(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['pandas', 'and'] ), @@ -154,7 +154,7 @@ def test_basic_one(self): def test_basic_two(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['foxes', 'are'] ), @@ -163,7 +163,7 @@ def test_basic_two(self): def test_basic_none(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['badgers', 'are'], 'Badgers are none? What?' @@ -173,7 +173,7 @@ def test_basic_none(self): def test_dict(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['foxes'] ), @@ -182,7 +182,7 @@ def test_dict(self): def test_default_single_depth(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['nothing'] ), @@ -191,7 +191,7 @@ def test_default_single_depth(self): def test_user_default_single_depth(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['nothing'], 'Nobody here but us chickens' @@ -201,7 +201,7 @@ def test_user_default_single_depth(self): def test_default_multi_depth(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['pandas', 'bad'] ), @@ -210,7 +210,7 @@ def test_default_multi_depth(self): def test_user_default_multi_depth(self): self.assertEqual( - dict_lookup( + traverse_tree( self.theDict, ['pandas', 'bad'], 'NO, THAT\'S A DAMN DIRTY LIE' diff --git a/test_srv.py b/test_srv.py index 662600b..3fcb3ed 100644 --- a/test_srv.py +++ b/test_srv.py @@ -121,14 +121,6 @@ def setup(self): self.mapping.lookup_container = mock_lookup_container self.resolver = DockerResolver(self.mapping) - def test_srv(self): - dns.Record_SRV(priority=100, weight=100, port=123, target='', ttl=None) - - mock_mapping = {} - resolver = DockerResolver(mock_mapping) - res = resolver.lookupService("_8888._tcp.jboss63.docker") - return res - def test_nat_all(self): host, port = "foo.docker", 8080 ret = self.mapping.get_nat(host, port) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..485e128 --- /dev/null +++ b/utils.py @@ -0,0 +1,76 @@ +"""utilities.py +""" + +import socket +def get_preferred_ip(): + """Return the ip associated to the default gw""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + # connecting to a UDP address doesn't send packets + s.connect(('8.8.8.8', 0)) + return s.getsockname()[0] + except Exception as e: + return socket.getfqdn() + + +from functools import partial +class memoize(object): + + """cache the return value of a method + + This class is meant to be used as a decorator of methods. The return value + from a given method invocation will be cached on the instance whose method + was invoked. All arguments passed to a method decorated with memoize must + be hashable. + + If a memoized method is invoked directly on its class the result will not + be cached. Instead the method will be invoked like a static method: + class Obj(object): + @memoize + def add_to(self, arg): + return self + arg + Obj.add_to(1) # not enough arguments + Obj.add_to(1, 2) # returns 3, result is not cached + """ + + def __init__(self, func): + self.func = func + + def __get__(self, obj, objtype=None): + if obj is None: + return self.func + return partial(self, obj) + + def __call__(self, *args, **kw): + obj = args[0] + try: + cache = obj.__cache + except AttributeError: + cache = obj.__cache = {} + key = (self.func, args[1:], frozenset(kw.items())) + try: + res = cache[key] + except KeyError: + res = cache[key] = self.func(*args, **kw) + return res + +# FIXME replace with a more generic solution like operator.attrgetter +def traverse_tree(haystack, key_path, default=None): + """ + Look up value in a nested dict + + Args: + dic: The dictionary to search + key_path: An iterable containing an ordered list of dict keys to + traverse + default: Value to return in case nothing is found + Returns: + Value of the dict at the nested location given, or default if no value + was found + """ + for k in key_path: + if k in haystack: + haystack = dic[k] + else: + return default + return haystack From e673c9e69b1924f9d684f4c483ee563f6d1ba996 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sun, 2 Nov 2014 16:51:12 +0100 Subject: [PATCH 18/51] use in-addr arpa name to return the associated ip. --- utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 485e128..293b73c 100644 --- a/utils.py +++ b/utils.py @@ -3,12 +3,14 @@ import socket def get_preferred_ip(): - """Return the ip associated to the default gw""" + """Return the in-addr name associated to the + ip used to contact the default gw""" try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # connecting to a UDP address doesn't send packets s.connect(('8.8.8.8', 0)) - return s.getsockname()[0] + ip = s.getsockname()[0] + return '.'.join(list(reversed(s.getsockname()[0].split(".")))) + ".in-addr.arpa" except Exception as e: return socket.getfqdn() From 3a413df020b367981780abc5d63309f8779c20b6 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sun, 2 Nov 2014 16:59:10 +0100 Subject: [PATCH 19/51] fix twisted version to 14. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b849f5e..93d2110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -twisted==12.0.0 +twisted==14.0.0 docker-py==0.4.0 From 8e858fd92ecc77a3573c49f6f62d07faa410748c Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 19 Jan 2015 18:05:12 +0100 Subject: [PATCH 20/51] use docker events interface to get infos --- README.bind | 18 +++++ docker_dns.py | 64 +++++++++++++---- docker_dns_test.py | 9 ++- docker_events.py | 139 ++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +- test_srv.py | 170 ++++++++++++++++++++++----------------------- utils.py | 8 ++- 7 files changed, 307 insertions(+), 104 deletions(-) create mode 100644 README.bind create mode 100644 docker_events.py diff --git a/README.bind b/README.bind new file mode 100644 index 0000000..f41dffc --- /dev/null +++ b/README.bind @@ -0,0 +1,18 @@ +Configure with bind +=================== + + +If you use a local bind instance you can configure twistd dns with bind adding this stanza to named.conf. For now this requires to disable EDNS, which is not actuallyimplemented in docker_dns. + + +zone "docker" { + type forward; + forwarders { + 127.0.0.64; + }; +}; + + +And binding your local docker_dns server on the following +loopback ip: 127.0.0.64 + diff --git a/docker_dns.py b/docker_dns.py index 14c0549..7189c04 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -8,7 +8,7 @@ - 'A' record query a container NAME that will match a container with a docker inspect command with '.d' as the TLD. eg: mysql_server1.d - 'SRV' record query to _port._srv.container.docker will return the natted address. - eg. _3306._tcp.mysql_server1.docker returns + eg. _3306._tcp.mysql_server1.docker returns _18080._tcp.compassionate_poincare.docker. 10 IN SRV 100 100 8080 192.168.42.126. @@ -30,6 +30,7 @@ from twisted.python import failure, log from utils import get_preferred_ip, memoize +from docker_events import UpdateDB # Merge user config over defaults CONFIG = DEFAULT_CONFIG = { @@ -57,12 +58,12 @@ class DockerMapping(object): XXX Should it be dns-agnostic, and just a wrapper around docker.api """ - def __init__(self, api=None): + def __init__(self, api=None, db=None): """ Args: api: Docker Client instance used to do API communication """ - + self.db = db self.api = api if api else docker.Client() log.msg("DockerMapping pointing to %r" % self.api.base_url) try: @@ -72,12 +73,35 @@ def __init__(self, api=None): log.err("Cannot instantiate docker api") raise ex - @memoize def lookup_container(self, name): """ Gets the container config from a DNS lookup name, or returns None if one could not be found + Args: + name: DNS query name to look up + + Returns: + Container config dict for the first matching container + """ + try: + key_path = 'Name' + name = name.replace('.docker', '') + log.msg('lookup container: %r' % name) + if name not in self.db.mappings_idx: + raise KeyError("Item %s not found in %s" % + (name, self.db.mappings_idx.keys())) + id = self.db.mappings_idx[name] + return self.db.mappings[id] + except KeyError as e: + # warn(str(e)) + return None + + def lookup_container_old(self, name): + """ + Gets the container config from a DNS lookup name, or returns None if + one could not be found + Args: name: DNS query name to look up @@ -85,7 +109,7 @@ def lookup_container(self, name): Container config dict for the first matching container """ key_path = 'Name' - name = name.strip('.d') + name = name.replace('.docker', '') log.msg('lookup container: %r' % name) try: @@ -95,7 +119,7 @@ def lookup_container(self, name): for cid in cid_all: cdic = self.api.inspect_container(cid) # as container names starts with "/" we should strip it - cname = str(cdic[key_path].strip('/')) + cname = str(cdic[key_path].strip('/')) if cname == name: container_id = cid break @@ -154,6 +178,10 @@ def get_nat(self, container_name, sport=0, sproto=None): """ sport = int(sport) container = self.lookup_container(container_name) + if not container: + log.err("Bad network information for docker") + return + try: for local, remote in container['NetworkSettings']['Ports'].items(): port, proto = local.split("/") @@ -183,16 +211,17 @@ class DockerResolver(common.ResolverBase): Twisted Names just uses the lookupXXX method """ mock_records = [dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=19999, target='name', ttl=None), - auth=True), - dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, dns.Record_SRV( priority=100, weight=100, port=18080, target='name', ttl=None), auth=True) ] + def __init__(self, mapping): """ Args: @@ -241,10 +270,10 @@ def lookupAddress(self, name, timeout=None): records = self._a_records(name) return defer.succeed((records, (), ())) - # We need to catch everything. Uncaught exceptian will make the server + # We need to catch everything. Uncaught exceptions will make the server # stop responding except DomainError as e: - log.err() + log.msg("DomainError: %r " % e) return defer.fail(failure.Failure(e)) except Exception as e: # pylint:disable=bare-except import traceback @@ -306,7 +335,7 @@ def main(): log.msg("Connecting to docker instance: %r" % docker_client.info()) # Create our custom mapping and resolver - mapping = DockerMapping(docker_client) + mapping = DockerMapping(docker_client, db=UpdateDB) resolver = DockerResolver(mapping) # Create twistd stuff to tie in our custom components @@ -335,9 +364,14 @@ def main(): ) svc.setServiceParent(ret) + # Add the event Loop + from docker_events import docker_event_monitor + docker_event_monitor.setServiceParent(ret) + # DO IT NOW ret.setServiceParent(service.IServiceCollection(application)) + # # This is the effective twisted application # diff --git a/docker_dns_test.py b/docker_dns_test.py index 60ca980..3db8c7a 100755 --- a/docker_dns_test.py +++ b/docker_dns_test.py @@ -67,6 +67,8 @@ def x_back(result): class MockDockerClient(object): + base_url = 'http://localhost:5000' + version = lambda x: {'ApiVersion': '1.0'} inspect_container_pandas = { 'ID': 'cidpandaslong', 'Same': 'Value', @@ -118,6 +120,7 @@ def inspect_container(self, cid): try: return self.inspect_container_returns[cid] except KeyError: + # Mocks a Docker Client Exception response = fudge.Fake() response.has_attr(status_code=404, content='PANDAS!') @@ -440,7 +443,8 @@ def test_lookupAddress_invalid_nxdomain(self): result = check_deferred(deferred, False) self.assertNotEqual(result, False) - self.assertEqual(result.type, DomainError) # noqa pylint:disable=maybe-no-member + self.assertEqual( + result.type, DomainError) # noqa pylint:disable=maybe-no-member def test_lookupAddress_invalid_no_nxdomain(self): docker_dns.CONFIG['no_nxdomain'] = True @@ -448,7 +452,8 @@ def test_lookupAddress_invalid_no_nxdomain(self): result = check_deferred(deferred, False) self.assertNotEqual(result, False) - self.assertEqual(result.type, DNSQueryTimeoutError) # noqa pylint:disable=maybe-no-member + self.assertEqual(result.type, DNSQueryTimeoutError) + # noqa pylint:disable=maybe-no-member def main(): diff --git a/docker_events.py b/docker_events.py new file mode 100644 index 0000000..05f54d7 --- /dev/null +++ b/docker_events.py @@ -0,0 +1,139 @@ +from __future__ import print_function + +from twisted.internet import reactor +from twisted.web.client import Agent, HTTPConnectionPool +from twisted.web.http_headers import Headers +import os +from config import CONFIG +import simplejson as json +from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.protocol import Protocol, ReconnectingClientFactory +from twisted.application import service, internet +from twisted.python import log + +agent1 = Agent(reactor) +agent2 = Agent(reactor, HTTPConnectionPool(reactor)) + + +class UpdateDB(Protocol): + """Update docker ip store""" + mappings = {} + mappings_idx = {} + + def __init__(self, deferred): + self.deferred = deferred + + def dataReceived(self, bytes_): + if bytes_: + item = json.loads(bytes_) + print("Get container %r" % item) + UpdateDB.updatedb(item) + + def connectionLost(self, reason): + self.deferred.callback(None) + + @staticmethod + def updatedb(item): + assert 'Id' in item, "Entry has no Id" + UpdateDB.mappings_idx.update({item['Name'][1:]: item['Id']}) + UpdateDB.mappings.update({item['Id']: item}) + + @staticmethod + def add_container(item): + UpdateDB.updatedb(item) + + @staticmethod + def del_container(id): + name = UpdateDB.mappings[id]['Name'][1:] + del UpdateDB.mappings[id] + del UpdateDB.mappings_idx[name] + + @staticmethod + def get_by_name(name): + if name not in UpdateDB.mappings_idx: + raise KeyError("%r not in %r" % (name, UpdateDB.mappings_idx)) + + cid = UpdateDB.mappings_idx[name] + if cid not in UpdateDB.mappings: + raise KeyError("%r not in %r" % (cid, UpdateDB.mappings.keys())) + return UpdateDB.mappings[id] + + +class UpdateDockerMapping(Protocol): + """Parse docker events and update item store""" + def __init__(self, agent): + self.remaining = 1024 * 10 + self.buff = "" + self.agent = agent + + def update_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + d = self.agent.request('GET', + os.path.join(CONFIG['docker_url'], + 'containers', item['id'], 'json') + ) + d.addCallback( + lambda response: response.deliverBody(UpdateDB(Deferred()))) + + def delete_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + log.msg("removing item: %r" % item) + UpdateDB.del_container(item['id']) + + def dataReceived(self, bytes_): + """Get the container id and calls the updater""" + try: + if self.remaining: + display = bytes_[:self.remaining] + print('Some data received:', display) + self.remaining -= len(display) + item = json.loads(display) + print("Parsed: %r" % item) + if item['status'] == 'start': + self.update_record(item) + elif item['status'] in ('stop', 'die'): + self.delete_record(item) + except KeyError: + log.exc() + + def connectionLost(self, reason): + print('Finished receiving body:', reason.type, reason.value) + Deferred().callback(None) + + +class EventFactory(ReconnectingClientFactory): + # protocol = UpdateDockerMapping + + def __init__(self, agent, db=UpdateDB): + log.msg("Initializing factory") + self.agent = agent + self.db = db + d = self.agent.request( + 'GET', + os.path.join(CONFIG['docker_url'], 'events'), + Headers({'User-Agent': ['Twisted Web Client Example'], + 'Content-Type': ['text/x-greeting']}), + None) + d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) + + def cbResponse(self, response): + try: + log.msg('Response received: %r' % response) + finished = Deferred() + response.deliverBody(UpdateDockerMapping(agent=agent2)) + except: + log.err() + + def buildProtocol(self, addr): + log.msg("addr: %r" % addr) + return UpdateDockerMapping(agent=self.agent) + + +e_url = os.path.join(CONFIG['docker_url'], 'events') +efactory = EventFactory(agent=agent1) +log.msg("Created event Factory") +docker_event_monitor = internet.TCPClient('10.0.8.162', 5000, efactory) diff --git a/requirements.txt b/requirements.txt index 93d2110..5621d86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ twisted==14.0.0 -docker-py==0.4.0 +docker-py>=0.4.0 +simplejson diff --git a/test_srv.py b/test_srv.py index 3fcb3ed..649109e 100644 --- a/test_srv.py +++ b/test_srv.py @@ -33,81 +33,81 @@ mock_lookup_container = lambda name: { u'Args': [], - u'Config': {u'AttachStderr': True, - u'AttachStdin': True, - u'AttachStdout': True, - u'Cmd': [u'/bin/bash'], - u'CpuShares': 0, - u'Cpuset': u'', - u'Domainname': u'', - u'Entrypoint': None, - u'Env': [u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], - u'ExposedPorts': {u'8080/tcp': {}, - u'8443/tcp': {}, - u'8787/tcp': {}, - u'9990/tcp': {}, - u'9999/tcp': {}}, - u'Hostname': u'7d564ceb891b', - u'Image': u'eap63_tracer:v6.3.1', - u'Memory': 0, - u'MemorySwap': 0, - u'NetworkDisabled': False, - u'OnBuild': None, - u'OpenStdin': True, - u'PortSpecs': None, - u'StdinOnce': True, - u'Tty': True, - u'User': u'', - u'Volumes': {}, - u'WorkingDir': u''}, - u'Created': u'2014-10-27T17:16:15.261857884Z', - u'Driver': u'devicemapper', - u'ExecDriver': u'native-0.2', - u'HostConfig': {u'Binds': [u'/home/rpolli/Downloads/:/mnt/tmp'], - u'CapAdd': None, - u'CapDrop': None, - u'ContainerIDFile': u'', - u'Devices': [], - u'Dns': None, - u'DnsSearch': None, - u'Links': None, - u'LxcConf': [], - u'NetworkMode': u'bridge', - u'PortBindings': {u'8080/tcp': [{u'HostIp': u'', u'HostPort': u'18080'}], - u'8787/tcp': [{u'HostIp': u'', u'HostPort': u'8787'}], - u'9999/tcp': [{u'HostIp': u'', u'HostPort': u'19999'}]}, - u'Privileged': False, - u'PublishAllPorts': False, - u'RestartPolicy': {u'MaximumRetryCount': 0, u'Name': u''}, - u'VolumesFrom': None}, - u'HostnamePath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hostname', - u'HostsPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hosts', - u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', - u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', - u'MountLabel': u'', - u'Name': u'/jboss631', - u'NetworkSettings': {u'Bridge': u'docker0', - u'Gateway': u'172.17.42.1', - u'IPAddress': u'172.17.0.10', - u'IPPrefixLen': 16, - u'PortMapping': None, - u'Ports': {u'8080/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'18080'}], - u'8443/tcp': None, - u'8787/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'8787'}], - u'9990/tcp': None, - u'9999/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'19999'}]}}, - u'Path': u'/bin/bash', - u'ProcessLabel': u'', - u'ResolvConfPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/resolv.conf', - u'State': {u'ExitCode': 0, - u'FinishedAt': u'0001-01-01T00:00:00Z', - u'Paused': False, - u'Pid': 8662, - u'Restarting': False, - u'Running': True, - u'StartedAt': u'2014-10-27T17:16:15.756653452Z'}, - u'Volumes': {u'/mnt/tmp': u'/home/rpolli/Downloads'}, - u'VolumesRW': {u'/mnt/tmp': True} + u'Config': {u'AttachStderr': True, + u'AttachStdin': True, + u'AttachStdout': True, + u'Cmd': [u'/bin/bash'], + u'CpuShares': 0, + u'Cpuset': u'', + u'Domainname': u'', + u'Entrypoint': None, + u'Env': [u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], + u'ExposedPorts': {u'8080/tcp': {}, + u'8443/tcp': {}, + u'8787/tcp': {}, + u'9990/tcp': {}, + u'9999/tcp': {}}, + u'Hostname': u'7d564ceb891b', + u'Image': u'eap63_tracer:v6.3.1', + u'Memory': 0, + u'MemorySwap': 0, + u'NetworkDisabled': False, + u'OnBuild': None, + u'OpenStdin': True, + u'PortSpecs': None, + u'StdinOnce': True, + u'Tty': True, + u'User': u'', + u'Volumes': {}, + u'WorkingDir': u''}, + u'Created': u'2014-10-27T17:16:15.261857884Z', + u'Driver': u'devicemapper', + u'ExecDriver': u'native-0.2', + u'HostConfig': {u'Binds': [u'/home/rpolli/Downloads/:/mnt/tmp'], + u'CapAdd': None, + u'CapDrop': None, + u'ContainerIDFile': u'', + u'Devices': [], + u'Dns': None, + u'DnsSearch': None, + u'Links': None, + u'LxcConf': [], + u'NetworkMode': u'bridge', + u'PortBindings': {u'8080/tcp': [{u'HostIp': u'', u'HostPort': u'18080'}], + u'8787/tcp': [{u'HostIp': u'', u'HostPort': u'8787'}], + u'9999/tcp': [{u'HostIp': u'', u'HostPort': u'19999'}]}, + u'Privileged': False, + u'PublishAllPorts': False, + u'RestartPolicy': {u'MaximumRetryCount': 0, u'Name': u''}, + u'VolumesFrom': None}, + u'HostnamePath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hostname', + u'HostsPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hosts', + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', + u'MountLabel': u'', + u'Name': u'/jboss631', + u'NetworkSettings': {u'Bridge': u'docker0', + u'Gateway': u'172.17.42.1', + u'IPAddress': u'172.17.0.10', + u'IPPrefixLen': 16, + u'PortMapping': None, + u'Ports': {u'8080/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'18080'}], + u'8443/tcp': None, + u'8787/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'8787'}], + u'9990/tcp': None, + u'9999/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'19999'}]}}, + u'Path': u'/bin/bash', + u'ProcessLabel': u'', + u'ResolvConfPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/resolv.conf', + u'State': {u'ExitCode': 0, + u'FinishedAt': u'0001-01-01T00:00:00Z', + u'Paused': False, + u'Pid': 8662, + u'Restarting': False, + u'Running': True, + u'StartedAt': u'2014-10-27T17:16:15.756653452Z'}, + u'Volumes': {u'/mnt/tmp': u'/home/rpolli/Downloads'}, + u'VolumesRW': {u'/mnt/tmp': True} } SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" @@ -160,13 +160,13 @@ def test_mapping(self): print(SRV_FMT.format( svc=port, - proto=proto, - container=container['Name'][1:], - cclass="IN", - priority=100, - weight=100, - port=remote['HostPort'], - target=remote['HostIp'] if remote[ - 'HostIp'] != '0.0.0.0' else "localhost" - ) - ) + proto=proto, + container=container['Name'][1:], + cclass="IN", + priority=100, + weight=100, + port=remote['HostPort'], + target=remote['HostIp'] if remote[ + 'HostIp'] != '0.0.0.0' else "localhost" + ) + ) diff --git a/utils.py b/utils.py index 293b73c..eaedd3a 100644 --- a/utils.py +++ b/utils.py @@ -2,8 +2,10 @@ """ import socket + + def get_preferred_ip(): - """Return the in-addr name associated to the + """Return the in-addr name associated to the ip used to contact the default gw""" try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -16,6 +18,8 @@ def get_preferred_ip(): from functools import partial + + class memoize(object): """cache the return value of a method @@ -57,6 +61,8 @@ def __call__(self, *args, **kw): return res # FIXME replace with a more generic solution like operator.attrgetter + + def traverse_tree(haystack, key_path, default=None): """ Look up value in a nested dict From b8562dbac343073c3436778eb21a157cc48d5174 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 19 Jan 2015 18:36:43 +0100 Subject: [PATCH 21/51] fix: instantiate event manager in .tac file --- docker_dns.py | 6 +++++- docker_events.py | 39 +++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index 7189c04..b62182d 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -365,7 +365,11 @@ def main(): svc.setServiceParent(ret) # Add the event Loop - from docker_events import docker_event_monitor + from docker_events import EventFactory + from urlparse import urlparse + u = urlparse(CONFIG['docker_url']) + efactory = EventFactory(config=CONFIG) + docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) docker_event_monitor.setServiceParent(ret) # DO IT NOW diff --git a/docker_events.py b/docker_events.py index 05f54d7..807df57 100644 --- a/docker_events.py +++ b/docker_events.py @@ -11,8 +11,6 @@ from twisted.application import service, internet from twisted.python import log -agent1 = Agent(reactor) -agent2 = Agent(reactor, HTTPConnectionPool(reactor)) class UpdateDB(Protocol): @@ -61,17 +59,19 @@ def get_by_name(name): class UpdateDockerMapping(Protocol): """Parse docker events and update item store""" - def __init__(self, agent): + def __init__(self, agent, config): + self.agent = agent + self.config = config + self.remaining = 1024 * 10 self.buff = "" - self.agent = agent def update_record(self, item): """Update docker mapping item = {'status': ..., 'id': ...} """ d = self.agent.request('GET', - os.path.join(CONFIG['docker_url'], + os.path.join(self.config['docker_url'], 'containers', item['id'], 'json') ) d.addCallback( @@ -107,16 +107,25 @@ def connectionLost(self, reason): class EventFactory(ReconnectingClientFactory): # protocol = UpdateDockerMapping + agent = Agent(reactor) + agent2 = Agent(reactor, HTTPConnectionPool(reactor)) - def __init__(self, agent, db=UpdateDB): + def __init__(self, config, db=UpdateDB): log.msg("Initializing factory") - self.agent = agent self.db = db + # + # Create a protocol handler to parse docker container data + # + self.dockerUpdater = UpdateDockerMapping(agent=self.agent2, config=config) + + # + # Poll to the docker event interface + # d = self.agent.request( 'GET', - os.path.join(CONFIG['docker_url'], 'events'), - Headers({'User-Agent': ['Twisted Web Client Example'], - 'Content-Type': ['text/x-greeting']}), + os.path.join(config['docker_url'], 'events'), + Headers({'User-Agent': ['Twisted Web Client for Docker Event'], + 'Content-Type': ['application/json']}), None) d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) @@ -124,16 +133,14 @@ def cbResponse(self, response): try: log.msg('Response received: %r' % response) finished = Deferred() - response.deliverBody(UpdateDockerMapping(agent=agent2)) + response.deliverBody(self.dockerUpdater) except: log.err() def buildProtocol(self, addr): log.msg("addr: %r" % addr) - return UpdateDockerMapping(agent=self.agent) + return self.dockerUpdater + + -e_url = os.path.join(CONFIG['docker_url'], 'events') -efactory = EventFactory(agent=agent1) -log.msg("Created event Factory") -docker_event_monitor = internet.TCPClient('10.0.8.162', 5000, efactory) From 5c481aac16bde627247b1cb8c1896ae335de1011 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Thu, 22 Jan 2015 13:12:39 +0100 Subject: [PATCH 22/51] add existing containers --- docker_dns.py | 8 ++++++++ docker_events.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index b62182d..a541d2a 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -73,6 +73,14 @@ def __init__(self, api=None, db=None): log.err("Cannot instantiate docker api") raise ex + def populate_db(self): + """ + Get all containers and populates the db + """ + self.db.mappings = self.api.containers(all=True) + for x in self.db.mappings.items(): + self.db.update + def lookup_container(self, name): """ Gets the container config from a DNS lookup name, or returns None if diff --git a/docker_events.py b/docker_events.py index 807df57..4d8e38f 100644 --- a/docker_events.py +++ b/docker_events.py @@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred, DeferredList from twisted.internet.protocol import Protocol, ReconnectingClientFactory from twisted.application import service, internet -from twisted.python import log +from twisted.python import log, failure @@ -98,7 +98,12 @@ def dataReceived(self, bytes_): elif item['status'] in ('stop', 'die'): self.delete_record(item) except KeyError: - log.exc() + log.err("Container not found") + except json.scanner.JSONDecodeError: + log.err("Error reading data") + except Exception: + log.err("Generic Error") + def connectionLost(self, reason): print('Finished receiving body:', reason.type, reason.value) @@ -118,6 +123,8 @@ def __init__(self, config, db=UpdateDB): # self.dockerUpdater = UpdateDockerMapping(agent=self.agent2, config=config) + # Populate existing containers (this is ok to be blocking) + # # Poll to the docker event interface # @@ -130,6 +137,7 @@ def __init__(self, config, db=UpdateDB): d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) def cbResponse(self, response): + """Manages the response using a Protocol class defined in __init__""" try: log.msg('Response received: %r' % response) finished = Deferred() From 866f377b6798d6be960597f4026d40b339ada7ff Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Thu, 22 Jan 2015 14:19:31 +0100 Subject: [PATCH 23/51] initialize db --- README.bind | 11 +++++++++++ docker_dns.py | 13 +++++-------- docker_events.py | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/README.bind b/README.bind index f41dffc..a98fc89 100644 --- a/README.bind +++ b/README.bind @@ -16,3 +16,14 @@ zone "docker" { And binding your local docker_dns server on the following loopback ip: 127.0.0.64 + +Configure with dnsmasq +====================== + +You can use twistd dns with dnsmasq using the following file: + +# Config file for /etc/dnsmasq.d/dockerdns.conf +port=53 + +# Forward only .docker requests... +server=/docker/127.0.0.64#10053 diff --git a/docker_dns.py b/docker_dns.py index a541d2a..ecf48c2 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -73,13 +73,8 @@ def __init__(self, api=None, db=None): log.err("Cannot instantiate docker api") raise ex - def populate_db(self): - """ - Get all containers and populates the db - """ - self.db.mappings = self.api.containers(all=True) - for x in self.db.mappings.items(): - self.db.update + db.populate(self.api.containers(all=True)) + def lookup_container(self, name): """ @@ -104,6 +99,9 @@ def lookup_container(self, name): except KeyError as e: # warn(str(e)) return None + except Exception as e: + log.exc("Unmanaged error") + return None def lookup_container_old(self, name): """ @@ -173,7 +171,6 @@ def get_a(self, name): return addr - #@memoize def get_nat(self, container_name, sport=0, sproto=None): """ @return - a generator of natted maps (local, nat, ip) diff --git a/docker_events.py b/docker_events.py index 4d8e38f..9776ee3 100644 --- a/docker_events.py +++ b/docker_events.py @@ -4,12 +4,10 @@ from twisted.web.client import Agent, HTTPConnectionPool from twisted.web.http_headers import Headers import os -from config import CONFIG import simplejson as json from twisted.internet.defer import Deferred, DeferredList from twisted.internet.protocol import Protocol, ReconnectingClientFactory -from twisted.application import service, internet -from twisted.python import log, failure +from twisted.python import log @@ -18,8 +16,9 @@ class UpdateDB(Protocol): mappings = {} mappings_idx = {} - def __init__(self, deferred): - self.deferred = deferred + def __init__(self, onLost): + """Initialize the Docker host database""" + self.onLost = onLost def dataReceived(self, bytes_): if bytes_: @@ -28,7 +27,15 @@ def dataReceived(self, bytes_): UpdateDB.updatedb(item) def connectionLost(self, reason): - self.deferred.callback(None) + self.onLost.callback(None) + + @staticmethod + def populate(mappings): + if mappings: + print("Populating db with data: %r" % mappings) + UpdateDB.mappings = mappings + UpdateDB.mappings_idx = {item['Names'][0][1:]: item['Id'] for item in mappings} + @staticmethod def updatedb(item): From 78dd9b5785425db7be154d7eb2006731dbb20ac1 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Thu, 29 Jan 2015 21:47:47 +0100 Subject: [PATCH 24/51] refactoring: starts working --- README.md | 5 +- docker_dns.py | 307 +----------------- dockerdns/__init__.py | 7 + docker_events.py => dockerdns/events.py | 143 ++++---- dockerdns/mappings.py | 112 +++++++ dockerdns/resolver.py | 151 +++++++++ utils.py => dockerdns/utils.py | 17 +- test_srv.py => test/__init__.py | 121 +++---- docker_dns_test.py => test/docker_dns_test.py | 135 +------- test/test_events.py | 32 ++ test/test_old.py | 0 test/test_srv.py | 71 ++++ 12 files changed, 547 insertions(+), 554 deletions(-) create mode 100644 dockerdns/__init__.py rename docker_events.py => dockerdns/events.py (52%) create mode 100644 dockerdns/mappings.py create mode 100644 dockerdns/resolver.py rename utils.py => dockerdns/utils.py (79%) rename test_srv.py => test/__init__.py (67%) rename docker_dns_test.py => test/docker_dns_test.py (72%) create mode 100644 test/test_events.py create mode 100644 test/test_old.py create mode 100644 test/test_srv.py diff --git a/README.md b/README.md index dbbdc07..c71c106 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,13 @@ Docker DNS ========== [![Build Status](https://travis-ci.org/infoxchange/docker_dns.png?branch=master)](https://travis-ci.org/infoxchange/docker_dns) -A simple Twisted DNS server using custom TLD and Docker as the back end for IP +A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP resolution. To look up a container: - 'A' record query a container NAME that will match a container with a docker inspect - command with '.d' as the TLD. eg: mysql_server1.d + command with '.docker' as the TLD. eg: mysql_server1.docker + - 'SRV' record query exposing the NAT informations Install/Run ----------- diff --git a/docker_dns.py b/docker_dns.py index ecf48c2..fd384f1 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -19,18 +19,14 @@ Author: Roberto Polli """ -import docker -from warnings import warn -from requests.exceptions import RequestException - from twisted.application import internet, service -from twisted.internet import defer -from twisted.names import common, dns, server -from twisted.names.error import DNSQueryTimeoutError, DomainError -from twisted.python import failure, log +from twisted.names import dns, server +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.events import DockerDB +from dockerdns.resolver import DockerResolver -from utils import get_preferred_ip, memoize -from docker_events import UpdateDB # Merge user config over defaults CONFIG = DEFAULT_CONFIG = { @@ -40,7 +36,7 @@ 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], 'no_nxdomain': True, - 'authoritive': True, + 'authoritative': True, } # Load the config @@ -51,296 +47,21 @@ appcfg = {} -class DockerMapping(object): - """ - Look up docker container data via docker.api. - - XXX Should it be dns-agnostic, and just a wrapper around docker.api - """ - - def __init__(self, api=None, db=None): - """ - Args: - api: Docker Client instance used to do API communication - """ - self.db = db - self.api = api if api else docker.Client() - log.msg("DockerMapping pointing to %r" % self.api.base_url) - try: - print('connected to docker instance running api version %s' % - self.api.version()['ApiVersion']) - except docker.errors.APIError as ex: - log.err("Cannot instantiate docker api") - raise ex - - db.populate(self.api.containers(all=True)) - - - def lookup_container(self, name): - """ - Gets the container config from a DNS lookup name, or returns None if - one could not be found - - Args: - name: DNS query name to look up - - Returns: - Container config dict for the first matching container - """ - try: - key_path = 'Name' - name = name.replace('.docker', '') - log.msg('lookup container: %r' % name) - if name not in self.db.mappings_idx: - raise KeyError("Item %s not found in %s" % - (name, self.db.mappings_idx.keys())) - id = self.db.mappings_idx[name] - return self.db.mappings[id] - except KeyError as e: - # warn(str(e)) - return None - except Exception as e: - log.exc("Unmanaged error") - return None - - def lookup_container_old(self, name): - """ - Gets the container config from a DNS lookup name, or returns None if - one could not be found - - Args: - name: DNS query name to look up - - Returns: - Container config dict for the first matching container - """ - key_path = 'Name' - name = name.replace('.docker', '') - log.msg('lookup container: %r' % name) - - try: - cid_all = (c['Id'] for c in self.api.containers(all=True) if c) - log.msg('found containers: %r ' % cid_all) - - for cid in cid_all: - cdic = self.api.inspect_container(cid) - # as container names starts with "/" we should strip it - cname = str(cdic[key_path].strip('/')) - if cname == name: - container_id = cid - break - else: - container_id = None - - log.msg('container matching %s: %s' % (name, container_id)) - - return self.api.inspect_container(container_id) - - except docker.errors.APIError as ex: - # 404 is valid, others aren't - if ex.response.status_code != 404: - warn(str(ex)) - return None - - except RequestException as ex: - log.err() - warn(str(ex)) - return None - - def get_a(self, name): - """ - Get an IPv4 address from a query name to be used in A record lookups - - Args: - name: DNS query name to look up - - Returns: - IPv4 address for the query name given - """ - - container = self.lookup_container(name) - - if container is None: - print("No container found") - return None - - addr = container['NetworkSettings']['IPAddress'] - - if not addr: - return None - - return addr - - def get_nat(self, container_name, sport=0, sproto=None): - """ @return - a generator of natted maps (local, nat, ip) - - @param sport: the port to search - @param sproto: the protocol to search - - eg. [ (8080, 'tcp', 18080, '0.0.0.0'), - (8787, 'tcp', 8787, '0.0.0.0'), - ] - """ - sport = int(sport) - container = self.lookup_container(container_name) - if not container: - log.err("Bad network information for docker") - return - - try: - for local, remote in container['NetworkSettings']['Ports'].items(): - port, proto = local.split("/") - port = int(port) - if sport and sport != port: - continue - if sproto and sproto != proto: - continue - if not remote: - continue - - for r in remote: - try: - yield (port, proto, int(r['HostPort']), r['HostIp']) - except (ValueError, KeyError) as e: - log.err() - continue - except KeyError as e: - log.err("Bad network information from docker") - - -# pylint:disable=too-many-public-methods -class DockerResolver(common.ResolverBase): - """ - DNS resolver to resolve queries with a DockerMapping instance. - - Twisted Names just uses the lookupXXX method - """ - mock_records = [dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=19999, target='name', ttl=None), - auth=True), - dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=18080, target='name', ttl=None), - auth=True) - ] - - def __init__(self, mapping): - """ - Args: - mapping: DockerMapping instance for lookups - """ - - self.mapping = mapping - - # Change to this ASAP when Twisted uses object base - # super(DockerResolver, self).__init__() - common.ResolverBase.__init__(self) - self.ttl = 10 - self.my_preferred_ip = get_preferred_ip() - - def _a_records(self, name): - """ - Get A records from a query name - - Args: - name: DNS query name to look up - - Returns: - Tuple of formatted DNS replies - """ - - addr = self.mapping.get_a(name) - if not addr: - raise DomainError(name) - - return tuple([ - dns.RRHeader(name, dns.A, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - CONFIG['authoritive']) - ]) - - def _srv_records(self, name): - print("getting srv: %r" + name) - return tuple([ - dns.RRHeader(name, dns.SRV, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - CONFIG['authoritive']) - ]) - - def lookupAddress(self, name, timeout=None): - try: - records = self._a_records(name) - return defer.succeed((records, (), ())) - - # We need to catch everything. Uncaught exceptions will make the server - # stop responding - except DomainError as e: - log.msg("DomainError: %r " % e) - return defer.fail(failure.Failure(e)) - except Exception as e: # pylint:disable=bare-except - import traceback - traceback.print_exc() - - if CONFIG['no_nxdomain']: - log.err() - # FIXME surely there's a better way to give SERVFAIL - exception = DNSQueryTimeoutError(name) - else: - exception = DomainError(name) - - return defer.fail(failure.Failure(exception)) - - def lookupService(self, name, timeout=None): - """ Lookup a docker natted service of - the form: NATTEDPORT._tcp.CONTAINERNAME.docker. - and returns a srv record of the for: - _service._proto.name. TTL class SRV priority weight port target. - - @returns - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. - The first element of the tuple gives answers. - The second element of the tuple gives authorities. - The third element of the tuple gives additional information. - The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) - """ - #return defer.succeed((self.mock_records, (), ())) - if not name.endswith(".docker"): - log.err("Domain not ending with .docker: %r" % name) - return defer.fail(failure.Failure(DomainError("not ending with docker"))) - try: - port, proto, container, _ = name.split(".") - port = int(port.strip("_")) - except (IndexError, TypeError, ValueError) as e: - log.err("Domain not of the right form: %r" % name) - return defer.fail(failure.Failure(DomainError("not of the right form"))) - - records = [dns.RRHeader( - name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip, ttl=None), - auth=True) - for c_port, protocol, c_nat_port, target - in self.mapping.get_nat(container) - if c_port == port # eventually filter - ] - return defer.succeed((records, (), ())) - - def main(): """ Set everything up """ + import docker # Create docker: by default dict.get returns None on missing keys docker_client = docker.Client(CONFIG.get('docker_url')) - + infos = docker_client.info() # Test docker connectivity before starting - log.msg("Connecting to docker instance: %r" % docker_client.info()) + log.msg("Connecting to docker instance: %r" % infos) + db = DockerDB(api=docker_client) # Create our custom mapping and resolver - mapping = DockerMapping(docker_client, db=UpdateDB) + mapping = DockerMapping(db=db) resolver = DockerResolver(mapping) # Create twistd stuff to tie in our custom components @@ -370,10 +91,10 @@ def main(): svc.setServiceParent(ret) # Add the event Loop - from docker_events import EventFactory + from dockerdns.events import EventFactory from urlparse import urlparse u = urlparse(CONFIG['docker_url']) - efactory = EventFactory(config=CONFIG) + efactory = EventFactory(config=CONFIG, db=db) docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) docker_event_monitor.setServiceParent(ret) diff --git a/dockerdns/__init__.py b/dockerdns/__init__.py new file mode 100644 index 0000000..f16daaa --- /dev/null +++ b/dockerdns/__init__.py @@ -0,0 +1,7 @@ +""" + + - DnsServer --> DBParser --> DB + - EventManager -http-> docker-server + 2 protocols: /events/, /containers/ + +""" diff --git a/docker_events.py b/dockerdns/events.py similarity index 52% rename from docker_events.py rename to dockerdns/events.py index 9776ee3..01e8728 100644 --- a/docker_events.py +++ b/dockerdns/events.py @@ -1,95 +1,109 @@ from __future__ import print_function +""" + Populate dns database getting info from docker via: + - docker-py api + - docker events interface +""" from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from twisted.web.http_headers import Headers -import os +from os.path import join as pjoin import simplejson as json -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol, ReconnectingClientFactory from twisted.python import log +class DockerDB(object): + """Update docker ip store connecting via docker-py + """ -class UpdateDB(Protocol): - """Update docker ip store""" - mappings = {} - mappings_idx = {} + def __init__(self, api=None): + self.api = api + self.mappings = {} + self.mappings_idx = {} + # + # Initialize db (TODO get initialization timestamp) + for c in self.api.containers(all=True): + item = self.api.inspect_container(c['Id']) + self.updatedb(item) + + def updatedb(self, item): + assert 'Id' in item, "Entry has no Id" + self.mappings_idx.update({item['Name'][1:]: item['Id']}) + self.mappings.update({item['Id']: item}) + + def add_container(self, item): + self.updatedb(item) + + def del_container(self, cid): + name = self.mappings[cid]['Name'][1:] + del self.mappings[cid] + del self.mappings_idx[name] - def __init__(self, onLost): + def get_by_name(self, name): + if name not in self.mappings_idx: + raise KeyError("%r not in %r" % (name, self.mappings_idx)) + + cid = self.mappings_idx[name] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + +class ContainerManager(Protocol): + """Manage the response to /containers/{id}/json + updating the network infos associated to the container + """ + + def __init__(self, onLost, db): """Initialize the Docker host database""" self.onLost = onLost + self.db = db def dataReceived(self, bytes_): if bytes_: item = json.loads(bytes_) print("Get container %r" % item) - UpdateDB.updatedb(item) + self.db.updatedb(item) def connectionLost(self, reason): self.onLost.callback(None) - @staticmethod - def populate(mappings): - if mappings: - print("Populating db with data: %r" % mappings) - UpdateDB.mappings = mappings - UpdateDB.mappings_idx = {item['Names'][0][1:]: item['Id'] for item in mappings} +class EventManager(Protocol): + """Manage the response to /events/json + - start: triggers a further request with a ContainerManager cb + - stop|die: removes the associations from IPs + """ - @staticmethod - def updatedb(item): - assert 'Id' in item, "Entry has no Id" - UpdateDB.mappings_idx.update({item['Name'][1:]: item['Id']}) - UpdateDB.mappings.update({item['Id']: item}) - - @staticmethod - def add_container(item): - UpdateDB.updatedb(item) - - @staticmethod - def del_container(id): - name = UpdateDB.mappings[id]['Name'][1:] - del UpdateDB.mappings[id] - del UpdateDB.mappings_idx[name] - - @staticmethod - def get_by_name(name): - if name not in UpdateDB.mappings_idx: - raise KeyError("%r not in %r" % (name, UpdateDB.mappings_idx)) - - cid = UpdateDB.mappings_idx[name] - if cid not in UpdateDB.mappings: - raise KeyError("%r not in %r" % (cid, UpdateDB.mappings.keys())) - return UpdateDB.mappings[id] - - -class UpdateDockerMapping(Protocol): - """Parse docker events and update item store""" - def __init__(self, agent, config): - self.agent = agent + def __init__(self, http_agent, config, db): + self.http_agent = http_agent self.config = config self.remaining = 1024 * 10 self.buff = "" + # Create an event_manager for parsing updates + self.event_manager = ContainerManager(Deferred(), db=db) def update_record(self, item): """Update docker mapping item = {'status': ..., 'id': ...} """ - d = self.agent.request('GET', - os.path.join(self.config['docker_url'], - 'containers', item['id'], 'json') - ) + d = self.http_agent.request('GET', + pjoin(self.config['docker_url'], + 'containers', item['id'], 'json') + ) d.addCallback( - lambda response: response.deliverBody(UpdateDB(Deferred()))) + lambda response: response.deliverBody(self.event_manager)) def delete_record(self, item): """Update docker mapping item = {'status': ..., 'id': ...} """ log.msg("removing item: %r" % item) - UpdateDB.del_container(item['id']) + self.db.del_container(item['id']) def dataReceived(self, bytes_): """Get the container id and calls the updater""" @@ -108,9 +122,8 @@ def dataReceived(self, bytes_): log.err("Container not found") except json.scanner.JSONDecodeError: log.err("Error reading data") - except Exception: - log.err("Generic Error") - + except Exception as e: + log.err("Generic Error %r" % e) def connectionLost(self, reason): print('Finished receiving body:', reason.type, reason.value) @@ -118,17 +131,27 @@ def connectionLost(self, reason): class EventFactory(ReconnectingClientFactory): - # protocol = UpdateDockerMapping + """Factory to connect to docker api interface and trigger + the first /events/ call. + + """ agent = Agent(reactor) agent2 = Agent(reactor, HTTPConnectionPool(reactor)) - def __init__(self, config, db=UpdateDB): + def __init__(self, config, db): + """ + + :param config: contains the docker url to poll + :param db: the mappings between containers and + :return: + """ log.msg("Initializing factory") self.db = db # # Create a protocol handler to parse docker container data # - self.dockerUpdater = UpdateDockerMapping(agent=self.agent2, config=config) + self.dockerUpdater = EventManager( + http_agent=self.agent2, config=config, db=db) # Populate existing containers (this is ok to be blocking) @@ -137,7 +160,7 @@ def __init__(self, config, db=UpdateDB): # d = self.agent.request( 'GET', - os.path.join(config['docker_url'], 'events'), + pjoin(config['docker_url'], 'events'), Headers({'User-Agent': ['Twisted Web Client for Docker Event'], 'Content-Type': ['application/json']}), None) @@ -155,7 +178,3 @@ def cbResponse(self, response): def buildProtocol(self, addr): log.msg("addr: %r" % addr) return self.dockerUpdater - - - - diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py new file mode 100644 index 0000000..dc38da2 --- /dev/null +++ b/dockerdns/mappings.py @@ -0,0 +1,112 @@ +""" + Maps DNS requests to host +""" +from twisted.python import log + + +class DockerMapping(object): + """ + Look up docker info via docker.api and + + XXX Should it be dns-agnostic, and just a wrapper around docker.api + + TODO: + + """ + + def __init__(self, db=None): + """ + Args: + db: a DockerDB instance containing infos + """ + self.db = db + + def lookup_container(self, name): + """ + Gets the container config from a DNS lookup name, or returns None if + one could not be found + + Args: + name: DNS query name to look up + + Returns: + Container config dict for the first matching container + """ + try: + name = name.replace('.docker', '') + log.msg('lookup container: %r' % name) + + return self.db.get_by_name(name) + except KeyError as e: + # warn(str(e)) + log.msg("Container not found: %r" % name) + except Exception as e: + log.err("Unmanaged error: %r" % e) + + return None + + def get_a(self, name): + """ + Get an IPv4 address from a query name to be used in A record lookups + + Args: + name: DNS query name to look up + + Returns: + IPv4 address for the query name given + """ + + container = self.lookup_container(name) + + if container is None: + log.msg("No container found") + return + + addr = container['NetworkSettings']['IPAddress'] + + if not addr: + return None + + return addr + + def get_nat(self, container_name, sport=0, sproto=None): + """ @return - a generator of natted maps (local, nat, ip) + + @param sport: the port to search + @param sproto: the protocol to search + + eg. [ (8080, 'tcp', 18080, '0.0.0.0'), + (8787, 'tcp', 8787, '0.0.0.0'), + ] + """ + sport = int(sport) + container = self.lookup_container(container_name) + if container is None: + log.msg("No container found") + return + + try: + nats = container['NetworkSettings']['Ports'].items() + except (KeyError, AttributeError) as e: + # container infos set and not None + log.err("Bad network information for container: %r" % container) + + for local, remote in nats: + if not remote: + continue + # + # filter for port and protocol + # + port, proto = local.split("/") + port = int(port) + if sport and sport != port: + continue + if sproto and sproto != proto: + continue + + for r in remote: + try: + yield (port, proto, int(r['HostPort']), r['HostIp']) + except (ValueError, KeyError) as e: + log.err() + continue diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py new file mode 100644 index 0000000..3ea806f --- /dev/null +++ b/dockerdns/resolver.py @@ -0,0 +1,151 @@ + + +from twisted.python import log +from twisted.internet import defer +from twisted.internet.defer import failure +from twisted.names import common, dns +from twisted.names.error import DomainError, DNSQueryTimeoutError +from dockerdns.utils import get_preferred_ip + + +# pylint:disable=too-many-public-methods +class DockerResolver(common.ResolverBase): + """ + DNS resolver to resolve queries with a DockerMapping instance. + + Twisted Names just uses the lookupXXX methods + """ + mock_records = [dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) + ] + + def __init__(self, mapping, config=None): + """ + :param mapping: DockerMapping instance for lookups + """ + + self.mapping = mapping + self.config = config or {} + + # Change to this ASAP when Twisted uses object base + # super(DockerResolver, self).__init__() + common.ResolverBase.__init__(self) + self.ttl = 10 + self.my_preferred_ip, self.my_preferred_ip_ptr_value = get_preferred_ip() + + # define authority and additional records + # an authority record defines the name IN NS TIMEOUT + # socket. + import socket + self.authority = [dns.RRHeader(name="docker.", type=dns.NS, cls=dns.IN, + payload=dns.Record_NS( + name=socket.gethostname()) + )] + self.additional = [dns.RRHeader( + name=socket.gethostname(), type=dns.A, cls=dns.IN, + payload=dns.Record_A(address=self.my_preferred_ip)) + ] + + def _a_records(self, name): + """ + Get A records from a query name + + :param name: DNS query name to look up + + :return: Tuple of formatted DNS replies + """ + + addr = self.mapping.get_a(name) + if not addr: + raise DomainError(name) + + return tuple([ + dns.RRHeader(name, dns.A, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def _srv_records(self, name): + print("getting srv: %r" + name) + addr = self.mapping.get_a(name) + return tuple([ + dns.RRHeader(name, dns.SRV, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def lookupAddress(self, name, timeout=None): + """ + + :param name: + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error + or with NotImplementedError. (type: Deferred) + + """ + try: + records = self._a_records(name) + return defer.succeed((records, self.authority, self.additional)) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as e: + log.msg("DomainError: %r " % e) + return defer.fail(failure.Failure(e)) + except Exception as e: # pylint:disable=bare-except + import traceback + traceback.print_exc() + + if self.config.get('no_nxdomain'): + log.err() + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + + def lookupService(self, name, timeout=None): + """ Lookup a docker natted service of + the form: NATTEDPORT._tcp.CONTAINERNAME.docker. + and returns a srv record of the for: + _service._proto.name. TTL class SRV priority weight port target. + + :return: - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) + """ + if not name.endswith(".docker"): + log.err("Domain not ending with .docker: %r" % name) + return defer.fail(failure.Failure(DomainError("not ending with docker"))) + try: + port, proto, container, _ = name.split(".") + port = int(port.strip("_")) + except (IndexError, TypeError, ValueError) as e: + log.err("Domain not of the right form: %r" % name) + return defer.fail(failure.Failure(DomainError("not of the right form"))) + + records = [dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + dns.Record_SRV( + priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip_ptr_value, ttl=None), + auth=True) + for c_port, protocol, c_nat_port, target + in self.mapping.get_nat(container) + if c_port == port # eventually filter + ] + return defer.succeed((records, self.authority, self.additional)) diff --git a/utils.py b/dockerdns/utils.py similarity index 79% rename from utils.py rename to dockerdns/utils.py index eaedd3a..86a4379 100644 --- a/utils.py +++ b/dockerdns/utils.py @@ -12,7 +12,7 @@ def get_preferred_ip(): # connecting to a UDP address doesn't send packets s.connect(('8.8.8.8', 0)) ip = s.getsockname()[0] - return '.'.join(list(reversed(s.getsockname()[0].split(".")))) + ".in-addr.arpa" + return ip, '.'.join(list(reversed(ip.split(".")))) + ".in-addr.arpa" except Exception as e: return socket.getfqdn() @@ -65,20 +65,19 @@ def __call__(self, *args, **kw): def traverse_tree(haystack, key_path, default=None): """ - Look up value in a nested dict + Find an element in a nested dict, eg. + traverse_tree({'Net': {'IP': '1.1.1.1'}}, ['Net', 'IP']) == '1.1.1.1' - Args: - dic: The dictionary to search - key_path: An iterable containing an ordered list of dict keys to + :param haystack: The nested dictionary to search + :param key_path: An iterable containing an ordered list of dict keys to traverse - default: Value to return in case nothing is found - Returns: - Value of the dict at the nested location given, or default if no value + :param default: Value to return in case nothing is found + :return:Value of the dict at the nested location given, or default if no value was found """ for k in key_path: if k in haystack: - haystack = dic[k] + haystack = haystack[k] else: return default return haystack diff --git a/test_srv.py b/test/__init__.py similarity index 67% rename from test_srv.py rename to test/__init__.py index 649109e..fc1c771 100644 --- a/test_srv.py +++ b/test/__init__.py @@ -1,13 +1,57 @@ -""" - Creating srv records +__author__ = 'rpolli' -""" -from docker_dns import DockerResolver, DockerMapping -from twisted.names import dns -from docker_dns_test import check_deferred -import docker +inspect_container_pandas = { + 'Id': 'cidpandaslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + 'Name': '/cidpandas' +} +inspect_container_foxes = { + 'Id': 'cidfoxeslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + }, + 'Name': '/cidfoxes' +} +inspect_container_sloths = { + 'Id': 'cidslothslong', + 'Config': { + 'Hostname': 'stopped-sloths', + }, + 'NetworkSettings': { + 'IPAddress': '' + }, + 'Name': '/cidsloths' +} +inspect_container_returns = { + 'cidpandas': inspect_container_pandas, + 'cidpandaslong': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidfoxeslong': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, + 'cidslothslong': inspect_container_sloths, +} +containers_return = [ + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, +] -mock_list_containers = lambda self, name: [ +mock_list_containers_2 = lambda *a, **k: [ + inspect_container_foxes, inspect_container_pandas, inspect_container_sloths +] +mock_inspect_containers_2 = lambda cid, **k: inspect_container_returns[cid] + +mock_list_containers = lambda *a, **k: [ {u'Command': u'/bin/bash', u'Created': 1414430175, u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', @@ -31,7 +75,7 @@ ] -mock_lookup_container = lambda name: { +mock_lookup_container = lambda *a, **k: { u'Args': [], u'Config': {u'AttachStderr': True, u'AttachStdin': True, @@ -111,62 +155,3 @@ } SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" - - -class Test(object): - - def setup(self): - docker_client = docker.Client() - self.mapping = DockerMapping(api=docker_client) - self.mapping.lookup_container = mock_lookup_container - self.resolver = DockerResolver(self.mapping) - - def test_nat_all(self): - host, port = "foo.docker", 8080 - ret = self.mapping.get_nat(host, port) - assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret - - def test_nat_ports(self): - expected = [(8080, 18080), - (9999, 19999), (8787, 8787)] - for pin, pout in expected: - ret = self.mapping.get_nat("foo", pin) - _, _, port, _ = next(ret) - assert port == pout, "unexpected value in %r" % ret - - def test_lookupService_ko(self): - expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( - ) - for n in expect_fail: - ret = self.resolver.lookupService(n) - check_deferred(ret, False) - - def test_lookupService_ok(self): - ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") - ret = check_deferred(ret, True) - print("resolved: %r" % [ret]) - - def test_mapping(self): - container = self.mapping.lookup_container("foo") - - for local, remote in container['NetworkSettings']['Ports'].items(): - port, proto = local.split("/") - if not remote: - continue - try: - remote = remote[0] - except IndexError: - continue - - print(SRV_FMT.format( - svc=port, - proto=proto, - container=container['Name'][1:], - cclass="IN", - priority=100, - weight=100, - port=remote['HostPort'], - target=remote['HostIp'] if remote[ - 'HostIp'] != '0.0.0.0' else "localhost" - ) - ) diff --git a/docker_dns_test.py b/test/docker_dns_test.py similarity index 72% rename from docker_dns_test.py rename to test/docker_dns_test.py index 3db8c7a..a4ad2b7 100755 --- a/docker_dns_test.py +++ b/test/docker_dns_test.py @@ -9,19 +9,17 @@ # Do not care...... # noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name -import docker_dns - -import docker -import fudge import itertools import unittest -from utils import traverse_tree -from docker_dns import (DEFAULT_CONFIG, - DockerMapping, - DockerResolver) +import docker +import fudge from twisted.names import dns from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log +from dockerdns.utils import traverse_tree +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver # FIXME I can not believe how disgusting this is @@ -40,6 +38,7 @@ def check_record(record, **kwargs): real_value = real_value.name if real_value != kwargs[k]: + log.err("Expected: %s vs %s" % (kwargs[k], real_value)) return False return True @@ -222,117 +221,13 @@ def test_user_default_multi_depth(self): ) -class DockerMappingTest(unittest.TestCase): - - def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) - - # - # TEST _ids_from_prop - # - def test__ids_from_prop_single_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['ID'], - 'cidpandaslong' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidpandaslong' - ) - - def test__ids_from_prop_multi_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['NetworkSettings', 'IPAddress'], - '8.8.8.8' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidfoxeslong' - ) - - def test__ids_from_prop_multi_match(self): - # FIXME I can not believe how disgusting this is - ids_gen1, ids_gen2, ids_gen3 = itertools.tee( - self.mapping._ids_from_prop( - ['Same'], - 'Value' - ), 3 - ) - self.assertEqual(sum(1 for _ in ids_gen1), 2) - self.assertTrue(in_generator(ids_gen2, 'cidpandaslong')) - self.assertTrue(in_generator(ids_gen3, 'cidfoxeslong')) - - # - # TEST lookup_container - # - def test_lookup_container_hostname(self): - self.assertEqual( - self.mapping.lookup_container('cuddly-pandas'), - self.client.inspect_container_pandas - ) - - def test_lookup_container_id(self): - self.assertEqual( - self.mapping.lookup_container('cidfoxes.docker'), - self.client.inspect_container_foxes - ) - - def test_lookup_container_hostname_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid'), - None - ) - - def test_lookup_container_id_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid.docker'), - None - ) - - # - # TEST get_a - # - def test_get_a_hostname(self): - self.assertEqual( - self.mapping.get_a('sneaky-foxes'), - '8.8.8.8' - ) - - def test_get_a_id(self): - self.assertEqual( - self.mapping.get_a('cidpandas.docker'), - '127.0.0.1' - ) - - def test_get_a_hostname_none(self): - self.assertEqual( - self.mapping.get_a('invalid'), - None - ) - - def test_get_a_id_none(self): - self.assertEqual( - self.mapping.get_a('invalid.docker'), - None - ) - - class DockerResolverTest(unittest.TestCase): def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) + self.CONFIG = {} + from test.test_events import create_mock_db2 + self.db = create_mock_db2() + self.mapping = DockerMapping(self.db) self.resolver = DockerResolver(self.mapping) # @@ -384,7 +279,7 @@ def test__a_records_blank_query(self): ) def test__a_records_authoritive(self): - docker_dns.CONFIG['authoritive'] = True + self.resolver.config['authoritive'] = True rec = self.resolver._a_records('cidpandas.docker') self.assertEqual(len(rec), 1) @@ -397,7 +292,7 @@ def test__a_records_authoritive(self): )) def test__a_records_non_authoritive(self): - docker_dns.CONFIG['authoritive'] = False + self.resolver.config['authoritive'] = False rec = self.resolver._a_records('cidpandas.docker') self.assertEqual(len(rec), 1) @@ -438,7 +333,7 @@ def test_lookupAddress_invalid(self): self.assertNotEqual(result, False) def test_lookupAddress_invalid_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = False + self.resolver.config['no_nxdomain'] = False deferred = self.resolver.lookupAddress('invalid') result = check_deferred(deferred, False) @@ -447,7 +342,7 @@ def test_lookupAddress_invalid_nxdomain(self): result.type, DomainError) # noqa pylint:disable=maybe-no-member def test_lookupAddress_invalid_no_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = True + self.resolver.config['no_nxdomain'] = True deferred = self.resolver.lookupAddress('invalid') result = check_deferred(deferred, False) diff --git a/test/test_events.py b/test/test_events.py new file mode 100644 index 0000000..374f514 --- /dev/null +++ b/test/test_events.py @@ -0,0 +1,32 @@ +from dockerdns.events import DockerDB +import docker +from test import mock_list_containers, mock_lookup_container, mock_list_containers_2, mock_inspect_containers_2 +from twisted.python import log + + +def create_mock_db(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers + api.inspect_container = mock_lookup_container + return DockerDB(api=api) + + +def create_mock_db2(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers_2 + api.inspect_container = mock_inspect_containers_2 + return DockerDB(api=api) + + +def test_init(): + db = create_mock_db() + ret = db.get_by_name('jboss631') + assert ret, "Missing container" + try: + assert ret['NetworkSettings'][ + 'IPAddress'] == '172.17.0.10', "item %r" % ret + except KeyError as e: + log.err("Bad container %r" % ret) + raise diff --git a/test/test_old.py b/test/test_old.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_srv.py b/test/test_srv.py new file mode 100644 index 0000000..ddc65e9 --- /dev/null +++ b/test/test_srv.py @@ -0,0 +1,71 @@ +""" + Creating srv records + +""" +import docker + +from dockerdns.resolver import DockerResolver +from dockerdns.mappings import DockerMapping +from test.docker_dns_test import check_deferred +from test import mock_lookup_container, SRV_FMT + + +class Test(object): + + def setup(self): + self.mapping = DockerMapping(db=None) + self.mapping.lookup_container = mock_lookup_container + self.resolver = DockerResolver(self.mapping) + + def test_service_image(self): + raise NotImplementedError("implement skydns-like") + + def test_nat_all(self): + host, port = "foo.docker", 8080 + ret = self.mapping.get_nat(host, port) + assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret + + def test_nat_ports(self): + expected = [(8080, 18080), + (9999, 19999), (8787, 8787)] + for pin, pout in expected: + ret = self.mapping.get_nat("foo", pin) + _, _, port, _ = next(ret) + assert port == pout, "unexpected value in %r" % ret + + def test_lookupService_ko(self): + expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( + ) + for n in expect_fail: + ret = self.resolver.lookupService(n) + check_deferred(ret, False) + + def test_lookupService_ok(self): + ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_mapping(self): + container = self.mapping.lookup_container("foo") + + for local, remote in container['NetworkSettings']['Ports'].items(): + port, proto = local.split("/") + if not remote: + continue + try: + remote = remote[0] + except IndexError: + continue + + print(SRV_FMT.format( + svc=port, + proto=proto, + container=container['Name'][1:], + cclass="IN", + priority=100, + weight=100, + port=remote['HostPort'], + target=remote['HostIp'] if remote[ + 'HostIp'] != '0.0.0.0' else "localhost" + ) + ) From ff480e6cc7d4fd7465b1dd6af83802fe5a4fb3ad Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Thu, 29 Jan 2015 21:47:47 +0100 Subject: [PATCH 25/51] refactor: tests --- .travis.yml | 4 +- README.md | 9 +- docker_dns.py | 308 ++------------------- docker_dns_test.py | 464 -------------------------------- docker_events.py | 161 ----------- dockerdns/__init__.py | 7 + dockerdns/events.py | 221 +++++++++++++++ dockerdns/mappings.py | 130 +++++++++ dockerdns/resolver.py | 175 ++++++++++++ utils.py => dockerdns/utils.py | 17 +- requirements.txt | 1 + test_srv.py => test/__init__.py | 146 +++++----- test/test_events.py | 76 ++++++ test/test_resolver.py | 277 +++++++++++++++++++ test/test_srv.py | 87 ++++++ test/test_utils.py | 51 ++++ 16 files changed, 1130 insertions(+), 1004 deletions(-) delete mode 100755 docker_dns_test.py delete mode 100644 docker_events.py create mode 100644 dockerdns/__init__.py create mode 100644 dockerdns/events.py create mode 100644 dockerdns/mappings.py create mode 100644 dockerdns/resolver.py rename utils.py => dockerdns/utils.py (79%) rename test_srv.py => test/__init__.py (67%) create mode 100644 test/test_events.py create mode 100755 test/test_resolver.py create mode 100644 test/test_srv.py create mode 100644 test/test_utils.py diff --git a/.travis.yml b/.travis.yml index a5bfbe3..53343c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: python python: - "2.7" install: - - sudo pip install -r test_requirements.txt --use-mirrors + - sudo pip install -r test_requirements.txt before_script: - pep8 *.py - pylint --rcfile=pylint.conf *.py script: - - ./docker_dns_test.py \ No newline at end of file + - nosetests -v -w test diff --git a/README.md b/README.md index dbbdc07..84c515d 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,13 @@ Docker DNS ========== [![Build Status](https://travis-ci.org/infoxchange/docker_dns.png?branch=master)](https://travis-ci.org/infoxchange/docker_dns) -A simple Twisted DNS server using custom TLD and Docker as the back end for IP +A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP resolution. To look up a container: - 'A' record query a container NAME that will match a container with a docker inspect - command with '.d' as the TLD. eg: mysql_server1.d + command with '.docker' as the TLD. eg: mysql_server1.docker + - 'SRV' record query exposing the NAT informations Install/Run ----------- @@ -140,8 +141,8 @@ configuration is rather limited. # NXDOMAIN so secondary DNS is used 'no_nxdomain': True, - # Makes successful requests authoritive - 'authoritive': True, + # Makes successful requests authoritative + 'authoritative': True, } Contributing diff --git a/docker_dns.py b/docker_dns.py index ecf48c2..de727bd 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -19,18 +19,14 @@ Author: Roberto Polli """ -import docker -from warnings import warn -from requests.exceptions import RequestException - from twisted.application import internet, service -from twisted.internet import defer -from twisted.names import common, dns, server -from twisted.names.error import DNSQueryTimeoutError, DomainError -from twisted.python import failure, log +from twisted.names import dns, server +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.events import DockerDB +from dockerdns.resolver import DockerResolver -from utils import get_preferred_ip, memoize -from docker_events import UpdateDB # Merge user config over defaults CONFIG = DEFAULT_CONFIG = { @@ -40,7 +36,8 @@ 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], 'no_nxdomain': True, - 'authoritive': True, + 'authoritative': True, + 'domain': 'docker' } # Load the config @@ -51,296 +48,21 @@ appcfg = {} -class DockerMapping(object): - """ - Look up docker container data via docker.api. - - XXX Should it be dns-agnostic, and just a wrapper around docker.api - """ - - def __init__(self, api=None, db=None): - """ - Args: - api: Docker Client instance used to do API communication - """ - self.db = db - self.api = api if api else docker.Client() - log.msg("DockerMapping pointing to %r" % self.api.base_url) - try: - print('connected to docker instance running api version %s' % - self.api.version()['ApiVersion']) - except docker.errors.APIError as ex: - log.err("Cannot instantiate docker api") - raise ex - - db.populate(self.api.containers(all=True)) - - - def lookup_container(self, name): - """ - Gets the container config from a DNS lookup name, or returns None if - one could not be found - - Args: - name: DNS query name to look up - - Returns: - Container config dict for the first matching container - """ - try: - key_path = 'Name' - name = name.replace('.docker', '') - log.msg('lookup container: %r' % name) - if name not in self.db.mappings_idx: - raise KeyError("Item %s not found in %s" % - (name, self.db.mappings_idx.keys())) - id = self.db.mappings_idx[name] - return self.db.mappings[id] - except KeyError as e: - # warn(str(e)) - return None - except Exception as e: - log.exc("Unmanaged error") - return None - - def lookup_container_old(self, name): - """ - Gets the container config from a DNS lookup name, or returns None if - one could not be found - - Args: - name: DNS query name to look up - - Returns: - Container config dict for the first matching container - """ - key_path = 'Name' - name = name.replace('.docker', '') - log.msg('lookup container: %r' % name) - - try: - cid_all = (c['Id'] for c in self.api.containers(all=True) if c) - log.msg('found containers: %r ' % cid_all) - - for cid in cid_all: - cdic = self.api.inspect_container(cid) - # as container names starts with "/" we should strip it - cname = str(cdic[key_path].strip('/')) - if cname == name: - container_id = cid - break - else: - container_id = None - - log.msg('container matching %s: %s' % (name, container_id)) - - return self.api.inspect_container(container_id) - - except docker.errors.APIError as ex: - # 404 is valid, others aren't - if ex.response.status_code != 404: - warn(str(ex)) - return None - - except RequestException as ex: - log.err() - warn(str(ex)) - return None - - def get_a(self, name): - """ - Get an IPv4 address from a query name to be used in A record lookups - - Args: - name: DNS query name to look up - - Returns: - IPv4 address for the query name given - """ - - container = self.lookup_container(name) - - if container is None: - print("No container found") - return None - - addr = container['NetworkSettings']['IPAddress'] - - if not addr: - return None - - return addr - - def get_nat(self, container_name, sport=0, sproto=None): - """ @return - a generator of natted maps (local, nat, ip) - - @param sport: the port to search - @param sproto: the protocol to search - - eg. [ (8080, 'tcp', 18080, '0.0.0.0'), - (8787, 'tcp', 8787, '0.0.0.0'), - ] - """ - sport = int(sport) - container = self.lookup_container(container_name) - if not container: - log.err("Bad network information for docker") - return - - try: - for local, remote in container['NetworkSettings']['Ports'].items(): - port, proto = local.split("/") - port = int(port) - if sport and sport != port: - continue - if sproto and sproto != proto: - continue - if not remote: - continue - - for r in remote: - try: - yield (port, proto, int(r['HostPort']), r['HostIp']) - except (ValueError, KeyError) as e: - log.err() - continue - except KeyError as e: - log.err("Bad network information from docker") - - -# pylint:disable=too-many-public-methods -class DockerResolver(common.ResolverBase): - """ - DNS resolver to resolve queries with a DockerMapping instance. - - Twisted Names just uses the lookupXXX method - """ - mock_records = [dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=19999, target='name', ttl=None), - auth=True), - dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=18080, target='name', ttl=None), - auth=True) - ] - - def __init__(self, mapping): - """ - Args: - mapping: DockerMapping instance for lookups - """ - - self.mapping = mapping - - # Change to this ASAP when Twisted uses object base - # super(DockerResolver, self).__init__() - common.ResolverBase.__init__(self) - self.ttl = 10 - self.my_preferred_ip = get_preferred_ip() - - def _a_records(self, name): - """ - Get A records from a query name - - Args: - name: DNS query name to look up - - Returns: - Tuple of formatted DNS replies - """ - - addr = self.mapping.get_a(name) - if not addr: - raise DomainError(name) - - return tuple([ - dns.RRHeader(name, dns.A, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - CONFIG['authoritive']) - ]) - - def _srv_records(self, name): - print("getting srv: %r" + name) - return tuple([ - dns.RRHeader(name, dns.SRV, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - CONFIG['authoritive']) - ]) - - def lookupAddress(self, name, timeout=None): - try: - records = self._a_records(name) - return defer.succeed((records, (), ())) - - # We need to catch everything. Uncaught exceptions will make the server - # stop responding - except DomainError as e: - log.msg("DomainError: %r " % e) - return defer.fail(failure.Failure(e)) - except Exception as e: # pylint:disable=bare-except - import traceback - traceback.print_exc() - - if CONFIG['no_nxdomain']: - log.err() - # FIXME surely there's a better way to give SERVFAIL - exception = DNSQueryTimeoutError(name) - else: - exception = DomainError(name) - - return defer.fail(failure.Failure(exception)) - - def lookupService(self, name, timeout=None): - """ Lookup a docker natted service of - the form: NATTEDPORT._tcp.CONTAINERNAME.docker. - and returns a srv record of the for: - _service._proto.name. TTL class SRV priority weight port target. - - @returns - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. - The first element of the tuple gives answers. - The second element of the tuple gives authorities. - The third element of the tuple gives additional information. - The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) - """ - #return defer.succeed((self.mock_records, (), ())) - if not name.endswith(".docker"): - log.err("Domain not ending with .docker: %r" % name) - return defer.fail(failure.Failure(DomainError("not ending with docker"))) - try: - port, proto, container, _ = name.split(".") - port = int(port.strip("_")) - except (IndexError, TypeError, ValueError) as e: - log.err("Domain not of the right form: %r" % name) - return defer.fail(failure.Failure(DomainError("not of the right form"))) - - records = [dns.RRHeader( - name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip, ttl=None), - auth=True) - for c_port, protocol, c_nat_port, target - in self.mapping.get_nat(container) - if c_port == port # eventually filter - ] - return defer.succeed((records, (), ())) - - def main(): """ Set everything up """ + import docker # Create docker: by default dict.get returns None on missing keys docker_client = docker.Client(CONFIG.get('docker_url')) - + infos = docker_client.info() # Test docker connectivity before starting - log.msg("Connecting to docker instance: %r" % docker_client.info()) + log.msg("Connecting to docker instance: %r" % infos) + db = DockerDB(api=docker_client) # Create our custom mapping and resolver - mapping = DockerMapping(docker_client, db=UpdateDB) + mapping = DockerMapping(db=db) resolver = DockerResolver(mapping) # Create twistd stuff to tie in our custom components @@ -370,10 +92,10 @@ def main(): svc.setServiceParent(ret) # Add the event Loop - from docker_events import EventFactory + from dockerdns.events import EventFactory from urlparse import urlparse u = urlparse(CONFIG['docker_url']) - efactory = EventFactory(config=CONFIG) + efactory = EventFactory(config=CONFIG, db=db) docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) docker_event_monitor.setServiceParent(ret) diff --git a/docker_dns_test.py b/docker_dns_test.py deleted file mode 100755 index 3db8c7a..0000000 --- a/docker_dns_test.py +++ /dev/null @@ -1,464 +0,0 @@ -#!/usr/bin/python - -""" -Tests for Docker DNS - -Author: Ricky Cook -""" - -# Do not care...... -# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name - -import docker_dns - -import docker -import fudge -import itertools -import unittest - -from utils import traverse_tree -from docker_dns import (DEFAULT_CONFIG, - DockerMapping, - DockerResolver) -from twisted.names import dns -from twisted.names.error import DNSQueryTimeoutError, DomainError - - -# FIXME I can not believe how disgusting this is -def in_generator(gen, val): - return reduce( - lambda old, new: old or new == val, - gen, - False - ) - - -def check_record(record, **kwargs): - for k in kwargs: - real_value = getattr(record, k) - if k is 'name': - real_value = real_value.name - - if real_value != kwargs[k]: - return False - - return True - - -def check_deferred(deferred, success): - completed = [] - - def gimme_x_back(is_success): - def x_back(result): - completed.append((is_success, result)) - - return x_back - - deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) - if len(completed) != 1: - return False - - status, result = completed[0] - if status != success: - raise AssertionError("Expected: %r, got %r" % (success, result)) - return False - - return result - - -class MockDockerClient(object): - base_url = 'http://localhost:5000' - version = lambda x: {'ApiVersion': '1.0'} - inspect_container_pandas = { - 'ID': 'cidpandaslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'cuddly-pandas', - }, - 'NetworkSettings': { - 'IPAddress': '127.0.0.1' - }, - } - inspect_container_foxes = { - 'ID': 'cidfoxeslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'sneaky-foxes', - }, - 'NetworkSettings': { - 'IPAddress': '8.8.8.8' - } - } - inspect_container_sloths = { - 'ID': 'cidslothslong', - 'Config': { - 'Hostname': 'stopped-sloths', - }, - 'NetworkSettings': { - 'IPAddress': '' - } - } - inspect_container_returns = { - 'cidpandas': inspect_container_pandas, - 'cidpandaslong': inspect_container_pandas, - 'cidfoxes': inspect_container_foxes, - 'cidfoxeslong': inspect_container_foxes, - 'cidsloths': inspect_container_sloths, - 'cidslothslong': inspect_container_sloths, - } - containers_return = [ - {'Id': 'cidpandas'}, - {'Id': 'cidfoxes'}, - {'Id': 'cidsloths'}, - ] - - inspect_container_id = None - - def inspect_container(self, cid): - self.inspect_container_id = cid - - try: - return self.inspect_container_returns[cid] - except KeyError: - # Mocks a Docker Client Exception - response = fudge.Fake() - response.has_attr(status_code=404, content='PANDAS!') - - exception = docker.client.APIError('bad', response) - raise exception - - def containers(self, *args, **kwargs): # pylint:disable=unused-argument - return self.containers_return - - -class DictLookupTest(unittest.TestCase): - theDict = { - 'pandas': { - 'are': 'cuddly', - 'and': 'awesome', - }, - 'foxes': { - 'are': 'sneaky', - 'and': 'orange', - }, - 'badgers': { - 'are': None, - }, - } - - def test_basic_one(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['pandas', 'and'] - ), - 'awesome' - ) - - def test_basic_two(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['foxes', 'are'] - ), - 'sneaky' - ) - - def test_basic_none(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['badgers', 'are'], - 'Badgers are none? What?' - ), - None - ) - - def test_dict(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['foxes'] - ), - self.theDict['foxes'] - ) - - def test_default_single_depth(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['nothing'] - ), - None - ) - - def test_user_default_single_depth(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['nothing'], - 'Nobody here but us chickens' - ), - 'Nobody here but us chickens' - ) - - def test_default_multi_depth(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['pandas', 'bad'] - ), - None - ) - - def test_user_default_multi_depth(self): - self.assertEqual( - traverse_tree( - self.theDict, - ['pandas', 'bad'], - 'NO, THAT\'S A DAMN DIRTY LIE' - ), - 'NO, THAT\'S A DAMN DIRTY LIE' - ) - - -class DockerMappingTest(unittest.TestCase): - - def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) - - # - # TEST _ids_from_prop - # - def test__ids_from_prop_single_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['ID'], - 'cidpandaslong' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidpandaslong' - ) - - def test__ids_from_prop_multi_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['NetworkSettings', 'IPAddress'], - '8.8.8.8' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidfoxeslong' - ) - - def test__ids_from_prop_multi_match(self): - # FIXME I can not believe how disgusting this is - ids_gen1, ids_gen2, ids_gen3 = itertools.tee( - self.mapping._ids_from_prop( - ['Same'], - 'Value' - ), 3 - ) - self.assertEqual(sum(1 for _ in ids_gen1), 2) - self.assertTrue(in_generator(ids_gen2, 'cidpandaslong')) - self.assertTrue(in_generator(ids_gen3, 'cidfoxeslong')) - - # - # TEST lookup_container - # - def test_lookup_container_hostname(self): - self.assertEqual( - self.mapping.lookup_container('cuddly-pandas'), - self.client.inspect_container_pandas - ) - - def test_lookup_container_id(self): - self.assertEqual( - self.mapping.lookup_container('cidfoxes.docker'), - self.client.inspect_container_foxes - ) - - def test_lookup_container_hostname_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid'), - None - ) - - def test_lookup_container_id_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid.docker'), - None - ) - - # - # TEST get_a - # - def test_get_a_hostname(self): - self.assertEqual( - self.mapping.get_a('sneaky-foxes'), - '8.8.8.8' - ) - - def test_get_a_id(self): - self.assertEqual( - self.mapping.get_a('cidpandas.docker'), - '127.0.0.1' - ) - - def test_get_a_hostname_none(self): - self.assertEqual( - self.mapping.get_a('invalid'), - None - ) - - def test_get_a_id_none(self): - self.assertEqual( - self.mapping.get_a('invalid.docker'), - None - ) - - -class DockerResolverTest(unittest.TestCase): - - def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) - self.resolver = DockerResolver(self.mapping) - - # - # TEST _a_records - # - def test__a_records_hostname(self): - rec = self.resolver._a_records('sneaky-foxes') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='sneaky-foxes', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') - - def test__a_records_id(self): - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '127.0.0.1') - - def test__a_records_shutdown(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'cidsloths.docker' - ) - - def test__a_records_invalid(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'invalid.docker' - ) - - def test__a_records_blank_query(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - '' - ) - - def test__a_records_authoritive(self): - docker_dns.CONFIG['authoritive'] = True - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - auth=True, - )) - - def test__a_records_non_authoritive(self): - docker_dns.CONFIG['authoritive'] = False - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - auth=False, - )) - - # - # TEST lookupAddress - # - def test_lookupAddress_id(self): - deferred = self.resolver.lookupAddress('cidfoxes.docker') - - result = check_deferred(deferred, True) - self.assertNotEqual(result, False) - - self.assertEqual(len(result), 3) - self.assertEqual(result[1], ()) - self.assertEqual(result[2], ()) - self.assertEqual(len(result[0]), 1) - - rec = result[0][0] - self.assertTrue(check_record( - rec, - name='cidfoxes.docker', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') - - def test_lookupAddress_invalid(self): - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - - def test_lookupAddress_invalid_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = False - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual( - result.type, DomainError) # noqa pylint:disable=maybe-no-member - - def test_lookupAddress_invalid_no_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = True - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual(result.type, DNSQueryTimeoutError) - # noqa pylint:disable=maybe-no-member - - -def main(): - unittest.main() - - -if __name__ == '__main__': - main() diff --git a/docker_events.py b/docker_events.py deleted file mode 100644 index 9776ee3..0000000 --- a/docker_events.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import print_function - -from twisted.internet import reactor -from twisted.web.client import Agent, HTTPConnectionPool -from twisted.web.http_headers import Headers -import os -import simplejson as json -from twisted.internet.defer import Deferred, DeferredList -from twisted.internet.protocol import Protocol, ReconnectingClientFactory -from twisted.python import log - - - -class UpdateDB(Protocol): - """Update docker ip store""" - mappings = {} - mappings_idx = {} - - def __init__(self, onLost): - """Initialize the Docker host database""" - self.onLost = onLost - - def dataReceived(self, bytes_): - if bytes_: - item = json.loads(bytes_) - print("Get container %r" % item) - UpdateDB.updatedb(item) - - def connectionLost(self, reason): - self.onLost.callback(None) - - @staticmethod - def populate(mappings): - if mappings: - print("Populating db with data: %r" % mappings) - UpdateDB.mappings = mappings - UpdateDB.mappings_idx = {item['Names'][0][1:]: item['Id'] for item in mappings} - - - @staticmethod - def updatedb(item): - assert 'Id' in item, "Entry has no Id" - UpdateDB.mappings_idx.update({item['Name'][1:]: item['Id']}) - UpdateDB.mappings.update({item['Id']: item}) - - @staticmethod - def add_container(item): - UpdateDB.updatedb(item) - - @staticmethod - def del_container(id): - name = UpdateDB.mappings[id]['Name'][1:] - del UpdateDB.mappings[id] - del UpdateDB.mappings_idx[name] - - @staticmethod - def get_by_name(name): - if name not in UpdateDB.mappings_idx: - raise KeyError("%r not in %r" % (name, UpdateDB.mappings_idx)) - - cid = UpdateDB.mappings_idx[name] - if cid not in UpdateDB.mappings: - raise KeyError("%r not in %r" % (cid, UpdateDB.mappings.keys())) - return UpdateDB.mappings[id] - - -class UpdateDockerMapping(Protocol): - """Parse docker events and update item store""" - def __init__(self, agent, config): - self.agent = agent - self.config = config - - self.remaining = 1024 * 10 - self.buff = "" - - def update_record(self, item): - """Update docker mapping - item = {'status': ..., 'id': ...} - """ - d = self.agent.request('GET', - os.path.join(self.config['docker_url'], - 'containers', item['id'], 'json') - ) - d.addCallback( - lambda response: response.deliverBody(UpdateDB(Deferred()))) - - def delete_record(self, item): - """Update docker mapping - item = {'status': ..., 'id': ...} - """ - log.msg("removing item: %r" % item) - UpdateDB.del_container(item['id']) - - def dataReceived(self, bytes_): - """Get the container id and calls the updater""" - try: - if self.remaining: - display = bytes_[:self.remaining] - print('Some data received:', display) - self.remaining -= len(display) - item = json.loads(display) - print("Parsed: %r" % item) - if item['status'] == 'start': - self.update_record(item) - elif item['status'] in ('stop', 'die'): - self.delete_record(item) - except KeyError: - log.err("Container not found") - except json.scanner.JSONDecodeError: - log.err("Error reading data") - except Exception: - log.err("Generic Error") - - - def connectionLost(self, reason): - print('Finished receiving body:', reason.type, reason.value) - Deferred().callback(None) - - -class EventFactory(ReconnectingClientFactory): - # protocol = UpdateDockerMapping - agent = Agent(reactor) - agent2 = Agent(reactor, HTTPConnectionPool(reactor)) - - def __init__(self, config, db=UpdateDB): - log.msg("Initializing factory") - self.db = db - # - # Create a protocol handler to parse docker container data - # - self.dockerUpdater = UpdateDockerMapping(agent=self.agent2, config=config) - - # Populate existing containers (this is ok to be blocking) - - # - # Poll to the docker event interface - # - d = self.agent.request( - 'GET', - os.path.join(config['docker_url'], 'events'), - Headers({'User-Agent': ['Twisted Web Client for Docker Event'], - 'Content-Type': ['application/json']}), - None) - d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) - - def cbResponse(self, response): - """Manages the response using a Protocol class defined in __init__""" - try: - log.msg('Response received: %r' % response) - finished = Deferred() - response.deliverBody(self.dockerUpdater) - except: - log.err() - - def buildProtocol(self, addr): - log.msg("addr: %r" % addr) - return self.dockerUpdater - - - - diff --git a/dockerdns/__init__.py b/dockerdns/__init__.py new file mode 100644 index 0000000..f16daaa --- /dev/null +++ b/dockerdns/__init__.py @@ -0,0 +1,7 @@ +""" + + - DnsServer --> DBParser --> DB + - EventManager -http-> docker-server + 2 protocols: /events/, /containers/ + +""" diff --git a/dockerdns/events.py b/dockerdns/events.py new file mode 100644 index 0000000..141736c --- /dev/null +++ b/dockerdns/events.py @@ -0,0 +1,221 @@ +from __future__ import print_function + +""" + Populate dns database getting info from docker via: + - docker-py api + - docker events interface +""" +from twisted.internet import reactor +from twisted.web.client import Agent, HTTPConnectionPool +from twisted.web.http_headers import Headers +from os.path import join as pjoin +import simplejson as json +from twisted.internet.defer import Deferred +from twisted.internet.protocol import Protocol, ReconnectingClientFactory +from twisted.python import log + + +class DockerDB(object): + """Update docker ip store connecting via docker-py + """ + + def __init__(self, api=None): + """ + + :param api: a docker.Client instance + :return: + """ + self.api = api + # container list and some indexes: + # - name: id + # - image: names + # - hostname: id + self.mappings = {} + self.mappings_name = {} + self.mappings_image = {} + self.mappings_hostname = {} + # + # Initialize db (TODO get initialization timestamp) + for c in self.api.containers(all=True): + item = self.api.inspect_container(c['Id']) + self.updatedb(item) + + def updatedb(self, item): + assert 'Id' in item, "Entry has no Id" + + name = item['Name'][1:] + hostname = item['Config']['Hostname'] + self.mappings_name.update({name: item['Id']}) + self.mappings_hostname.update({hostname: item['Id']}) + self.mappings.update({item['Id']: item}) + self.mappings_image.setdefault(item['Config']['Image'], []).append(name) + + def add_container(self, item): + self.updatedb(item) + + def del_container(self, cid): + name = self.mappings[cid]['Name'][1:] + image = self.mappings[cid]['Config']['Image'] + hostname = self.mappings[cid]['Config']['Hostname'] + self.mappings_image.get(image, []).remove(name) + del self.mappings[cid] + del self.mappings_name[name] + del self.mappings_hostname[hostname] + + def get_by_name(self, name): + if name not in self.mappings_name: + raise KeyError("%r not in %r" % (name, self.mappings_name)) + + cid = self.mappings_name[name] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + def get_by_hostname(self, name): + if name not in self.mappings_hostname: + raise KeyError("%r not in %r" % (name, self.mappings_hostname)) + + cid = self.mappings_hostname[name] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + def get_by_image(self, image): + """ + + :param image: + :return: an generator of container dicts + """ + if image not in self.mappings_image: + raise KeyError("%r not in %r" % (image, self.mappings_hostname)) + for cid in self.mappings_image[image]: + yield self.mappings[cid] + + +class ContainerManager(Protocol): + """Manage the response to /containers/{id}/json + updating the network infos associated to the container + """ + + def __init__(self, onLost, db): + """Initialize the Docker host database""" + self.onLost = onLost + self.db = db + + def dataReceived(self, bytes_): + if bytes_: + item = json.loads(bytes_) + print("Get container %r" % item) + self.db.updatedb(item) + + def connectionLost(self, reason): + self.onLost.callback(None) + + +class EventManager(Protocol): + """Manage the response to /events/json + - start: triggers a further request with a ContainerManager cb + - stop|die: removes the associations from IPs + """ + + def __init__(self, http_agent, config, db): + self.http_agent = http_agent + self.config = config + self.db = db + + self.remaining = 1024 * 10 + self.buff = "" + # Create an event_manager for parsing updates + self.event_manager = ContainerManager(Deferred(), db=db) + + def update_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + d = self.http_agent.request('GET', + pjoin(self.config['docker_url'], + 'containers', item['id'], 'json') + ) + d.addCallback( + lambda response: response.deliverBody(self.event_manager)) + + def delete_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + log.msg("removing item: %r" % item) + self.db.del_container(item['id']) + + def dataReceived(self, bytes_): + """Get the container id and calls the updater""" + try: + if self.remaining: + display = bytes_[:self.remaining] + print('Some data received:', display) + self.remaining -= len(display) + item = json.loads(display) + print("Parsed: %r" % item) + if item['status'] == 'start': + self.update_record(item) + elif item['status'] in ('stop', 'die'): + self.delete_record(item) + except KeyError: + log.err("Container not found") + except json.scanner.JSONDecodeError: + log.err("Error reading data") + except Exception as e: + log.err("Generic Error %r" % e) + + def connectionLost(self, reason): + print('Finished receiving body:', reason.type, reason.value) + Deferred().callback(None) + + +class EventFactory(ReconnectingClientFactory): + """Factory to connect to docker api interface and trigger + the first /events/ call. + + """ + agent = Agent(reactor) + agent2 = Agent(reactor, HTTPConnectionPool(reactor)) + + def __init__(self, config, db): + """ + + :param config: contains the docker url to poll + :param db: the mappings between containers and + :return: + """ + log.msg("Initializing factory") + self.db = db + # + # Create a protocol handler to parse docker container data + # + self.dockerUpdater = EventManager( + http_agent=self.agent2, config=config, db=db) + + # Populate existing containers (this is ok to be blocking) + + # + # Poll to the docker event interface + # + d = self.agent.request( + 'GET', + pjoin(config['docker_url'], 'events'), + Headers({'User-Agent': ['Twisted Web Client for Docker Event'], + 'Content-Type': ['application/json']}), + None) + d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) + + def cbResponse(self, response): + """Manages the response using a Protocol class defined in __init__""" + try: + log.msg('Response received: %r' % response) + finished = Deferred() + response.deliverBody(self.dockerUpdater) + except: + log.err() + + def buildProtocol(self, addr): + log.msg("addr: %r" % addr) + return self.dockerUpdater diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py new file mode 100644 index 0000000..c9a6ad7 --- /dev/null +++ b/dockerdns/mappings.py @@ -0,0 +1,130 @@ +""" + Maps DNS requests to host +""" +import re +from twisted.python import log + + +class DockerMapping(object): + """ + Look up docker info via docker.api and + + XXX Should it be dns-agnostic, and just a wrapper around docker.api + + TODO: + + """ + + def __init__(self, db=None): + """ + Args: + db: a DockerDB instance containing infos + """ + self.db = db + + def lookup_container(self, name): + """ + Gets the container config from container name or hostname + + Args: + name: DNS query name to look up + + Returns: + Container config dict for the first matching container + """ + for search_f in (self.db.get_by_name, self.db.get_by_hostname): + try: + log.msg('lookup container: %r' % name) + + return search_f(name) + except KeyError as e: + # warn(str(e)) + log.msg("Container not found: %r" % name) + except Exception as e: + log.err("Unmanaged error: %r" % e) + + return None + + def get_a(self, name): + """ + Get an IPv4 address from a query name to be used in A record lookups + + Args: + name: DNS query name to look up + + Returns: + IPv4 address for the query name given + """ + + container = self.lookup_container(name) + + if container is None: + log.msg("No container found") + return + + addr = container['NetworkSettings']['IPAddress'] + + if not addr: + return None + + return addr + + def get_a_multi(self, name_multi): + """ + + :param name_multi: [(addr1, name1), .., (addrX, nameX)] + :return: a tuple of addresses + """ + name_multi, count = re.subn(r'\.\*$', '', name_multi, 1) + if not count: + log.err("Not a multihost search: %r" % (name_multi,)) + return + return tuple( + (container['NetworkSettings']['IPAddress'], container['Name'][1:]) + for container + in self.db.get_by_image(name_multi) + if 'NetworkSettings' in container + ) + + + def get_nat(self, container_name, sport=0, sproto=None): + """ @return - a generator of natted maps (local, proto, nat, ip) + + @param sport: the port to search + @param sproto: the protocol to search + + eg. [ (8080, 'tcp', 18080, '0.0.0.0'), + (8787, 'tcp', 8787, '0.0.0.0'), + ] + """ + sport = int(sport) + container = self.lookup_container(container_name) + if container is None: + log.msg("No container found") + return + + try: + nats = container['NetworkSettings']['Ports'].items() + except (KeyError, AttributeError) as e: + # container infos set and not None + log.err("Bad network information for container: %r" % container) + + for local, remote in nats: + if not remote: + continue + # + # filter for port and protocol + # + port, proto = local.split("/") + port = int(port) + if sport and sport != port: + continue + if sproto and sproto != proto: + continue + + for r in remote: + try: + yield (port, proto, int(r['HostPort']), r['HostIp']) + except (ValueError, KeyError) as e: + log.err() + continue diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py new file mode 100644 index 0000000..10eb472 --- /dev/null +++ b/dockerdns/resolver.py @@ -0,0 +1,175 @@ +import re +from twisted.python import log +from twisted.internet import defer +from twisted.internet.defer import failure +from twisted.names import common, dns +from twisted.names.error import DomainError, DNSQueryTimeoutError +from dockerdns.utils import get_preferred_ip + + +# pylint:disable=too-many-public-methods +NO_NXDOMAIN = 'no_nxdomain' + + +class DockerResolver(common.ResolverBase): + """ + DNS resolver to resolve queries with a DockerMapping instance. + + Twisted Names just uses the lookupXXX methods + """ + mock_records = [dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) + ] + + def __init__(self, mapping, config=None): + """ + :param mapping: DockerMapping instance for lookups + """ + + self.mapping = mapping + self.config = config or {} + + # Change to this ASAP when Twisted uses object base + # super(DockerResolver, self).__init__() + common.ResolverBase.__init__(self) + self.ttl = 10 + self.my_preferred_ip, self.my_preferred_ip_ptr_value = get_preferred_ip() + + # define authority and additional records + # an authority record defines the name IN NS TIMEOUT + # socket. + import socket + + self.authority = [dns.RRHeader(name="docker.", type=dns.NS, cls=dns.IN, + payload=dns.Record_NS( + name=socket.gethostname()) + )] + self.additional = [dns.RRHeader( + name=socket.gethostname(), type=dns.A, cls=dns.IN, + payload=dns.Record_A(address=self.my_preferred_ip)) + ] + + def _a_records(self, name): + """ + Get A records from a query name + + :param name: DNS query name to look up + + :return: Tuple of formatted DNS replies + """ + + addr = self.mapping.get_a(name) + if not addr: + raise DomainError(name) + + return tuple([ + dns.RRHeader('.'.join((name, 'docker')), dns.A, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def _srv_records(self, name): + print("getting srv: %r" + name) + addr = self.mapping.get_a(name) + return tuple([ + dns.RRHeader('.'.join((name, 'docker')), dns.SRV, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def lookupAddress(self, name, timeout=None): + """ + + :param name: + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error + or with NotImplementedError. (type: Deferred) + + """ + name, occurrences = re.subn(r'.docker$', '', name) + if not occurrences: + log.err("Domain not ending with .docker: %r" % name) + return defer.fail(failure.Failure(DomainError("not ending with docker"))) + + try: + if name.endswith(".*"): + records = tuple( + dns.RRHeader('.'.join((name_, 'docker')), dns.A, dns.IN, self.ttl, + dns.Record_A(addr_, self.ttl), + auth=self.config.get('authoritative') + ) + for addr_, name_ + in self.mapping.get_a_multi(name) + ) + else: + records = self._a_records(name) + return defer.succeed((records, self.authority, self.additional)) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as e: + log.msg("DomainError: %r " % e) + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + e = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(e)) + except Exception as e: # pylint:disable=bare-except + import traceback + + traceback.print_exc() + + if self.config.get(NO_NXDOMAIN): + log.err() + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + + def lookupService(self, name, timeout=None): + """ Lookup a docker natted service of + the form: NATTEDPORT._tcp.CONTAINERNAME.docker. + and returns a srv record of the for: + _service._proto.name. TTL class SRV priority weight port target. + + If _service == _nat: + :return: - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) + """ + name, occurrences = re.subn(r'.docker$', '', name) + if not occurrences: + log.err("Domain not ending with .docker: %r" % name) + return defer.fail(failure.Failure(DomainError("not ending with docker"))) + try: + port, proto, container = name.split(".") + port = int(port.strip("_")) + except (IndexError, TypeError, ValueError) as e: + log.err("Domain not of the right form: %r" % name) + return defer.fail(failure.Failure(DomainError("not of the right form"))) + + records = [dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + dns.Record_SRV( + priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip_ptr_value, ttl=None), + auth=True) + for c_port, protocol, c_nat_port, target + in self.mapping.get_nat(container) + if c_port == port # eventually filter + ] + return defer.succeed((records, self.authority, self.additional)) diff --git a/utils.py b/dockerdns/utils.py similarity index 79% rename from utils.py rename to dockerdns/utils.py index eaedd3a..86a4379 100644 --- a/utils.py +++ b/dockerdns/utils.py @@ -12,7 +12,7 @@ def get_preferred_ip(): # connecting to a UDP address doesn't send packets s.connect(('8.8.8.8', 0)) ip = s.getsockname()[0] - return '.'.join(list(reversed(s.getsockname()[0].split(".")))) + ".in-addr.arpa" + return ip, '.'.join(list(reversed(ip.split(".")))) + ".in-addr.arpa" except Exception as e: return socket.getfqdn() @@ -65,20 +65,19 @@ def __call__(self, *args, **kw): def traverse_tree(haystack, key_path, default=None): """ - Look up value in a nested dict + Find an element in a nested dict, eg. + traverse_tree({'Net': {'IP': '1.1.1.1'}}, ['Net', 'IP']) == '1.1.1.1' - Args: - dic: The dictionary to search - key_path: An iterable containing an ordered list of dict keys to + :param haystack: The nested dictionary to search + :param key_path: An iterable containing an ordered list of dict keys to traverse - default: Value to return in case nothing is found - Returns: - Value of the dict at the nested location given, or default if no value + :param default: Value to return in case nothing is found + :return:Value of the dict at the nested location given, or default if no value was found """ for k in key_path: if k in haystack: - haystack = dic[k] + haystack = haystack[k] else: return default return haystack diff --git a/requirements.txt b/requirements.txt index 5621d86..7a5ebb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ twisted==14.0.0 docker-py>=0.4.0 simplejson +nose \ No newline at end of file diff --git a/test_srv.py b/test/__init__.py similarity index 67% rename from test_srv.py rename to test/__init__.py index 649109e..f595b3f 100644 --- a/test_srv.py +++ b/test/__init__.py @@ -1,13 +1,76 @@ -""" - Creating srv records +__author__ = 'rpolli' -""" -from docker_dns import DockerResolver, DockerMapping -from twisted.names import dns -from docker_dns_test import check_deferred -import docker +inspect_container_pandas_0 = { + 'Id': 'cidpandas0', + 'Same': 'Value', + 'Config': { + 'Hostname': 'furby-pandas', + 'Image': 'impandas' + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + 'Name': '/cidpandas0', + 'Image': 'imgid_pandas' +} +inspect_container_pandas = { + 'Id': 'cidpandas', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + 'Image': 'impandas' + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + 'Name': '/cidpandas', + 'Image': 'imgid_pandas' +} +inspect_container_foxes = { + 'Id': 'cidfoxes', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + 'Image': 'imfoxes' + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + }, + 'Name': '/cidfoxes', + 'Image': 'imgid_foxes' +} +inspect_container_sloths = { + 'Id': 'cidsloths', + 'Config': { + 'Hostname': 'stopped-sloths', + 'Image': 'imsloths' + }, + 'NetworkSettings': { + 'IPAddress': '' + }, + 'Name': '/cidsloths', + 'Image': 'imgid_sloths' +} +inspect_container_returns = { + 'cidpandas0': inspect_container_pandas_0, + 'cidpandas': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, +} +containers_return = [ + {'Id': 'cidpandas0'}, + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, +] -mock_list_containers = lambda self, name: [ +mock_list_containers_2 = lambda *a, **k: [ + inspect_container_foxes, inspect_container_pandas, inspect_container_sloths, + inspect_container_pandas_0 +] +mock_inspect_containers_2 = lambda cid, **k: inspect_container_returns[cid] + +mock_list_containers = lambda *a, **k: [ {u'Command': u'/bin/bash', u'Created': 1414430175, u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', @@ -31,7 +94,10 @@ ] -mock_lookup_container = lambda name: { +mock_lookup_container = lambda *a, **k: { + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', + u'Name': u'/jboss631', u'Args': [], u'Config': {u'AttachStderr': True, u'AttachStdin': True, @@ -82,10 +148,7 @@ u'VolumesFrom': None}, u'HostnamePath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hostname', u'HostsPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hosts', - u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', - u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', u'MountLabel': u'', - u'Name': u'/jboss631', u'NetworkSettings': {u'Bridge': u'docker0', u'Gateway': u'172.17.42.1', u'IPAddress': u'172.17.0.10', @@ -111,62 +174,3 @@ } SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" - - -class Test(object): - - def setup(self): - docker_client = docker.Client() - self.mapping = DockerMapping(api=docker_client) - self.mapping.lookup_container = mock_lookup_container - self.resolver = DockerResolver(self.mapping) - - def test_nat_all(self): - host, port = "foo.docker", 8080 - ret = self.mapping.get_nat(host, port) - assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret - - def test_nat_ports(self): - expected = [(8080, 18080), - (9999, 19999), (8787, 8787)] - for pin, pout in expected: - ret = self.mapping.get_nat("foo", pin) - _, _, port, _ = next(ret) - assert port == pout, "unexpected value in %r" % ret - - def test_lookupService_ko(self): - expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( - ) - for n in expect_fail: - ret = self.resolver.lookupService(n) - check_deferred(ret, False) - - def test_lookupService_ok(self): - ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") - ret = check_deferred(ret, True) - print("resolved: %r" % [ret]) - - def test_mapping(self): - container = self.mapping.lookup_container("foo") - - for local, remote in container['NetworkSettings']['Ports'].items(): - port, proto = local.split("/") - if not remote: - continue - try: - remote = remote[0] - except IndexError: - continue - - print(SRV_FMT.format( - svc=port, - proto=proto, - container=container['Name'][1:], - cclass="IN", - priority=100, - weight=100, - port=remote['HostPort'], - target=remote['HostIp'] if remote[ - 'HostIp'] != '0.0.0.0' else "localhost" - ) - ) diff --git a/test/test_events.py b/test/test_events.py new file mode 100644 index 0000000..a1ecc20 --- /dev/null +++ b/test/test_events.py @@ -0,0 +1,76 @@ +from dockerdns.events import DockerDB, EventManager +import docker + +from test import mock_list_containers, mock_lookup_container, mock_list_containers_2, mock_inspect_containers_2 +from twisted.python import log +from twisted.web.client import Agent, Response, ResponseFailed +from twisted.internet import defer +from twisted.test.proto_helpers import MemoryReactor + + +def create_mock_db(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers + api.inspect_container = mock_lookup_container + return DockerDB(api=api) + + +def create_mock_db2(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers_2 + api.inspect_container = mock_inspect_containers_2 + return DockerDB(api=api) + + +def test_init(): + db = create_mock_db() + ret = db.get_by_name('jboss631') + assert ret, "Missing container" + try: + assert ret['NetworkSettings'][ + 'IPAddress'] == '172.17.0.10', "item %r" % ret + except KeyError as e: + log.err("Bad container %r" % ret) + raise + + +def test_init_and_get_images(): + db = create_mock_db2() + # images are correctly added to the indexes + assert 'impandas' in db.mappings_image, "%r, %r" % (db.mappings_image, db.mappings) + assert 'cidpandas' in db.mappings_image['impandas'], "%r, %r" % (db.mappings_image, db.mappings) + + # images are correctly retrieved + pandas_container = [x['Id'] for x in db.get_by_image('impandas')] + assert set(pandas_container) == set(('cidpandas', 'cidpandas0')), pandas_container + + + +class MockAgent(Agent): + reasons = ['No Reason'] + + def request(self, method, uri, **kw): + #return defer.fail(ResponseFailed(reasons=self.reasons, response=None)) + d = defer.Deferred() + d.addCallback(lambda *args: "ciao") + return d + +class TestEventManager(object): + def setup(self): + self.config = dict(docker_url='http://localhost:5000') + self.db = create_mock_db() + self.reactor = MemoryReactor() + self.http_agent = MockAgent(self.reactor) + + def test_delete_record(self): + em = EventManager(http_agent=self.http_agent, config=self.config, db=self.db) + assert em + em.delete_record({'status': 'stop', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) + + @defer.inlineCallbacks + def test_update_record(self): + em = EventManager(http_agent=self.http_agent, config=self.config, db=self.db) + resp = yield em.update_record({'status': 'start', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) + assert resp diff --git a/test/test_resolver.py b/test/test_resolver.py new file mode 100755 index 0000000..af384a9 --- /dev/null +++ b/test/test_resolver.py @@ -0,0 +1,277 @@ +#!/usr/bin/python + +""" +Tests for Docker DNS + +Author: Ricky Cook +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + +import itertools +import unittest + +import docker +import fudge +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN + + +# FIXME I can not believe how disgusting this is +def in_generator(gen, val): + return reduce( + lambda old, new: old or new == val, + gen, + False + ) + + +def check_record(record, **expected): + """ + Compare a record with the values of the kwargs + :param record: + :param expected: + :return: + """ + for k in expected: + real_value = getattr(record, k) + if k is 'name': + real_value = real_value.name + + if real_value != expected[k]: + log.err("Expected %s: %s vs %s" % (k, expected[k], real_value)) + return False + + return True + + +def check_deferred(deferred, success): + completed = [] + + def gimme_x_back(is_success): + def x_back(result): + completed.append((is_success, result)) + + return x_back + + deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) + if len(completed) != 1: + return False + + status, result = completed[0] + if status != success: + raise AssertionError("Expected: %r, got %r" % (success, result)) + return False + + return result + + +class MockDockerClient(object): + base_url = 'http://localhost:5000' + version = lambda x: {'ApiVersion': '1.0'} + inspect_container_pandas = { + 'ID': 'cidpandaslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + } + inspect_container_foxes = { + 'ID': 'cidfoxeslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + } + } + inspect_container_sloths = { + 'ID': 'cidslothslong', + 'Config': { + 'Hostname': 'stopped-sloths', + }, + 'NetworkSettings': { + 'IPAddress': '' + } + } + inspect_container_returns = { + 'cidpandas': inspect_container_pandas, + 'cidpandaslong': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidfoxeslong': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, + 'cidslothslong': inspect_container_sloths, + } + containers_return = [ + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, + ] + + inspect_container_id = None + + def inspect_container(self, cid): + self.inspect_container_id = cid + + try: + return self.inspect_container_returns[cid] + except KeyError: + # Mocks a Docker Client Exception + response = fudge.Fake() + response.has_attr(status_code=404, content='PANDAS!') + + exception = docker.client.APIError('bad', response) + raise exception + + def containers(self, *args, **kwargs): # pylint:disable=unused-argument + return self.containers_return + + +class DockerResolverTest(unittest.TestCase): + def setUp(self): + self.CONFIG = {} + from test.test_events import create_mock_db2 + + self.db = create_mock_db2() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + self.assertEqual(len(rec), 1) + rec = rec[0] + self.assertTrue(check_record(rec, **expected_record)) + return rec + + # + # TEST _a_records + # + def test__a_records_hostname(self): + name, expected_record = 'sneaky-foxes', {'name': 'sneaky-foxes.docker', 'type': dns.A} + rec = self.harn_expected(name, expected_record) + self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') + + def test__a_records_id(self): + name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A} + rec = self.harn_expected(name, expected_record) + self.assertEqual(rec.payload.dottedQuad(), '127.0.0.1') + + def test__a_records_shutdown(self): + self.assertRaises( + DomainError, + self.resolver._a_records, + 'cidsloths.docker' + ) + + def test__a_records_invalid(self): + self.assertRaises( + DomainError, + self.resolver._a_records, + 'invalid.docker' + ) + + def test__a_records_blank_query(self): + self.assertRaises( + DomainError, + self.resolver._a_records, + '' + ) + + def test__a_records_authoritative(self): + name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A, 'auth': True} + self.resolver.config['authoritative'] = True + self.harn_expected(name, expected_record) + + def test__a_records_non_authoritative(self): + name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A, 'auth': False} + self.resolver.config['authoritative'] = False + self.harn_expected(name, expected_record) + + # + # TEST lookupAddress + # + def test_lookupAddress_id(self): + expected_record, expected_authority, expected_additional = ( + {'name': 'cidfoxes.docker', 'type': dns.A}, + tuple(), + tuple() + ) + deferred = self.resolver.lookupAddress('cidfoxes.docker') + + result = check_deferred(deferred, True) + self.assertNotEqual(result, False) + + response_rr, authority_rr, additional_rr = result + # skip this tests as we're now populating + # the authority and additional section + self.assertEqual(len(response_rr), 1) + + rec = response_rr[0] + self.assertTrue(check_record( + rec, + **expected_record + )) + self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') + + def test_lookupAddress_invalid(self): + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + self.assertNotEqual(result, False) + + def test_lookupAddress_invalid_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = False + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + self.assertNotEqual(result, False) + self.assertEqual( + result.type, DomainError) # noqa pylint:disable=maybe-no-member + + def test_lookupAddress_invalid_no_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = True + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + self.assertNotEqual(result, False) + self.assertEqual(result.type, DNSQueryTimeoutError) + # noqa pylint:disable=maybe-no-member + + def test_lookupAddress_multi(self): + # search by image + # host -t a impandas.*.docker + # + expected_records = ( + {'name': 'cidpandas.docker'}, + {'name': 'cidpandas0.docker'} + ) + self.resolver.config[NO_NXDOMAIN] = False + + # retrieve hosts by image + deferred = self.resolver.lookupAddress('impandas.*.docker') + result = check_deferred(deferred, True) + self.assertNotEqual(result, False) + + response_rr, authority_rr, additional_rr = result + self.assertEqual(len(response_rr), len(expected_records)) + + for rec, expected_record in zip(response_rr, expected_records): + self.assertTrue(check_record( + rec, + **expected_record + )) + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/test/test_srv.py b/test/test_srv.py new file mode 100644 index 0000000..1e36c32 --- /dev/null +++ b/test/test_srv.py @@ -0,0 +1,87 @@ +""" + Creating srv records + +""" +import twisted +from dockerdns.resolver import DockerResolver +from dockerdns.mappings import DockerMapping +from test.docker_dns_test import check_deferred +from test import mock_lookup_container, SRV_FMT +from nose import SkipTest +from nose.tools import raises +from nose.twistedtools import deferred as nosedeferred + + +class TestSrv(object): + mapping = DockerMapping(db=None) + mapping.lookup_container = mock_lookup_container + + def check_equals(self, a, b, msg=None): + assert a == b, msg + + def setup(self): + self.resolver = DockerResolver(self.mapping) + + def test_nat_all(self): + host, port = "foo.docker", 8080 + ret = self.mapping.get_nat(host, port) + assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret + + def test_nat_ports(self): + expected = [(8080, 18080), + (9999, 19999), (8787, 8787)] + + for pin, pout in expected: + ret = self.mapping.get_nat("foo", pin) + _, _, port, _ = next(ret) + + yield self.check_equals, port, pout, "unexpected value in %r" % ret + + + @raises(twisted.names.error.DomainError) + @nosedeferred() + def harn_lookupService_ko(self, n): + """Harness to run lookup in a deferred + """ + return self.resolver.lookupService(n) + + def test_lookupService_ko(self): + expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( + ) + for n in expect_fail: + yield self.harn_lookupService_ko, n + + def test_lookupService_ok(self): + ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_mapping(self): + container = self.mapping.lookup_container("foo") + + for local, remote in container['NetworkSettings']['Ports'].items(): + port, proto = local.split("/") + if not remote: + continue + try: + remote = remote[0] + except IndexError: + continue + + print(SRV_FMT.format( + svc=port, + proto=proto, + container=container['Name'][1:], + cclass="IN", + priority=100, + weight=100, + port=remote['HostPort'], + target=remote['HostIp'] if remote[ + 'HostIp'] != '0.0.0.0' else "localhost" + ) + ) + + @SkipTest + def test_service_image(self): + host = "impandas.docker" + raise NotImplementedError("implement skydns-like") \ No newline at end of file diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..168b534 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,51 @@ +""" +Test the traverse_tree function +""" +from dockerdns.utils import traverse_tree + + +theDict = { + 'pandas': { + 'are': 'cuddly', + 'and': 'awesome', + }, + 'foxes': { + 'are': 'sneaky', + 'and': 'orange', + }, + 'badgers': { + 'are': None, + }, +} + + +def harn_basic_check(given, expected, default=None): + ret = traverse_tree(theDict, given, default) + assert ret == expected, "given: %s, expected: %s, actual: %s, default: %s" % (given, expected, ret, default) + + +def test_basic_one(): + for given, expected in [ + ('pandas and', 'awesome'), + ('foxes are', 'sneaky'), + ('nothing', None), + ('pandas bad', None), + ('foxes', theDict['foxes']), + + ]: + yield harn_basic_check, given.split(), expected + + +def test_user_default(): + for given, default in [ + ('nothing', 'Nobody here but us chickens'), + ('pandas bad', 'NO, THAT\'S A DAMN DIRTY LIE'), + ]: + yield harn_basic_check, given.split(), default, default + + +def test_ignore_default(): + for given, expected, default in [ + ('badgers are', None, 'Badgers are none? What?') + ]: + yield harn_basic_check, given.split(), expected, default From 785260275d4f703faaa70225e9b44c25729a8b2e Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 6 Feb 2015 10:22:31 +0100 Subject: [PATCH 26/51] fix: set docker api version --- docker_dns.py | 4 ++-- dockerdns/events.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker_dns.py b/docker_dns.py index fd384f1..d6bc51c 100755 --- a/docker_dns.py +++ b/docker_dns.py @@ -31,7 +31,7 @@ # Merge user config over defaults CONFIG = DEFAULT_CONFIG = { 'docker_url': 'unix://var/run/docker.sock', - 'version': '1.13', + 'version': '1.15', 'bind_interface': '', 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], @@ -54,7 +54,7 @@ def main(): import docker # Create docker: by default dict.get returns None on missing keys - docker_client = docker.Client(CONFIG.get('docker_url')) + docker_client = docker.Client(CONFIG.get('docker_url'), version=CONFIG['version']) infos = docker_client.info() # Test docker connectivity before starting log.msg("Connecting to docker instance: %r" % infos) diff --git a/dockerdns/events.py b/dockerdns/events.py index 01e8728..9f3036e 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -81,11 +81,12 @@ class EventManager(Protocol): def __init__(self, http_agent, config, db): self.http_agent = http_agent self.config = config + self.db = db self.remaining = 1024 * 10 self.buff = "" - # Create an event_manager for parsing updates - self.event_manager = ContainerManager(Deferred(), db=db) + # Create an container_manager for parsing updates + self.container_manager = ContainerManager(Deferred(), db=db) def update_record(self, item): """Update docker mapping @@ -96,7 +97,7 @@ def update_record(self, item): 'containers', item['id'], 'json') ) d.addCallback( - lambda response: response.deliverBody(self.event_manager)) + lambda response: response.deliverBody(self.container_manager)) def delete_record(self, item): """Update docker mapping From f1ba8163226c2cdd9b0ee143e835bb5558b1d4af Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 6 Feb 2015 10:26:23 +0100 Subject: [PATCH 27/51] fix: import in test_srv --- test/test_srv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_srv.py b/test/test_srv.py index 1e36c32..f2def20 100644 --- a/test/test_srv.py +++ b/test/test_srv.py @@ -5,11 +5,11 @@ import twisted from dockerdns.resolver import DockerResolver from dockerdns.mappings import DockerMapping -from test.docker_dns_test import check_deferred from test import mock_lookup_container, SRV_FMT from nose import SkipTest from nose.tools import raises from nose.twistedtools import deferred as nosedeferred +from test.test_resolver import check_deferred class TestSrv(object): From 10cf6e6ec027002d499578b00b35623da45fa9af Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sun, 8 Feb 2015 21:37:50 +0100 Subject: [PATCH 28/51] enh: using twistd plugin for passing arguments --- README.md | 16 +++- docker_dns.py | 122 ------------------------ dockerdns/events.py | 5 +- dockerdns/mappings.py | 8 +- dockerdns/resolver.py | 48 ++++++---- test/test_events.py | 18 ++-- test/test_resolver.py | 55 ++++++++++- test/test_srv.py | 3 +- twisted/plugins/dockerdns_plugin.py | 138 ++++++++++++++++++++++++++++ 9 files changed, 251 insertions(+), 162 deletions(-) delete mode 100755 docker_dns.py create mode 100644 twisted/plugins/dockerdns_plugin.py diff --git a/README.md b/README.md index 84c515d..6d15fc6 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,13 @@ Docker DNS A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP resolution. +Containers can be found by: + - container name + - hostname + - image name + To look up a container: - - 'A' record query a container NAME that will match a container with a docker inspect + - 'A' record: query a container NAME that will match a container with a docker inspect command with '.docker' as the TLD. eg: mysql_server1.docker - 'SRV' record query exposing the NAT informations @@ -15,17 +20,22 @@ Install/Run Just install from requirements (in a virtualenv if you'd like) - pip install -r requirements.txt --use-mirrors + pip install -r requirements.txt That's it! To run, remember that you may need to set user/group ids on the process - sudo twistd -gdocker -y docker_dns.py + sudo twistd -gdocker -y dockerdns -p 53 This will start a DNS server on port 53 (default DNS port). To make this useful, you probably want to combine it with your regular DNS in something like Dnsmasq. +You can get configuration parameters with + + sudo twistd dockerdns --help + + Examples -------- Dig output is shortened for brevity. We have Docker containers like this: diff --git a/docker_dns.py b/docker_dns.py deleted file mode 100755 index de727bd..0000000 --- a/docker_dns.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/python -#from __future__ import print_function, unicode_literals -""" -A simple TwistD DNS server using custom TLD and Docker as the back end for IP -resolution. - -To look up a container: - - 'A' record query a container NAME that will match a container with a docker inspect - command with '.d' as the TLD. eg: mysql_server1.d - - 'SRV' record query to _port._srv.container.docker will return the natted address. - eg. _3306._tcp.mysql_server1.docker returns - _18080._tcp.compassionate_poincare.docker. 10 IN SRV 100 100 8080 192.168.42.126. - - -Code modified from -https://github.com/infoxchange/docker_dns - -Author: Bradley Cicenas -Author: Roberto Polli -""" - -from twisted.application import internet, service -from twisted.names import dns, server -from twisted.python import log - -from dockerdns.mappings import DockerMapping -from dockerdns.events import DockerDB -from dockerdns.resolver import DockerResolver - - -# Merge user config over defaults -CONFIG = DEFAULT_CONFIG = { - 'docker_url': 'unix://var/run/docker.sock', - 'version': '1.13', - 'bind_interface': '', - 'bind_port': 53, - 'bind_protocols': ['tcp', 'udp'], - 'no_nxdomain': True, - 'authoritative': True, - 'domain': 'docker' -} - -# Load the config -try: - from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error - CONFIG.update(appcfg) -except ImportError: - appcfg = {} - - -def main(): - """ - Set everything up - """ - import docker - - # Create docker: by default dict.get returns None on missing keys - docker_client = docker.Client(CONFIG.get('docker_url')) - infos = docker_client.info() - # Test docker connectivity before starting - log.msg("Connecting to docker instance: %r" % infos) - - db = DockerDB(api=docker_client) - # Create our custom mapping and resolver - mapping = DockerMapping(db=db) - resolver = DockerResolver(mapping) - - # Create twistd stuff to tie in our custom components - factory = server.DNSServerFactory(clients=[resolver]) - factory.noisy = False - - # Protocols to bind - bind_list = [] - if 'tcp' in CONFIG['bind_protocols']: - bind_list.append( - (internet.TCPServer, factory)) # noqa pylint:disable=no-member - - if 'udp' in CONFIG['bind_protocols']: - proto = dns.DNSDatagramProtocol(factory) - proto.noisy = False - bind_list.append( - (internet.UDPServer, proto)) # noqa pylint:disable=no-member - - # Register the service - ret = service.MultiService() - for (InternetServerKlass, arg) in bind_list: - svc = InternetServerKlass( - CONFIG['bind_port'], - arg, - interface=CONFIG['bind_interface'] - ) - svc.setServiceParent(ret) - - # Add the event Loop - from dockerdns.events import EventFactory - from urlparse import urlparse - u = urlparse(CONFIG['docker_url']) - efactory = EventFactory(config=CONFIG, db=db) - docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) - docker_event_monitor.setServiceParent(ret) - - # DO IT NOW - ret.setServiceParent(service.IServiceCollection(application)) - - -# -# This is the effective twisted application -# - - -# -# Run it with twisted... it should be named .tac, not .py -# -application = service.Application( - 'dnsserver', 1, 1) # noqa pylint:disable=invalid-name -main() - - -# Doin' it wrong -if __name__ == '__main__': - import sys - print "Usage: twistd -y %s" % sys.argv[0] diff --git a/dockerdns/events.py b/dockerdns/events.py index 141736c..e502827 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -48,7 +48,8 @@ def updatedb(self, item): self.mappings_name.update({name: item['Id']}) self.mappings_hostname.update({hostname: item['Id']}) self.mappings.update({item['Id']: item}) - self.mappings_image.setdefault(item['Config']['Image'], []).append(name) + self.mappings_image.setdefault( + item['Config']['Image'], []).append(item['Id']) def add_container(self, item): self.updatedb(item) @@ -57,7 +58,7 @@ def del_container(self, cid): name = self.mappings[cid]['Name'][1:] image = self.mappings[cid]['Config']['Image'] hostname = self.mappings[cid]['Config']['Hostname'] - self.mappings_image.get(image, []).remove(name) + self.mappings_image.get(image, []).remove(cid) del self.mappings[cid] del self.mappings_name[name] del self.mappings_hostname[hostname] diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index c9a6ad7..3218483 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -80,12 +80,12 @@ def get_a_multi(self, name_multi): log.err("Not a multihost search: %r" % (name_multi,)) return return tuple( - (container['NetworkSettings']['IPAddress'], container['Name'][1:]) - for container + (container['NetworkSettings'][ + 'IPAddress'], container['Name'][1:]) + for container in self.db.get_by_image(name_multi) if 'NetworkSettings' in container - ) - + ) def get_nat(self, container_name, sport=0, sproto=None): """ @return - a generator of natted maps (local, proto, nat, ip) diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index 10eb472..c86b4c1 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -22,8 +22,8 @@ class DockerResolver(common.ResolverBase): dns.Record_SRV( priority=100, weight=100, port=19999, target='name', ttl=None), auth=True), - dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, dns.Record_SRV( priority=100, weight=100, port=18080, target='name', ttl=None), auth=True) @@ -35,8 +35,8 @@ def __init__(self, mapping, config=None): """ self.mapping = mapping - self.config = config or {} - + self.config = config or {'domain': 'docker'} + self.re_domain = re.compile(r'\.' + self.config['domain'] + '$') # Change to this ASAP when Twisted uses object base # super(DockerResolver, self).__init__() common.ResolverBase.__init__(self) @@ -48,7 +48,8 @@ def __init__(self, mapping, config=None): # socket. import socket - self.authority = [dns.RRHeader(name="docker.", type=dns.NS, cls=dns.IN, + self.authority = [dns.RRHeader( + name=self.config['domain'] + ".", type=dns.NS, cls=dns.IN, payload=dns.Record_NS( name=socket.gethostname()) )] @@ -71,7 +72,9 @@ def _a_records(self, name): raise DomainError(name) return tuple([ - dns.RRHeader('.'.join((name, 'docker')), dns.A, dns.IN, self.ttl, + dns.RRHeader( + '.'.join( + (name, self.config['domain'])), dns.A, dns.IN, self.ttl, dns.Record_A(addr, self.ttl), auth=self.config.get('authoritative')) ]) @@ -80,7 +83,9 @@ def _srv_records(self, name): print("getting srv: %r" + name) addr = self.mapping.get_a(name) return tuple([ - dns.RRHeader('.'.join((name, 'docker')), dns.SRV, dns.IN, self.ttl, + dns.RRHeader( + '.'.join( + (name, self.config['domain'])), dns.SRV, dns.IN, self.ttl, dns.Record_A(addr, self.ttl), auth=self.config.get('authoritative')) ]) @@ -88,30 +93,36 @@ def _srv_records(self, name): def lookupAddress(self, name, timeout=None): """ - :param name: + :param name: a name like container_name.docker, hostname.docker, image_name.*.docker :param timeout: :return: A deferred firing a 3-tuple - The first element of the tuple gives answers. + The first element of the tuple gives answers. The second element of the tuple gives authorities. The third element of the tuple gives additional information. The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) """ - name, occurrences = re.subn(r'.docker$', '', name) + name, occurrences = self.re_domain.subn('', name) if not occurrences: - log.err("Domain not ending with .docker: %r" % name) + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) return defer.fail(failure.Failure(DomainError("not ending with docker"))) try: if name.endswith(".*"): + a_multi = self.mapping.get_a_multi(name) + log.msg(a_multi) records = tuple( - dns.RRHeader('.'.join((name_, 'docker')), dns.A, dns.IN, self.ttl, + dns.RRHeader( + '.'.join((name_, self.config[ + 'domain'])), dns.A, dns.IN, self.ttl, dns.Record_A(addr_, self.ttl), auth=self.config.get('authoritative') - ) + ) for addr_, name_ - in self.mapping.get_a_multi(name) + in a_multi + if addr_ and name_ # skip empty entries ) else: records = self._a_records(name) @@ -152,10 +163,11 @@ def lookupService(self, name, timeout=None): The third element of the tuple gives additional information. The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) """ - name, occurrences = re.subn(r'.docker$', '', name) + name, occurrences = self.re_domain.subn('', name) if not occurrences: - log.err("Domain not ending with .docker: %r" % name) - return defer.fail(failure.Failure(DomainError("not ending with docker"))) + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure(DomainError("not ending with {domain}".format(**self.config)))) try: port, proto, container = name.split(".") port = int(port.strip("_")) @@ -168,7 +180,7 @@ def lookupService(self, name, timeout=None): dns.Record_SRV( priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip_ptr_value, ttl=None), auth=True) - for c_port, protocol, c_nat_port, target + for c_port, protocol, c_nat_port, target in self.mapping.get_nat(container) if c_port == port # eventually filter ] diff --git a/test/test_events.py b/test/test_events.py index a428662..474c4e5 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -9,6 +9,7 @@ from nose import SkipTest + def create_mock_db(): """Create a mock DockerDB""" api = docker.Client() @@ -40,13 +41,15 @@ def test_init(): def test_init_and_get_images(): db = create_mock_db2() # images are correctly added to the indexes - assert 'impandas' in db.mappings_image, "%r, %r" % (db.mappings_image, db.mappings) - assert 'cidpandas' in db.mappings_image['impandas'], "%r, %r" % (db.mappings_image, db.mappings) + assert 'impandas' in db.mappings_image, "%r, %r" % ( + db.mappings_image, db.mappings) + assert 'cidpandas' in db.mappings_image['impandas'], "%r, %r" % ( + db.mappings_image, db.mappings) # images are correctly retrieved pandas_container = [x['Id'] for x in db.get_by_image('impandas')] - assert set(pandas_container) == set(('cidpandas', 'cidpandas0')), pandas_container - + assert set(pandas_container) == set( + ('cidpandas', 'cidpandas0')), pandas_container class MockAgent(Agent): @@ -58,6 +61,7 @@ def request(self, method, uri, **kw): d.addCallback(lambda *args: "ciao") return d + class TestEventManager(object): def setup(self): self.config = dict(docker_url='http://localhost:5000') @@ -66,12 +70,14 @@ def setup(self): self.http_agent = MockAgent(self.reactor) def test_delete_record(self): - em = EventManager(http_agent=self.http_agent, config=self.config, db=self.db) + em = EventManager( + http_agent=self.http_agent, config=self.config, db=self.db) assert em em.delete_record({'status': 'stop', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) @SkipTest def test_update_record(self): - em = EventManager(http_agent=self.http_agent, config=self.config, db=self.db) + em = EventManager( + http_agent=self.http_agent, config=self.config, db=self.db) resp = yield em.update_record({'status': 'start', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) assert resp diff --git a/test/test_resolver.py b/test/test_resolver.py index af384a9..6d7634f 100755 --- a/test/test_resolver.py +++ b/test/test_resolver.py @@ -19,6 +19,7 @@ from twisted.python import log from dockerdns.mappings import DockerMapping from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2, create_mock_db # FIXME I can not believe how disgusting this is @@ -138,7 +139,6 @@ def containers(self, *args, **kwargs): # pylint:disable=unused-argument class DockerResolverTest(unittest.TestCase): def setUp(self): self.CONFIG = {} - from test.test_events import create_mock_db2 self.db = create_mock_db2() self.mapping = DockerMapping(self.db) @@ -155,12 +155,14 @@ def harn_expected(self, name, expected_record): # TEST _a_records # def test__a_records_hostname(self): - name, expected_record = 'sneaky-foxes', {'name': 'sneaky-foxes.docker', 'type': dns.A} + name, expected_record = 'sneaky-foxes', {'name': + 'sneaky-foxes.docker', 'type': dns.A} rec = self.harn_expected(name, expected_record) self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') def test__a_records_id(self): - name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A} + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A} rec = self.harn_expected(name, expected_record) self.assertEqual(rec.payload.dottedQuad(), '127.0.0.1') @@ -186,12 +188,14 @@ def test__a_records_blank_query(self): ) def test__a_records_authoritative(self): - name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A, 'auth': True} + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A, 'auth': True} self.resolver.config['authoritative'] = True self.harn_expected(name, expected_record) def test__a_records_non_authoritative(self): - name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A, 'auth': False} + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A, 'auth': False} self.resolver.config['authoritative'] = False self.harn_expected(name, expected_record) @@ -269,6 +273,47 @@ def test_lookupAddress_multi(self): **expected_record )) + +class DockerResolver2Test(unittest.TestCase): + def setUp(self): + self.CONFIG = {} + + self.db = create_mock_db() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + self.assertEqual(len(rec), 1) + rec = rec[0] + self.assertTrue(check_record(rec, **expected_record)) + return rec + + def test_lookupAddress_multi(self): + # search by image + # host -t a impandas.*.docker + # + expected_records = ( + {'name': 'jboss631.docker'}, + # {'name': 'cidpandas0.docker'} + ) + self.resolver.config[NO_NXDOMAIN] = False + + # retrieve hosts by image + deferred = self.resolver.lookupAddress('eap63_tracer:v6.3.1.*.docker') + result = check_deferred(deferred, True) + self.assertNotEqual(result, False) + + response_rr, authority_rr, additional_rr = result + self.assertEqual(len(response_rr), len(expected_records)) + + for rec, expected_record in zip(response_rr, expected_records): + self.assertTrue(check_record( + rec, + **expected_record + )) + + def main(): unittest.main() diff --git a/test/test_srv.py b/test/test_srv.py index f2def20..04ff737 100644 --- a/test/test_srv.py +++ b/test/test_srv.py @@ -37,7 +37,6 @@ def test_nat_ports(self): yield self.check_equals, port, pout, "unexpected value in %r" % ret - @raises(twisted.names.error.DomainError) @nosedeferred() def harn_lookupService_ko(self, n): @@ -84,4 +83,4 @@ def test_mapping(self): @SkipTest def test_service_image(self): host = "impandas.docker" - raise NotImplementedError("implement skydns-like") \ No newline at end of file + raise NotImplementedError("implement skydns-like") diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py new file mode 100644 index 0000000..c988cf1 --- /dev/null +++ b/twisted/plugins/dockerdns_plugin.py @@ -0,0 +1,138 @@ +#!/usr/bin/python +#from __future__ import print_function, unicode_literals +""" +A simple TwistD DNS server using custom TLD and Docker as the back end for IP +resolution. + +To look up a container: + - 'A' record query a container NAME that will match a container with a docker inspect + command with '.d' as the TLD. eg: mysql_server1.d + - 'SRV' record query to _port._srv.container.docker will return the natted address. + eg. _3306._tcp.mysql_server1.docker returns + _18080._tcp.compassionate_poincare.docker. 10 IN SRV 100 100 8080 192.168.42.126. + + +Code modified from +https://github.com/infoxchange/docker_dns + +Author: Bradley Cicenas +Author: Roberto Polli +""" + +from twisted.application import internet, service +from twisted.names import dns, server +from twisted.python import log +from zope.interface import implements + +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.application.service import IServiceMaker +from twisted.application import internet + + +from dockerdns.mappings import DockerMapping +from dockerdns.events import DockerDB +from dockerdns.resolver import DockerResolver + +import docker + + +# Merge user config over defaults +CONFIG = DEFAULT_CONFIG = { + 'docker_url': 'unix://var/run/docker.sock', + 'version': '1.13', + 'bind_interface': '', + 'bind_port': 53, + 'bind_protocols': ['tcp', 'udp'], + 'no_nxdomain': True, + 'authoritative': True, + 'domain': 'docker' +} + +# Load the config +try: + from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error + CONFIG.update(appcfg) +except ImportError: + appcfg = {} + + +class Options(usage.Options): + optParameters = [ + ["bind_port", "p", 10053, "The port number to listen on."], + ["bind_interface", "h", "127.0.0.1", "The host address to bind to"], + ["domain", "d", "docker", "The default domain"], + ["config", "c", "docker_dns.json", "Configuration file"], + ] + + +class MyServiceMaker(object): + """ + Define a MultiService running: + - dns server for tcp and udp + - http client for retrieving docker events + """ + implements(IServiceMaker, IPlugin) + tapname = "dockerdns" + description = "Run this! It'll make your dog happy." + options = Options + + def makeService(self, options): + """ + Set everything up + """ + # Update config stuff with command line params + CONFIG.update(options) + log.err("config: %r" % CONFIG) + # Create docker: by default dict.get returns None on missing keys + docker_client = docker.Client(CONFIG.get('docker_url')) + infos = docker_client.info() + # Test docker connectivity before starting + log.msg("Connecting to docker instance: %r" % infos) + + db = DockerDB(api=docker_client) + # Create our custom mapping and resolver + mapping = DockerMapping(db=db) + resolver = DockerResolver(mapping, config=CONFIG) + + # Create twistd stuff to tie in our custom components + factory = server.DNSServerFactory(clients=[resolver]) + factory.noisy = False + + # Protocols to bind + bind_list = [] + if 'tcp' in CONFIG['bind_protocols']: + bind_list.append( + (internet.TCPServer, factory)) # noqa pylint:disable=no-member + + if 'udp' in CONFIG['bind_protocols']: + proto = dns.DNSDatagramProtocol(factory) + proto.noisy = False + bind_list.append( + (internet.UDPServer, proto)) # noqa pylint:disable=no-member + + # Register the service + ret = service.MultiService() + for (InternetServerKlass, arg) in bind_list: + svc = InternetServerKlass( + int(CONFIG['bind_port']), + arg, + interface=CONFIG['bind_interface'] + ) + svc.setServiceParent(ret) + + # Add the event Loop + from dockerdns.events import EventFactory + from urlparse import urlparse + u = urlparse(CONFIG['docker_url']) + efactory = EventFactory(config=CONFIG, db=db) + docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) + docker_event_monitor.setServiceParent(ret) + + return ret + + +# +# Create the MultiService +# +serviceMaker = MyServiceMaker() From 165955ed1c620497bd9377b3566625d833d7e62d Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 9 Feb 2015 11:53:12 +0100 Subject: [PATCH 29/51] fix: nose moved to test_requirements.txt --- requirements.txt | 1 - test_requirements.txt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a5ebb4..5621d86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ twisted==14.0.0 docker-py>=0.4.0 simplejson -nose \ No newline at end of file diff --git a/test_requirements.txt b/test_requirements.txt index 731c279..2d6def5 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,5 @@ -r requirements.txt fudge==1.0.3 pep8==1.4.6 -pylint==1.0.0 \ No newline at end of file +pylint==1.0.0 +nose From cc52cefa7abcd96802cd54e2ba36c73c73b13fd2 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 9 Feb 2015 13:02:33 +0100 Subject: [PATCH 30/51] feat: json config file --- README.md | 96 +++++++++-------------------- config.py.sample | 8 --- dockerdns.json.sample | 16 +++++ dockerdns/events.py | 10 ++- requirements.txt | 2 +- twisted/plugins/dockerdns_plugin.py | 56 ++++++++--------- 6 files changed, 80 insertions(+), 108 deletions(-) delete mode 100644 config.py.sample create mode 100644 dockerdns.json.sample diff --git a/README.md b/README.md index 6d15fc6..b93cab3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ Docker DNS ========== -[![Build Status](https://travis-ci.org/infoxchange/docker_dns.png?branch=master)](https://travis-ci.org/infoxchange/docker_dns) A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP resolution. @@ -15,6 +14,8 @@ To look up a container: command with '.docker' as the TLD. eg: mysql_server1.docker - 'SRV' record query exposing the NAT informations +Note: This fork of docker_dns requires *always* to specify the TLD (by default .docker) + Install/Run ----------- @@ -40,9 +41,9 @@ Examples -------- Dig output is shortened for brevity. We have Docker containers like this: - ID IMAGE COMMAND CREATED STATUS PORTS - 26ed50b1bf59 ubuntu:12.04 /bin/bash 4 seconds ago Up 4 seconds - 0949efde23bf ubuntu:12.04 /bin/bash 18 hours ago Up 18 hours + ID IMAGE STATUS Names + 26ed50b1bf59 ubuntu:12.04 Up 1 hour sad_turing + 0949efde23bf ubuntu:12.04 Up 18 hours happy_bohr 0949efde23bf has: @@ -56,36 +57,22 @@ Dig output is shortened for brevity. We have Docker containers like this: - IP: 172.17.0.3 - Hostname: my-thing -Container IDs are variable length. You use the same input as the Docker `inspect` command, so they can be long: - - dig 0949efde23bf017.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 51840 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 +Search by Hostname + dig +short 26ed50b1bf59.docker + 172.17.0.2 - ;; ANSWER SECTION: - 0949efde23bf017.docker. 10 IN A 172.17.0.2 +Search by Names (works only the first Name) + dig +short sad_turing.docker + 172.17.0.2 +Search by Hostname + dig +short my-thing.docker + 172.17.0.3 -Or they can be short: +Search by Names (works only the first Name) + dig +short happy_bohr.docker + 172.17.0.3 - dig 0949.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 42797 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - - ;; ANSWER SECTION: - 0949.docker. 10 IN A 172.17.0.2 - -And the other container: - - dig 26ed50b1bf59.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 25901 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - - ;; ANSWER SECTION: - 26ed50b1bf59.docker. 10 IN A 172.17.0.3 When a container doesn't exist, no answer is given: @@ -94,34 +81,11 @@ When a container doesn't exist, no answer is given: ;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 24269 ;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0 -You can look up by hostname by removing the .docker TLD: - - dig 0949efde23bf - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 61822 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - - ;; ANSWER SECTION: - 0949efde23bf. 10 IN A 172.17.0.2 - -Here's a manually defined hostname: - - dig my-thing - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3355 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - - ;; ANSWER SECTION: - my-thing. 10 IN A 172.17.0.3 - -And the host name that would have been automatically assigned for the above -container: - - dig 26ed50b1bf59 - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 12687 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0 +You can search by image, like skydock: + dig +short ubuntu.*.docker + 172.17.0.2 + 172.17.0.3 Nat discovery: you can discover natted ports with queries like this one @@ -131,27 +95,23 @@ Nat discovery: you can discover natted ports with queries like this one Configuration ------------- -Config is done in the `config.py` file. There's a skeleton in -`config.py.sample`. Below are the default config values. Currently, +Config is done in the `dockerdns.json` file. There's a skeleton in +`dockerdns.json.sample`. Below are the default config values. Currently, configuration is rather limited. - CONFIG = { - # URL to connect to the Docker API. docker-py defaults to - # unix://var/run/docker.sock + { + "#": "# URL to connect to the Docker API. docker-py defaults to unix://var/run/docker.sock", 'docker_url': None - # socket.bind defaults to 0.0.0.0 + "#": "# socket.bind defaults to 0.0.0.0", 'bind_interface': '', 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], - # When the request matches no docker container, we should return - # NXDOMAIN, however the OS interprets this as "doesn't exist anywhere" - # so things like google.com fail. This will return SERVFAIL rather than - # NXDOMAIN so secondary DNS is used + "#": "Return SERVFAIL instead of NXDOMAIN if no matching container found" 'no_nxdomain': True, - # Makes successful requests authoritative + "#": "Makes successful requests authoritative", 'authoritative': True, } diff --git a/config.py.sample b/config.py.sample deleted file mode 100644 index a2bbdfb..0000000 --- a/config.py.sample +++ /dev/null @@ -1,8 +0,0 @@ -""" -Configuration for Docker DNS server. - -Author: Ricky Cook -""" - -CONFIG = { -} diff --git a/dockerdns.json.sample b/dockerdns.json.sample new file mode 100644 index 0000000..1666639 --- /dev/null +++ b/dockerdns.json.sample @@ -0,0 +1,16 @@ +{ + "#": "# URL to connect to the Docker API. docker-py defaults to unix://var/run/docker.sock", + 'docker_url': None, + + "#": "# socket.bind defaults to 0.0.0.0", + 'bind_interface': '', + 'bind_port': 53, + 'bind_protocols': ['tcp', 'udp'], + + "#": "Return SERVFAIL instead of NXDOMAIN if no matching container found", + 'no_nxdomain': True, + + "#": "Makes successful requests authoritative", + 'authoritative': True, +} + diff --git a/dockerdns/events.py b/dockerdns/events.py index e502827..6be4ce4 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -45,11 +45,15 @@ def updatedb(self, item): name = item['Name'][1:] hostname = item['Config']['Hostname'] + image = item['Config']['Image'] self.mappings_name.update({name: item['Id']}) self.mappings_hostname.update({hostname: item['Id']}) self.mappings.update({item['Id']: item}) + image_notag = image[:image.find(":")] self.mappings_image.setdefault( - item['Config']['Image'], []).append(item['Id']) + image_notag, []).append(item['Id']) + self.mappings_image.setdefault( + image, []).append(item['Id']) def add_container(self, item): self.updatedb(item) @@ -58,7 +62,11 @@ def del_container(self, cid): name = self.mappings[cid]['Name'][1:] image = self.mappings[cid]['Config']['Image'] hostname = self.mappings[cid]['Config']['Hostname'] + image_notag = image[:image.find(":")] + self.mappings_image.get(image_notag, []).remove(cid) self.mappings_image.get(image, []).remove(cid) + + del self.mappings[cid] del self.mappings_name[name] del self.mappings_hostname[hostname] diff --git a/requirements.txt b/requirements.txt index 5621d86..c4b5bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -twisted==14.0.0 +twisted>=14.0.0 docker-py>=0.4.0 simplejson diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index c988cf1..7e86b75 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -23,7 +23,7 @@ from twisted.names import dns, server from twisted.python import log from zope.interface import implements - +import json from twisted.python import usage from twisted.plugin import IPlugin from twisted.application.service import IServiceMaker @@ -37,24 +37,6 @@ import docker -# Merge user config over defaults -CONFIG = DEFAULT_CONFIG = { - 'docker_url': 'unix://var/run/docker.sock', - 'version': '1.13', - 'bind_interface': '', - 'bind_port': 53, - 'bind_protocols': ['tcp', 'udp'], - 'no_nxdomain': True, - 'authoritative': True, - 'domain': 'docker' -} - -# Load the config -try: - from config import CONFIG as appcfg # pylint:disable=no-name-in-module,import-error - CONFIG.update(appcfg) -except ImportError: - appcfg = {} class Options(usage.Options): @@ -62,7 +44,12 @@ class Options(usage.Options): ["bind_port", "p", 10053, "The port number to listen on."], ["bind_interface", "h", "127.0.0.1", "The host address to bind to"], ["domain", "d", "docker", "The default domain"], - ["config", "c", "docker_dns.json", "Configuration file"], + ["config", "c", "dockerdns.json", "Configuration file"], + ["docker_url", "u", 'unix://var/run/docker.sock', "Docker URL"], + ['no_nxdomain', "x", True, "Return SERVFAIL instead of NXDOMAIN if container not found"], + ["authoritative", "A", True, "Return authoritative replies"], + ['version', "v", '1.13', "Docker API version"], + ['bind_protocols', "B", ['tcp', 'udp'], "Bind protocols"] ] @@ -81,11 +68,20 @@ def makeService(self, options): """ Set everything up """ + try: + with open(options['config']) as fh: + appcfg = json.load(fh) + appcfg = {k: v for k, v in appcfg.items() if k[0] is not '#'} + except IOError as e: + if options['config'] != "dockerdns.json": + raise + log.err("File {config} not found. Using default values".format(options)) + # Update config stuff with command line params - CONFIG.update(options) - log.err("config: %r" % CONFIG) + appcfg.update(options) + log.err("config: %r" % appcfg) # Create docker: by default dict.get returns None on missing keys - docker_client = docker.Client(CONFIG.get('docker_url')) + docker_client = docker.Client(appcfg.get('docker_url')) infos = docker_client.info() # Test docker connectivity before starting log.msg("Connecting to docker instance: %r" % infos) @@ -93,7 +89,7 @@ def makeService(self, options): db = DockerDB(api=docker_client) # Create our custom mapping and resolver mapping = DockerMapping(db=db) - resolver = DockerResolver(mapping, config=CONFIG) + resolver = DockerResolver(mapping, config=appcfg) # Create twistd stuff to tie in our custom components factory = server.DNSServerFactory(clients=[resolver]) @@ -101,11 +97,11 @@ def makeService(self, options): # Protocols to bind bind_list = [] - if 'tcp' in CONFIG['bind_protocols']: + if 'tcp' in appcfg['bind_protocols']: bind_list.append( (internet.TCPServer, factory)) # noqa pylint:disable=no-member - if 'udp' in CONFIG['bind_protocols']: + if 'udp' in appcfg['bind_protocols']: proto = dns.DNSDatagramProtocol(factory) proto.noisy = False bind_list.append( @@ -115,17 +111,17 @@ def makeService(self, options): ret = service.MultiService() for (InternetServerKlass, arg) in bind_list: svc = InternetServerKlass( - int(CONFIG['bind_port']), + int(appcfg['bind_port']), arg, - interface=CONFIG['bind_interface'] + interface=appcfg['bind_interface'] ) svc.setServiceParent(ret) # Add the event Loop from dockerdns.events import EventFactory from urlparse import urlparse - u = urlparse(CONFIG['docker_url']) - efactory = EventFactory(config=CONFIG, db=db) + u = urlparse(appcfg['docker_url']) + efactory = EventFactory(config=appcfg, db=db) docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) docker_event_monitor.setServiceParent(ret) From 0fa303bd336f20de264177b2db1f9f41915c9869 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 9 Feb 2015 13:18:36 +0100 Subject: [PATCH 31/51] enh: refactoring tests --- test/__init__.py | 68 ++++++++++++++ test/test_resolver.py | 174 ++++++------------------------------ test/test_resolver_multi.py | 76 ++++++++++++++++ 3 files changed, 171 insertions(+), 147 deletions(-) create mode 100755 test/test_resolver_multi.py diff --git a/test/__init__.py b/test/__init__.py index f595b3f..a439f23 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,3 +1,6 @@ +import docker +import fudge + __author__ = 'rpolli' inspect_container_pandas_0 = { @@ -174,3 +177,68 @@ } SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" + + +class MockDockerClient(object): + base_url = 'http://localhost:5000' + version = lambda x: {'ApiVersion': '1.0'} + inspect_container_pandas = { + 'ID': 'cidpandaslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + } + inspect_container_foxes = { + 'ID': 'cidfoxeslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + } + } + inspect_container_sloths = { + 'ID': 'cidslothslong', + 'Config': { + 'Hostname': 'stopped-sloths', + }, + 'NetworkSettings': { + 'IPAddress': '' + } + } + inspect_container_returns = { + 'cidpandas': inspect_container_pandas, + 'cidpandaslong': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidfoxeslong': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, + 'cidslothslong': inspect_container_sloths, + } + containers_return = [ + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, + ] + + inspect_container_id = None + + def inspect_container(self, cid): + self.inspect_container_id = cid + + try: + return self.inspect_container_returns[cid] + except KeyError: + # Mocks a Docker Client Exception + response = fudge.Fake() + response.has_attr(status_code=404, content='PANDAS!') + + exception = docker.client.APIError('bad', response) + raise exception + + def containers(self, *args, **kwargs): # pylint:disable=unused-argument + return self.containers_return \ No newline at end of file diff --git a/test/test_resolver.py b/test/test_resolver.py index 6d7634f..02a4056 100755 --- a/test/test_resolver.py +++ b/test/test_resolver.py @@ -9,18 +9,15 @@ # Do not care...... # noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name -import itertools -import unittest -import docker -import fudge from twisted.names import dns from twisted.names.error import DNSQueryTimeoutError, DomainError from twisted.python import log + from dockerdns.mappings import DockerMapping from dockerdns.resolver import DockerResolver, NO_NXDOMAIN -from test.test_events import create_mock_db2, create_mock_db - +from test.test_events import create_mock_db2 +from nose.tools import * # FIXME I can not believe how disgusting this is def in_generator(gen, val): @@ -71,72 +68,7 @@ def x_back(result): return result -class MockDockerClient(object): - base_url = 'http://localhost:5000' - version = lambda x: {'ApiVersion': '1.0'} - inspect_container_pandas = { - 'ID': 'cidpandaslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'cuddly-pandas', - }, - 'NetworkSettings': { - 'IPAddress': '127.0.0.1' - }, - } - inspect_container_foxes = { - 'ID': 'cidfoxeslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'sneaky-foxes', - }, - 'NetworkSettings': { - 'IPAddress': '8.8.8.8' - } - } - inspect_container_sloths = { - 'ID': 'cidslothslong', - 'Config': { - 'Hostname': 'stopped-sloths', - }, - 'NetworkSettings': { - 'IPAddress': '' - } - } - inspect_container_returns = { - 'cidpandas': inspect_container_pandas, - 'cidpandaslong': inspect_container_pandas, - 'cidfoxes': inspect_container_foxes, - 'cidfoxeslong': inspect_container_foxes, - 'cidsloths': inspect_container_sloths, - 'cidslothslong': inspect_container_sloths, - } - containers_return = [ - {'Id': 'cidpandas'}, - {'Id': 'cidfoxes'}, - {'Id': 'cidsloths'}, - ] - - inspect_container_id = None - - def inspect_container(self, cid): - self.inspect_container_id = cid - - try: - return self.inspect_container_returns[cid] - except KeyError: - # Mocks a Docker Client Exception - response = fudge.Fake() - response.has_attr(status_code=404, content='PANDAS!') - - exception = docker.client.APIError('bad', response) - raise exception - - def containers(self, *args, **kwargs): # pylint:disable=unused-argument - return self.containers_return - - -class DockerResolverTest(unittest.TestCase): +class TestDockerResolver(object): def setUp(self): self.CONFIG = {} @@ -146,9 +78,9 @@ def setUp(self): def harn_expected(self, name, expected_record): rec = self.resolver._a_records(name) - self.assertEqual(len(rec), 1) + assert_equal(len(rec), 1) rec = rec[0] - self.assertTrue(check_record(rec, **expected_record)) + assert_true(check_record(rec, **expected_record)) return rec # @@ -158,34 +90,26 @@ def test__a_records_hostname(self): name, expected_record = 'sneaky-foxes', {'name': 'sneaky-foxes.docker', 'type': dns.A} rec = self.harn_expected(name, expected_record) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') + assert_equal(rec.payload.dottedQuad(), '8.8.8.8') def test__a_records_id(self): name, expected_record = 'cidpandas', {'name': 'cidpandas.docker', 'type': dns.A} rec = self.harn_expected(name, expected_record) - self.assertEqual(rec.payload.dottedQuad(), '127.0.0.1') + assert_equal(rec.payload.dottedQuad(), '127.0.0.1') + @raises(DomainError) def test__a_records_shutdown(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'cidsloths.docker' - ) + self.resolver._a_records('cidsloths.docker') + @raises(DomainError) def test__a_records_invalid(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'invalid.docker' - ) + self.resolver._a_records('invalid.docker') + + @raises(DomainError) def test__a_records_blank_query(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - '' - ) + self.resolver._a_records("") def test__a_records_authoritative(self): name, expected_record = 'cidpandas', {'name': @@ -211,33 +135,33 @@ def test_lookupAddress_id(self): deferred = self.resolver.lookupAddress('cidfoxes.docker') result = check_deferred(deferred, True) - self.assertNotEqual(result, False) + assert_not_equal(result, False) response_rr, authority_rr, additional_rr = result # skip this tests as we're now populating # the authority and additional section - self.assertEqual(len(response_rr), 1) + assert_equal(len(response_rr), 1) rec = response_rr[0] - self.assertTrue(check_record( + assert_true(check_record( rec, **expected_record )) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') + assert_equal(rec.payload.dottedQuad(), '8.8.8.8') def test_lookupAddress_invalid(self): deferred = self.resolver.lookupAddress('invalid.docker') result = check_deferred(deferred, False) - self.assertNotEqual(result, False) + assert_not_equal(result, False) def test_lookupAddress_invalid_nxdomain(self): self.resolver.config[NO_NXDOMAIN] = False deferred = self.resolver.lookupAddress('invalid.docker') result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual( + assert_not_equal(result, False) + assert_equal( result.type, DomainError) # noqa pylint:disable=maybe-no-member def test_lookupAddress_invalid_no_nxdomain(self): @@ -245,8 +169,8 @@ def test_lookupAddress_invalid_no_nxdomain(self): deferred = self.resolver.lookupAddress('invalid.docker') result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual(result.type, DNSQueryTimeoutError) + assert_not_equal(result, False) + assert_equal(result.type, DNSQueryTimeoutError) # noqa pylint:disable=maybe-no-member def test_lookupAddress_multi(self): @@ -262,61 +186,17 @@ def test_lookupAddress_multi(self): # retrieve hosts by image deferred = self.resolver.lookupAddress('impandas.*.docker') result = check_deferred(deferred, True) - self.assertNotEqual(result, False) - - response_rr, authority_rr, additional_rr = result - self.assertEqual(len(response_rr), len(expected_records)) - - for rec, expected_record in zip(response_rr, expected_records): - self.assertTrue(check_record( - rec, - **expected_record - )) - - -class DockerResolver2Test(unittest.TestCase): - def setUp(self): - self.CONFIG = {} - - self.db = create_mock_db() - self.mapping = DockerMapping(self.db) - self.resolver = DockerResolver(self.mapping) - - def harn_expected(self, name, expected_record): - rec = self.resolver._a_records(name) - self.assertEqual(len(rec), 1) - rec = rec[0] - self.assertTrue(check_record(rec, **expected_record)) - return rec - - def test_lookupAddress_multi(self): - # search by image - # host -t a impandas.*.docker - # - expected_records = ( - {'name': 'jboss631.docker'}, - # {'name': 'cidpandas0.docker'} - ) - self.resolver.config[NO_NXDOMAIN] = False - - # retrieve hosts by image - deferred = self.resolver.lookupAddress('eap63_tracer:v6.3.1.*.docker') - result = check_deferred(deferred, True) - self.assertNotEqual(result, False) + assert_not_equal(result, False) response_rr, authority_rr, additional_rr = result - self.assertEqual(len(response_rr), len(expected_records)) + assert_equal(len(response_rr), len(expected_records)) for rec, expected_record in zip(response_rr, expected_records): - self.assertTrue(check_record( + assert_true(check_record( rec, **expected_record )) -def main(): - unittest.main() -if __name__ == '__main__': - main() diff --git a/test/test_resolver_multi.py b/test/test_resolver_multi.py new file mode 100755 index 0000000..6059e79 --- /dev/null +++ b/test/test_resolver_multi.py @@ -0,0 +1,76 @@ +#!/usr/bin/python + +""" +Tests for Docker DNS + +Author: Ricky Cook +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + +import unittest + +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2, create_mock_db +from test.test_resolver import check_record, in_generator, check_deferred + +from nose.tools import * + +class TestDockerResolver(object): + def setup(self): + self.CONFIG = {} + + self.db = create_mock_db() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + assert_equal(len(rec), 1) + rec = rec[0] + assert_true(check_record(rec, **expected_record)) + return rec + + def harn_lookupAddress_multi(self, image_name, expected_records): + # search by image + # host -t a impandas.*.docker + # + + # retrieve hosts by image + deferred = self.resolver.lookupAddress(image_name) + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + assert_equal(len(response_rr), len(expected_records)) + + for rec, expected_record in zip(response_rr, expected_records): + assert_true(check_record( + rec, + **expected_record + )) + + def test_lookupAddress_multi(self): + self.resolver.config[NO_NXDOMAIN] = False + expected_records = ( + {'name': 'jboss631.docker'}, + # {'name': 'cidpandas0.docker'} + ) + self.harn_lookupAddress_multi('eap63_tracer:v6.3.1.*.docker', expected_records) + + def test_lookupAddress_multi_notag(self): + self.resolver.config[NO_NXDOMAIN] = False + expected_records = ( + {'name': 'jboss631.docker'}, + # {'name': 'cidpandas0.docker'} + ) + self.harn_lookupAddress_multi('eap63_tracer.*.docker', expected_records) + + + From 032034d39a7bd061d97becbef688823aa419f7be Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 13 Feb 2015 15:01:50 +0100 Subject: [PATCH 32/51] added a small REST console to check database --- dockerdns/console.py | 48 +++++++++++++++++++++++++++++ twisted/plugins/dockerdns_plugin.py | 7 ++++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 dockerdns/console.py diff --git a/dockerdns/console.py b/dockerdns/console.py new file mode 100644 index 0000000..f1edd97 --- /dev/null +++ b/dockerdns/console.py @@ -0,0 +1,48 @@ +""" + Management Console +""" +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.web.http import Request +from events import DockerDB +import simplejson +import time + + +class RestConsole(Resource): + isLeaf = True + + def __init__(self, db): + """ + + :param db: + :return: + """ + assert isinstance(db, DockerDB) + self.db = db + + def render_GET(self, request): + """ + + :param request: + :type twisted.web.http.Request + :return: + """ + serialize = lambda x: simplejson.dumps(x, indent=True) + assert isinstance(request, Request) + if 'ping' in request.path: + return "%s" % (time.ctime(),) + elif 'hostname/' in request.path: + return serialize(self.db.mappings_hostname) + elif 'image/' in request.path: + return serialize(self.db.mappings_image) + elif 'dump/' in request.path: + return serialize(self.db.mappings) + else: + return serialize(dict(error='command not found', + msg='try http://localhost:8080/{hostname,image,dump}/')) + +class ConsoleFactory(Site): + def __init__(self, db): + Site.__init__(self, RestConsole(db)) + diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 7e86b75..05c19f7 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -33,7 +33,7 @@ from dockerdns.mappings import DockerMapping from dockerdns.events import DockerDB from dockerdns.resolver import DockerResolver - +from dockerdns.console import ConsoleFactory import docker @@ -125,6 +125,11 @@ def makeService(self, options): docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) docker_event_monitor.setServiceParent(ret) + # Add the console + consoleFactory = ConsoleFactory(db) + console = internet.TCPServer(8080, consoleFactory) + console.setServiceParent(ret) + return ret From 13f21bc3ba21ab6f704dc69bfa9b06d861987e5b Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 13 Feb 2015 15:05:01 +0100 Subject: [PATCH 33/51] enh: more logs --- dockerdns/events.py | 2 +- twisted/plugins/dockerdns_plugin.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dockerdns/events.py b/dockerdns/events.py index c789fbd..8c532ba 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -118,7 +118,7 @@ def dataReceived(self, bytes_): self.db.updatedb(item) def connectionLost(self, reason): - self.onLost.callback(None) + Deferred().callback(None) class EventManager(Protocol): diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 7e86b75..15f024e 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -48,7 +48,7 @@ class Options(usage.Options): ["docker_url", "u", 'unix://var/run/docker.sock', "Docker URL"], ['no_nxdomain', "x", True, "Return SERVFAIL instead of NXDOMAIN if container not found"], ["authoritative", "A", True, "Return authoritative replies"], - ['version', "v", '1.13', "Docker API version"], + ['version', "v", '1.15', "Docker API version"], ['bind_protocols', "B", ['tcp', 'udp'], "Bind protocols"] ] @@ -75,13 +75,14 @@ def makeService(self, options): except IOError as e: if options['config'] != "dockerdns.json": raise - log.err("File {config} not found. Using default values".format(options)) + log.err("File {config} not found. Using default values".format(**options)) + appcfg = {} # Update config stuff with command line params appcfg.update(options) log.err("config: %r" % appcfg) # Create docker: by default dict.get returns None on missing keys - docker_client = docker.Client(appcfg.get('docker_url')) + docker_client = docker.Client(appcfg.get('docker_url'), version=appcfg.get('version')) infos = docker_client.info() # Test docker connectivity before starting log.msg("Connecting to docker instance: %r" % infos) From 8fecd575619c4ae6e69a897ee835d1c9271d128e Mon Sep 17 00:00:00 2001 From: root Date: Fri, 13 Feb 2015 15:48:14 +0100 Subject: [PATCH 34/51] add: console on port 8080 --- README.md | 10 +++++++--- dockerdns/console.py | 32 ++++++++++++++++++++++---------- dockerdns/events.py | 2 +- dockerdns/mappings.py | 7 ++++--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b93cab3..26c0bb8 100644 --- a/README.md +++ b/README.md @@ -21,22 +21,26 @@ Install/Run Just install from requirements (in a virtualenv if you'd like) - pip install -r requirements.txt + #pip install -r requirements.txt That's it! To run, remember that you may need to set user/group ids on the process - sudo twistd -gdocker -y dockerdns -p 53 + #sudo twistd -gdocker -y dockerdns -p 53 This will start a DNS server on port 53 (default DNS port). To make this useful, you probably want to combine it with your regular DNS in something like Dnsmasq. You can get configuration parameters with - sudo twistd dockerdns --help + #sudo twistd dockerdns --help +There's a simple HTTP console to check the internal mappings. You can curl it with + + #curl -v http://localhost:8080/{hostname,image,name,id,ping}/{optional_key} + Examples -------- Dig output is shortened for brevity. We have Docker containers like this: diff --git a/dockerdns/console.py b/dockerdns/console.py index f1edd97..a9fee93 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -21,6 +21,20 @@ def __init__(self, db): assert isinstance(db, DockerDB) self.db = db + def dump(self, table, k=None): + """ + Return a value from a given mapping of DB. This should be moved to DockerDB + """ + try: + d = getattr(self.db, 'mappings_' + table if table != "id" else "mappings" ) + except AttributeError: + return "Table not found %r" % table + + if k: + return d[k] + + return d + def render_GET(self, request): """ @@ -30,17 +44,15 @@ def render_GET(self, request): """ serialize = lambda x: simplejson.dumps(x, indent=True) assert isinstance(request, Request) + r = request.path.strip("/").split("/") + action = r[0] + + if 'ping' in request.path: - return "%s" % (time.ctime(),) - elif 'hostname/' in request.path: - return serialize(self.db.mappings_hostname) - elif 'image/' in request.path: - return serialize(self.db.mappings_image) - elif 'dump/' in request.path: - return serialize(self.db.mappings) - else: - return serialize(dict(error='command not found', - msg='try http://localhost:8080/{hostname,image,dump}/')) + return "%s" % [time.ctime(), request.path] + + return serialize(self.dump(action, *r[1:])) + # return serialize(dict(error='command not found', msg='try http://localhost:8080/{hostname,image,dump}/')) class ConsoleFactory(Site): def __init__(self, db): diff --git a/dockerdns/events.py b/dockerdns/events.py index 8c532ba..fa88e9b 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -36,7 +36,7 @@ def __init__(self, api=None): self.mappings_hostname = {} # # Initialize db (TODO get initialization timestamp) - for c in self.api.containers(all=True): + for c in self.api.containers(all=False): item = self.api.inspect_container(c['Id']) self.updatedb(item) diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index 3218483..0fb3822 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -32,14 +32,14 @@ def lookup_container(self, name): Returns: Container config dict for the first matching container """ - for search_f in (self.db.get_by_name, self.db.get_by_hostname): + for map_name, search_f in (('name', self.db.get_by_name), ('hostname', self.db.get_by_hostname)): try: - log.msg('lookup container: %r' % name) + log.msg('lookup container by %r: %r' % (map_name, name)) return search_f(name) except KeyError as e: # warn(str(e)) - log.msg("Container not found: %r" % name) + log.msg("Container %r not found: %r" % (map_name, name)) except Exception as e: log.err("Unmanaged error: %r" % e) @@ -65,6 +65,7 @@ def get_a(self, name): addr = container['NetworkSettings']['IPAddress'] if not addr: + log.msg("No IPAddress associated with container %r" % container) return None return addr From 62127230318caaa5be37668ac940159ba78b262b Mon Sep 17 00:00:00 2001 From: root Date: Fri, 13 Feb 2015 16:18:34 +0100 Subject: [PATCH 35/51] fix: big limiting received events --- dockerdns/events.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dockerdns/events.py b/dockerdns/events.py index fa88e9b..9d1aca8 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -132,8 +132,6 @@ def __init__(self, http_agent, config, db): self.config = config self.db = db - self.remaining = 1024 * 10 - self.buff = "" # Create an container_manager for parsing updates self.container_manager = ContainerManager(Deferred(), db=db) @@ -158,16 +156,14 @@ def delete_record(self, item): def dataReceived(self, bytes_): """Get the container id and calls the updater""" try: - if self.remaining: - display = bytes_[:self.remaining] - print('Some data received:', display) - self.remaining -= len(display) - item = json.loads(display) - print("Parsed: %r" % item) - if item['status'] == 'start': - self.update_record(item) - elif item['status'] in ('stop', 'die'): - self.delete_record(item) + display = bytes_ + print('Some data received:', display) + item = json.loads(display) + print("Parsed: %r" % item) + if item['status'] == 'start': + self.update_record(item) + elif item['status'] in ('stop', 'die'): + self.delete_record(item) except KeyError: log.err("Container not found") except json.scanner.JSONDecodeError: From 8cab1c50aa06f414db67f588919383c73fa813ff Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Fri, 20 Feb 2015 17:07:51 +0100 Subject: [PATCH 36/51] refresh hosts via console --- dockerdns/console.py | 30 ++++++++++++++++++++++++++++-- dockerdns/events.py | 9 +++++++++ test/test_events.py | 18 ++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/dockerdns/console.py b/dockerdns/console.py index a9fee93..409555a 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -47,12 +47,38 @@ def render_GET(self, request): r = request.path.strip("/").split("/") action = r[0] - if 'ping' in request.path: return "%s" % [time.ctime(), request.path] + if action == 'help': + return """ +Allowed commands: + +Retrieve container names \t\t\tcurl 'http://localhost:8080/name' +Retrieve container hostnames \t\t\tcurl 'http://localhost:8080/hostname' +Retrieve container ids\t\t\tcurl 'http://localhost:8080/id' +Refresh dns mapping\t\t\tcurl -XPOST 'http://localhost:8080/refresh' + +""" return serialize(self.dump(action, *r[1:])) - # return serialize(dict(error='command not found', msg='try http://localhost:8080/{hostname,image,dump}/')) + + def render_POST(self, request): + """ + + :param request: + :type twisted.web.http.Request + :return: + """ + serialize = lambda x: simplejson.dumps(x, indent=True) + assert isinstance(request, Request) + r = request.path.strip("/").split("/") + action = r[0] + + if action == 'refresh': + db.cleanup() + db.load_container() + return serialize(dict(status="ok",action="refresh")) + raise ValueError("Not Found") class ConsoleFactory(Site): def __init__(self, db): diff --git a/dockerdns/events.py b/dockerdns/events.py index 9d1aca8..e975a7c 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -35,6 +35,15 @@ def __init__(self, api=None): self.mappings_image = {} self.mappings_hostname = {} # + self.load_containers() + + def cleandb(self): + self.mappings = {} + self.mappings_name = {} + self.mappings_image = {} + self.mappings_hostname = {} + + def load_containers(self): # Initialize db (TODO get initialization timestamp) for c in self.api.containers(all=False): item = self.api.inspect_container(c['Id']) diff --git a/test/test_events.py b/test/test_events.py index 474c4e5..a3a1edd 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -51,6 +51,24 @@ def test_init_and_get_images(): assert set(pandas_container) == set( ('cidpandas', 'cidpandas0')), pandas_container +def test_clean_db(): + db = create_mock_db2() + db.cleandb() + assert not db.mappings, "Mappings not clean" + assert not db.mappings_name, "Mappings not clean" + assert not db.mappings_hostname, "Mappings not clean" + assert not db.mappings_image, "Mappings not clean" + +def test_reload_container(): + db = create_mock_db2() + db.mappings = {} + db.load_containers() + assert db.mappings + assert 'impandas' in db.mappings_image, "%r, %r" % ( + db.mappings_image, db.mappings) + + + class MockAgent(Agent): reasons = ['No Reason'] From 00884d0c25be07fcf9d9b2d035f445116d5db04b Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sat, 21 Feb 2015 22:10:13 +0100 Subject: [PATCH 37/51] autopep8 --- dockerdns/console.py | 11 ++++++----- dockerdns/events.py | 1 - test/__init__.py | 2 +- test/test_events.py | 4 ++-- test/test_resolver.py | 7 ++----- test/test_resolver_multi.py | 10 +++++----- twisted/plugins/dockerdns_plugin.py | 11 ++++++----- 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/dockerdns/console.py b/dockerdns/console.py index 409555a..bebe1ee 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -26,13 +26,14 @@ def dump(self, table, k=None): Return a value from a given mapping of DB. This should be moved to DockerDB """ try: - d = getattr(self.db, 'mappings_' + table if table != "id" else "mappings" ) + d = getattr(self.db, 'mappings_' + + table if table != "id" else "mappings") except AttributeError: return "Table not found %r" % table if k: return d[k] - + return d def render_GET(self, request): @@ -46,7 +47,7 @@ def render_GET(self, request): assert isinstance(request, Request) r = request.path.strip("/").split("/") action = r[0] - + if 'ping' in request.path: return "%s" % [time.ctime(), request.path] if action == 'help': @@ -77,10 +78,10 @@ def render_POST(self, request): if action == 'refresh': db.cleanup() db.load_container() - return serialize(dict(status="ok",action="refresh")) + return serialize(dict(status="ok", action="refresh")) raise ValueError("Not Found") + class ConsoleFactory(Site): def __init__(self, db): Site.__init__(self, RestConsole(db)) - diff --git a/dockerdns/events.py b/dockerdns/events.py index e975a7c..2cfcaf0 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -75,7 +75,6 @@ def del_container(self, cid): self.mappings_image.get(image_notag, []).remove(cid) self.mappings_image.get(image, []).remove(cid) - del self.mappings[cid] del self.mappings_name[name] del self.mappings_hostname[hostname] diff --git a/test/__init__.py b/test/__init__.py index a439f23..8e66d58 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -241,4 +241,4 @@ def inspect_container(self, cid): raise exception def containers(self, *args, **kwargs): # pylint:disable=unused-argument - return self.containers_return \ No newline at end of file + return self.containers_return diff --git a/test/test_events.py b/test/test_events.py index a3a1edd..d94d3ce 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -51,6 +51,7 @@ def test_init_and_get_images(): assert set(pandas_container) == set( ('cidpandas', 'cidpandas0')), pandas_container + def test_clean_db(): db = create_mock_db2() db.cleandb() @@ -59,6 +60,7 @@ def test_clean_db(): assert not db.mappings_hostname, "Mappings not clean" assert not db.mappings_image, "Mappings not clean" + def test_reload_container(): db = create_mock_db2() db.mappings = {} @@ -67,8 +69,6 @@ def test_reload_container(): assert 'impandas' in db.mappings_image, "%r, %r" % ( db.mappings_image, db.mappings) - - class MockAgent(Agent): reasons = ['No Reason'] diff --git a/test/test_resolver.py b/test/test_resolver.py index 02a4056..b38e529 100755 --- a/test/test_resolver.py +++ b/test/test_resolver.py @@ -20,6 +20,8 @@ from nose.tools import * # FIXME I can not believe how disgusting this is + + def in_generator(gen, val): return reduce( lambda old, new: old or new == val, @@ -106,7 +108,6 @@ def test__a_records_shutdown(self): def test__a_records_invalid(self): self.resolver._a_records('invalid.docker') - @raises(DomainError) def test__a_records_blank_query(self): self.resolver._a_records("") @@ -196,7 +197,3 @@ def test_lookupAddress_multi(self): rec, **expected_record )) - - - - diff --git a/test/test_resolver_multi.py b/test/test_resolver_multi.py index 6059e79..44cb997 100755 --- a/test/test_resolver_multi.py +++ b/test/test_resolver_multi.py @@ -22,6 +22,7 @@ from nose.tools import * + class TestDockerResolver(object): def setup(self): self.CONFIG = {} @@ -62,7 +63,8 @@ def test_lookupAddress_multi(self): {'name': 'jboss631.docker'}, # {'name': 'cidpandas0.docker'} ) - self.harn_lookupAddress_multi('eap63_tracer:v6.3.1.*.docker', expected_records) + self.harn_lookupAddress_multi( + 'eap63_tracer:v6.3.1.*.docker', expected_records) def test_lookupAddress_multi_notag(self): self.resolver.config[NO_NXDOMAIN] = False @@ -70,7 +72,5 @@ def test_lookupAddress_multi_notag(self): {'name': 'jboss631.docker'}, # {'name': 'cidpandas0.docker'} ) - self.harn_lookupAddress_multi('eap63_tracer.*.docker', expected_records) - - - + self.harn_lookupAddress_multi( + 'eap63_tracer.*.docker', expected_records) diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 58df4eb..4cbb111 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -37,8 +37,6 @@ import docker - - class Options(usage.Options): optParameters = [ ["bind_port", "p", 10053, "The port number to listen on."], @@ -46,7 +44,8 @@ class Options(usage.Options): ["domain", "d", "docker", "The default domain"], ["config", "c", "dockerdns.json", "Configuration file"], ["docker_url", "u", 'unix://var/run/docker.sock', "Docker URL"], - ['no_nxdomain', "x", True, "Return SERVFAIL instead of NXDOMAIN if container not found"], + ['no_nxdomain', "x", True, + "Return SERVFAIL instead of NXDOMAIN if container not found"], ["authoritative", "A", True, "Return authoritative replies"], ['version', "v", '1.15', "Docker API version"], ['bind_protocols', "B", ['tcp', 'udp'], "Bind protocols"] @@ -75,14 +74,16 @@ def makeService(self, options): except IOError as e: if options['config'] != "dockerdns.json": raise - log.err("File {config} not found. Using default values".format(**options)) + log.err("File {config} not found. Using default values".format( + **options)) appcfg = {} # Update config stuff with command line params appcfg.update(options) log.err("config: %r" % appcfg) # Create docker: by default dict.get returns None on missing keys - docker_client = docker.Client(appcfg.get('docker_url'), version=appcfg.get('version')) + docker_client = docker.Client( + appcfg.get('docker_url'), version=appcfg.get('version')) infos = docker_client.info() # Test docker connectivity before starting log.msg("Connecting to docker instance: %r" % infos) From 9e9be7a09b9390cb54d14df80a983357192eea46 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sun, 22 Feb 2015 21:15:02 +0100 Subject: [PATCH 38/51] cleanup code --- .travis.yml | 4 +-- dockerdns/console.py | 44 +++++++++++++---------- dockerdns/events.py | 44 +++++++++-------------- dockerdns/mappings.py | 4 +-- dockerdns/resolver.py | 28 +++++++++------ dockerdns/utils.py | 55 +++-------------------------- pylint.conf | 15 ++++---- test/test_resolver.py | 0 test/test_resolver_multi.py | 0 twisted/plugins/dockerdns_plugin.py | 8 ++--- 10 files changed, 81 insertions(+), 121 deletions(-) mode change 100755 => 100644 test/test_resolver.py mode change 100755 => 100644 test/test_resolver_multi.py diff --git a/.travis.yml b/.travis.yml index 53343c5..b9b2982 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: install: - sudo pip install -r test_requirements.txt before_script: - - pep8 *.py - - pylint --rcfile=pylint.conf *.py + - pep8 dockerdns/*.py twisted/plugins/*py + - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py script: - nosetests -v -w test diff --git a/dockerdns/console.py b/dockerdns/console.py index bebe1ee..6d49d18 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -1,13 +1,28 @@ """ + author: robipolli@gmail.com Management Console """ from twisted.web.server import Site from twisted.web.resource import Resource from twisted.web.http import Request -from events import DockerDB +from dockerdns.events import DockerDB import simplejson import time +HELP_STR = """ +Allowed commands: + +Retrieve container names \t\t\tcurl 'http://localhost:8080/name' +Retrieve container hostnames \t\t\tcurl 'http://localhost:8080/hostname' +Retrieve container ids\t\t\tcurl 'http://localhost:8080/id' +Refresh dns mapping\t\t\tcurl -XPOST 'http://localhost:8080/refresh' + +""" + + +def serialize(item): + return simplejson.dumps(item, indent=True) + class RestConsole(Resource): isLeaf = True @@ -20,6 +35,7 @@ def __init__(self, db): """ assert isinstance(db, DockerDB) self.db = db + Resource.__init__(self) def dump(self, table, k=None): """ @@ -43,25 +59,16 @@ def render_GET(self, request): :type twisted.web.http.Request :return: """ - serialize = lambda x: simplejson.dumps(x, indent=True) assert isinstance(request, Request) - r = request.path.strip("/").split("/") - action = r[0] + rpath = request.path.strip("/").split("/") + action = rpath[0] if 'ping' in request.path: return "%s" % [time.ctime(), request.path] if action == 'help': - return """ -Allowed commands: - -Retrieve container names \t\t\tcurl 'http://localhost:8080/name' -Retrieve container hostnames \t\t\tcurl 'http://localhost:8080/hostname' -Retrieve container ids\t\t\tcurl 'http://localhost:8080/id' -Refresh dns mapping\t\t\tcurl -XPOST 'http://localhost:8080/refresh' - -""" + return HELP_STR - return serialize(self.dump(action, *r[1:])) + return serialize(self.dump(action, *rpath[1:])) def render_POST(self, request): """ @@ -70,14 +77,13 @@ def render_POST(self, request): :type twisted.web.http.Request :return: """ - serialize = lambda x: simplejson.dumps(x, indent=True) assert isinstance(request, Request) - r = request.path.strip("/").split("/") - action = r[0] + rpath = request.path.strip("/").split("/") + action = rpath[0] if action == 'refresh': - db.cleanup() - db.load_container() + self.db.cleanup() + self.db.load_container() return serialize(dict(status="ok", action="refresh")) raise ValueError("Not Found") diff --git a/dockerdns/events.py b/dockerdns/events.py index 2cfcaf0..2bd3261 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -1,16 +1,15 @@ -from __future__ import print_function - """ Populate dns database getting info from docker via: - docker-py api - docker events interface """ +from __future__ import print_function +from os.path import join as pjoin +from logging import DEBUG +import simplejson as json from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from twisted.web.http_headers import Headers -from os.path import join as pjoin -import simplejson as json -from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol, ReconnectingClientFactory from twisted.python import log @@ -114,20 +113,16 @@ class ContainerManager(Protocol): updating the network infos associated to the container """ - def __init__(self, onLost, db): + def __init__(self, db): """Initialize the Docker host database""" - self.onLost = onLost self.db = db def dataReceived(self, bytes_): if bytes_: item = json.loads(bytes_) - print("Get container %r" % item) + log.msg("Get container %r" % item) self.db.updatedb(item) - def connectionLost(self, reason): - Deferred().callback(None) - class EventManager(Protocol): """Manage the response to /events/json @@ -141,7 +136,7 @@ def __init__(self, http_agent, config, db): self.db = db # Create an container_manager for parsing updates - self.container_manager = ContainerManager(Deferred(), db=db) + self.container_manager = ContainerManager(db=db) def update_record(self, item): """Update docker mapping @@ -165,9 +160,9 @@ def dataReceived(self, bytes_): """Get the container id and calls the updater""" try: display = bytes_ - print('Some data received:', display) + log.msg('Some data received:', display) item = json.loads(display) - print("Parsed: %r" % item) + log.msg("Parsed: %r" % item) if item['status'] == 'start': self.update_record(item) elif item['status'] in ('stop', 'die'): @@ -176,12 +171,8 @@ def dataReceived(self, bytes_): log.err("Container not found") except json.scanner.JSONDecodeError: log.err("Error reading data") - except Exception as e: - log.err("Generic Error %r" % e) - - def connectionLost(self, reason): - print('Finished receiving body:', reason.type, reason.value) - Deferred().callback(None) + except Exception as ex: + log.err("Generic Error %r" % ex) class EventFactory(ReconnectingClientFactory): @@ -204,7 +195,7 @@ def __init__(self, config, db): # # Create a protocol handler to parse docker container data # - self.dockerUpdater = EventManager( + self.update_event_manager = EventManager( http_agent=self.agent2, config=config, db=db) # Populate existing containers (this is ok to be blocking) @@ -212,23 +203,22 @@ def __init__(self, config, db): # # Poll to the docker event interface # - d = self.agent.request( + deferred = self.agent.request( 'GET', pjoin(config['docker_url'], 'events'), Headers({'User-Agent': ['Twisted Web Client for Docker Event'], 'Content-Type': ['application/json']}), None) - d.addCallbacks(self.cbResponse, lambda failure: print(str(failure))) + deferred.addCallbacks(self.cbResponse, lambda failure: log.msg(str(failure), logLevel=DEBUG)) def cbResponse(self, response): """Manages the response using a Protocol class defined in __init__""" try: log.msg('Response received: %r' % response) - finished = Deferred() - response.deliverBody(self.dockerUpdater) - except: + response.deliverBody(self.update_event_manager) + except Exception as ex: log.err() def buildProtocol(self, addr): log.msg("addr: %r" % addr) - return self.dockerUpdater + return self.update_event_manager diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index 0fb3822..2d1c274 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -106,7 +106,7 @@ def get_nat(self, container_name, sport=0, sproto=None): try: nats = container['NetworkSettings']['Ports'].items() - except (KeyError, AttributeError) as e: + except (KeyError, AttributeError) as ex: # container infos set and not None log.err("Bad network information for container: %r" % container) @@ -126,6 +126,6 @@ def get_nat(self, container_name, sport=0, sproto=None): for r in remote: try: yield (port, proto, int(r['HostPort']), r['HostIp']) - except (ValueError, KeyError) as e: + except (ValueError, KeyError) as ex: log.err() continue diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index c86b4c1..6846107 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -1,3 +1,11 @@ +""" +author: robipolli@gmail.com + +Extending twisted.names.common.ResolverBase to +reply with the informations provided by +dockerdns.mappings.DockerMapping + +""" import re from twisted.python import log from twisted.internet import defer @@ -7,10 +15,10 @@ from dockerdns.utils import get_preferred_ip -# pylint:disable=too-many-public-methods NO_NXDOMAIN = 'no_nxdomain' - +# pylint:disable=too-many-public-methods +# pylint:disable=too-many-instance-attributes class DockerResolver(common.ResolverBase): """ DNS resolver to resolve queries with a DockerMapping instance. @@ -130,13 +138,13 @@ def lookupAddress(self, name, timeout=None): # We need to catch everything. Uncaught exceptions will make the server # stop responding - except DomainError as e: - log.msg("DomainError: %r " % e) + except DomainError as ex: + log.msg("DomainError: %r " % ex) if self.config.get(NO_NXDOMAIN): # FIXME surely there's a better way to give SERVFAIL - e = DNSQueryTimeoutError(name) - return defer.fail(failure.Failure(e)) - except Exception as e: # pylint:disable=bare-except + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) + except Exception as ex: # pylint:disable=bare-except import traceback traceback.print_exc() @@ -171,7 +179,7 @@ def lookupService(self, name, timeout=None): try: port, proto, container = name.split(".") port = int(port.strip("_")) - except (IndexError, TypeError, ValueError) as e: + except (IndexError, TypeError, ValueError) as ex: log.err("Domain not of the right form: %r" % name) return defer.fail(failure.Failure(DomainError("not of the right form"))) @@ -181,7 +189,7 @@ def lookupService(self, name, timeout=None): priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip_ptr_value, ttl=None), auth=True) for c_port, protocol, c_nat_port, target - in self.mapping.get_nat(container) - if c_port == port # eventually filter + in self.mapping.get_nat(container) + if c_port == port # eventually filter ] return defer.succeed((records, self.authority, self.additional)) diff --git a/dockerdns/utils.py b/dockerdns/utils.py index 86a4379..8bd35f9 100644 --- a/dockerdns/utils.py +++ b/dockerdns/utils.py @@ -1,6 +1,5 @@ """utilities.py """ - import socket @@ -8,61 +7,15 @@ def get_preferred_ip(): """Return the in-addr name associated to the ip used to contact the default gw""" try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # connecting to a UDP address doesn't send packets - s.connect(('8.8.8.8', 0)) - ip = s.getsockname()[0] + sock.connect(('8.8.8.8', 0)) + ip = sock.getsockname()[0] return ip, '.'.join(list(reversed(ip.split(".")))) + ".in-addr.arpa" - except Exception as e: + except Exception as ex: return socket.getfqdn() -from functools import partial - - -class memoize(object): - - """cache the return value of a method - - This class is meant to be used as a decorator of methods. The return value - from a given method invocation will be cached on the instance whose method - was invoked. All arguments passed to a method decorated with memoize must - be hashable. - - If a memoized method is invoked directly on its class the result will not - be cached. Instead the method will be invoked like a static method: - class Obj(object): - @memoize - def add_to(self, arg): - return self + arg - Obj.add_to(1) # not enough arguments - Obj.add_to(1, 2) # returns 3, result is not cached - """ - - def __init__(self, func): - self.func = func - - def __get__(self, obj, objtype=None): - if obj is None: - return self.func - return partial(self, obj) - - def __call__(self, *args, **kw): - obj = args[0] - try: - cache = obj.__cache - except AttributeError: - cache = obj.__cache = {} - key = (self.func, args[1:], frozenset(kw.items())) - try: - res = cache[key] - except KeyError: - res = cache[key] = self.func(*args, **kw) - return res - -# FIXME replace with a more generic solution like operator.attrgetter - - def traverse_tree(haystack, key_path, default=None): """ Find an element in a nested dict, eg. diff --git a/pylint.conf b/pylint.conf index c6af89c..f5c61c1 100644 --- a/pylint.conf +++ b/pylint.conf @@ -150,16 +150,19 @@ class-rgx=[A-Z_][a-zA-Z0-9]+$ function-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# modified for twisted camelized method names +method-rgx=[a-z_][a-zA-Z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +# two-chars variables and attrs are ok +attr-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +argument-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# including single-letter variables +variable-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct attribute names in class # bodies @@ -170,10 +173,10 @@ class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +good-names=a,b,c,d,e,s,f,i,j,k,v,ex,Run,_ # Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +bad-names=foo,bar,baz,toto,tutu,tata,l,o,I # Regular expression which should only match function or class names that do # not require a docstring. diff --git a/test/test_resolver.py b/test/test_resolver.py old mode 100755 new mode 100644 diff --git a/test/test_resolver_multi.py b/test/test_resolver_multi.py old mode 100755 new mode 100644 diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 4cbb111..b807c41 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -47,8 +47,8 @@ class Options(usage.Options): ['no_nxdomain', "x", True, "Return SERVFAIL instead of NXDOMAIN if container not found"], ["authoritative", "A", True, "Return authoritative replies"], - ['version', "v", '1.15', "Docker API version"], - ['bind_protocols', "B", ['tcp', 'udp'], "Bind protocols"] + ["docker-version", "V", '1.15', "Docker API version"], + ["bind_protocols", "B", ['tcp', 'udp'], "Bind protocols"] ] @@ -60,7 +60,7 @@ class MyServiceMaker(object): """ implements(IServiceMaker, IPlugin) tapname = "dockerdns" - description = "Run this! It'll make your dog happy." + description = "Run this! It'll make your docker happy." options = Options def makeService(self, options): @@ -83,7 +83,7 @@ def makeService(self, options): log.err("config: %r" % appcfg) # Create docker: by default dict.get returns None on missing keys docker_client = docker.Client( - appcfg.get('docker_url'), version=appcfg.get('version')) + appcfg.get('docker_url'), version=appcfg.get('docker-version')) infos = docker_client.info() # Test docker connectivity before starting log.msg("Connecting to docker instance: %r" % infos) From 8da174b7b3fdbbf76fcd37afce4b4e410b23cc3d Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Sun, 22 Feb 2015 21:35:28 +0100 Subject: [PATCH 39/51] added pypy --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b9b2982..488bbea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "pypy" install: - sudo pip install -r test_requirements.txt before_script: From 76c512918433916f05f3d1cba01e0e426fde7b22 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 23 Feb 2015 14:38:34 +0100 Subject: [PATCH 40/51] fix: pep8 --- dockerdns/console.py | 10 +-- dockerdns/events.py | 5 +- dockerdns/mappings.py | 7 +- dockerdns/resolver.py | 101 ++++++++++++++++------------ dockerdns/utils.py | 4 +- twisted/plugins/dockerdns_plugin.py | 26 +++---- 6 files changed, 85 insertions(+), 68 deletions(-) diff --git a/dockerdns/console.py b/dockerdns/console.py index 6d49d18..0de846c 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -39,7 +39,8 @@ def __init__(self, db): def dump(self, table, k=None): """ - Return a value from a given mapping of DB. This should be moved to DockerDB + Return a value from a given mapping of DB. + TODO This should be moved to DockerDB """ try: d = getattr(self.db, 'mappings_' + @@ -64,7 +65,8 @@ def render_GET(self, request): action = rpath[0] if 'ping' in request.path: - return "%s" % [time.ctime(), request.path] + return "{0:s}".format([ + time.ctime(), request.path]) if action == 'help': return HELP_STR @@ -82,8 +84,8 @@ def render_POST(self, request): action = rpath[0] if action == 'refresh': - self.db.cleanup() - self.db.load_container() + self.db.cleandb() + self.db.load_containers() return serialize(dict(status="ok", action="refresh")) raise ValueError("Not Found") diff --git a/dockerdns/events.py b/dockerdns/events.py index 2bd3261..6f5247f 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -203,13 +203,14 @@ def __init__(self, config, db): # # Poll to the docker event interface # - deferred = self.agent.request( + d = self.agent.request( 'GET', pjoin(config['docker_url'], 'events'), Headers({'User-Agent': ['Twisted Web Client for Docker Event'], 'Content-Type': ['application/json']}), None) - deferred.addCallbacks(self.cbResponse, lambda failure: log.msg(str(failure), logLevel=DEBUG)) + d.addCallbacks(self.cbResponse, + lambda failure: log.msg(str(failure), logLevel=DEBUG)) def cbResponse(self, response): """Manages the response using a Protocol class defined in __init__""" diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index 2d1c274..342dcb9 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -32,7 +32,8 @@ def lookup_container(self, name): Returns: Container config dict for the first matching container """ - for map_name, search_f in (('name', self.db.get_by_name), ('hostname', self.db.get_by_hostname)): + for map_name, search_f in (('name', self.db.get_by_name), + ('hostname', self.db.get_by_hostname)): try: log.msg('lookup container by %r: %r' % (map_name, name)) @@ -84,8 +85,8 @@ def get_a_multi(self, name_multi): (container['NetworkSettings'][ 'IPAddress'], container['Name'][1:]) for container - in self.db.get_by_image(name_multi) - if 'NetworkSettings' in container + in self.db.get_by_image(name_multi) + if 'NetworkSettings' in container ) def get_nat(self, container_name, sport=0, sproto=None): diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index 6846107..fd9613c 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -7,9 +7,10 @@ """ import re +import socket from twisted.python import log from twisted.internet import defer -from twisted.internet.defer import failure +from twisted.internet.defer.failure import Failure from twisted.names import common, dns from twisted.names.error import DomainError, DNSQueryTimeoutError from dockerdns.utils import get_preferred_ip @@ -17,6 +18,7 @@ NO_NXDOMAIN = 'no_nxdomain' + # pylint:disable=too-many-public-methods # pylint:disable=too-many-instance-attributes class DockerResolver(common.ResolverBase): @@ -25,16 +27,17 @@ class DockerResolver(common.ResolverBase): Twisted Names just uses the lookupXXX methods """ - mock_records = [dns.RRHeader( - "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=19999, target='name', ttl=None), - auth=True), + mock_records = [ + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + payload=dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), dns.RRHeader( "mock_name", dns.SRV, dns.IN, 86400, - dns.Record_SRV( - priority=100, weight=100, port=18080, target='name', ttl=None), - auth=True) + payload=dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) ] def __init__(self, mapping, config=None): @@ -49,17 +52,15 @@ def __init__(self, mapping, config=None): # super(DockerResolver, self).__init__() common.ResolverBase.__init__(self) self.ttl = 10 - self.my_preferred_ip, self.my_preferred_ip_ptr_value = get_preferred_ip() + self.my_preferred_ip, self.my_preferred_ip_ptr_value \ + = get_preferred_ip() # define authority and additional records # an authority record defines the name IN NS TIMEOUT # socket. - import socket - self.authority = [dns.RRHeader( name=self.config['domain'] + ".", type=dns.NS, cls=dns.IN, - payload=dns.Record_NS( - name=socket.gethostname()) + payload=dns.Record_NS(name=socket.gethostname()) )] self.additional = [dns.RRHeader( name=socket.gethostname(), type=dns.A, cls=dns.IN, @@ -81,10 +82,10 @@ def _a_records(self, name): return tuple([ dns.RRHeader( - '.'.join( - (name, self.config['domain'])), dns.A, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - auth=self.config.get('authoritative')) + '.'.join((name, self.config['domain'])), + dns.A, dns.IN, self.ttl, + payload=dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) ]) def _srv_records(self, name): @@ -92,30 +93,33 @@ def _srv_records(self, name): addr = self.mapping.get_a(name) return tuple([ dns.RRHeader( - '.'.join( - (name, self.config['domain'])), dns.SRV, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - auth=self.config.get('authoritative')) + '.'.join((name, self.config['domain'])), + dns.SRV, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) ]) def lookupAddress(self, name, timeout=None): """ - :param name: a name like container_name.docker, hostname.docker, image_name.*.docker + :param name: a name like container_name.docker, hostname.docker, + image_name.*.docker :param timeout: :return: A deferred firing a 3-tuple - The first element of the tuple gives answers. - The second element of the tuple gives authorities. - The third element of the tuple gives additional information. - The Deferred may instead fail with one of the exceptions defined in twisted.names.error - or with NotImplementedError. (type: Deferred) + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred """ name, occurrences = self.re_domain.subn('', name) if not occurrences: log.err("Domain not ending with {domain}: {name}".format( name=name, **self.config)) - return defer.fail(failure.Failure(DomainError("not ending with docker"))) + return defer.fail(Failure(DomainError("not ending with docker"))) try: if name.endswith(".*"): @@ -123,10 +127,10 @@ def lookupAddress(self, name, timeout=None): log.msg(a_multi) records = tuple( dns.RRHeader( - '.'.join((name_, self.config[ - 'domain'])), dns.A, dns.IN, self.ttl, - dns.Record_A(addr_, self.ttl), - auth=self.config.get('authoritative') + '.'.join((name_, self.config['domain'])), + dns.A, dns.IN, self.ttl, + dns.Record_A(addr_, self.ttl), + auth=self.config.get('authoritative') ) for addr_, name_ in a_multi @@ -143,7 +147,7 @@ def lookupAddress(self, name, timeout=None): if self.config.get(NO_NXDOMAIN): # FIXME surely there's a better way to give SERVFAIL ex = DNSQueryTimeoutError(name) - return defer.fail(failure.Failure(ex)) + return defer.fail(Failure(ex)) except Exception as ex: # pylint:disable=bare-except import traceback @@ -156,7 +160,7 @@ def lookupAddress(self, name, timeout=None): else: exception = DomainError(name) - return defer.fail(failure.Failure(exception)) + return defer.fail(Failure(exception)) def lookupService(self, name, timeout=None): """ Lookup a docker natted service of @@ -165,28 +169,37 @@ def lookupService(self, name, timeout=None): _service._proto.name. TTL class SRV priority weight port target. If _service == _nat: - :return: - A Deferred which fires with a three-tuple of lists of twisted.names.dns.RRHeader instances. - The first element of the tuple gives answers. - The second element of the tuple gives authorities. - The third element of the tuple gives additional information. - The Deferred may instead fail with one of the exceptions defined in twisted.names.error or with NotImplementedError. (type: Deferred) + :return: A Deferred firing a three-tuple of lists of + twisted.names.dns.RRHeader instances. + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or with NotImplementedError. + :type: Deferred """ name, occurrences = self.re_domain.subn('', name) if not occurrences: log.err("Domain not ending with {domain}: {name}".format( name=name, **self.config)) - return defer.fail(failure.Failure(DomainError("not ending with {domain}".format(**self.config)))) + return defer.fail(Failure(DomainError( + "not ending with {domain}".format(**self.config))) + ) try: port, proto, container = name.split(".") port = int(port.strip("_")) except (IndexError, TypeError, ValueError) as ex: log.err("Domain not of the right form: %r" % name) - return defer.fail(failure.Failure(DomainError("not of the right form"))) + return defer.fail(Failure(DomainError("not of the right form"))) records = [dns.RRHeader( name, dns.SRV, dns.IN, self.ttl, - dns.Record_SRV( - priority=100, weight=100, port=c_nat_port, target=self.my_preferred_ip_ptr_value, ttl=None), + payload=dns.Record_SRV( + priority=100, + weight=100, + port=c_nat_port, + target=self.my_preferred_ip_ptr_value, + ttl=None), auth=True) for c_port, protocol, c_nat_port, target in self.mapping.get_nat(container) diff --git a/dockerdns/utils.py b/dockerdns/utils.py index 8bd35f9..5622e5c 100644 --- a/dockerdns/utils.py +++ b/dockerdns/utils.py @@ -25,8 +25,8 @@ def traverse_tree(haystack, key_path, default=None): :param key_path: An iterable containing an ordered list of dict keys to traverse :param default: Value to return in case nothing is found - :return:Value of the dict at the nested location given, or default if no value - was found + :return:Value of the dict at the nested location given, + or default if no value was found """ for k in key_path: if k in haystack: diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index b807c41..0965eab 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -1,16 +1,16 @@ #!/usr/bin/python -#from __future__ import print_function, unicode_literals """ A simple TwistD DNS server using custom TLD and Docker as the back end for IP resolution. To look up a container: - - 'A' record query a container NAME that will match a container with a docker inspect - command with '.d' as the TLD. eg: mysql_server1.d - - 'SRV' record query to _port._srv.container.docker will return the natted address. - eg. _3306._tcp.mysql_server1.docker returns - _18080._tcp.compassionate_poincare.docker. 10 IN SRV 100 100 8080 192.168.42.126. - + - 'A' record query a container NAME that will match + a container with a docker inspect + command with '.d' as the TLD. eg: mysql_server1.d + - 'SRV' record query to _port._srv.container.docker will return + the natted address. + eg. _3306._tcp.mysql_server1.docker returns + _18080._tcp.nice_bohr.docker. 10 IN SRV 100 100 8080 192.168.42.126. Code modified from https://github.com/infoxchange/docker_dns @@ -18,23 +18,23 @@ Author: Bradley Cicenas Author: Roberto Polli """ +from __future__ import print_function, unicode_literals -from twisted.application import internet, service -from twisted.names import dns, server -from twisted.python import log from zope.interface import implements import json + +from twisted.application import service, internet +from twisted.names import dns, server +from twisted.python import log from twisted.python import usage from twisted.plugin import IPlugin from twisted.application.service import IServiceMaker -from twisted.application import internet - +import docker from dockerdns.mappings import DockerMapping from dockerdns.events import DockerDB from dockerdns.resolver import DockerResolver from dockerdns.console import ConsoleFactory -import docker class Options(usage.Options): From 60c591f4fde63833aaac39cb63250cf2910b7703 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 23 Feb 2015 14:42:24 +0100 Subject: [PATCH 41/51] fix: pep8 --- dockerdns/resolver.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index fd9613c..53eba72 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -10,7 +10,7 @@ import socket from twisted.python import log from twisted.internet import defer -from twisted.internet.defer.failure import Failure +from twisted.internet.defer import failure from twisted.names import common, dns from twisted.names.error import DomainError, DNSQueryTimeoutError from dockerdns.utils import get_preferred_ip @@ -119,7 +119,9 @@ def lookupAddress(self, name, timeout=None): if not occurrences: log.err("Domain not ending with {domain}: {name}".format( name=name, **self.config)) - return defer.fail(Failure(DomainError("not ending with docker"))) + return defer.fail(failure.Failure( + DomainError("not ending with docker")) + ) try: if name.endswith(".*"): @@ -147,7 +149,7 @@ def lookupAddress(self, name, timeout=None): if self.config.get(NO_NXDOMAIN): # FIXME surely there's a better way to give SERVFAIL ex = DNSQueryTimeoutError(name) - return defer.fail(Failure(ex)) + return defer.fail(failure.Failure(ex)) except Exception as ex: # pylint:disable=bare-except import traceback @@ -160,7 +162,7 @@ def lookupAddress(self, name, timeout=None): else: exception = DomainError(name) - return defer.fail(Failure(exception)) + return defer.fail(failure.Failure(exception)) def lookupService(self, name, timeout=None): """ Lookup a docker natted service of @@ -182,7 +184,7 @@ def lookupService(self, name, timeout=None): if not occurrences: log.err("Domain not ending with {domain}: {name}".format( name=name, **self.config)) - return defer.fail(Failure(DomainError( + return defer.fail(failure.Failure(DomainError( "not ending with {domain}".format(**self.config))) ) try: @@ -190,7 +192,9 @@ def lookupService(self, name, timeout=None): port = int(port.strip("_")) except (IndexError, TypeError, ValueError) as ex: log.err("Domain not of the right form: %r" % name) - return defer.fail(Failure(DomainError("not of the right form"))) + return defer.fail(failure.Failure( + DomainError("not of the right form")) + ) records = [dns.RRHeader( name, dns.SRV, dns.IN, self.ttl, From 3ba6a8db0bb1253e1197a52cb82aeabb3256d848 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 23 Feb 2015 14:48:53 +0100 Subject: [PATCH 42/51] ci: removed pylint --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 488bbea..2b60e7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,6 @@ install: - sudo pip install -r test_requirements.txt before_script: - pep8 dockerdns/*.py twisted/plugins/*py - - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py +# - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py script: - nosetests -v -w test From 3cb5054f296ed0df6b0f3d457c72b69f4f6ea8cd Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 23 Feb 2015 19:36:02 +0100 Subject: [PATCH 43/51] moved test requirements outside module --- .travis.yml | 1 - test/__init__.py | 68 ------------------------------------------- test/test_utils.py | 67 ++++++++++++++++++++++++++++++++++++++++++ test_requirements.txt | 1 + 4 files changed, 68 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b60e7e..f7d5ec8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - "2.7" - - "pypy" install: - sudo pip install -r test_requirements.txt before_script: diff --git a/test/__init__.py b/test/__init__.py index 8e66d58..f595b3f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,6 +1,3 @@ -import docker -import fudge - __author__ = 'rpolli' inspect_container_pandas_0 = { @@ -177,68 +174,3 @@ } SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" - - -class MockDockerClient(object): - base_url = 'http://localhost:5000' - version = lambda x: {'ApiVersion': '1.0'} - inspect_container_pandas = { - 'ID': 'cidpandaslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'cuddly-pandas', - }, - 'NetworkSettings': { - 'IPAddress': '127.0.0.1' - }, - } - inspect_container_foxes = { - 'ID': 'cidfoxeslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'sneaky-foxes', - }, - 'NetworkSettings': { - 'IPAddress': '8.8.8.8' - } - } - inspect_container_sloths = { - 'ID': 'cidslothslong', - 'Config': { - 'Hostname': 'stopped-sloths', - }, - 'NetworkSettings': { - 'IPAddress': '' - } - } - inspect_container_returns = { - 'cidpandas': inspect_container_pandas, - 'cidpandaslong': inspect_container_pandas, - 'cidfoxes': inspect_container_foxes, - 'cidfoxeslong': inspect_container_foxes, - 'cidsloths': inspect_container_sloths, - 'cidslothslong': inspect_container_sloths, - } - containers_return = [ - {'Id': 'cidpandas'}, - {'Id': 'cidfoxes'}, - {'Id': 'cidsloths'}, - ] - - inspect_container_id = None - - def inspect_container(self, cid): - self.inspect_container_id = cid - - try: - return self.inspect_container_returns[cid] - except KeyError: - # Mocks a Docker Client Exception - response = fudge.Fake() - response.has_attr(status_code=404, content='PANDAS!') - - exception = docker.client.APIError('bad', response) - raise exception - - def containers(self, *args, **kwargs): # pylint:disable=unused-argument - return self.containers_return diff --git a/test/test_utils.py b/test/test_utils.py index 168b534..7aa9161 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,6 +1,8 @@ """ Test the traverse_tree function """ +import docker +import fudge from dockerdns.utils import traverse_tree @@ -49,3 +51,68 @@ def test_ignore_default(): ('badgers are', None, 'Badgers are none? What?') ]: yield harn_basic_check, given.split(), expected, default + + +class MockDockerClient(object): + base_url = 'http://localhost:5000' + version = lambda x: {'ApiVersion': '1.0'} + inspect_container_pandas = { + 'ID': 'cidpandaslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + } + inspect_container_foxes = { + 'ID': 'cidfoxeslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + } + } + inspect_container_sloths = { + 'ID': 'cidslothslong', + 'Config': { + 'Hostname': 'stopped-sloths', + }, + 'NetworkSettings': { + 'IPAddress': '' + } + } + inspect_container_returns = { + 'cidpandas': inspect_container_pandas, + 'cidpandaslong': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidfoxeslong': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, + 'cidslothslong': inspect_container_sloths, + } + containers_return = [ + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, + ] + + inspect_container_id = None + + def inspect_container(self, cid): + self.inspect_container_id = cid + + try: + return self.inspect_container_returns[cid] + except KeyError: + # Mocks a Docker Client Exception + response = fudge.Fake() + response.has_attr(status_code=404, content='PANDAS!') + + exception = docker.client.APIError('bad', response) + raise exception + + def containers(self, *args, **kwargs): # pylint:disable=unused-argument + return self.containers_return diff --git a/test_requirements.txt b/test_requirements.txt index 2d6def5..69afa44 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,3 +3,4 @@ fudge==1.0.3 pep8==1.4.6 pylint==1.0.0 nose +docker From e47b6dfef8b211d8ad33d5476ab626158fcdb9e9 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 23 Feb 2015 20:03:29 +0100 Subject: [PATCH 44/51] removed pypy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f7d5ec8..f65d758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python python: - "2.7" install: - - sudo pip install -r test_requirements.txt + - pip install -r test_requirements.txt before_script: - pep8 dockerdns/*.py twisted/plugins/*py # - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py From a4d6d170031ef97a118351e1fd0d04178e6c95f0 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Tue, 24 Feb 2015 09:41:34 +0100 Subject: [PATCH 45/51] fix: test_requirements --- dockerdns/events.py | 32 ++++++++++++++++---- dockerdns/mappings.py | 32 ++++++++++++-------- dockerdns/resolver.py | 69 ++++++++++++++++++++++++++++++++++++++++++- test/test_events.py | 12 ++++++++ test_requirements.txt | 1 - 5 files changed, 126 insertions(+), 20 deletions(-) diff --git a/dockerdns/events.py b/dockerdns/events.py index 6f5247f..fbf05f4 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -33,6 +33,7 @@ def __init__(self, api=None): self.mappings_name = {} self.mappings_image = {} self.mappings_hostname = {} + self.mappings_ip = {} # self.load_containers() @@ -41,6 +42,8 @@ def cleandb(self): self.mappings_name = {} self.mappings_image = {} self.mappings_hostname = {} + self.mappings_ip = {} + def load_containers(self): # Initialize db (TODO get initialization timestamp) @@ -54,14 +57,17 @@ def updatedb(self, item): name = item['Name'][1:] hostname = item['Config']['Hostname'] image = item['Config']['Image'] - self.mappings_name.update({name: item['Id']}) - self.mappings_hostname.update({hostname: item['Id']}) - self.mappings.update({item['Id']: item}) + ip = item['NetworkSettings'].get('IPAddress') + id_ = item['Id'] + self.mappings_name.update({name: id_}) + self.mappings_hostname.update({hostname: id_}) + self.mappings_ip.update({ip: id_}) + self.mappings.update({id_: item}) image_notag = image[:image.find(":")] self.mappings_image.setdefault( - image_notag, []).append(item['Id']) + image_notag, []).append(id_) self.mappings_image.setdefault( - image, []).append(item['Id']) + image, []).append(id_) def add_container(self, item): self.updatedb(item) @@ -70,6 +76,7 @@ def del_container(self, cid): name = self.mappings[cid]['Name'][1:] image = self.mappings[cid]['Config']['Image'] hostname = self.mappings[cid]['Config']['Hostname'] + ip = self.mappings[cid]['NetworkSettings']['IPAddress'] image_notag = image[:image.find(":")] self.mappings_image.get(image_notag, []).remove(cid) self.mappings_image.get(image, []).remove(cid) @@ -77,6 +84,7 @@ def del_container(self, cid): del self.mappings[cid] del self.mappings_name[name] del self.mappings_hostname[hostname] + del self.mappings_ip[ip] def get_by_name(self, name): if name not in self.mappings_name: @@ -107,6 +115,20 @@ def get_by_image(self, image): for cid in self.mappings_image[image]: yield self.mappings[cid] + def get_by_ip(self, ip): + """ + + :param ip: + :return: an generator of container dicts + """ + if ip not in self.mappings_ip: + raise KeyError("%r not in %r" % (ip, self.mappings_ip)) + cid = self.mappings_ip[ip] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + class ContainerManager(Protocol): """Manage the response to /containers/{id}/json diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index 342dcb9..a09ec12 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -50,11 +50,8 @@ def get_a(self, name): """ Get an IPv4 address from a query name to be used in A record lookups - Args: - name: DNS query name to look up - - Returns: - IPv4 address for the query name given + :name: DNS query name to look up + :return: IPv4 address for the query name given """ container = self.lookup_container(name) @@ -71,21 +68,21 @@ def get_a(self, name): return addr - def get_a_multi(self, name_multi): + def get_a_multi(self, image): """ - - :param name_multi: [(addr1, name1), .., (addrX, nameX)] - :return: a tuple of addresses + Return the IPs matching the given image name + :param image: + :return: a tuple of {(addr1, name1), .., (addrX, nameX)} """ - name_multi, count = re.subn(r'\.\*$', '', name_multi, 1) + image, count = re.subn(r'\.\*$', '', image, 1) if not count: - log.err("Not a multihost search: %r" % (name_multi,)) + log.err("Not a multihost search: %r" % (image,)) return return tuple( (container['NetworkSettings'][ 'IPAddress'], container['Name'][1:]) for container - in self.db.get_by_image(name_multi) + in self.db.get_by_image(image) if 'NetworkSettings' in container ) @@ -97,7 +94,7 @@ def get_nat(self, container_name, sport=0, sproto=None): eg. [ (8080, 'tcp', 18080, '0.0.0.0'), (8787, 'tcp', 8787, '0.0.0.0'), - ] + ]rfiutato """ sport = int(sport) container = self.lookup_container(container_name) @@ -130,3 +127,12 @@ def get_nat(self, container_name, sport=0, sproto=None): except (ValueError, KeyError) as ex: log.err() continue + + def get_ptr(self, ip): + """ + Return the Container with a given IP + :param ip: + :return: + """ + c = self.db.get_by_ip(ip) + return c['Name'] \ No newline at end of file diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index 53eba72..f8735e5 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -43,11 +43,14 @@ class DockerResolver(common.ResolverBase): def __init__(self, mapping, config=None): """ :param mapping: DockerMapping instance for lookups + + TODO: configurable --bip """ self.mapping = mapping - self.config = config or {'domain': 'docker'} + self.config = config or {'domain': 'docker', 'bip': '172.17.0.0/16'} self.re_domain = re.compile(r'\.' + self.config['domain'] + '$') + self.re_ptr = re.compile(r'[0-9]+\.[0-9]+\.17\.172\.in-addr\.arpa$') # Change to this ASAP when Twisted uses object base # super(DockerResolver, self).__init__() common.ResolverBase.__init__(self) @@ -99,6 +102,28 @@ def _srv_records(self, name): auth=self.config.get('authoritative')) ]) + def _a_ptr(self, name): + """ + Get PTR records from a query name + + :param name: DNS query name to look up + + :return: Tuple of formatted DNS replies + """ + # convert x.y.z.q.in-addr.arpa -> q.z.y.x + ip_addr = '.'.join(reversed(name.split(".")[:4])) + addr = self.mapping.get_ptr(ip_addr) + if not addr: + raise DomainError(name) + addr = '.'.join(name, self.config['domain']) + return tuple([ + dns.RRHeader( + name, + dns.PTR, dns.IN, self.ttl, + payload=dns.Record_PTR(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + def lookupAddress(self, name, timeout=None): """ @@ -210,3 +235,45 @@ def lookupService(self, name, timeout=None): if c_port == port # eventually filter ] return defer.succeed((records, self.authority, self.additional)) + + def lookupPointer(self, name, timeout=None): + """ + + :param name: a ptr name like 1.0.17.172.in-addr.arpa + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred + + """ + if not self.re_ptr.match(name): + log.err("Domain not in docker network {bip}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure( + DomainError("not in docker network")) + ) + + try: + records = self._ptr_record(name) + return defer.succeed((records, self.authority, self.additional)) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as ex: + log.msg("DomainError: %r " % ex) + except Exception as ex: # pylint:disable=bare-except + import traceback + traceback.print_exc() + ex = DomainError(name) + # + # With NO_NXDOMAIN we mask everything with a Timeout + # + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) diff --git a/test/test_events.py b/test/test_events.py index d94d3ce..ede7d53 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -38,6 +38,18 @@ def test_init(): raise +def test_get_by_ip(): + db = create_mock_db() + ret = db.get_by_ip('172.17.0.10') + assert ret, "Missing container" + try: + assert ret['NetworkSettings'][ + 'IPAddress'] == '172.17.0.10', "item %r" % ret + except KeyError as e: + log.err("Bad container %r" % ret) + raise + + def test_init_and_get_images(): db = create_mock_db2() # images are correctly added to the indexes diff --git a/test_requirements.txt b/test_requirements.txt index 69afa44..2d6def5 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,4 +3,3 @@ fudge==1.0.3 pep8==1.4.6 pylint==1.0.0 nose -docker From 69d24a9f6a0b29316685a457b91ec219995e2e6f Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Tue, 24 Feb 2015 14:09:16 +0100 Subject: [PATCH 46/51] feat: PTR record support --- README.md | 36 +++++++++++-------- dockerdns/console.py | 2 +- dockerdns/events.py | 6 ++-- dockerdns/mappings.py | 4 +-- dockerdns/resolver.py | 4 +-- test/__init__.py | 1 + test/test_ptr.py | 55 +++++++++++++++++++++++++++++ test/test_srv.py | 5 +++ twisted/plugins/dockerdns_plugin.py | 15 +++++--- 9 files changed, 101 insertions(+), 27 deletions(-) create mode 100644 test/test_ptr.py diff --git a/README.md b/README.md index 26c0bb8..faf8564 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,24 @@ A simple Twisted DNS server using custom TLD and Docker Event interface as the b resolution. Containers can be found by: + - image name - container name - hostname - - image name + - ip + +eg: here are some examples -To look up a container: - - 'A' record: query a container NAME that will match a container with a docker inspect + #host busybox.*.docker # search all busybox containers + #host 26ed50b1bf59.docker # search a container by hostname (not by ID!) + #host nice_bohr.docker # search a container by name + +You can lookup different records: + - 'A' record: query a container NAME or HOSTNAME that will match a container with a docker inspect command with '.docker' as the TLD. eg: mysql_server1.docker - - 'SRV' record query exposing the NAT informations + - 'SRV' record query exposing the NAT informations (more to come!) + - 'PTR' record, with reverse pointer -Note: This fork of docker_dns requires *always* to specify the TLD (by default .docker) +Note: This fork of docker_dns *always* requires to query using a TLD (by default .docker) Install/Run ----------- @@ -39,7 +47,7 @@ You can get configuration parameters with There's a simple HTTP console to check the internal mappings. You can curl it with - #curl -v http://localhost:8080/{hostname,image,name,id,ping}/{optional_key} + #curl -v http://localhost:8080/{hostname,image,name,id,ping,help,ip}/{optional_key} Examples -------- @@ -61,20 +69,20 @@ Dig output is shortened for brevity. We have Docker containers like this: - IP: 172.17.0.3 - Hostname: my-thing -Search by Hostname - dig +short 26ed50b1bf59.docker - 172.17.0.2 +Search by Hostname (uses default or explicit hostname) -Search by Names (works only the first Name) - dig +short sad_turing.docker + #dig +short 26ed50b1bf59.docker 172.17.0.2 -Search by Hostname - dig +short my-thing.docker + #dig +short my-thing.docker 172.17.0.3 Search by Names (works only the first Name) - dig +short happy_bohr.docker + + #dig +short sad_turing.docker + 172.17.0.2 + + #dig +short happy_bohr.docker 172.17.0.3 diff --git a/dockerdns/console.py b/dockerdns/console.py index 0de846c..cc0bad3 100644 --- a/dockerdns/console.py +++ b/dockerdns/console.py @@ -67,7 +67,7 @@ def render_GET(self, request): if 'ping' in request.path: return "{0:s}".format([ time.ctime(), request.path]) - if action == 'help': + if action in ('help', 'refresh'): return HELP_STR return serialize(self.dump(action, *rpath[1:])) diff --git a/dockerdns/events.py b/dockerdns/events.py index fbf05f4..6b26658 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -2,6 +2,8 @@ Populate dns database getting info from docker via: - docker-py api - docker events interface + + XXX Not using unicode_literals in Twisted """ from __future__ import print_function from os.path import join as pjoin @@ -44,7 +46,6 @@ def cleandb(self): self.mappings_hostname = {} self.mappings_ip = {} - def load_containers(self): # Initialize db (TODO get initialization timestamp) for c in self.api.containers(all=False): @@ -129,7 +130,6 @@ def get_by_ip(self, ip): return self.mappings[cid] - class ContainerManager(Protocol): """Manage the response to /containers/{id}/json updating the network infos associated to the container @@ -227,7 +227,7 @@ def __init__(self, config, db): # d = self.agent.request( 'GET', - pjoin(config['docker_url'], 'events'), + pjoin(config['docker_url'].encode(), 'events'.encode()), Headers({'User-Agent': ['Twisted Web Client for Docker Event'], 'Content-Type': ['application/json']}), None) diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py index a09ec12..aff6d53 100644 --- a/dockerdns/mappings.py +++ b/dockerdns/mappings.py @@ -130,9 +130,9 @@ def get_nat(self, container_name, sport=0, sproto=None): def get_ptr(self, ip): """ - Return the Container with a given IP + Return the Hostname of the Container with the given IP :param ip: :return: """ c = self.db.get_by_ip(ip) - return c['Name'] \ No newline at end of file + return c['Config']['Hostname'] diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index f8735e5..0e1f12f 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -102,7 +102,7 @@ def _srv_records(self, name): auth=self.config.get('authoritative')) ]) - def _a_ptr(self, name): + def _ptr_record(self, name): """ Get PTR records from a query name @@ -115,7 +115,7 @@ def _a_ptr(self, name): addr = self.mapping.get_ptr(ip_addr) if not addr: raise DomainError(name) - addr = '.'.join(name, self.config['domain']) + addr = '.'.join((addr, self.config['domain'])) return tuple([ dns.RRHeader( name, diff --git a/test/__init__.py b/test/__init__.py index f595b3f..659266f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -172,5 +172,6 @@ u'Volumes': {u'/mnt/tmp': u'/home/rpolli/Downloads'}, u'VolumesRW': {u'/mnt/tmp': True} } +mock_get_ptr = lambda *a, **k: mock_lookup_container()['Name'][1:] SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" diff --git a/test/test_ptr.py b/test/test_ptr.py new file mode 100644 index 0000000..35366ed --- /dev/null +++ b/test/test_ptr.py @@ -0,0 +1,55 @@ +""" + Creating Ptr records + +""" +import twisted +from dockerdns.resolver import DockerResolver +from dockerdns.mappings import DockerMapping +from test import mock_lookup_container, mock_get_ptr +from nose import SkipTest +from nose.tools import raises +from nose.twistedtools import deferred as nosedeferred +from test.test_resolver import check_deferred + + +class TestPtr(object): + """ + Test lookupPointer record mocking DockerMapping functions. + + This class doesn't test DockerDB + """ + mapping = DockerMapping(db=None) + mapping.lookup_container = mock_lookup_container + mapping.get_ptr = mock_get_ptr + + def check_equals(self, a, b, msg=None): + assert a == b, msg + + def setup(self): + self.resolver = DockerResolver(self.mapping) + + @raises(twisted.names.error.DomainError) + @nosedeferred() + def harn_lookupPtr_ko(self, n): + """Harness to run lookup in a deferred + """ + return self.resolver.lookupPointer(n) + + def test_lookupPtr_ko(self): + expect_fail = ('166.192.in-addr.arpa').split() + for n in expect_fail: + yield self.harn_lookupPtr_ko, n + + def test_lookupPtr_ok(self): + ret = self.resolver.lookupPointer("10.0.17.172.in-addr.arpa") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_ptr_ok(self): + ret, = self.resolver._ptr_record("10.0.17.172.in-addr.arpa") + assert ret + + def test_setup(self): + assert self.resolver + assert self.resolver.mapping + assert self.resolver.mapping.db is None diff --git a/test/test_srv.py b/test/test_srv.py index 04ff737..e9a5867 100644 --- a/test/test_srv.py +++ b/test/test_srv.py @@ -80,6 +80,11 @@ def test_mapping(self): ) ) + def test_setup(self): + assert self.resolver + assert self.resolver.mapping + assert self.resolver.mapping.db is None + @SkipTest def test_service_image(self): host = "impandas.docker" diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 0965eab..befae25 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -122,11 +122,16 @@ def makeService(self, options): # Add the event Loop from dockerdns.events import EventFactory from urlparse import urlparse - u = urlparse(appcfg['docker_url']) - efactory = EventFactory(config=appcfg, db=db) - docker_event_monitor = internet.TCPClient(u.hostname, u.port, efactory) - docker_event_monitor.setServiceParent(ret) - + if appcfg['docker_url'].startswith("http"): + u = urlparse(appcfg['docker_url']) + efactory = EventFactory(config=appcfg, db=db) + docker_event_monitor = internet.TCPClient( + u.hostname, u.port, efactory) + docker_event_monitor.setServiceParent(ret) + else: + log.err("Cannot load the Event interface: " + "requires an http endpoint, " + "not {docker_url}".format(**appcfg)) # Add the console consoleFactory = ConsoleFactory(db) console = internet.TCPServer(8080, consoleFactory) From d7c9bfa40bc299f0ba031dcaa664f3c083bb65ca Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 2 Mar 2015 00:41:51 +0100 Subject: [PATCH 47/51] first sftp access to volumes --- dockerdns/sftp/__init__.py | 0 dockerdns/sftp/checkers.py | 59 ++++ dockerdns/sftp/factory.py | 73 ++++ dockerdns/sftp/unix.py | 497 +++++++++++++++++++++++++++ test/test_sftpserver_volumes.py | 44 +++ twisted/plugins/dockersftp_plugin.py | 109 ++++++ 6 files changed, 782 insertions(+) create mode 100644 dockerdns/sftp/__init__.py create mode 100644 dockerdns/sftp/checkers.py create mode 100644 dockerdns/sftp/factory.py create mode 100644 dockerdns/sftp/unix.py create mode 100644 test/test_sftpserver_volumes.py create mode 100644 twisted/plugins/dockersftp_plugin.py diff --git a/dockerdns/sftp/__init__.py b/dockerdns/sftp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockerdns/sftp/checkers.py b/dockerdns/sftp/checkers.py new file mode 100644 index 0000000..bfa7d45 --- /dev/null +++ b/dockerdns/sftp/checkers.py @@ -0,0 +1,59 @@ +# -*- test-case-name: twisted.conch.test.test_checkers -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Provide L{ICredentialsChecker} implementations to be used in Conch protocols. +""" + +try: + import pwd +except ImportError: + pwd = None +else: + import crypt + +try: + # Python 2.5 got spwd to interface with shadow passwords + import spwd +except ImportError: + spwd = None + try: + import shadow + except ImportError: + shadow = None +else: + shadow = None + +try: + from twisted.cred import pamauth +except ImportError: + pamauth = None + +from zope.interface import implementer + +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class PermitChecker: + """ + A checker which validates users out of the UNIX password databases, or + databases of a compatible format. + + @ivar _getByNameFunctions: a C{list} of functions which are called in order + to valid a user. The default value is such that the /etc/passwd + database will be tried first, followed by the /etc/shadow database. + """ + credentialInterfaces = IUsernamePassword, + + def __init__(self, getByNameFunctions=None): + pass + + def requestAvatarId(self, credentials): + return defer.succeed(credentials.username) + # fallback + return defer.fail(UnauthorizedLogin("unable to verify password")) diff --git a/dockerdns/sftp/factory.py b/dockerdns/sftp/factory.py new file mode 100644 index 0000000..ea06e4a --- /dev/null +++ b/dockerdns/sftp/factory.py @@ -0,0 +1,73 @@ +# -*- test-case-name: twisted.conch.test.test_openssh_compat -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Factory for reading openssh configuration files: public keys, private keys, and +moduli file. +""" + +import os, errno + +from twisted.python import log +from twisted.python.util import runAsEffectiveUser + +from twisted.conch.ssh import keys, factory, common +from twisted.conch.openssh_compat import primes + + + +class OpenSSHFactory(factory.SSHFactory): + dataRoot = '.' + moduliRoot = '.' # for openbsd which puts moduli in a different + # directory from keys + + + def getPublicKeys(self): + """ + Return the server public keys. + """ + ks = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-8:]=='_key.pub': + try: + k = keys.Key.fromFile( + os.path.join(self.dataRoot, filename)) + t = common.getNS(k.blob())[0] + ks[t] = k + except Exception, e: + log.msg('bad public key file %s: %s' % (filename, e)) + return ks + + + def getPrivateKeys(self): + """ + Return the server private keys. + """ + privateKeys = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-4:]=='_key': + fullPath = os.path.join(self.dataRoot, filename) + try: + key = keys.Key.fromFile(fullPath) + except IOError, e: + if e.errno == errno.EACCES: + # Not allowed, let's switch to root + key = runAsEffectiveUser(0, 0, keys.Key.fromFile, fullPath) + keyType = keys.objectType(key.keyObject) + privateKeys[keyType] = key + else: + raise + except Exception, e: + log.msg('bad private key file %s: %s' % (filename, e)) + else: + keyType = keys.objectType(key.keyObject) + privateKeys[keyType] = key + return privateKeys + + + def getPrimes(self): + try: + return primes.parseModuliFile(self.moduliRoot+'/moduli') + except IOError: + return None diff --git a/dockerdns/sftp/unix.py b/dockerdns/sftp/unix.py new file mode 100644 index 0000000..419d002 --- /dev/null +++ b/dockerdns/sftp/unix.py @@ -0,0 +1,497 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +import fcntl +import grp +import os +import pty +import re +import socket +import struct +import time +import tty + +from zope.interface import implementer + +from twisted.conch import ttymodes +from twisted.conch.avatar import ConchUser +from twisted.conch.error import ConchError +from twisted.conch.ls import lsLine +from twisted.conch.ssh import session, forwarding, filetransfer +from twisted.conch.ssh.filetransfer import ( + FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL +) +from twisted.conch.interfaces import ISession, ISFTPServer, ISFTPFile +from twisted.cred import portal +from twisted.internet.error import ProcessExitedAlready +from twisted.python import components, log + +try: + import utmp +except ImportError: + utmp = None + +container_database = dict( + # container_name: volume_path + foo={"Volumes": { + "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/var/log": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + + #"/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" + + } + } +) +@implementer(portal.IRealm) +class DockerRealm: + def requestAvatar(self, username, mind, *interfaces): + user = DockerVolumeConchUser(username) + return interfaces[0], user, user.logout + + +class DockerVolumeConchUser(ConchUser): + + def __init__(self, container_name): + ConchUser.__init__(self) + self.container_name = container_name + self.volumes = container_database[container_name]["Volumes"] + self.listeners = {} # dict mapping (interface, port) -> listener + self.channelLookup.update( + {"session": session.SSHSession, + "direct-tcpip": forwarding.openConnectForwardingClient}) + + self.subsystemLookup.update( + {"sftp": filetransfer.FileTransferServer}) + + def getUserGroupId(self): + return os.getuid(), os.getgid() + + def getOtherGroups(self): + return tuple() + + def getHomeDir(self): + return "/" + + def getShell(self): + return "/sbin/nologin" + + def global_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + from twisted.internet import reactor + try: listener = self._runAsUser( + reactor.listenTCP, portToBind, + forwarding.SSHListenForwardingFactory(self.conn, + (hostToBind, portToBind), + forwarding.SSHListenServerForwardingChannel), + interface = hostToBind) + except: + return 0 + else: + self.listeners[(hostToBind, portToBind)] = listener + if portToBind == 0: + portToBind = listener.getHost()[2] # the port + return 1, struct.pack('>L', portToBind) + else: + return 1 + + def global_cancel_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + listener = self.listeners.get((hostToBind, portToBind), None) + if not listener: + return 0 + del self.listeners[(hostToBind, portToBind)] + self._runAsUser(listener.stopListening) + return 1 + + def logout(self): + # remove all listeners + for listener in self.listeners.itervalues(): + self._runAsUser(listener.stopListening) + log.msg('avatar %s logging out (%i)' % (self.container_name, len(self.listeners))) + + def _runAsUser(self, f, *args, **kw): + euid = os.geteuid() + egid = os.getegid() + groups = os.getgroups() + uid, gid = self.getUserGroupId() + # os.setegid(0) + # os.seteuid(0) + # os.setgroups(self.getOtherGroups()) + # os.setegid(gid) + # os.seteuid(uid) + try: + f = iter(f) + except TypeError: + f = [(f, args, kw)] + try: + for i in f: + func = i[0] + args = len(i)>1 and i[1] or () + kw = len(i)>2 and i[2] or {} + if func in (os.lstat,): + for from_, to_ in self.volumes.items(): + args = [re.sub(from_, to_, x) for x in args] + r = func(*args, **kw) + finally: + # os.setegid(0) + # os.seteuid(0) + # os.setgroups(groups) + # os.setegid(egid) + # os.seteuid(euid) + pass + return r + + + +@implementer(ISession) +class SSHSessionForDockerVolumeConchUser: + def __init__(self, avatar): + self.avatar = avatar + self.environ = {'PATH':'/bin:/usr/bin:/usr/local/bin'} + self.pty = None + self.ptyTuple = 0 + + def addUTMPEntry(self, loggedIn=1): + if not utmp: + return + ipAddress = self.avatar.conn.transport.transport.getPeer().host + packedIp ,= struct.unpack('L', socket.inet_aton(ipAddress)) + ttyName = self.ptyTuple[2][5:] + t = time.time() + t1 = int(t) + t2 = int((t-t1) * 1e6) + entry = utmp.UtmpEntry() + entry.ut_type = loggedIn and utmp.USER_PROCESS or utmp.DEAD_PROCESS + entry.ut_pid = self.pty.pid + entry.ut_line = ttyName + entry.ut_id = ttyName[-4:] + entry.ut_tv = (t1,t2) + if loggedIn: + entry.ut_user = self.avatar.username + entry.ut_host = socket.gethostbyaddr(ipAddress)[0] + entry.ut_addr_v6 = (packedIp, 0, 0, 0) + a = utmp.UtmpRecord(utmp.UTMP_FILE) + a.pututline(entry) + a.endutent() + b = utmp.UtmpRecord(utmp.WTMP_FILE) + b.pututline(entry) + b.endutent() + + + def getPty(self, term, windowSize, modes): + self.environ['TERM'] = term + self.winSize = windowSize + self.modes = modes + master, slave = pty.openpty() + ttyname = os.ttyname(slave) + self.environ['SSH_TTY'] = ttyname + self.ptyTuple = (master, slave, ttyname) + + def openShell(self, proto): + from twisted.internet import reactor + if not self.ptyTuple: # we didn't get a pty-req + log.msg('tried to get shell without pty, failing') + raise ConchError("no pty") + uid, gid = self.avatar.getUserGroupId() + homeDir = self.avatar.getHomeDir() + shell = self.avatar.getShell() + self.environ['USER'] = self.avatar.username + self.environ['HOME'] = homeDir + self.environ['SHELL'] = shell + shellExec = os.path.basename(shell) + peer = self.avatar.conn.transport.transport.getPeer() + host = self.avatar.conn.transport.transport.getHost() + self.environ['SSH_CLIENT'] = '%s %s %s' % (peer.host, peer.port, host.port) + self.getPtyOwnership() + self.pty = reactor.spawnProcess(proto, \ + shell, ['-%s' % shellExec], self.environ, homeDir, uid, gid, + usePTY = self.ptyTuple) + self.addUTMPEntry() + fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, + struct.pack('4H', *self.winSize)) + if self.modes: + self.setModes() + self.oldWrite = proto.transport.write + proto.transport.write = self._writeHack + self.avatar.conn.transport.transport.setTcpNoDelay(1) + + def execCommand(self, proto, cmd): + from twisted.internet import reactor + uid, gid = self.avatar.getUserGroupId() + homeDir = self.avatar.getHomeDir() + shell = self.avatar.getShell() or '/bin/sh' + command = (shell, '-c', cmd) + peer = self.avatar.conn.transport.transport.getPeer() + host = self.avatar.conn.transport.transport.getHost() + self.environ['SSH_CLIENT'] = '%s %s %s' % (peer.host, peer.port, host.port) + if self.ptyTuple: + self.getPtyOwnership() + self.pty = reactor.spawnProcess(proto, \ + shell, command, self.environ, homeDir, + uid, gid, usePTY = self.ptyTuple or 0) + if self.ptyTuple: + self.addUTMPEntry() + if self.modes: + self.setModes() +# else: +# tty.setraw(self.pty.pipes[0].fileno(), tty.TCSANOW) + self.avatar.conn.transport.transport.setTcpNoDelay(1) + + def getPtyOwnership(self): + ttyGid = os.stat(self.ptyTuple[2])[5] + uid, gid = self.avatar.getUserGroupId() + euid, egid = os.geteuid(), os.getegid() + os.setegid(0) + os.seteuid(0) + try: + os.chown(self.ptyTuple[2], uid, ttyGid) + finally: + os.setegid(egid) + os.seteuid(euid) + + def setModes(self): + pty = self.pty + attr = tty.tcgetattr(pty.fileno()) + for mode, modeValue in self.modes: + if not ttymodes.TTYMODES.has_key(mode): continue + ttyMode = ttymodes.TTYMODES[mode] + if len(ttyMode) == 2: # flag + flag, ttyAttr = ttyMode + if not hasattr(tty, ttyAttr): continue + ttyval = getattr(tty, ttyAttr) + if modeValue: + attr[flag] = attr[flag]|ttyval + else: + attr[flag] = attr[flag]&~ttyval + elif ttyMode == 'OSPEED': + attr[tty.OSPEED] = getattr(tty, 'B%s'%modeValue) + elif ttyMode == 'ISPEED': + attr[tty.ISPEED] = getattr(tty, 'B%s'%modeValue) + else: + if not hasattr(tty, ttyMode): continue + ttyval = getattr(tty, ttyMode) + attr[tty.CC][ttyval] = chr(modeValue) + tty.tcsetattr(pty.fileno(), tty.TCSANOW, attr) + + def eofReceived(self): + if self.pty: + self.pty.closeStdin() + + def closed(self): + if self.ptyTuple and os.path.exists(self.ptyTuple[2]): + ttyGID = os.stat(self.ptyTuple[2])[5] + os.chown(self.ptyTuple[2], 0, ttyGID) + if self.pty: + try: + self.pty.signalProcess('HUP') + except (OSError,ProcessExitedAlready): + pass + self.pty.loseConnection() + self.addUTMPEntry(0) + log.msg('shell closed') + + def windowChanged(self, winSize): + self.winSize = winSize + fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, + struct.pack('4H', *self.winSize)) + + def _writeHack(self, data): + """ + Hack to send ignore messages when we aren't echoing. + """ + if self.pty is not None: + attr = tty.tcgetattr(self.pty.fileno())[3] + if not attr & tty.ECHO and attr & tty.ICANON: # no echo + self.avatar.conn.transport.sendIgnore('\x00'*(8+len(data))) + self.oldWrite(data) + + + +@implementer(ISFTPServer) +class SFTPServerForDockerVolumeConchUser: + def __init__(self, avatar): + self.avatar = avatar + + + def _setAttrs(self, path, attrs): + """ + NOTE: this function assumes it runs as the logged-in user: + i.e. under _runAsUser() + """ + if "uid" in attrs and "gid" in attrs: + os.chown(path, attrs["uid"], attrs["gid"]) + if "permissions" in attrs: + os.chmod(path, attrs["permissions"]) + if "atime" in attrs and "mtime" in attrs: + os.utime(path, (attrs["atime"], attrs["mtime"])) + + def _getAttrs(self, s): + return { + "size" : s.st_size, + "uid" : s.st_uid, + "gid" : s.st_gid, + "permissions" : s.st_mode, + "atime" : int(s.st_atime), + "mtime" : int(s.st_mtime) + } + + def _absPath(self, path): + import re + allowed_paths = tuple(self.avatar.volumes.keys()) + (".", "/") + home = self.avatar.getHomeDir() + if path.startswith(allowed_paths): + apath = os.path.abspath(os.path.join(home, path)) + if apath == "/": + return apath + + for v, folder in self.avatar.volumes.items(): + apath = re.sub('^'+v, folder, apath) + return apath + + log.msg("not outside container path: %r" % [ + path, + self.avatar.volumes.keys()] + ) + os.stat("/dev/File not found") + + def gotVersion(self, otherVersion, extData): + return {} + + def openFile(self, filename, flags, attrs): + return UnixSFTPFile(self, self._absPath(filename), flags, attrs) + + def removeFile(self, filename): + filename = self._absPath(filename) + return self.avatar._runAsUser(os.remove, filename) + + def renameFile(self, oldpath, newpath): + oldpath = self._absPath(oldpath) + newpath = self._absPath(newpath) + return self.avatar._runAsUser(os.rename, oldpath, newpath) + + def makeDirectory(self, path, attrs): + path = self._absPath(path) + return self.avatar._runAsUser([(os.mkdir, (path,)), + (self._setAttrs, (path, attrs))]) + + def removeDirectory(self, path): + path = self._absPath(path) + self.avatar._runAsUser(os.rmdir, path) + + def openDirectory(self, path): + return UnixSFTPDirectory(self, self._absPath(path)) + + def getAttrs(self, path, followLinks): + path = self._absPath(path) + if followLinks: + s = self.avatar._runAsUser(os.stat, path) + else: + s = self.avatar._runAsUser(os.lstat, path) + return self._getAttrs(s) + + def setAttrs(self, path, attrs): + path = self._absPath(path) + self.avatar._runAsUser(self._setAttrs, path, attrs) + + def readLink(self, path): + path = self._absPath(path) + return self.avatar._runAsUser(os.readlink, path) + + def makeLink(self, linkPath, targetPath): + linkPath = self._absPath(linkPath) + targetPath = self._absPath(targetPath) + return self.avatar._runAsUser(os.symlink, targetPath, linkPath) + + def realPath(self, path): + return os.path.realpath(self._absPath(path)) + + def extendedRequest(self, extName, extData): + raise NotImplementedError + + + +@implementer(ISFTPFile) +class UnixSFTPFile: + def __init__(self, server, filename, flags, attrs): + self.server = server + openFlags = 0 + if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0: + openFlags = os.O_RDONLY + if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0: + openFlags = os.O_WRONLY + if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ: + openFlags = os.O_RDWR + if flags & FXF_APPEND == FXF_APPEND: + openFlags |= os.O_APPEND + if flags & FXF_CREAT == FXF_CREAT: + openFlags |= os.O_CREAT + if flags & FXF_TRUNC == FXF_TRUNC: + openFlags |= os.O_TRUNC + if flags & FXF_EXCL == FXF_EXCL: + openFlags |= os.O_EXCL + if "permissions" in attrs: + mode = attrs["permissions"] + del attrs["permissions"] + else: + mode = 0777 + fd = server.avatar._runAsUser(os.open, filename, openFlags, mode) + if attrs: + server.avatar._runAsUser(server._setAttrs, filename, attrs) + self.fd = fd + + def close(self): + return self.server.avatar._runAsUser(os.close, self.fd) + + def readChunk(self, offset, length): + return self.server.avatar._runAsUser([ (os.lseek, (self.fd, offset, 0)), + (os.read, (self.fd, length)) ]) + + def writeChunk(self, offset, data): + return self.server.avatar._runAsUser([(os.lseek, (self.fd, offset, 0)), + (os.write, (self.fd, data))]) + + def getAttrs(self): + s = self.server.avatar._runAsUser(os.fstat, self.fd) + return self.server._getAttrs(s) + + def setAttrs(self, attrs): + raise NotImplementedError + + +class UnixSFTPDirectory: + + def __init__(self, server, directory): + self.server = server + if directory == "/": + self.files = [x.split("/")[1] + for x + in self.server.avatar.volumes.keys()] + elif any(x for x in self.server.avatar.volumes.keys() if x.startswith(directory)): + self.files = [x.replace(directory, "").split("/")[1] + for x + in self.server.avatar.volumes.keys() + if x.startswith(directory)] + else: + self.files = server.avatar._runAsUser(os.listdir, directory) + self.dir = directory + + def __iter__(self): + return self + + def next(self): + try: + f = self.files.pop(0) + except IndexError: + raise StopIteration + else: + s = self.server.avatar._runAsUser(os.lstat, os.path.join(self.dir, f)) + longname = lsLine(f, s) + attrs = self.server._getAttrs(s) + return (f, longname, attrs) + + def close(self): + self.files = [] + + +components.registerAdapter(SFTPServerForDockerVolumeConchUser, DockerVolumeConchUser, filetransfer.ISFTPServer) +components.registerAdapter(SSHSessionForDockerVolumeConchUser, DockerVolumeConchUser, session.ISession) diff --git a/test/test_sftpserver_volumes.py b/test/test_sftpserver_volumes.py new file mode 100644 index 0000000..78f0dc3 --- /dev/null +++ b/test/test_sftpserver_volumes.py @@ -0,0 +1,44 @@ +from dockerdns.sftp import unix +from twisted.python import log +from nose.tools import assert_true, assert_equal +unix.container_database = dict( + # container_name: volume_path + foo={"Volumes": { + "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/var/log/sql": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" + + } + } +) +MOCK_CONTAINER = "foo" +user = unix.DockerVolumeConchUser(MOCK_CONTAINER) +srv = unix.SFTPServerForDockerVolumeConchUser(user) + + +def test_sftpserver_subpat(): + user = unix.DockerVolumeConchUser(MOCK_CONTAINER) + assert user.getHomeDir() == "/" + + +def test_server(): + user = unix.DockerVolumeConchUser(MOCK_CONTAINER) + srv = unix.SFTPServerForDockerVolumeConchUser(user) + for p in ( + '/', '/shared', '/shared/foo' + ): + log.msg(srv._absPath(p)) + yield assert_true, srv._absPath(p).startswith("/data/docker") + + +def test_unixdirectory_1(): + directory = unix.UnixSFTPDirectory(srv, '/') + assert_equal(set(directory.files), set(['shared', 'var', 'shared1'])) + +def test_unixdir_2(): + directory = unix.UnixSFTPDirectory(srv, '/var') + assert_equal(set(directory.files), set(["log"])) + +def test_unixdir_3(): + directory = unix.UnixSFTPDirectory(srv, '/var/log') + assert_equal(set(directory.files), set(["sql"])) diff --git a/twisted/plugins/dockersftp_plugin.py b/twisted/plugins/dockersftp_plugin.py new file mode 100644 index 0000000..5a07c65 --- /dev/null +++ b/twisted/plugins/dockersftp_plugin.py @@ -0,0 +1,109 @@ +# -*- test-case-name: twisted.conch.test.test_tap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support module for making SSH servers with twistd. +""" +from zope.interface.declarations import implements +from twisted.application.service import IServiceMaker + +from twisted.conch import checkers as conch_checkers +from twisted.cred import portal, checkers, strcred +from twisted.plugin import IPlugin +from twisted.python import usage +from twisted.application import strports +try: + from twisted.cred import pamauth +except ImportError: + pamauth = None + +from dockerdns.sftp import factory +from dockerdns.sftp import unix, checkers as docker_checkers +class Options(usage.Options, strcred.AuthOptionMixin): + synopsis = "[-i ] [-p ] [-d ] " + longdesc = ("Makes a Conch SSH server. If no authentication methods are " + "specified, the default authentication methods are UNIX passwords, " + "SSH public keys, and PAM if it is available. If --auth options are " + "passed, only the measures specified will be used.") + optParameters = [ + ["interface", "i", "", "local interface to which we listen"], + ["port", "p", "tcp:22", "Port on which to listen"], + ["data", "d", "/etc", "directory to look for host keys in"], + ["moduli", "", None, "directory to look for moduli in " + "(if different from --data)"] + ] + compData = usage.Completions( + optActions={"data": usage.CompleteDirs(descr="data directory"), + "moduli": usage.CompleteDirs(descr="moduli directory"), + "interface": usage.CompleteNetInterfaces()} + ) + + + def __init__(self, *a, **kw): + usage.Options.__init__(self, *a, **kw) + + # call the default addCheckers (for backwards compatibility) that will + # be used if no --auth option is provided - note that conch's + # UNIXPasswordDatabase is used, instead of twisted.plugins.cred_unix's + # checker + super(Options, self).addChecker(conch_checkers.UNIXPasswordDatabase()) + super(Options, self).addChecker(conch_checkers.SSHPublicKeyChecker( + conch_checkers.UNIXAuthorizedKeysFiles())) + if pamauth is not None: + super(Options, self).addChecker( + checkers.PluggableAuthenticationModulesChecker()) + self._usingDefaultAuth = True + + + def addChecker(self, checker): + """ + Add the checker specified. If any checkers are added, the default + checkers are automatically cleared and the only checkers will be the + specified one(s). + """ + if self._usingDefaultAuth: + self['credCheckers'] = [] + self['credInterfaces'] = {} + self._usingDefaultAuth = False + super(Options, self).addChecker(checker) + + +class MyServiceMaker(object): + """ + Define a MultiService running: + - dns server for tcp and udp + - http client for retrieving docker events + """ + implements(IServiceMaker, IPlugin) + tapname = "dockersftp" + description = "Run this! It'll make your docker happy." + options = Options + + def makeService(self, options): + """ + Construct a service for operating a SSH server. + + @param options: An L{Options} instance specifying server options, including + where server keys are stored and what authentication methods to use. + + @return: An L{IService} provider which contains the requested SSH server. + """ + # The factory just sets the ssh keys + t = factory.OpenSSHFactory() + + r = unix.DockerRealm() + t.portal = portal.Portal(r, [docker_checkers.PermitChecker()]) + t.dataRoot = options['data'] + t.moduliRoot = options['moduli'] or options['data'] + + port = options['port'] + if options['interface']: + # Add warning here + port += ':interface=' + options['interface'] + return strports.service(port, t) + +# +# Create the MultiService +# +serviceMaker = MyServiceMaker() From 760d90dceefa2aecdec6a7de93c048cec44ae4d6 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 2 Mar 2015 10:31:06 +0100 Subject: [PATCH 48/51] feat: working sftp access to volumes --- dockerdns/events.py | 9 +- dockerdns/sftp/factory.py | 18 +- dockerdns/sftp/unix.py | 425 ++++----------------------- test/test_events.py | 6 + test/test_sftpserver_volumes.py | 8 +- twisted/plugins/dockerdns_plugin.py | 39 ++- twisted/plugins/dockersftp_plugin.py | 109 ------- 7 files changed, 126 insertions(+), 488 deletions(-) delete mode 100644 twisted/plugins/dockersftp_plugin.py diff --git a/dockerdns/events.py b/dockerdns/events.py index 6b26658..b1d6bff 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -8,6 +8,7 @@ from __future__ import print_function from os.path import join as pjoin from logging import DEBUG +import re import simplejson as json from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool @@ -19,6 +20,7 @@ class DockerDB(object): """Update docker ip store connecting via docker-py """ + re_image = re.compile(r"([^/]+/)?([^:]+)(:[^:]+)?") def __init__(self, api=None): """ @@ -64,7 +66,7 @@ def updatedb(self, item): self.mappings_hostname.update({hostname: id_}) self.mappings_ip.update({ip: id_}) self.mappings.update({id_: item}) - image_notag = image[:image.find(":")] + _, image_notag, _ = DockerDB.re_image.match(image).groups() self.mappings_image.setdefault( image_notag, []).append(id_) self.mappings_image.setdefault( @@ -78,7 +80,7 @@ def del_container(self, cid): image = self.mappings[cid]['Config']['Image'] hostname = self.mappings[cid]['Config']['Hostname'] ip = self.mappings[cid]['NetworkSettings']['IPAddress'] - image_notag = image[:image.find(":")] + _, image_notag, _ = DockerDB.re_image.match(image).groups() self.mappings_image.get(image_notag, []).remove(cid) self.mappings_image.get(image, []).remove(cid) @@ -112,7 +114,8 @@ def get_by_image(self, image): :return: an generator of container dicts """ if image not in self.mappings_image: - raise KeyError("%r not in %r" % (image, self.mappings_hostname)) + log.err("%r not in %r" % (image, self.mappings_image)) + return for cid in self.mappings_image[image]: yield self.mappings[cid] diff --git a/dockerdns/sftp/factory.py b/dockerdns/sftp/factory.py index ea06e4a..11ab198 100644 --- a/dockerdns/sftp/factory.py +++ b/dockerdns/sftp/factory.py @@ -7,7 +7,8 @@ moduli file. """ -import os, errno +import os +import errno from twisted.python import log from twisted.python.util import runAsEffectiveUser @@ -16,12 +17,10 @@ from twisted.conch.openssh_compat import primes - class OpenSSHFactory(factory.SSHFactory): dataRoot = '.' - moduliRoot = '.' # for openbsd which puts moduli in a different - # directory from keys - + # for openbsd which puts moduli in a different directory from keys + moduliRoot = '.' def getPublicKeys(self): """ @@ -29,7 +28,7 @@ def getPublicKeys(self): """ ks = {} for filename in os.listdir(self.dataRoot): - if filename[:9] == 'ssh_host_' and filename[-8:]=='_key.pub': + if filename[:9] == 'ssh_host_' and filename[-8:] == '_key.pub': try: k = keys.Key.fromFile( os.path.join(self.dataRoot, filename)) @@ -39,21 +38,21 @@ def getPublicKeys(self): log.msg('bad public key file %s: %s' % (filename, e)) return ks - def getPrivateKeys(self): """ Return the server private keys. """ privateKeys = {} for filename in os.listdir(self.dataRoot): - if filename[:9] == 'ssh_host_' and filename[-4:]=='_key': + if filename[:9] == 'ssh_host_' and filename[-4:] == '_key': fullPath = os.path.join(self.dataRoot, filename) try: key = keys.Key.fromFile(fullPath) except IOError, e: if e.errno == errno.EACCES: # Not allowed, let's switch to root - key = runAsEffectiveUser(0, 0, keys.Key.fromFile, fullPath) + key = runAsEffectiveUser( + 0, 0, keys.Key.fromFile, fullPath) keyType = keys.objectType(key.keyObject) privateKeys[keyType] = key else: @@ -65,7 +64,6 @@ def getPrivateKeys(self): privateKeys[keyType] = key return privateKeys - def getPrimes(self): try: return primes.parseModuliFile(self.moduliRoot+'/moduli') diff --git a/dockerdns/sftp/unix.py b/dockerdns/sftp/unix.py index 419d002..34ad86f 100644 --- a/dockerdns/sftp/unix.py +++ b/dockerdns/sftp/unix.py @@ -1,67 +1,46 @@ # Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. -import fcntl -import grp import os -import pty import re -import socket import struct -import time -import tty - from zope.interface import implementer -from twisted.conch import ttymodes from twisted.conch.avatar import ConchUser -from twisted.conch.error import ConchError -from twisted.conch.ls import lsLine from twisted.conch.ssh import session, forwarding, filetransfer -from twisted.conch.ssh.filetransfer import ( - FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL -) -from twisted.conch.interfaces import ISession, ISFTPServer, ISFTPFile +from twisted.conch.unix import (UnixSFTPFile, SSHSessionForUnixConchUser, + SFTPServerForUnixConchUser, UnixSFTPDirectory) from twisted.cred import portal -from twisted.internet.error import ProcessExitedAlready from twisted.python import components, log -try: - import utmp -except ImportError: - utmp = None - -container_database = dict( - # container_name: volume_path - foo={"Volumes": { - "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", - "/var/log": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", - - #"/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" - } - } -) @implementer(portal.IRealm) class DockerRealm: + def __init__(self, container_db): + self.container_db = container_db + def requestAvatar(self, username, mind, *interfaces): - user = DockerVolumeConchUser(username) + user = DockerVolumeConchUser(username, self.container_db) return interfaces[0], user, user.logout class DockerVolumeConchUser(ConchUser): - - def __init__(self, container_name): + def __init__(self, container_name, container_db): ConchUser.__init__(self) self.container_name = container_name - self.volumes = container_database[container_name]["Volumes"] + container = container_db.get_by_name(container_name) + self.volumes = { + k.encode('utf-8'): v.encode('utf-8') + for k, v + in container.get("Volumes", {}).items() + } self.listeners = {} # dict mapping (interface, port) -> listener self.channelLookup.update( - {"session": session.SSHSession, - "direct-tcpip": forwarding.openConnectForwardingClient}) + {"session": session.SSHSession, + "direct-tcpip": forwarding.openConnectForwardingClient}) self.subsystemLookup.update( - {"sftp": filetransfer.FileTransferServer}) + {"sftp": filetransfer.FileTransferServer}) def getUserGroupId(self): return os.getuid(), os.getgid() @@ -78,18 +57,22 @@ def getShell(self): def global_tcpip_forward(self, data): hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) from twisted.internet import reactor - try: listener = self._runAsUser( - reactor.listenTCP, portToBind, - forwarding.SSHListenForwardingFactory(self.conn, - (hostToBind, portToBind), - forwarding.SSHListenServerForwardingChannel), - interface = hostToBind) + + try: + listener = self._runAsUser( + reactor.listenTCP, portToBind, + forwarding.SSHListenForwardingFactory( + self.conn, + (hostToBind, portToBind), + forwarding.SSHListenServerForwardingChannel + ), + interface=hostToBind) except: return 0 else: self.listeners[(hostToBind, portToBind)] = listener if portToBind == 0: - portToBind = listener.getHost()[2] # the port + portToBind = listener.getHost()[2] # the port return 1, struct.pack('>L', portToBind) else: return 1 @@ -107,7 +90,8 @@ def logout(self): # remove all listeners for listener in self.listeners.itervalues(): self._runAsUser(listener.stopListening) - log.msg('avatar %s logging out (%i)' % (self.container_name, len(self.listeners))) + log.msg('avatar %s logging out (%i)' % ( + self.container_name, len(self.listeners))) def _runAsUser(self, f, *args, **kw): euid = os.geteuid() @@ -126,8 +110,8 @@ def _runAsUser(self, f, *args, **kw): try: for i in f: func = i[0] - args = len(i)>1 and i[1] or () - kw = len(i)>2 and i[2] or {} + args = len(i) > 1 and i[1] or () + kw = len(i) > 2 and i[2] or {} if func in (os.lstat,): for from_, to_ in self.volumes.items(): args = [re.sub(from_, to_, x) for x in args] @@ -142,201 +126,14 @@ def _runAsUser(self, f, *args, **kw): return r - -@implementer(ISession) -class SSHSessionForDockerVolumeConchUser: - def __init__(self, avatar): - self.avatar = avatar - self.environ = {'PATH':'/bin:/usr/bin:/usr/local/bin'} - self.pty = None - self.ptyTuple = 0 - - def addUTMPEntry(self, loggedIn=1): - if not utmp: - return - ipAddress = self.avatar.conn.transport.transport.getPeer().host - packedIp ,= struct.unpack('L', socket.inet_aton(ipAddress)) - ttyName = self.ptyTuple[2][5:] - t = time.time() - t1 = int(t) - t2 = int((t-t1) * 1e6) - entry = utmp.UtmpEntry() - entry.ut_type = loggedIn and utmp.USER_PROCESS or utmp.DEAD_PROCESS - entry.ut_pid = self.pty.pid - entry.ut_line = ttyName - entry.ut_id = ttyName[-4:] - entry.ut_tv = (t1,t2) - if loggedIn: - entry.ut_user = self.avatar.username - entry.ut_host = socket.gethostbyaddr(ipAddress)[0] - entry.ut_addr_v6 = (packedIp, 0, 0, 0) - a = utmp.UtmpRecord(utmp.UTMP_FILE) - a.pututline(entry) - a.endutent() - b = utmp.UtmpRecord(utmp.WTMP_FILE) - b.pututline(entry) - b.endutent() - - - def getPty(self, term, windowSize, modes): - self.environ['TERM'] = term - self.winSize = windowSize - self.modes = modes - master, slave = pty.openpty() - ttyname = os.ttyname(slave) - self.environ['SSH_TTY'] = ttyname - self.ptyTuple = (master, slave, ttyname) - - def openShell(self, proto): - from twisted.internet import reactor - if not self.ptyTuple: # we didn't get a pty-req - log.msg('tried to get shell without pty, failing') - raise ConchError("no pty") - uid, gid = self.avatar.getUserGroupId() - homeDir = self.avatar.getHomeDir() - shell = self.avatar.getShell() - self.environ['USER'] = self.avatar.username - self.environ['HOME'] = homeDir - self.environ['SHELL'] = shell - shellExec = os.path.basename(shell) - peer = self.avatar.conn.transport.transport.getPeer() - host = self.avatar.conn.transport.transport.getHost() - self.environ['SSH_CLIENT'] = '%s %s %s' % (peer.host, peer.port, host.port) - self.getPtyOwnership() - self.pty = reactor.spawnProcess(proto, \ - shell, ['-%s' % shellExec], self.environ, homeDir, uid, gid, - usePTY = self.ptyTuple) - self.addUTMPEntry() - fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, - struct.pack('4H', *self.winSize)) - if self.modes: - self.setModes() - self.oldWrite = proto.transport.write - proto.transport.write = self._writeHack - self.avatar.conn.transport.transport.setTcpNoDelay(1) - - def execCommand(self, proto, cmd): - from twisted.internet import reactor - uid, gid = self.avatar.getUserGroupId() - homeDir = self.avatar.getHomeDir() - shell = self.avatar.getShell() or '/bin/sh' - command = (shell, '-c', cmd) - peer = self.avatar.conn.transport.transport.getPeer() - host = self.avatar.conn.transport.transport.getHost() - self.environ['SSH_CLIENT'] = '%s %s %s' % (peer.host, peer.port, host.port) - if self.ptyTuple: - self.getPtyOwnership() - self.pty = reactor.spawnProcess(proto, \ - shell, command, self.environ, homeDir, - uid, gid, usePTY = self.ptyTuple or 0) - if self.ptyTuple: - self.addUTMPEntry() - if self.modes: - self.setModes() -# else: -# tty.setraw(self.pty.pipes[0].fileno(), tty.TCSANOW) - self.avatar.conn.transport.transport.setTcpNoDelay(1) - - def getPtyOwnership(self): - ttyGid = os.stat(self.ptyTuple[2])[5] - uid, gid = self.avatar.getUserGroupId() - euid, egid = os.geteuid(), os.getegid() - os.setegid(0) - os.seteuid(0) - try: - os.chown(self.ptyTuple[2], uid, ttyGid) - finally: - os.setegid(egid) - os.seteuid(euid) - - def setModes(self): - pty = self.pty - attr = tty.tcgetattr(pty.fileno()) - for mode, modeValue in self.modes: - if not ttymodes.TTYMODES.has_key(mode): continue - ttyMode = ttymodes.TTYMODES[mode] - if len(ttyMode) == 2: # flag - flag, ttyAttr = ttyMode - if not hasattr(tty, ttyAttr): continue - ttyval = getattr(tty, ttyAttr) - if modeValue: - attr[flag] = attr[flag]|ttyval - else: - attr[flag] = attr[flag]&~ttyval - elif ttyMode == 'OSPEED': - attr[tty.OSPEED] = getattr(tty, 'B%s'%modeValue) - elif ttyMode == 'ISPEED': - attr[tty.ISPEED] = getattr(tty, 'B%s'%modeValue) - else: - if not hasattr(tty, ttyMode): continue - ttyval = getattr(tty, ttyMode) - attr[tty.CC][ttyval] = chr(modeValue) - tty.tcsetattr(pty.fileno(), tty.TCSANOW, attr) - - def eofReceived(self): - if self.pty: - self.pty.closeStdin() - - def closed(self): - if self.ptyTuple and os.path.exists(self.ptyTuple[2]): - ttyGID = os.stat(self.ptyTuple[2])[5] - os.chown(self.ptyTuple[2], 0, ttyGID) - if self.pty: - try: - self.pty.signalProcess('HUP') - except (OSError,ProcessExitedAlready): - pass - self.pty.loseConnection() - self.addUTMPEntry(0) - log.msg('shell closed') - - def windowChanged(self, winSize): - self.winSize = winSize - fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, - struct.pack('4H', *self.winSize)) - - def _writeHack(self, data): - """ - Hack to send ignore messages when we aren't echoing. - """ - if self.pty is not None: - attr = tty.tcgetattr(self.pty.fileno())[3] - if not attr & tty.ECHO and attr & tty.ICANON: # no echo - self.avatar.conn.transport.sendIgnore('\x00'*(8+len(data))) - self.oldWrite(data) - - - -@implementer(ISFTPServer) -class SFTPServerForDockerVolumeConchUser: +class SFTPServerForDockerVolumeConchUser(SFTPServerForUnixConchUser): def __init__(self, avatar): - self.avatar = avatar - - - def _setAttrs(self, path, attrs): - """ - NOTE: this function assumes it runs as the logged-in user: - i.e. under _runAsUser() - """ - if "uid" in attrs and "gid" in attrs: - os.chown(path, attrs["uid"], attrs["gid"]) - if "permissions" in attrs: - os.chmod(path, attrs["permissions"]) - if "atime" in attrs and "mtime" in attrs: - os.utime(path, (attrs["atime"], attrs["mtime"])) - - def _getAttrs(self, s): - return { - "size" : s.st_size, - "uid" : s.st_uid, - "gid" : s.st_gid, - "permissions" : s.st_mode, - "atime" : int(s.st_atime), - "mtime" : int(s.st_mtime) - } + SFTPServerForUnixConchUser.__init__(self, + avatar) def _absPath(self, path): import re + allowed_paths = tuple(self.avatar.volumes.keys()) + (".", "/") home = self.avatar.getHomeDir() if path.startswith(allowed_paths): @@ -345,153 +142,57 @@ def _absPath(self, path): return apath for v, folder in self.avatar.volumes.items(): - apath = re.sub('^'+v, folder, apath) + apath = re.sub('^' + v, folder, apath) return apath log.msg("not outside container path: %r" % [ - path, - self.avatar.volumes.keys()] + path, + self.avatar.volumes.keys()] ) os.stat("/dev/File not found") - def gotVersion(self, otherVersion, extData): - return {} - def openFile(self, filename, flags, attrs): return UnixSFTPFile(self, self._absPath(filename), flags, attrs) - def removeFile(self, filename): - filename = self._absPath(filename) - return self.avatar._runAsUser(os.remove, filename) - - def renameFile(self, oldpath, newpath): - oldpath = self._absPath(oldpath) - newpath = self._absPath(newpath) - return self.avatar._runAsUser(os.rename, oldpath, newpath) - - def makeDirectory(self, path, attrs): - path = self._absPath(path) - return self.avatar._runAsUser([(os.mkdir, (path,)), - (self._setAttrs, (path, attrs))]) - - def removeDirectory(self, path): - path = self._absPath(path) - self.avatar._runAsUser(os.rmdir, path) - def openDirectory(self, path): - return UnixSFTPDirectory(self, self._absPath(path)) + return DockerVolumeDirectory(self, self._absPath(path)) - def getAttrs(self, path, followLinks): - path = self._absPath(path) - if followLinks: - s = self.avatar._runAsUser(os.stat, path) - else: - s = self.avatar._runAsUser(os.lstat, path) - return self._getAttrs(s) - - def setAttrs(self, path, attrs): - path = self._absPath(path) - self.avatar._runAsUser(self._setAttrs, path, attrs) - - def readLink(self, path): - path = self._absPath(path) - return self.avatar._runAsUser(os.readlink, path) - - def makeLink(self, linkPath, targetPath): - linkPath = self._absPath(linkPath) - targetPath = self._absPath(targetPath) - return self.avatar._runAsUser(os.symlink, targetPath, linkPath) - - def realPath(self, path): - return os.path.realpath(self._absPath(path)) - - def extendedRequest(self, extName, extData): - raise NotImplementedError - - - -@implementer(ISFTPFile) -class UnixSFTPFile: - def __init__(self, server, filename, flags, attrs): - self.server = server - openFlags = 0 - if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0: - openFlags = os.O_RDONLY - if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0: - openFlags = os.O_WRONLY - if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ: - openFlags = os.O_RDWR - if flags & FXF_APPEND == FXF_APPEND: - openFlags |= os.O_APPEND - if flags & FXF_CREAT == FXF_CREAT: - openFlags |= os.O_CREAT - if flags & FXF_TRUNC == FXF_TRUNC: - openFlags |= os.O_TRUNC - if flags & FXF_EXCL == FXF_EXCL: - openFlags |= os.O_EXCL - if "permissions" in attrs: - mode = attrs["permissions"] - del attrs["permissions"] - else: - mode = 0777 - fd = server.avatar._runAsUser(os.open, filename, openFlags, mode) - if attrs: - server.avatar._runAsUser(server._setAttrs, filename, attrs) - self.fd = fd - def close(self): - return self.server.avatar._runAsUser(os.close, self.fd) +class DockerVolumeDirectory(UnixSFTPDirectory): + """ + Expose volumes of an host as tree. - def readChunk(self, offset, length): - return self.server.avatar._runAsUser([ (os.lseek, (self.fd, offset, 0)), - (os.read, (self.fd, length)) ]) + """ - def writeChunk(self, offset, data): - return self.server.avatar._runAsUser([(os.lseek, (self.fd, offset, 0)), - (os.write, (self.fd, data))]) - - def getAttrs(self): - s = self.server.avatar._runAsUser(os.fstat, self.fd) - return self.server._getAttrs(s) - - def setAttrs(self, attrs): - raise NotImplementedError - - -class UnixSFTPDirectory: + @staticmethod + def _dirname(path): + try: + return path.split("/")[1] + except IndexError, TypeError: + return path def __init__(self, server, directory): self.server = server + allowed_volumes, allowed_paths = zip( + *self.server.avatar.volumes.items()) if directory == "/": - self.files = [x.split("/")[1] + self.files = [self._dirname(x) for x - in self.server.avatar.volumes.keys()] - elif any(x for x in self.server.avatar.volumes.keys() if x.startswith(directory)): - self.files = [x.replace(directory, "").split("/")[1] + in allowed_volumes] + elif any(x for x in allowed_volumes if x.startswith(directory)): + self.files = [self._dirname(x.replace(directory, "")) for x - in self.server.avatar.volumes.keys() + in allowed_volumes if x.startswith(directory)] else: + log.err("Accessing %r" % directory) + if not directory.startswith(tuple(allowed_paths)): + raise OSError(2, "No such file or directory", directory) self.files = server.avatar._runAsUser(os.listdir, directory) self.dir = directory - def __iter__(self): - return self - - def next(self): - try: - f = self.files.pop(0) - except IndexError: - raise StopIteration - else: - s = self.server.avatar._runAsUser(os.lstat, os.path.join(self.dir, f)) - longname = lsLine(f, s) - attrs = self.server._getAttrs(s) - return (f, longname, attrs) - - def close(self): - self.files = [] - -components.registerAdapter(SFTPServerForDockerVolumeConchUser, DockerVolumeConchUser, filetransfer.ISFTPServer) -components.registerAdapter(SSHSessionForDockerVolumeConchUser, DockerVolumeConchUser, session.ISession) +components.registerAdapter(SFTPServerForDockerVolumeConchUser, + DockerVolumeConchUser, filetransfer.ISFTPServer) +components.registerAdapter( + SSHSessionForUnixConchUser, DockerVolumeConchUser, session.ISession) diff --git a/test/test_events.py b/test/test_events.py index ede7d53..27f7e18 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -82,6 +82,12 @@ def test_reload_container(): db.mappings_image, db.mappings) +def test_check_volumes(): + db = create_mock_db() + ret = db.get_by_name('jboss631') + assert '/mnt/tmp' in ret['Volumes'] + + class MockAgent(Agent): reasons = ['No Reason'] diff --git a/test/test_sftpserver_volumes.py b/test/test_sftpserver_volumes.py index 78f0dc3..bfbe8ca 100644 --- a/test/test_sftpserver_volumes.py +++ b/test/test_sftpserver_volumes.py @@ -32,13 +32,15 @@ def test_server(): def test_unixdirectory_1(): - directory = unix.UnixSFTPDirectory(srv, '/') + directory = unix.DockerVolumeDirectory(srv, '/') assert_equal(set(directory.files), set(['shared', 'var', 'shared1'])) + def test_unixdir_2(): - directory = unix.UnixSFTPDirectory(srv, '/var') + directory = unix.DockerVolumeDirectory(srv, '/var') assert_equal(set(directory.files), set(["log"])) + def test_unixdir_3(): - directory = unix.UnixSFTPDirectory(srv, '/var/log') + directory = unix.DockerVolumeDirectory(srv, '/var/log') assert_equal(set(directory.files), set(["sql"])) diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index befae25..936a466 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -24,6 +24,7 @@ import json from twisted.application import service, internet +from twisted.cred.checkers import AllowAnonymousAccess from twisted.names import dns, server from twisted.python import log from twisted.python import usage @@ -48,7 +49,13 @@ class Options(usage.Options): "Return SERVFAIL instead of NXDOMAIN if container not found"], ["authoritative", "A", True, "Return authoritative replies"], ["docker-version", "V", '1.15', "Docker API version"], - ["bind_protocols", "B", ['tcp', 'udp'], "Bind protocols"] + ["bind_protocols", "B", ['tcp', 'udp'], "Bind protocols"], + ["sftp-bind", "S", "10022", "SFTP port to expose Volume access" + " eg. 10022. To specify an ip you can" + " use eg. 22:interface=127.0.0.1"], + ["sftp-keypath", "", "./", "Path to ssh_host_*key"], + ["sftp-moduli", "", None, "Path to moduli directory"] + ] @@ -63,6 +70,33 @@ class MyServiceMaker(object): description = "Run this! It'll make your docker happy." options = Options + def sftpVolumeService(self, options, db): + """ + Construct a service for operating a SSH server. + + @param options: An L{Options} instance specifying server options, + including where server keys are stored. + + @param db: A L{DockerDB} instance with container information + + @return: An L{IService} provider which contains the requested + SSH server. + """ + from dockerdns.sftp import factory + from dockerdns.sftp import unix, checkers as docker_checkers + from twisted.cred import portal + from twisted.application import strports + # The factory just sets the ssh keys + t = factory.OpenSSHFactory() + + r = unix.DockerRealm(db) + t.portal = portal.Portal(r, [docker_checkers.PermitChecker()]) + t.dataRoot = options['sftp-keypath'] + t.moduliRoot = options['sftp-moduli'] or options['sftp-keypath'] + + port = options['sftp-bind'] + return strports.service(port, t) + def makeService(self, options): """ Set everything up @@ -137,6 +171,9 @@ def makeService(self, options): console = internet.TCPServer(8080, consoleFactory) console.setServiceParent(ret) + # sftp Volume + sftp_volumes = self.sftpVolumeService(options, db) + sftp_volumes.setServiceParent(ret) return ret diff --git a/twisted/plugins/dockersftp_plugin.py b/twisted/plugins/dockersftp_plugin.py deleted file mode 100644 index 5a07c65..0000000 --- a/twisted/plugins/dockersftp_plugin.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- test-case-name: twisted.conch.test.test_tap -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Support module for making SSH servers with twistd. -""" -from zope.interface.declarations import implements -from twisted.application.service import IServiceMaker - -from twisted.conch import checkers as conch_checkers -from twisted.cred import portal, checkers, strcred -from twisted.plugin import IPlugin -from twisted.python import usage -from twisted.application import strports -try: - from twisted.cred import pamauth -except ImportError: - pamauth = None - -from dockerdns.sftp import factory -from dockerdns.sftp import unix, checkers as docker_checkers -class Options(usage.Options, strcred.AuthOptionMixin): - synopsis = "[-i ] [-p ] [-d ] " - longdesc = ("Makes a Conch SSH server. If no authentication methods are " - "specified, the default authentication methods are UNIX passwords, " - "SSH public keys, and PAM if it is available. If --auth options are " - "passed, only the measures specified will be used.") - optParameters = [ - ["interface", "i", "", "local interface to which we listen"], - ["port", "p", "tcp:22", "Port on which to listen"], - ["data", "d", "/etc", "directory to look for host keys in"], - ["moduli", "", None, "directory to look for moduli in " - "(if different from --data)"] - ] - compData = usage.Completions( - optActions={"data": usage.CompleteDirs(descr="data directory"), - "moduli": usage.CompleteDirs(descr="moduli directory"), - "interface": usage.CompleteNetInterfaces()} - ) - - - def __init__(self, *a, **kw): - usage.Options.__init__(self, *a, **kw) - - # call the default addCheckers (for backwards compatibility) that will - # be used if no --auth option is provided - note that conch's - # UNIXPasswordDatabase is used, instead of twisted.plugins.cred_unix's - # checker - super(Options, self).addChecker(conch_checkers.UNIXPasswordDatabase()) - super(Options, self).addChecker(conch_checkers.SSHPublicKeyChecker( - conch_checkers.UNIXAuthorizedKeysFiles())) - if pamauth is not None: - super(Options, self).addChecker( - checkers.PluggableAuthenticationModulesChecker()) - self._usingDefaultAuth = True - - - def addChecker(self, checker): - """ - Add the checker specified. If any checkers are added, the default - checkers are automatically cleared and the only checkers will be the - specified one(s). - """ - if self._usingDefaultAuth: - self['credCheckers'] = [] - self['credInterfaces'] = {} - self._usingDefaultAuth = False - super(Options, self).addChecker(checker) - - -class MyServiceMaker(object): - """ - Define a MultiService running: - - dns server for tcp and udp - - http client for retrieving docker events - """ - implements(IServiceMaker, IPlugin) - tapname = "dockersftp" - description = "Run this! It'll make your docker happy." - options = Options - - def makeService(self, options): - """ - Construct a service for operating a SSH server. - - @param options: An L{Options} instance specifying server options, including - where server keys are stored and what authentication methods to use. - - @return: An L{IService} provider which contains the requested SSH server. - """ - # The factory just sets the ssh keys - t = factory.OpenSSHFactory() - - r = unix.DockerRealm() - t.portal = portal.Portal(r, [docker_checkers.PermitChecker()]) - t.dataRoot = options['data'] - t.moduliRoot = options['moduli'] or options['data'] - - port = options['port'] - if options['interface']: - # Add warning here - port += ':interface=' + options['interface'] - return strports.service(port, t) - -# -# Create the MultiService -# -serviceMaker = MyServiceMaker() From 8f13cb0a7fed04a482549e000ebae82b59b6384e Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 2 Mar 2015 16:16:27 +0100 Subject: [PATCH 49/51] fix: test sftp volumes --- .travis.yml | 2 +- dockerdns/events.py | 12 ++-- dockerdns/sftp/unix.py | 47 ++++++++++--- test/test_sftpserver_volumes.py | 115 +++++++++++++++++++++++++------- 4 files changed, 137 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index f65d758..922f81b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: install: - pip install -r test_requirements.txt before_script: - - pep8 dockerdns/*.py twisted/plugins/*py + - pep8 dockerdns/*.py twisted/plugins/*py dockerdns/sftp/*py # - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py script: - nosetests -v -w test diff --git a/dockerdns/events.py b/dockerdns/events.py index b1d6bff..9b91409 100644 --- a/dockerdns/events.py +++ b/dockerdns/events.py @@ -67,10 +67,14 @@ def updatedb(self, item): self.mappings_ip.update({ip: id_}) self.mappings.update({id_: item}) _, image_notag, _ = DockerDB.re_image.match(image).groups() - self.mappings_image.setdefault( - image_notag, []).append(id_) - self.mappings_image.setdefault( - image, []).append(id_) + l = self.mappings_image.setdefault( + image_notag, []) + if id_ not in l: + l.append(id_) + l = self.mappings_image.setdefault( + image, []) + if id_ not in l: + l.append(id_) def add_container(self, item): self.updatedb(item) diff --git a/dockerdns/sftp/unix.py b/dockerdns/sftp/unix.py index 34ad86f..8994c41 100644 --- a/dockerdns/sftp/unix.py +++ b/dockerdns/sftp/unix.py @@ -130,26 +130,53 @@ class SFTPServerForDockerVolumeConchUser(SFTPServerForUnixConchUser): def __init__(self, avatar): SFTPServerForUnixConchUser.__init__(self, avatar) + self.allowed_volumes, self.allowed_paths = zip( + *self.avatar.volumes.items()) + + def _fits(self, volume, path): + should_empty, expected, rest = path.partition(volume) + if should_empty is not '': + return False + if expected is not volume: + return False + if rest == '': + return True + if rest[0] == '/': + return True + return False + + def _best_fit(self, path): + """ Return the volume containing directory + """ + matching_volumes = ( + x + for x in self.allowed_volumes + if self._fits(x, path) + ) + return sorted(matching_volumes, key=len, reverse=True) def _absPath(self, path): - import re - + # You can only visit volumes allowed_paths = tuple(self.avatar.volumes.keys()) + (".", "/") home = self.avatar.getHomeDir() + if path.startswith(allowed_paths): apath = os.path.abspath(os.path.join(home, path)) if apath == "/": return apath - - for v, folder in self.avatar.volumes.items(): - apath = re.sub('^' + v, folder, apath) - return apath - + best_fit = self._best_fit(apath) + if best_fit: + volume = best_fit[0] + folder = self.avatar.volumes[volume] + log.msg("Volume: %r, %r" % (volume, folder)) + apath = re.sub('^' + volume, folder, apath) + return apath + # By default you're not authorized log.msg("not outside container path: %r" % [ path, self.avatar.volumes.keys()] ) - os.stat("/dev/File not found") + raise OSError(2, "File not found: %r" % path, path) def openFile(self, filename, flags, attrs): return UnixSFTPFile(self, self._absPath(filename), flags, attrs) @@ -185,8 +212,8 @@ def __init__(self, server, directory): in allowed_volumes if x.startswith(directory)] else: - log.err("Accessing %r" % directory) - if not directory.startswith(tuple(allowed_paths)): + log.err("Accessing %r" % [directory]) + if not directory.startswith(allowed_paths): raise OSError(2, "No such file or directory", directory) self.files = server.avatar._runAsUser(os.listdir, directory) self.dir = directory diff --git a/test/test_sftpserver_volumes.py b/test/test_sftpserver_volumes.py index bfbe8ca..032f48d 100644 --- a/test/test_sftpserver_volumes.py +++ b/test/test_sftpserver_volumes.py @@ -1,46 +1,113 @@ +""" + author: roberto.polli@par-tec.it + + This class tests the SFTP Volume access. + + To access: + - the container jboss63 + - with the volumes "/shared", "/var/log" + + # sftp -P 10022 jboss63@localhost # no password for now + # pwd + / + # ls + /shared + /var + +""" from dockerdns.sftp import unix from twisted.python import log -from nose.tools import assert_true, assert_equal -unix.container_database = dict( - # container_name: volume_path - foo={"Volumes": { - "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", - "/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", - "/var/log/sql": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" +from nose.tools import (assert_true, assert_equal, raises, + assert_false, assert_is_instance) - } - } -) MOCK_CONTAINER = "foo" -user = unix.DockerVolumeConchUser(MOCK_CONTAINER) + + +class MockDB(): + """Mocking L{DockerDB}""" + + def get_by_name(self, *a, **kw): + return { + "Volumes": { + "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/var/log/sql": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" + } + } + +# initialize +db = MockDB() +user = unix.DockerVolumeConchUser(MOCK_CONTAINER, db) srv = unix.SFTPServerForDockerVolumeConchUser(user) -def test_sftpserver_subpat(): - user = unix.DockerVolumeConchUser(MOCK_CONTAINER) +def test_get_homedir_is_rootdir(): assert user.getHomeDir() == "/" -def test_server(): - user = unix.DockerVolumeConchUser(MOCK_CONTAINER) - srv = unix.SFTPServerForDockerVolumeConchUser(user) +def test_abspath_volumes_to_real_folder(): for p in ( - '/', '/shared', '/shared/foo' + '/shared', '/shared/foo.txt', '/shared1', '/var/log/sql', + '/var/log/sql/db.out' ): - log.msg(srv._absPath(p)) - yield assert_true, srv._absPath(p).startswith("/data/docker") + real_path = srv._absPath(p) + log.msg("real path is %r" % real_path) + yield assert_true, real_path.startswith("/data/docker"), \ + "Not starting with /data/docker: %r" % real_path + + +def test_abspath_files_outside_volumes(): + for p in '/var/log/messages /shared2 /shared2/'.split(): + ex = None + try: + real_path = srv._absPath(p) + print("real path is %r" % real_path) + except Exception as ex: + pass + yield assert_is_instance, ex, OSError def test_unixdirectory_1(): directory = unix.DockerVolumeDirectory(srv, '/') - assert_equal(set(directory.files), set(['shared', 'var', 'shared1'])) + expected_files = {'shared', 'var', 'shared1'} + assert_equal(set(directory.files), expected_files) def test_unixdir_2(): - directory = unix.DockerVolumeDirectory(srv, '/var') - assert_equal(set(directory.files), set(["log"])) + test_cases = [ + ('/var', {'log'}), + ('/var/log', {'sql'}) + ] + for dir, expected in test_cases: + directory = unix.DockerVolumeDirectory(srv, dir) + assert_equal, set(directory.files), expected, "%r" % [set(directory.files)] +@raises(OSError) def test_unixdir_3(): - directory = unix.DockerVolumeDirectory(srv, '/var/log') - assert_equal(set(directory.files), set(["sql"])) + forbidden_dirs = "/root".split() + for dir in forbidden_dirs: + unix.DockerVolumeDirectory(srv, dir) + + +def test_fits(): + cases_true = [ + ("/var", "/var /var/log /var/log/messages".split()), + ("/s", "/s /s/a.out".split()), + ] + for v, paths in cases_true: + for p in paths: + yield harn_fits, assert_true, v, p + + +def test_fits_false(): + cases_true = [ + ('/s', "/s1 /s1/ /s1/a.out".split()) + ] + for v, paths in cases_true: + for p in paths: + yield harn_fits, assert_false, v, p + + +def harn_fits(checker, v, p): + checker(srv._fits(v, p)) From 24dae0406184bc52c014e68e2f651875c7f1d448 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Mon, 2 Mar 2015 22:03:37 +0100 Subject: [PATCH 50/51] added sftp doc --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index faf8564..2f4bff3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Docker DNS ========== A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP -resolution. +resolution. As a plus you get a sperimental SFTP server to access Docker Volumes. Containers can be found by: - image name @@ -49,7 +49,7 @@ There's a simple HTTP console to check the internal mappings. You can curl it wi #curl -v http://localhost:8080/{hostname,image,name,id,ping,help,ip}/{optional_key} -Examples +DNS Examples -------- Dig output is shortened for brevity. We have Docker containers like this: @@ -105,6 +105,17 @@ Nat discovery: you can discover natted ports with queries like this one ;; ANSWER SECTION: _8080._tcp.jboss631.docker. 10 IN SRV 100 100 18080 192.168.204.17. + +SFTP Examples +------------- +To access the /myshare volume on the container jboss631, just: + + #sftp -P10022 jboss631@localhost # empty password + #ls / + /myshare + + + Configuration ------------- Config is done in the `dockerdns.json` file. There's a skeleton in From dd1bcb61b4646ea1bcd0c96bb39994e1900dc9d7 Mon Sep 17 00:00:00 2001 From: Roberto Polli Date: Wed, 8 Apr 2015 12:10:45 +0200 Subject: [PATCH 51/51] fix: config file now overrides command lines, return empty records on AAAA --- dockerdns/resolver.py | 61 ++++++++++++- test/test_aaaa.py | 130 ++++++++++++++++++++++++++++ twisted/plugins/dockerdns_plugin.py | 5 +- 3 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 test/test_aaaa.py diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py index 0e1f12f..2a7d2f8 100644 --- a/dockerdns/resolver.py +++ b/dockerdns/resolver.py @@ -48,13 +48,17 @@ def __init__(self, mapping, config=None): """ self.mapping = mapping - self.config = config or {'domain': 'docker', 'bip': '172.17.0.0/16'} + self.config = config or { + 'domain': 'docker', + 'bip': '172.17.0.0/16', + 'ttl': 10 + } self.re_domain = re.compile(r'\.' + self.config['domain'] + '$') self.re_ptr = re.compile(r'[0-9]+\.[0-9]+\.17\.172\.in-addr\.arpa$') # Change to this ASAP when Twisted uses object base # super(DockerResolver, self).__init__() common.ResolverBase.__init__(self) - self.ttl = 10 + self.ttl = int(self.config['ttl']) self.my_preferred_ip, self.my_preferred_ip_ptr_value \ = get_preferred_ip() @@ -189,6 +193,59 @@ def lookupAddress(self, name, timeout=None): return defer.fail(failure.Failure(exception)) + def lookupIPV6Address(self, name, timeout=None): + """ + + :param name: a name like container_name.docker, hostname.docker, + image_name.*.docker + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred + + """ + name, occurrences = self.re_domain.subn('', name) + if not occurrences: + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure( + DomainError("not ending with docker")) + ) + + # Raise exception if host not found + try: + records = self._a_records(name) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as ex: + log.msg("DomainError: %r " % ex) + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) + except Exception as ex: # pylint:disable=bare-except + import traceback + + traceback.print_exc() + + if self.config.get(NO_NXDOMAIN): + log.err() + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + # Otherwise no RR -> Answer RRs: 0 + empty_record = tuple() + return defer.succeed((empty_record, self.authority, self.additional)) + def lookupService(self, name, timeout=None): """ Lookup a docker natted service of the form: NATTEDPORT._tcp.CONTAINERNAME.docker. diff --git a/test/test_aaaa.py b/test/test_aaaa.py new file mode 100644 index 0000000..729659e --- /dev/null +++ b/test/test_aaaa.py @@ -0,0 +1,130 @@ +#!/usr/bin/python + +""" +Tests for AAAA queries + +Author: Roberto Polli +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + + +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2 +from nose.tools import * + +# FIXME I can not believe how disgusting this is + + +def in_generator(gen, val): + return reduce( + lambda old, new: old or new == val, + gen, + False + ) + + +def check_record(record, **expected): + """ + Compare a record with the values of the kwargs + :param record: + :param expected: + :return: + """ + for k in expected: + real_value = getattr(record, k) + if k is 'name': + real_value = real_value.name + + if real_value != expected[k]: + log.err("Expected %s: %s vs %s" % (k, expected[k], real_value)) + return False + + return True + + +def check_deferred(deferred, success): + completed = [] + + def gimme_x_back(is_success): + def x_back(result): + completed.append((is_success, result)) + + return x_back + + deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) + if len(completed) != 1: + return False + + status, result = completed[0] + if status != success: + raise AssertionError("Expected: %r, got %r" % (success, result)) + return False + + return result + + +class TestAAAA(object): + def setUp(self): + self.CONFIG = {} + + self.db = create_mock_db2() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + assert_equal(len(rec), 1) + rec = rec[0] + assert_true(check_record(rec, **expected_record)) + return rec + + + # + # TEST lookupIPV6Address + # + def test_lookupIPV6Address_id_empty_response(self): + # skip testing authority_rr, additional_rr + # as we're now populating the authority and additional section + expected_record = tuple() + deferred = self.resolver.lookupIPV6Address('cidfoxes.docker') + + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + + # We are returning an empty reply + # because we're not supporting IPV6 + assert_equal(len(response_rr), 0) + + def test_lookupIPV6Address_invalid(self): + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + + def test_lookupIPV6Address_invalid_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = False + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal( + result.type, DomainError) # noqa pylint:disable=maybe-no-member + + def test_lookupIPV6Address_invalid_no_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = True + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal(result.type, DNSQueryTimeoutError) + # noqa pylint:disable=maybe-no-member + diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py index 936a466..2fa6447 100644 --- a/twisted/plugins/dockerdns_plugin.py +++ b/twisted/plugins/dockerdns_plugin.py @@ -48,6 +48,7 @@ class Options(usage.Options): ['no_nxdomain', "x", True, "Return SERVFAIL instead of NXDOMAIN if container not found"], ["authoritative", "A", True, "Return authoritative replies"], + ["ttl", "", 10, "The default TTL for those entries"], ["docker-version", "V", '1.15', "Docker API version"], ["bind_protocols", "B", ['tcp', 'udp'], "Bind protocols"], ["sftp-bind", "S", "10022", "SFTP port to expose Volume access" @@ -113,9 +114,11 @@ def makeService(self, options): appcfg = {} # Update config stuff with command line params - appcfg.update(options) + options.update(appcfg) + appcfg = options log.err("config: %r" % appcfg) # Create docker: by default dict.get returns None on missing keys + # TODO: we could source more than one docker server docker_client = docker.Client( appcfg.get('docker_url'), version=appcfg.get('docker-version')) infos = docker_client.info()