diff --git a/.travis.yml b/.travis.yml index a5bfbe3..922f81b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: python python: - "2.7" install: - - sudo pip install -r test_requirements.txt --use-mirrors + - pip install -r test_requirements.txt before_script: - - pep8 *.py - - pylint --rcfile=pylint.conf *.py + - pep8 dockerdns/*.py twisted/plugins/*py dockerdns/sftp/*py +# - pylint --rcfile=pylint.conf dockerdns/*.py twisted/plugins/*py script: - - ./docker_dns_test.py \ No newline at end of file + - nosetests -v -w test diff --git a/README.bind b/README.bind new file mode 100644 index 0000000..a98fc89 --- /dev/null +++ b/README.bind @@ -0,0 +1,29 @@ +Configure with bind +=================== + + +If you use a local bind instance you can configure twistd dns with bind adding this stanza to named.conf. For now this requires to disable EDNS, which is not actuallyimplemented in docker_dns. + + +zone "docker" { + type forward; + forwarders { + 127.0.0.64; + }; +}; + + +And binding your local docker_dns server on the following +loopback ip: 127.0.0.64 + + +Configure with dnsmasq +====================== + +You can use twistd dns with dnsmasq using the following file: + +# Config file for /etc/dnsmasq.d/dockerdns.conf +port=53 + +# Forward only .docker requests... +server=/docker/127.0.0.64#10053 diff --git a/README.md b/README.md index df7f605..2f4bff3 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,61 @@ Docker DNS ========== -[![Build Status](https://travis-ci.org/infoxchange/docker_dns.png?branch=master)](https://travis-ci.org/infoxchange/docker_dns) -A simple Twisted DNS server using custom TLD and Docker as the back end for IP -resolution. +A simple Twisted DNS server using custom TLD and Docker Event interface as the back end for IP +resolution. As a plus you get a sperimental SFTP server to access Docker Volumes. -To look up a container: - - 'A' record query container's hostname with no TLD. Must be an exact match - - 'A' record query an ID that will match a container with a docker inspect - command with '.docker' as the TLD. eg: 0949efde23b.docker +Containers can be found by: + - image name + - container name + - hostname + - ip + +eg: here are some examples + + #host busybox.*.docker # search all busybox containers + #host 26ed50b1bf59.docker # search a container by hostname (not by ID!) + #host nice_bohr.docker # search a container by name + +You can lookup different records: + - 'A' record: query a container NAME or HOSTNAME that will match a container with a docker inspect + command with '.docker' as the TLD. eg: mysql_server1.docker + - 'SRV' record query exposing the NAT informations (more to come!) + - 'PTR' record, with reverse pointer + +Note: This fork of docker_dns *always* requires to query using a TLD (by default .docker) Install/Run ----------- Just install from requirements (in a virtualenv if you'd like) - pip install -r requirements.txt --use-mirrors + #pip install -r requirements.txt -That's it! To run, just +That's it! To run, remember that you may need to set user/group ids on +the process - twistd -y docker_dns.py + + #sudo twistd -gdocker -y dockerdns -p 53 This will start a DNS server on port 53 (default DNS port). To make this -useful, you probably want to combine it with your regular DNS in something like -Dnsmasq. +useful, you probably want to combine it with your regular DNS in something like Dnsmasq. + +You can get configuration parameters with + + #sudo twistd dockerdns --help + -Examples +There's a simple HTTP console to check the internal mappings. You can curl it with + + #curl -v http://localhost:8080/{hostname,image,name,id,ping,help,ip}/{optional_key} + +DNS Examples -------- Dig output is shortened for brevity. We have Docker containers like this: - ID IMAGE COMMAND CREATED STATUS PORTS - 26ed50b1bf59 ubuntu:12.04 /bin/bash 4 seconds ago Up 4 seconds - 0949efde23bf ubuntu:12.04 /bin/bash 18 hours ago Up 18 hours + ID IMAGE STATUS Names + 26ed50b1bf59 ubuntu:12.04 Up 1 hour sad_turing + 0949efde23bf ubuntu:12.04 Up 18 hours happy_bohr 0949efde23bf has: @@ -45,36 +69,22 @@ Dig output is shortened for brevity. We have Docker containers like this: - IP: 172.17.0.3 - Hostname: my-thing -Container IDs are variable length. You use the same input as the Docker `inspect` command, so they can be long: - - dig 0949efde23bf017.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 51840 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - - ;; ANSWER SECTION: - 0949efde23bf017.docker. 10 IN A 172.17.0.2 - +Search by Hostname (uses default or explicit hostname) -Or they can be short: + #dig +short 26ed50b1bf59.docker + 172.17.0.2 - dig 0949.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 42797 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 + #dig +short my-thing.docker + 172.17.0.3 - ;; ANSWER SECTION: - 0949.docker. 10 IN A 172.17.0.2 +Search by Names (works only the first Name) -And the other container: + #dig +short sad_turing.docker + 172.17.0.2 - dig 26ed50b1bf59.docker - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 25901 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 + #dig +short happy_bohr.docker + 172.17.0.3 - ;; ANSWER SECTION: - 26ed50b1bf59.docker. 10 IN A 172.17.0.3 When a container doesn't exist, no answer is given: @@ -83,58 +93,49 @@ When a container doesn't exist, no answer is given: ;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 24269 ;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0 -You can look up by hostname by removing the .docker TLD: +You can search by image, like skydock: - dig 0949efde23bf - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 61822 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 + dig +short ubuntu.*.docker + 172.17.0.2 + 172.17.0.3 - ;; ANSWER SECTION: - 0949efde23bf. 10 IN A 172.17.0.2 +Nat discovery: you can discover natted ports with queries like this one -Here's a manually defined hostname: + dig _8080._tcp.my-thing.docker srv + ;; ANSWER SECTION: + _8080._tcp.jboss631.docker. 10 IN SRV 100 100 18080 192.168.204.17. - dig my-thing - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3355 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 - ;; ANSWER SECTION: - my-thing. 10 IN A 172.17.0.3 +SFTP Examples +------------- +To access the /myshare volume on the container jboss631, just: -And the host name that would have been automatically assigned for the above -container: + #sftp -P10022 jboss631@localhost # empty password + #ls / + /myshare + - dig 26ed50b1bf59 - ;; Got answer: - ;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 12687 - ;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0 Configuration ------------- -Config is done in the `config.py` file. There's a skeleton in -`config.py.sample`. Below are the default config values. Currently, +Config is done in the `dockerdns.json` file. There's a skeleton in +`dockerdns.json.sample`. Below are the default config values. Currently, configuration is rather limited. - CONFIG = { - # URL to connect to the Docker API. docker-py defaults to - # unix://var/run/docker.sock + { + "#": "# URL to connect to the Docker API. docker-py defaults to unix://var/run/docker.sock", 'docker_url': None - # socket.bind defaults to 0.0.0.0 + "#": "# socket.bind defaults to 0.0.0.0", 'bind_interface': '', 'bind_port': 53, 'bind_protocols': ['tcp', 'udp'], - # When the request matches no docker container, we should return - # NXDOMAIN, however the OS interprets this as "doesn't exist anywhere" - # so things like google.com fail. This will return SERVFAIL rather than - # NXDOMAIN so secondary DNS is used + "#": "Return SERVFAIL instead of NXDOMAIN if no matching container found" 'no_nxdomain': True, - # Makes successful requests authoritive - 'authoritive': True, + "#": "Makes successful requests authoritative", + 'authoritative': True, } Contributing diff --git a/config.py.sample b/config.py.sample deleted file mode 100644 index a2bbdfb..0000000 --- a/config.py.sample +++ /dev/null @@ -1,8 +0,0 @@ -""" -Configuration for Docker DNS server. - -Author: Ricky Cook -""" - -CONFIG = { -} diff --git a/docker_dns.py b/docker_dns.py deleted file mode 100755 index 6466b1a..0000000 --- a/docker_dns.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/python - -""" -A simple TwistD DNS server using custom TLD and Docker as the back end for IP -resolution. - -To look up a container: - - 'A' record query container's hostname with no TLD. Must be an exact match - - 'A' record query an ID that will match a container with a docker inspect - command with '.docker' as the TLD. eg: 0949efde23b.docker - -Code heavily modified from -http://stackoverflow.com/a/4401671/509043 - -Author: Ricky Cook -""" - -import docker -import re - -from requests.exceptions import RequestException -from twisted.application import internet, service -from twisted.internet import defer -from twisted.names import common, dns, server -from twisted.names.error import DNSQueryTimeoutError, DomainError -from twisted.python import failure -from warnings import warn - - -# FIXME replace with a more generic solution like operator.attrgetter -def dict_lookup(dic, key_path, default=None): - """ - Look up value in a nested dict - - Args: - dic: The dictionary to search - key_path: An iterable containing an ordered list of dict keys to - traverse - default: Value to return in case nothing is found - - Returns: - Value of the dict at the nested location given, or default if no value - was found - """ - - for k in key_path: - if k in dic: - dic = dic[k] - else: - return default - return dic - - -class DockerMapping(object): - """ - Look up docker container data - """ - - id_re = re.compile(r'([a-z0-9]+)\.docker') - - def __init__(self, api): - """ - Args: - api: Docker Client instance used to do API communication - """ - - self.api = api - - def _ids_from_prop(self, key_path, value): - """ - Get IDs of containers where their config matches a value - - Args: - key_path: An iterable containing an ordered list of container - config keys to traverse - value: What the value at key_path must match to qualify - - Returns: - Generator with a list of containers that match the config value - """ - - return ( - c['ID'] - for c in ( - self.api.inspect_container(c_lite['Id']) - for c_lite in self.api.containers(all=True) - ) - if dict_lookup(c, key_path, None) == value - if 'ID' in c - ) - - def lookup_container(self, name): - """ - Gets the container config from a DNS lookup name, or returns None if - one could not be found - - Args: - name: DNS query name to look up - - Returns: - Container config dict for the first matching container - """ - - match = self.id_re.match(name) - if match: - container_id = match.group(1) - else: - ids = self._ids_from_prop(('Config', 'Hostname'), unicode(name)) - # FIXME Should be able to support multiple - try: - container_id = ids.next() - except StopIteration: - return None - - try: - return self.api.inspect_container(container_id) - - except docker.client.APIError as ex: - # 404 is valid, others aren't - if ex.response.status_code != 404: - warn(ex) - - return None - - except RequestException as ex: - warn(ex) - return None - - def get_a(self, name): - """ - Get an IPv4 address from a query name to be used in A record lookups - - Args: - name: DNS query name to look up - - Returns: - IPv4 address for the query name given - """ - - container = self.lookup_container(name) - - if container is None: - return None - - addr = container['NetworkSettings']['IPAddress'] - - if addr is '': - return None - - return addr - - -# pylint:disable=too-many-public-methods -class DockerResolver(common.ResolverBase): - """ - DNS resolver to resolve queries with a DockerMapping instance. - """ - - def __init__(self, mapping): - """ - Args: - mapping: DockerMapping instance for lookups - """ - - self.mapping = mapping - - # Change to this ASAP when Twisted uses object base - # super(DockerResolver, self).__init__() - common.ResolverBase.__init__(self) - self.ttl = 10 - - def _a_records(self, name): - """ - Get A records from a query name - - Args: - name: DNS query name to look up - - Returns: - Tuple of formatted DNS replies - """ - - addr = self.mapping.get_a(name) - if not addr: - raise DomainError(name) - - return tuple([ - dns.RRHeader(name, dns.A, dns.IN, self.ttl, - dns.Record_A(addr, self.ttl), - CONFIG['authoritive']) - ]) - - def lookupAddress(self, name, timeout=None): - try: - records = self._a_records(name) - return defer.succeed((records, (), ())) - - # We need to catch everything. Uncaught exceptian will make the server - # stop responding - except: # pylint:disable=bare-except - if CONFIG['no_nxdomain']: - # FIXME surely there's a better way to give SERVFAIL - exception = DNSQueryTimeoutError(name) - else: - exception = DomainError(name) - - return defer.fail(failure.Failure(exception)) - - -def main(): - """ - Set everything up - """ - - # Create docker - if CONFIG['docker_url']: - docker_client = docker.Client(CONFIG['docker_url']) - else: - docker_client = docker.Client() - - # Create our custom mapping and resolver - mapping = DockerMapping(docker_client) - resolver = DockerResolver(mapping) - - # Create twistd stuff to tie in our custom components - factory = server.DNSServerFactory(clients=[resolver]) - factory.noisy = False - - # Protocols to bind - bind_list = [] - if 'tcp' in CONFIG['bind_protocols']: - bind_list.append((internet.TCPServer, factory)) # noqa pylint:disable=no-member - - if 'udp' in CONFIG['bind_protocols']: - proto = dns.DNSDatagramProtocol(factory) - proto.noisy = False - bind_list.append((internet.UDPServer, proto)) # noqa pylint:disable=no-member - - # Register the service - ret = service.MultiService() - for (klass, arg) in bind_list: - svc = klass( - CONFIG['bind_port'], - arg, - interface=CONFIG['bind_interface'] - ) - svc.setServiceParent(ret) - - # DO IT NOW - ret.setServiceParent(service.IServiceCollection(application)) - -# Load the config -try: - from config import CONFIG # pylint:disable=no-name-in-module,import-error -except ImportError: - CONFIG = {} - -# Merge user config over defaults -DEFAULT_CONFIG = { - 'docker_url': None, - 'bind_interface': '', - 'bind_port': 53, - 'bind_protocols': ['tcp', 'udp'], - 'no_nxdomain': True, - 'authoritive': True, -} -CONFIG = dict(DEFAULT_CONFIG.items() + CONFIG.items()) - -application = service.Application('dnsserver', 1, 1) # noqa pylint:disable=invalid-name -main() - - -# Doin' it wrong -if __name__ == '__main__': - import sys - print "Usage: twistd -y %s" % sys.argv[0] diff --git a/docker_dns_test.py b/docker_dns_test.py deleted file mode 100755 index 2f54b72..0000000 --- a/docker_dns_test.py +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/python - -""" -Tests for Docker DNS - -Author: Ricky Cook -""" - -# Do not care...... -# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name - -import docker_dns - -import docker -import fudge -import itertools -import unittest - -from docker_dns import (DEFAULT_CONFIG, - dict_lookup, - DockerMapping, - DockerResolver) -from twisted.names import dns -from twisted.names.error import DNSQueryTimeoutError, DomainError - - -# FIXME I can not believe how disgusting this is -def in_generator(gen, val): - return reduce( - lambda old, new: old or new == val, - gen, - False - ) - - -def check_record(record, **kwargs): - for k in kwargs: - real_value = getattr(record, k) - if k is 'name': - real_value = real_value.name - - if real_value != kwargs[k]: - return False - - return True - - -def check_deferred(deferred, success): - completed = [] - - def gimme_x_back(is_success): - def x_back(result): - completed.append((is_success, result)) - - return x_back - - deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) - if len(completed) != 1: - return False - - status, result = completed[0] - if status != success: - return False - - return result - - -class MockDockerClient(object): - inspect_container_pandas = { - 'ID': 'cidpandaslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'cuddly-pandas', - }, - 'NetworkSettings': { - 'IPAddress': '127.0.0.1' - }, - } - inspect_container_foxes = { - 'ID': 'cidfoxeslong', - 'Same': 'Value', - 'Config': { - 'Hostname': 'sneaky-foxes', - }, - 'NetworkSettings': { - 'IPAddress': '8.8.8.8' - } - } - inspect_container_sloths = { - 'ID': 'cidslothslong', - 'Config': { - 'Hostname': 'stopped-sloths', - }, - 'NetworkSettings': { - 'IPAddress': '' - } - } - inspect_container_returns = { - 'cidpandas': inspect_container_pandas, - 'cidpandaslong': inspect_container_pandas, - 'cidfoxes': inspect_container_foxes, - 'cidfoxeslong': inspect_container_foxes, - 'cidsloths': inspect_container_sloths, - 'cidslothslong': inspect_container_sloths, - } - containers_return = [ - {'Id': 'cidpandas'}, - {'Id': 'cidfoxes'}, - {'Id': 'cidsloths'}, - ] - - inspect_container_id = None - - def inspect_container(self, cid): - self.inspect_container_id = cid - - try: - return self.inspect_container_returns[cid] - except KeyError: - response = fudge.Fake() - response.has_attr(status_code=404, content='PANDAS!') - - exception = docker.client.APIError('bad', response) - raise exception - - def containers(self, *args, **kwargs): # pylint:disable=unused-argument - return self.containers_return - - -class DictLookupTest(unittest.TestCase): - theDict = { - 'pandas': { - 'are': 'cuddly', - 'and': 'awesome', - }, - 'foxes': { - 'are': 'sneaky', - 'and': 'orange', - }, - 'badgers': { - 'are': None, - }, - } - - def test_basic_one(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['pandas', 'and'] - ), - 'awesome' - ) - - def test_basic_two(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['foxes', 'are'] - ), - 'sneaky' - ) - - def test_basic_none(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['badgers', 'are'], - 'Badgers are none? What?' - ), - None - ) - - def test_dict(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['foxes'] - ), - self.theDict['foxes'] - ) - - def test_default_single_depth(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['nothing'] - ), - None - ) - - def test_user_default_single_depth(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['nothing'], - 'Nobody here but us chickens' - ), - 'Nobody here but us chickens' - ) - - def test_default_multi_depth(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['pandas', 'bad'] - ), - None - ) - - def test_user_default_multi_depth(self): - self.assertEqual( - dict_lookup( - self.theDict, - ['pandas', 'bad'], - 'NO, THAT\'S A DAMN DIRTY LIE' - ), - 'NO, THAT\'S A DAMN DIRTY LIE' - ) - - -class DockerMappingTest(unittest.TestCase): - - def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) - - # - # TEST _ids_from_prop - # - def test__ids_from_prop_single_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['ID'], - 'cidpandaslong' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidpandaslong' - ) - - def test__ids_from_prop_multi_depth(self): - ids_gen1, ids_gen2 = itertools.tee( - self.mapping._ids_from_prop( - ['NetworkSettings', 'IPAddress'], - '8.8.8.8' - ) - ) - self.assertEqual(sum(1 for _ in ids_gen1), 1) - self.assertEqual( - ids_gen2.next(), - 'cidfoxeslong' - ) - - def test__ids_from_prop_multi_match(self): - # FIXME I can not believe how disgusting this is - ids_gen1, ids_gen2, ids_gen3 = itertools.tee( - self.mapping._ids_from_prop( - ['Same'], - 'Value' - ), 3 - ) - self.assertEqual(sum(1 for _ in ids_gen1), 2) - self.assertTrue(in_generator(ids_gen2, 'cidpandaslong')) - self.assertTrue(in_generator(ids_gen3, 'cidfoxeslong')) - - # - # TEST lookup_container - # - def test_lookup_container_hostname(self): - self.assertEqual( - self.mapping.lookup_container('cuddly-pandas'), - self.client.inspect_container_pandas - ) - - def test_lookup_container_id(self): - self.assertEqual( - self.mapping.lookup_container('cidfoxes.docker'), - self.client.inspect_container_foxes - ) - - def test_lookup_container_hostname_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid'), - None - ) - - def test_lookup_container_id_none(self): - self.assertEqual( - # Raises an APIError 404 - self.mapping.lookup_container('invalid.docker'), - None - ) - - # - # TEST get_a - # - def test_get_a_hostname(self): - self.assertEqual( - self.mapping.get_a('sneaky-foxes'), - '8.8.8.8' - ) - - def test_get_a_id(self): - self.assertEqual( - self.mapping.get_a('cidpandas.docker'), - '127.0.0.1' - ) - - def test_get_a_hostname_none(self): - self.assertEqual( - self.mapping.get_a('invalid'), - None - ) - - def test_get_a_id_none(self): - self.assertEqual( - self.mapping.get_a('invalid.docker'), - None - ) - - -class DockerResolverTest(unittest.TestCase): - - def setUp(self): - docker_dns.CONFIG = DEFAULT_CONFIG - self.client = MockDockerClient() - self.mapping = DockerMapping(self.client) - self.resolver = DockerResolver(self.mapping) - - # - # TEST _a_records - # - def test__a_records_hostname(self): - rec = self.resolver._a_records('sneaky-foxes') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='sneaky-foxes', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') - - def test__a_records_id(self): - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '127.0.0.1') - - def test__a_records_shutdown(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'cidsloths.docker' - ) - - def test__a_records_invalid(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - 'invalid.docker' - ) - - def test__a_records_blank_query(self): - self.assertRaises( - DomainError, - self.resolver._a_records, - '' - ) - - def test__a_records_authoritive(self): - docker_dns.CONFIG['authoritive'] = True - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - auth=True, - )) - - def test__a_records_non_authoritive(self): - docker_dns.CONFIG['authoritive'] = False - rec = self.resolver._a_records('cidpandas.docker') - self.assertEqual(len(rec), 1) - - rec = rec[0] - self.assertTrue(check_record( - rec, - name='cidpandas.docker', - type=dns.A, - auth=False, - )) - - # - # TEST lookupAddress - # - def test_lookupAddress_id(self): - deferred = self.resolver.lookupAddress('cidfoxes.docker') - - result = check_deferred(deferred, True) - self.assertNotEqual(result, False) - - self.assertEqual(len(result), 3) - self.assertEqual(result[1], ()) - self.assertEqual(result[2], ()) - self.assertEqual(len(result[0]), 1) - - rec = result[0][0] - self.assertTrue(check_record( - rec, - name='cidfoxes.docker', - type=dns.A, - )) - self.assertEqual(rec.payload.dottedQuad(), '8.8.8.8') - - def test_lookupAddress_invalid(self): - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - - def test_lookupAddress_invalid_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = False - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual(result.type, DomainError) # noqa pylint:disable=maybe-no-member - - def test_lookupAddress_invalid_no_nxdomain(self): - docker_dns.CONFIG['no_nxdomain'] = True - deferred = self.resolver.lookupAddress('invalid') - - result = check_deferred(deferred, False) - self.assertNotEqual(result, False) - self.assertEqual(result.type, DNSQueryTimeoutError) # noqa pylint:disable=maybe-no-member - - -def main(): - unittest.main() - - -if __name__ == '__main__': - main() diff --git a/dockerdns.json.sample b/dockerdns.json.sample new file mode 100644 index 0000000..1666639 --- /dev/null +++ b/dockerdns.json.sample @@ -0,0 +1,16 @@ +{ + "#": "# URL to connect to the Docker API. docker-py defaults to unix://var/run/docker.sock", + 'docker_url': None, + + "#": "# socket.bind defaults to 0.0.0.0", + 'bind_interface': '', + 'bind_port': 53, + 'bind_protocols': ['tcp', 'udp'], + + "#": "Return SERVFAIL instead of NXDOMAIN if no matching container found", + 'no_nxdomain': True, + + "#": "Makes successful requests authoritative", + 'authoritative': True, +} + diff --git a/dockerdns/__init__.py b/dockerdns/__init__.py new file mode 100644 index 0000000..f16daaa --- /dev/null +++ b/dockerdns/__init__.py @@ -0,0 +1,7 @@ +""" + + - DnsServer --> DBParser --> DB + - EventManager -http-> docker-server + 2 protocols: /events/, /containers/ + +""" diff --git a/dockerdns/console.py b/dockerdns/console.py new file mode 100644 index 0000000..cc0bad3 --- /dev/null +++ b/dockerdns/console.py @@ -0,0 +1,95 @@ +""" + author: robipolli@gmail.com + Management Console +""" +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.web.http import Request +from dockerdns.events import DockerDB +import simplejson +import time + +HELP_STR = """ +Allowed commands: + +Retrieve container names \t\t\tcurl 'http://localhost:8080/name' +Retrieve container hostnames \t\t\tcurl 'http://localhost:8080/hostname' +Retrieve container ids\t\t\tcurl 'http://localhost:8080/id' +Refresh dns mapping\t\t\tcurl -XPOST 'http://localhost:8080/refresh' + +""" + + +def serialize(item): + return simplejson.dumps(item, indent=True) + + +class RestConsole(Resource): + isLeaf = True + + def __init__(self, db): + """ + + :param db: + :return: + """ + assert isinstance(db, DockerDB) + self.db = db + Resource.__init__(self) + + def dump(self, table, k=None): + """ + Return a value from a given mapping of DB. + TODO This should be moved to DockerDB + """ + try: + d = getattr(self.db, 'mappings_' + + table if table != "id" else "mappings") + except AttributeError: + return "Table not found %r" % table + + if k: + return d[k] + + return d + + def render_GET(self, request): + """ + + :param request: + :type twisted.web.http.Request + :return: + """ + assert isinstance(request, Request) + rpath = request.path.strip("/").split("/") + action = rpath[0] + + if 'ping' in request.path: + return "{0:s}".format([ + time.ctime(), request.path]) + if action in ('help', 'refresh'): + return HELP_STR + + return serialize(self.dump(action, *rpath[1:])) + + def render_POST(self, request): + """ + + :param request: + :type twisted.web.http.Request + :return: + """ + assert isinstance(request, Request) + rpath = request.path.strip("/").split("/") + action = rpath[0] + + if action == 'refresh': + self.db.cleandb() + self.db.load_containers() + return serialize(dict(status="ok", action="refresh")) + raise ValueError("Not Found") + + +class ConsoleFactory(Site): + def __init__(self, db): + Site.__init__(self, RestConsole(db)) diff --git a/dockerdns/events.py b/dockerdns/events.py new file mode 100644 index 0000000..9b91409 --- /dev/null +++ b/dockerdns/events.py @@ -0,0 +1,254 @@ +""" + Populate dns database getting info from docker via: + - docker-py api + - docker events interface + + XXX Not using unicode_literals in Twisted +""" +from __future__ import print_function +from os.path import join as pjoin +from logging import DEBUG +import re +import simplejson as json +from twisted.internet import reactor +from twisted.web.client import Agent, HTTPConnectionPool +from twisted.web.http_headers import Headers +from twisted.internet.protocol import Protocol, ReconnectingClientFactory +from twisted.python import log + + +class DockerDB(object): + """Update docker ip store connecting via docker-py + """ + re_image = re.compile(r"([^/]+/)?([^:]+)(:[^:]+)?") + + def __init__(self, api=None): + """ + + :param api: a docker.Client instance + :return: + """ + self.api = api + # container list and some indexes: + # - name: id + # - image: names + # - hostname: id + self.mappings = {} + self.mappings_name = {} + self.mappings_image = {} + self.mappings_hostname = {} + self.mappings_ip = {} + # + self.load_containers() + + def cleandb(self): + self.mappings = {} + self.mappings_name = {} + self.mappings_image = {} + self.mappings_hostname = {} + self.mappings_ip = {} + + def load_containers(self): + # Initialize db (TODO get initialization timestamp) + for c in self.api.containers(all=False): + item = self.api.inspect_container(c['Id']) + self.updatedb(item) + + def updatedb(self, item): + assert 'Id' in item, "Entry has no Id" + + name = item['Name'][1:] + hostname = item['Config']['Hostname'] + image = item['Config']['Image'] + ip = item['NetworkSettings'].get('IPAddress') + id_ = item['Id'] + self.mappings_name.update({name: id_}) + self.mappings_hostname.update({hostname: id_}) + self.mappings_ip.update({ip: id_}) + self.mappings.update({id_: item}) + _, image_notag, _ = DockerDB.re_image.match(image).groups() + l = self.mappings_image.setdefault( + image_notag, []) + if id_ not in l: + l.append(id_) + l = self.mappings_image.setdefault( + image, []) + if id_ not in l: + l.append(id_) + + def add_container(self, item): + self.updatedb(item) + + def del_container(self, cid): + name = self.mappings[cid]['Name'][1:] + image = self.mappings[cid]['Config']['Image'] + hostname = self.mappings[cid]['Config']['Hostname'] + ip = self.mappings[cid]['NetworkSettings']['IPAddress'] + _, image_notag, _ = DockerDB.re_image.match(image).groups() + self.mappings_image.get(image_notag, []).remove(cid) + self.mappings_image.get(image, []).remove(cid) + + del self.mappings[cid] + del self.mappings_name[name] + del self.mappings_hostname[hostname] + del self.mappings_ip[ip] + + def get_by_name(self, name): + if name not in self.mappings_name: + raise KeyError("%r not in %r" % (name, self.mappings_name)) + + cid = self.mappings_name[name] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + def get_by_hostname(self, name): + if name not in self.mappings_hostname: + raise KeyError("%r not in %r" % (name, self.mappings_hostname)) + + cid = self.mappings_hostname[name] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + def get_by_image(self, image): + """ + + :param image: + :return: an generator of container dicts + """ + if image not in self.mappings_image: + log.err("%r not in %r" % (image, self.mappings_image)) + return + for cid in self.mappings_image[image]: + yield self.mappings[cid] + + def get_by_ip(self, ip): + """ + + :param ip: + :return: an generator of container dicts + """ + if ip not in self.mappings_ip: + raise KeyError("%r not in %r" % (ip, self.mappings_ip)) + cid = self.mappings_ip[ip] + if cid not in self.mappings: + raise KeyError("%r not in %r" % (cid, self.mappings.keys())) + return self.mappings[cid] + + +class ContainerManager(Protocol): + """Manage the response to /containers/{id}/json + updating the network infos associated to the container + """ + + def __init__(self, db): + """Initialize the Docker host database""" + self.db = db + + def dataReceived(self, bytes_): + if bytes_: + item = json.loads(bytes_) + log.msg("Get container %r" % item) + self.db.updatedb(item) + + +class EventManager(Protocol): + """Manage the response to /events/json + - start: triggers a further request with a ContainerManager cb + - stop|die: removes the associations from IPs + """ + + def __init__(self, http_agent, config, db): + self.http_agent = http_agent + self.config = config + self.db = db + + # Create an container_manager for parsing updates + self.container_manager = ContainerManager(db=db) + + def update_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + d = self.http_agent.request('GET', + pjoin(self.config['docker_url'], + 'containers', item['id'], 'json') + ) + d.addCallback( + lambda response: response.deliverBody(self.container_manager)) + + def delete_record(self, item): + """Update docker mapping + item = {'status': ..., 'id': ...} + """ + log.msg("removing item: %r" % item) + self.db.del_container(item['id']) + + def dataReceived(self, bytes_): + """Get the container id and calls the updater""" + try: + display = bytes_ + log.msg('Some data received:', display) + item = json.loads(display) + log.msg("Parsed: %r" % item) + if item['status'] == 'start': + self.update_record(item) + elif item['status'] in ('stop', 'die'): + self.delete_record(item) + except KeyError: + log.err("Container not found") + except json.scanner.JSONDecodeError: + log.err("Error reading data") + except Exception as ex: + log.err("Generic Error %r" % ex) + + +class EventFactory(ReconnectingClientFactory): + """Factory to connect to docker api interface and trigger + the first /events/ call. + + """ + agent = Agent(reactor) + agent2 = Agent(reactor, HTTPConnectionPool(reactor)) + + def __init__(self, config, db): + """ + + :param config: contains the docker url to poll + :param db: the mappings between containers and + :return: + """ + log.msg("Initializing factory") + self.db = db + # + # Create a protocol handler to parse docker container data + # + self.update_event_manager = EventManager( + http_agent=self.agent2, config=config, db=db) + + # Populate existing containers (this is ok to be blocking) + + # + # Poll to the docker event interface + # + d = self.agent.request( + 'GET', + pjoin(config['docker_url'].encode(), 'events'.encode()), + Headers({'User-Agent': ['Twisted Web Client for Docker Event'], + 'Content-Type': ['application/json']}), + None) + d.addCallbacks(self.cbResponse, + lambda failure: log.msg(str(failure), logLevel=DEBUG)) + + def cbResponse(self, response): + """Manages the response using a Protocol class defined in __init__""" + try: + log.msg('Response received: %r' % response) + response.deliverBody(self.update_event_manager) + except Exception as ex: + log.err() + + def buildProtocol(self, addr): + log.msg("addr: %r" % addr) + return self.update_event_manager diff --git a/dockerdns/mappings.py b/dockerdns/mappings.py new file mode 100644 index 0000000..aff6d53 --- /dev/null +++ b/dockerdns/mappings.py @@ -0,0 +1,138 @@ +""" + Maps DNS requests to host +""" +import re +from twisted.python import log + + +class DockerMapping(object): + """ + Look up docker info via docker.api and + + XXX Should it be dns-agnostic, and just a wrapper around docker.api + + TODO: + + """ + + def __init__(self, db=None): + """ + Args: + db: a DockerDB instance containing infos + """ + self.db = db + + def lookup_container(self, name): + """ + Gets the container config from container name or hostname + + Args: + name: DNS query name to look up + + Returns: + Container config dict for the first matching container + """ + for map_name, search_f in (('name', self.db.get_by_name), + ('hostname', self.db.get_by_hostname)): + try: + log.msg('lookup container by %r: %r' % (map_name, name)) + + return search_f(name) + except KeyError as e: + # warn(str(e)) + log.msg("Container %r not found: %r" % (map_name, name)) + except Exception as e: + log.err("Unmanaged error: %r" % e) + + return None + + def get_a(self, name): + """ + Get an IPv4 address from a query name to be used in A record lookups + + :name: DNS query name to look up + :return: IPv4 address for the query name given + """ + + container = self.lookup_container(name) + + if container is None: + log.msg("No container found") + return + + addr = container['NetworkSettings']['IPAddress'] + + if not addr: + log.msg("No IPAddress associated with container %r" % container) + return None + + return addr + + def get_a_multi(self, image): + """ + Return the IPs matching the given image name + :param image: + :return: a tuple of {(addr1, name1), .., (addrX, nameX)} + """ + image, count = re.subn(r'\.\*$', '', image, 1) + if not count: + log.err("Not a multihost search: %r" % (image,)) + return + return tuple( + (container['NetworkSettings'][ + 'IPAddress'], container['Name'][1:]) + for container + in self.db.get_by_image(image) + if 'NetworkSettings' in container + ) + + def get_nat(self, container_name, sport=0, sproto=None): + """ @return - a generator of natted maps (local, proto, nat, ip) + + @param sport: the port to search + @param sproto: the protocol to search + + eg. [ (8080, 'tcp', 18080, '0.0.0.0'), + (8787, 'tcp', 8787, '0.0.0.0'), + ]rfiutato + """ + sport = int(sport) + container = self.lookup_container(container_name) + if container is None: + log.msg("No container found") + return + + try: + nats = container['NetworkSettings']['Ports'].items() + except (KeyError, AttributeError) as ex: + # container infos set and not None + log.err("Bad network information for container: %r" % container) + + for local, remote in nats: + if not remote: + continue + # + # filter for port and protocol + # + port, proto = local.split("/") + port = int(port) + if sport and sport != port: + continue + if sproto and sproto != proto: + continue + + for r in remote: + try: + yield (port, proto, int(r['HostPort']), r['HostIp']) + except (ValueError, KeyError) as ex: + log.err() + continue + + def get_ptr(self, ip): + """ + Return the Hostname of the Container with the given IP + :param ip: + :return: + """ + c = self.db.get_by_ip(ip) + return c['Config']['Hostname'] diff --git a/dockerdns/resolver.py b/dockerdns/resolver.py new file mode 100644 index 0000000..2a7d2f8 --- /dev/null +++ b/dockerdns/resolver.py @@ -0,0 +1,336 @@ +""" +author: robipolli@gmail.com + +Extending twisted.names.common.ResolverBase to +reply with the informations provided by +dockerdns.mappings.DockerMapping + +""" +import re +import socket +from twisted.python import log +from twisted.internet import defer +from twisted.internet.defer import failure +from twisted.names import common, dns +from twisted.names.error import DomainError, DNSQueryTimeoutError +from dockerdns.utils import get_preferred_ip + + +NO_NXDOMAIN = 'no_nxdomain' + + +# pylint:disable=too-many-public-methods +# pylint:disable=too-many-instance-attributes +class DockerResolver(common.ResolverBase): + """ + DNS resolver to resolve queries with a DockerMapping instance. + + Twisted Names just uses the lookupXXX methods + """ + mock_records = [ + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + payload=dns.Record_SRV( + priority=100, weight=100, port=19999, target='name', ttl=None), + auth=True), + dns.RRHeader( + "mock_name", dns.SRV, dns.IN, 86400, + payload=dns.Record_SRV( + priority=100, weight=100, port=18080, target='name', ttl=None), + auth=True) + ] + + def __init__(self, mapping, config=None): + """ + :param mapping: DockerMapping instance for lookups + + TODO: configurable --bip + """ + + self.mapping = mapping + self.config = config or { + 'domain': 'docker', + 'bip': '172.17.0.0/16', + 'ttl': 10 + } + self.re_domain = re.compile(r'\.' + self.config['domain'] + '$') + self.re_ptr = re.compile(r'[0-9]+\.[0-9]+\.17\.172\.in-addr\.arpa$') + # Change to this ASAP when Twisted uses object base + # super(DockerResolver, self).__init__() + common.ResolverBase.__init__(self) + self.ttl = int(self.config['ttl']) + self.my_preferred_ip, self.my_preferred_ip_ptr_value \ + = get_preferred_ip() + + # define authority and additional records + # an authority record defines the name IN NS TIMEOUT + # socket. + self.authority = [dns.RRHeader( + name=self.config['domain'] + ".", type=dns.NS, cls=dns.IN, + payload=dns.Record_NS(name=socket.gethostname()) + )] + self.additional = [dns.RRHeader( + name=socket.gethostname(), type=dns.A, cls=dns.IN, + payload=dns.Record_A(address=self.my_preferred_ip)) + ] + + def _a_records(self, name): + """ + Get A records from a query name + + :param name: DNS query name to look up + + :return: Tuple of formatted DNS replies + """ + + addr = self.mapping.get_a(name) + if not addr: + raise DomainError(name) + + return tuple([ + dns.RRHeader( + '.'.join((name, self.config['domain'])), + dns.A, dns.IN, self.ttl, + payload=dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def _srv_records(self, name): + print("getting srv: %r" + name) + addr = self.mapping.get_a(name) + return tuple([ + dns.RRHeader( + '.'.join((name, self.config['domain'])), + dns.SRV, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def _ptr_record(self, name): + """ + Get PTR records from a query name + + :param name: DNS query name to look up + + :return: Tuple of formatted DNS replies + """ + # convert x.y.z.q.in-addr.arpa -> q.z.y.x + ip_addr = '.'.join(reversed(name.split(".")[:4])) + addr = self.mapping.get_ptr(ip_addr) + if not addr: + raise DomainError(name) + addr = '.'.join((addr, self.config['domain'])) + return tuple([ + dns.RRHeader( + name, + dns.PTR, dns.IN, self.ttl, + payload=dns.Record_PTR(addr, self.ttl), + auth=self.config.get('authoritative')) + ]) + + def lookupAddress(self, name, timeout=None): + """ + + :param name: a name like container_name.docker, hostname.docker, + image_name.*.docker + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred + + """ + name, occurrences = self.re_domain.subn('', name) + if not occurrences: + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure( + DomainError("not ending with docker")) + ) + + try: + if name.endswith(".*"): + a_multi = self.mapping.get_a_multi(name) + log.msg(a_multi) + records = tuple( + dns.RRHeader( + '.'.join((name_, self.config['domain'])), + dns.A, dns.IN, self.ttl, + dns.Record_A(addr_, self.ttl), + auth=self.config.get('authoritative') + ) + for addr_, name_ + in a_multi + if addr_ and name_ # skip empty entries + ) + else: + records = self._a_records(name) + return defer.succeed((records, self.authority, self.additional)) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as ex: + log.msg("DomainError: %r " % ex) + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) + except Exception as ex: # pylint:disable=bare-except + import traceback + + traceback.print_exc() + + if self.config.get(NO_NXDOMAIN): + log.err() + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + + def lookupIPV6Address(self, name, timeout=None): + """ + + :param name: a name like container_name.docker, hostname.docker, + image_name.*.docker + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred + + """ + name, occurrences = self.re_domain.subn('', name) + if not occurrences: + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure( + DomainError("not ending with docker")) + ) + + # Raise exception if host not found + try: + records = self._a_records(name) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as ex: + log.msg("DomainError: %r " % ex) + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) + except Exception as ex: # pylint:disable=bare-except + import traceback + + traceback.print_exc() + + if self.config.get(NO_NXDOMAIN): + log.err() + # FIXME surely there's a better way to give SERVFAIL + exception = DNSQueryTimeoutError(name) + else: + exception = DomainError(name) + + return defer.fail(failure.Failure(exception)) + # Otherwise no RR -> Answer RRs: 0 + empty_record = tuple() + return defer.succeed((empty_record, self.authority, self.additional)) + + def lookupService(self, name, timeout=None): + """ Lookup a docker natted service of + the form: NATTEDPORT._tcp.CONTAINERNAME.docker. + and returns a srv record of the for: + _service._proto.name. TTL class SRV priority weight port target. + + If _service == _nat: + :return: A Deferred firing a three-tuple of lists of + twisted.names.dns.RRHeader instances. + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or with NotImplementedError. + :type: Deferred + """ + name, occurrences = self.re_domain.subn('', name) + if not occurrences: + log.err("Domain not ending with {domain}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure(DomainError( + "not ending with {domain}".format(**self.config))) + ) + try: + port, proto, container = name.split(".") + port = int(port.strip("_")) + except (IndexError, TypeError, ValueError) as ex: + log.err("Domain not of the right form: %r" % name) + return defer.fail(failure.Failure( + DomainError("not of the right form")) + ) + + records = [dns.RRHeader( + name, dns.SRV, dns.IN, self.ttl, + payload=dns.Record_SRV( + priority=100, + weight=100, + port=c_nat_port, + target=self.my_preferred_ip_ptr_value, + ttl=None), + auth=True) + for c_port, protocol, c_nat_port, target + in self.mapping.get_nat(container) + if c_port == port # eventually filter + ] + return defer.succeed((records, self.authority, self.additional)) + + def lookupPointer(self, name, timeout=None): + """ + + :param name: a ptr name like 1.0.17.172.in-addr.arpa + :param timeout: + :return: A deferred firing a 3-tuple + The first element of the tuple gives answers. + The second element of the tuple gives authorities. + The third element of the tuple gives additional information. + The Deferred may instead fail with one of the exceptions + defined in twisted.names.error or + with NotImplementedError. + :type: Deferred + + """ + if not self.re_ptr.match(name): + log.err("Domain not in docker network {bip}: {name}".format( + name=name, **self.config)) + return defer.fail(failure.Failure( + DomainError("not in docker network")) + ) + + try: + records = self._ptr_record(name) + return defer.succeed((records, self.authority, self.additional)) + + # We need to catch everything. Uncaught exceptions will make the server + # stop responding + except DomainError as ex: + log.msg("DomainError: %r " % ex) + except Exception as ex: # pylint:disable=bare-except + import traceback + traceback.print_exc() + ex = DomainError(name) + # + # With NO_NXDOMAIN we mask everything with a Timeout + # + if self.config.get(NO_NXDOMAIN): + # FIXME surely there's a better way to give SERVFAIL + ex = DNSQueryTimeoutError(name) + return defer.fail(failure.Failure(ex)) diff --git a/dockerdns/sftp/__init__.py b/dockerdns/sftp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockerdns/sftp/checkers.py b/dockerdns/sftp/checkers.py new file mode 100644 index 0000000..bfa7d45 --- /dev/null +++ b/dockerdns/sftp/checkers.py @@ -0,0 +1,59 @@ +# -*- test-case-name: twisted.conch.test.test_checkers -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Provide L{ICredentialsChecker} implementations to be used in Conch protocols. +""" + +try: + import pwd +except ImportError: + pwd = None +else: + import crypt + +try: + # Python 2.5 got spwd to interface with shadow passwords + import spwd +except ImportError: + spwd = None + try: + import shadow + except ImportError: + shadow = None +else: + shadow = None + +try: + from twisted.cred import pamauth +except ImportError: + pamauth = None + +from zope.interface import implementer + +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class PermitChecker: + """ + A checker which validates users out of the UNIX password databases, or + databases of a compatible format. + + @ivar _getByNameFunctions: a C{list} of functions which are called in order + to valid a user. The default value is such that the /etc/passwd + database will be tried first, followed by the /etc/shadow database. + """ + credentialInterfaces = IUsernamePassword, + + def __init__(self, getByNameFunctions=None): + pass + + def requestAvatarId(self, credentials): + return defer.succeed(credentials.username) + # fallback + return defer.fail(UnauthorizedLogin("unable to verify password")) diff --git a/dockerdns/sftp/factory.py b/dockerdns/sftp/factory.py new file mode 100644 index 0000000..11ab198 --- /dev/null +++ b/dockerdns/sftp/factory.py @@ -0,0 +1,71 @@ +# -*- test-case-name: twisted.conch.test.test_openssh_compat -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Factory for reading openssh configuration files: public keys, private keys, and +moduli file. +""" + +import os +import errno + +from twisted.python import log +from twisted.python.util import runAsEffectiveUser + +from twisted.conch.ssh import keys, factory, common +from twisted.conch.openssh_compat import primes + + +class OpenSSHFactory(factory.SSHFactory): + dataRoot = '.' + # for openbsd which puts moduli in a different directory from keys + moduliRoot = '.' + + def getPublicKeys(self): + """ + Return the server public keys. + """ + ks = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-8:] == '_key.pub': + try: + k = keys.Key.fromFile( + os.path.join(self.dataRoot, filename)) + t = common.getNS(k.blob())[0] + ks[t] = k + except Exception, e: + log.msg('bad public key file %s: %s' % (filename, e)) + return ks + + def getPrivateKeys(self): + """ + Return the server private keys. + """ + privateKeys = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-4:] == '_key': + fullPath = os.path.join(self.dataRoot, filename) + try: + key = keys.Key.fromFile(fullPath) + except IOError, e: + if e.errno == errno.EACCES: + # Not allowed, let's switch to root + key = runAsEffectiveUser( + 0, 0, keys.Key.fromFile, fullPath) + keyType = keys.objectType(key.keyObject) + privateKeys[keyType] = key + else: + raise + except Exception, e: + log.msg('bad private key file %s: %s' % (filename, e)) + else: + keyType = keys.objectType(key.keyObject) + privateKeys[keyType] = key + return privateKeys + + def getPrimes(self): + try: + return primes.parseModuliFile(self.moduliRoot+'/moduli') + except IOError: + return None diff --git a/dockerdns/sftp/unix.py b/dockerdns/sftp/unix.py new file mode 100644 index 0000000..8994c41 --- /dev/null +++ b/dockerdns/sftp/unix.py @@ -0,0 +1,225 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +import os +import re +import struct +from zope.interface import implementer + +from twisted.conch.avatar import ConchUser +from twisted.conch.ssh import session, forwarding, filetransfer +from twisted.conch.unix import (UnixSFTPFile, SSHSessionForUnixConchUser, + SFTPServerForUnixConchUser, UnixSFTPDirectory) +from twisted.cred import portal +from twisted.python import components, log + + +@implementer(portal.IRealm) +class DockerRealm: + def __init__(self, container_db): + self.container_db = container_db + + def requestAvatar(self, username, mind, *interfaces): + user = DockerVolumeConchUser(username, self.container_db) + return interfaces[0], user, user.logout + + +class DockerVolumeConchUser(ConchUser): + def __init__(self, container_name, container_db): + ConchUser.__init__(self) + self.container_name = container_name + container = container_db.get_by_name(container_name) + self.volumes = { + k.encode('utf-8'): v.encode('utf-8') + for k, v + in container.get("Volumes", {}).items() + } + self.listeners = {} # dict mapping (interface, port) -> listener + self.channelLookup.update( + {"session": session.SSHSession, + "direct-tcpip": forwarding.openConnectForwardingClient}) + + self.subsystemLookup.update( + {"sftp": filetransfer.FileTransferServer}) + + def getUserGroupId(self): + return os.getuid(), os.getgid() + + def getOtherGroups(self): + return tuple() + + def getHomeDir(self): + return "/" + + def getShell(self): + return "/sbin/nologin" + + def global_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + from twisted.internet import reactor + + try: + listener = self._runAsUser( + reactor.listenTCP, portToBind, + forwarding.SSHListenForwardingFactory( + self.conn, + (hostToBind, portToBind), + forwarding.SSHListenServerForwardingChannel + ), + interface=hostToBind) + except: + return 0 + else: + self.listeners[(hostToBind, portToBind)] = listener + if portToBind == 0: + portToBind = listener.getHost()[2] # the port + return 1, struct.pack('>L', portToBind) + else: + return 1 + + def global_cancel_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + listener = self.listeners.get((hostToBind, portToBind), None) + if not listener: + return 0 + del self.listeners[(hostToBind, portToBind)] + self._runAsUser(listener.stopListening) + return 1 + + def logout(self): + # remove all listeners + for listener in self.listeners.itervalues(): + self._runAsUser(listener.stopListening) + log.msg('avatar %s logging out (%i)' % ( + self.container_name, len(self.listeners))) + + def _runAsUser(self, f, *args, **kw): + euid = os.geteuid() + egid = os.getegid() + groups = os.getgroups() + uid, gid = self.getUserGroupId() + # os.setegid(0) + # os.seteuid(0) + # os.setgroups(self.getOtherGroups()) + # os.setegid(gid) + # os.seteuid(uid) + try: + f = iter(f) + except TypeError: + f = [(f, args, kw)] + try: + for i in f: + func = i[0] + args = len(i) > 1 and i[1] or () + kw = len(i) > 2 and i[2] or {} + if func in (os.lstat,): + for from_, to_ in self.volumes.items(): + args = [re.sub(from_, to_, x) for x in args] + r = func(*args, **kw) + finally: + # os.setegid(0) + # os.seteuid(0) + # os.setgroups(groups) + # os.setegid(egid) + # os.seteuid(euid) + pass + return r + + +class SFTPServerForDockerVolumeConchUser(SFTPServerForUnixConchUser): + def __init__(self, avatar): + SFTPServerForUnixConchUser.__init__(self, + avatar) + self.allowed_volumes, self.allowed_paths = zip( + *self.avatar.volumes.items()) + + def _fits(self, volume, path): + should_empty, expected, rest = path.partition(volume) + if should_empty is not '': + return False + if expected is not volume: + return False + if rest == '': + return True + if rest[0] == '/': + return True + return False + + def _best_fit(self, path): + """ Return the volume containing directory + """ + matching_volumes = ( + x + for x in self.allowed_volumes + if self._fits(x, path) + ) + return sorted(matching_volumes, key=len, reverse=True) + + def _absPath(self, path): + # You can only visit volumes + allowed_paths = tuple(self.avatar.volumes.keys()) + (".", "/") + home = self.avatar.getHomeDir() + + if path.startswith(allowed_paths): + apath = os.path.abspath(os.path.join(home, path)) + if apath == "/": + return apath + best_fit = self._best_fit(apath) + if best_fit: + volume = best_fit[0] + folder = self.avatar.volumes[volume] + log.msg("Volume: %r, %r" % (volume, folder)) + apath = re.sub('^' + volume, folder, apath) + return apath + # By default you're not authorized + log.msg("not outside container path: %r" % [ + path, + self.avatar.volumes.keys()] + ) + raise OSError(2, "File not found: %r" % path, path) + + def openFile(self, filename, flags, attrs): + return UnixSFTPFile(self, self._absPath(filename), flags, attrs) + + def openDirectory(self, path): + return DockerVolumeDirectory(self, self._absPath(path)) + + +class DockerVolumeDirectory(UnixSFTPDirectory): + """ + Expose volumes of an host as tree. + + """ + + @staticmethod + def _dirname(path): + try: + return path.split("/")[1] + except IndexError, TypeError: + return path + + def __init__(self, server, directory): + self.server = server + allowed_volumes, allowed_paths = zip( + *self.server.avatar.volumes.items()) + if directory == "/": + self.files = [self._dirname(x) + for x + in allowed_volumes] + elif any(x for x in allowed_volumes if x.startswith(directory)): + self.files = [self._dirname(x.replace(directory, "")) + for x + in allowed_volumes + if x.startswith(directory)] + else: + log.err("Accessing %r" % [directory]) + if not directory.startswith(allowed_paths): + raise OSError(2, "No such file or directory", directory) + self.files = server.avatar._runAsUser(os.listdir, directory) + self.dir = directory + + +components.registerAdapter(SFTPServerForDockerVolumeConchUser, + DockerVolumeConchUser, filetransfer.ISFTPServer) +components.registerAdapter( + SSHSessionForUnixConchUser, DockerVolumeConchUser, session.ISession) diff --git a/dockerdns/utils.py b/dockerdns/utils.py new file mode 100644 index 0000000..5622e5c --- /dev/null +++ b/dockerdns/utils.py @@ -0,0 +1,36 @@ +"""utilities.py +""" +import socket + + +def get_preferred_ip(): + """Return the in-addr name associated to the + ip used to contact the default gw""" + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + # connecting to a UDP address doesn't send packets + sock.connect(('8.8.8.8', 0)) + ip = sock.getsockname()[0] + return ip, '.'.join(list(reversed(ip.split(".")))) + ".in-addr.arpa" + except Exception as ex: + return socket.getfqdn() + + +def traverse_tree(haystack, key_path, default=None): + """ + Find an element in a nested dict, eg. + traverse_tree({'Net': {'IP': '1.1.1.1'}}, ['Net', 'IP']) == '1.1.1.1' + + :param haystack: The nested dictionary to search + :param key_path: An iterable containing an ordered list of dict keys to + traverse + :param default: Value to return in case nothing is found + :return:Value of the dict at the nested location given, + or default if no value was found + """ + for k in key_path: + if k in haystack: + haystack = haystack[k] + else: + return default + return haystack diff --git a/pylint.conf b/pylint.conf index c6af89c..f5c61c1 100644 --- a/pylint.conf +++ b/pylint.conf @@ -150,16 +150,19 @@ class-rgx=[A-Z_][a-zA-Z0-9]+$ function-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# modified for twisted camelized method names +method-rgx=[a-z_][a-zA-Z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +# two-chars variables and attrs are ok +attr-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +argument-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# including single-letter variables +variable-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct attribute names in class # bodies @@ -170,10 +173,10 @@ class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +good-names=a,b,c,d,e,s,f,i,j,k,v,ex,Run,_ # Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +bad-names=foo,bar,baz,toto,tutu,tata,l,o,I # Regular expression which should only match function or class names that do # not require a docstring. diff --git a/requirements.txt b/requirements.txt index b6b285a..c4b5bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -twisted==12.0.0 --e git+git://github.com/dotcloud/docker-py.git@3754edc2673996e8598b617f7d32d4ce035f81c5#egg=docker \ No newline at end of file +twisted>=14.0.0 +docker-py>=0.4.0 +simplejson diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..659266f --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,177 @@ +__author__ = 'rpolli' + +inspect_container_pandas_0 = { + 'Id': 'cidpandas0', + 'Same': 'Value', + 'Config': { + 'Hostname': 'furby-pandas', + 'Image': 'impandas' + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + 'Name': '/cidpandas0', + 'Image': 'imgid_pandas' +} +inspect_container_pandas = { + 'Id': 'cidpandas', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + 'Image': 'impandas' + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + 'Name': '/cidpandas', + 'Image': 'imgid_pandas' +} +inspect_container_foxes = { + 'Id': 'cidfoxes', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + 'Image': 'imfoxes' + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + }, + 'Name': '/cidfoxes', + 'Image': 'imgid_foxes' +} +inspect_container_sloths = { + 'Id': 'cidsloths', + 'Config': { + 'Hostname': 'stopped-sloths', + 'Image': 'imsloths' + }, + 'NetworkSettings': { + 'IPAddress': '' + }, + 'Name': '/cidsloths', + 'Image': 'imgid_sloths' +} +inspect_container_returns = { + 'cidpandas0': inspect_container_pandas_0, + 'cidpandas': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, +} +containers_return = [ + {'Id': 'cidpandas0'}, + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, +] + +mock_list_containers_2 = lambda *a, **k: [ + inspect_container_foxes, inspect_container_pandas, inspect_container_sloths, + inspect_container_pandas_0 +] +mock_inspect_containers_2 = lambda cid, **k: inspect_container_returns[cid] + +mock_list_containers = lambda *a, **k: [ + {u'Command': u'/bin/bash', + u'Created': 1414430175, + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'eap63_tracer:v6.3.1', + u'Names': [u'/jboss631'], + u'Ports': [{u'IP': u'0.0.0.0', + u'PrivatePort': 8080, + u'PublicPort': 18080, + u'Type': u'tcp'}, + {u'IP': u'0.0.0.0', + u'PrivatePort': 8787, + u'PublicPort': 8787, + u'Type': u'tcp'}, + {u'IP': u'0.0.0.0', + u'PrivatePort': 9999, + u'PublicPort': 19999, + u'Type': u'tcp'}, + {u'PrivatePort': 8443, u'Type': u'tcp'}, + {u'PrivatePort': 9990, u'Type': u'tcp'}], + u'Status': u'Up 2 days'} +] + + +mock_lookup_container = lambda *a, **k: { + u'Id': u'7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4', + u'Image': u'1fc3b15852c8cb8f5b195cee6c3c178b739b77411d9dbebbcbb3d5217f5a6ac6', + u'Name': u'/jboss631', + u'Args': [], + u'Config': {u'AttachStderr': True, + u'AttachStdin': True, + u'AttachStdout': True, + u'Cmd': [u'/bin/bash'], + u'CpuShares': 0, + u'Cpuset': u'', + u'Domainname': u'', + u'Entrypoint': None, + u'Env': [u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], + u'ExposedPorts': {u'8080/tcp': {}, + u'8443/tcp': {}, + u'8787/tcp': {}, + u'9990/tcp': {}, + u'9999/tcp': {}}, + u'Hostname': u'7d564ceb891b', + u'Image': u'eap63_tracer:v6.3.1', + u'Memory': 0, + u'MemorySwap': 0, + u'NetworkDisabled': False, + u'OnBuild': None, + u'OpenStdin': True, + u'PortSpecs': None, + u'StdinOnce': True, + u'Tty': True, + u'User': u'', + u'Volumes': {}, + u'WorkingDir': u''}, + u'Created': u'2014-10-27T17:16:15.261857884Z', + u'Driver': u'devicemapper', + u'ExecDriver': u'native-0.2', + u'HostConfig': {u'Binds': [u'/home/rpolli/Downloads/:/mnt/tmp'], + u'CapAdd': None, + u'CapDrop': None, + u'ContainerIDFile': u'', + u'Devices': [], + u'Dns': None, + u'DnsSearch': None, + u'Links': None, + u'LxcConf': [], + u'NetworkMode': u'bridge', + u'PortBindings': {u'8080/tcp': [{u'HostIp': u'', u'HostPort': u'18080'}], + u'8787/tcp': [{u'HostIp': u'', u'HostPort': u'8787'}], + u'9999/tcp': [{u'HostIp': u'', u'HostPort': u'19999'}]}, + u'Privileged': False, + u'PublishAllPorts': False, + u'RestartPolicy': {u'MaximumRetryCount': 0, u'Name': u''}, + u'VolumesFrom': None}, + u'HostnamePath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hostname', + u'HostsPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/hosts', + u'MountLabel': u'', + u'NetworkSettings': {u'Bridge': u'docker0', + u'Gateway': u'172.17.42.1', + u'IPAddress': u'172.17.0.10', + u'IPPrefixLen': 16, + u'PortMapping': None, + u'Ports': {u'8080/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'18080'}], + u'8443/tcp': None, + u'8787/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'8787'}], + u'9990/tcp': None, + u'9999/tcp': [{u'HostIp': u'0.0.0.0', u'HostPort': u'19999'}]}}, + u'Path': u'/bin/bash', + u'ProcessLabel': u'', + u'ResolvConfPath': u'/var/lib/docker/containers/7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4/resolv.conf', + u'State': {u'ExitCode': 0, + u'FinishedAt': u'0001-01-01T00:00:00Z', + u'Paused': False, + u'Pid': 8662, + u'Restarting': False, + u'Running': True, + u'StartedAt': u'2014-10-27T17:16:15.756653452Z'}, + u'Volumes': {u'/mnt/tmp': u'/home/rpolli/Downloads'}, + u'VolumesRW': {u'/mnt/tmp': True} +} +mock_get_ptr = lambda *a, **k: mock_lookup_container()['Name'][1:] + +SRV_FMT = "_{svc}._{proto}.{container}.docker TTL {cclass} SRV {priority} {weight} {port} {target}" diff --git a/test/test_aaaa.py b/test/test_aaaa.py new file mode 100644 index 0000000..729659e --- /dev/null +++ b/test/test_aaaa.py @@ -0,0 +1,130 @@ +#!/usr/bin/python + +""" +Tests for AAAA queries + +Author: Roberto Polli +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + + +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2 +from nose.tools import * + +# FIXME I can not believe how disgusting this is + + +def in_generator(gen, val): + return reduce( + lambda old, new: old or new == val, + gen, + False + ) + + +def check_record(record, **expected): + """ + Compare a record with the values of the kwargs + :param record: + :param expected: + :return: + """ + for k in expected: + real_value = getattr(record, k) + if k is 'name': + real_value = real_value.name + + if real_value != expected[k]: + log.err("Expected %s: %s vs %s" % (k, expected[k], real_value)) + return False + + return True + + +def check_deferred(deferred, success): + completed = [] + + def gimme_x_back(is_success): + def x_back(result): + completed.append((is_success, result)) + + return x_back + + deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) + if len(completed) != 1: + return False + + status, result = completed[0] + if status != success: + raise AssertionError("Expected: %r, got %r" % (success, result)) + return False + + return result + + +class TestAAAA(object): + def setUp(self): + self.CONFIG = {} + + self.db = create_mock_db2() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + assert_equal(len(rec), 1) + rec = rec[0] + assert_true(check_record(rec, **expected_record)) + return rec + + + # + # TEST lookupIPV6Address + # + def test_lookupIPV6Address_id_empty_response(self): + # skip testing authority_rr, additional_rr + # as we're now populating the authority and additional section + expected_record = tuple() + deferred = self.resolver.lookupIPV6Address('cidfoxes.docker') + + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + + # We are returning an empty reply + # because we're not supporting IPV6 + assert_equal(len(response_rr), 0) + + def test_lookupIPV6Address_invalid(self): + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + + def test_lookupIPV6Address_invalid_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = False + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal( + result.type, DomainError) # noqa pylint:disable=maybe-no-member + + def test_lookupIPV6Address_invalid_no_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = True + deferred = self.resolver.lookupIPV6Address('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal(result.type, DNSQueryTimeoutError) + # noqa pylint:disable=maybe-no-member + diff --git a/test/test_events.py b/test/test_events.py new file mode 100644 index 0000000..27f7e18 --- /dev/null +++ b/test/test_events.py @@ -0,0 +1,119 @@ +from dockerdns.events import DockerDB, EventManager +import docker + +from test import mock_list_containers, mock_lookup_container, mock_list_containers_2, mock_inspect_containers_2 +from twisted.python import log +from twisted.web.client import Agent, Response, ResponseFailed +from twisted.internet import defer +from twisted.test.proto_helpers import MemoryReactor + +from nose import SkipTest + + +def create_mock_db(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers + api.inspect_container = mock_lookup_container + return DockerDB(api=api) + + +def create_mock_db2(): + """Create a mock DockerDB""" + api = docker.Client() + api.containers = mock_list_containers_2 + api.inspect_container = mock_inspect_containers_2 + return DockerDB(api=api) + + +def test_init(): + db = create_mock_db() + ret = db.get_by_name('jboss631') + assert ret, "Missing container" + try: + assert ret['NetworkSettings'][ + 'IPAddress'] == '172.17.0.10', "item %r" % ret + except KeyError as e: + log.err("Bad container %r" % ret) + raise + + +def test_get_by_ip(): + db = create_mock_db() + ret = db.get_by_ip('172.17.0.10') + assert ret, "Missing container" + try: + assert ret['NetworkSettings'][ + 'IPAddress'] == '172.17.0.10', "item %r" % ret + except KeyError as e: + log.err("Bad container %r" % ret) + raise + + +def test_init_and_get_images(): + db = create_mock_db2() + # images are correctly added to the indexes + assert 'impandas' in db.mappings_image, "%r, %r" % ( + db.mappings_image, db.mappings) + assert 'cidpandas' in db.mappings_image['impandas'], "%r, %r" % ( + db.mappings_image, db.mappings) + + # images are correctly retrieved + pandas_container = [x['Id'] for x in db.get_by_image('impandas')] + assert set(pandas_container) == set( + ('cidpandas', 'cidpandas0')), pandas_container + + +def test_clean_db(): + db = create_mock_db2() + db.cleandb() + assert not db.mappings, "Mappings not clean" + assert not db.mappings_name, "Mappings not clean" + assert not db.mappings_hostname, "Mappings not clean" + assert not db.mappings_image, "Mappings not clean" + + +def test_reload_container(): + db = create_mock_db2() + db.mappings = {} + db.load_containers() + assert db.mappings + assert 'impandas' in db.mappings_image, "%r, %r" % ( + db.mappings_image, db.mappings) + + +def test_check_volumes(): + db = create_mock_db() + ret = db.get_by_name('jboss631') + assert '/mnt/tmp' in ret['Volumes'] + + +class MockAgent(Agent): + reasons = ['No Reason'] + + def request(self, method, uri, **kw): + #return defer.fail(ResponseFailed(reasons=self.reasons, response=None)) + d = defer.Deferred() + d.addCallback(lambda *args: "ciao") + return d + + +class TestEventManager(object): + def setup(self): + self.config = dict(docker_url='http://localhost:5000') + self.db = create_mock_db() + self.reactor = MemoryReactor() + self.http_agent = MockAgent(self.reactor) + + def test_delete_record(self): + em = EventManager( + http_agent=self.http_agent, config=self.config, db=self.db) + assert em + em.delete_record({'status': 'stop', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) + + @SkipTest + def test_update_record(self): + em = EventManager( + http_agent=self.http_agent, config=self.config, db=self.db) + resp = yield em.update_record({'status': 'start', 'id': '7d564ceb891bb0b2997210936392c1b893e4e438b4fae5b874aa7b5e6137f0d4'}) + assert resp diff --git a/test/test_ptr.py b/test/test_ptr.py new file mode 100644 index 0000000..35366ed --- /dev/null +++ b/test/test_ptr.py @@ -0,0 +1,55 @@ +""" + Creating Ptr records + +""" +import twisted +from dockerdns.resolver import DockerResolver +from dockerdns.mappings import DockerMapping +from test import mock_lookup_container, mock_get_ptr +from nose import SkipTest +from nose.tools import raises +from nose.twistedtools import deferred as nosedeferred +from test.test_resolver import check_deferred + + +class TestPtr(object): + """ + Test lookupPointer record mocking DockerMapping functions. + + This class doesn't test DockerDB + """ + mapping = DockerMapping(db=None) + mapping.lookup_container = mock_lookup_container + mapping.get_ptr = mock_get_ptr + + def check_equals(self, a, b, msg=None): + assert a == b, msg + + def setup(self): + self.resolver = DockerResolver(self.mapping) + + @raises(twisted.names.error.DomainError) + @nosedeferred() + def harn_lookupPtr_ko(self, n): + """Harness to run lookup in a deferred + """ + return self.resolver.lookupPointer(n) + + def test_lookupPtr_ko(self): + expect_fail = ('166.192.in-addr.arpa').split() + for n in expect_fail: + yield self.harn_lookupPtr_ko, n + + def test_lookupPtr_ok(self): + ret = self.resolver.lookupPointer("10.0.17.172.in-addr.arpa") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_ptr_ok(self): + ret, = self.resolver._ptr_record("10.0.17.172.in-addr.arpa") + assert ret + + def test_setup(self): + assert self.resolver + assert self.resolver.mapping + assert self.resolver.mapping.db is None diff --git a/test/test_resolver.py b/test/test_resolver.py new file mode 100644 index 0000000..b38e529 --- /dev/null +++ b/test/test_resolver.py @@ -0,0 +1,199 @@ +#!/usr/bin/python + +""" +Tests for Docker DNS + +Author: Ricky Cook +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + + +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2 +from nose.tools import * + +# FIXME I can not believe how disgusting this is + + +def in_generator(gen, val): + return reduce( + lambda old, new: old or new == val, + gen, + False + ) + + +def check_record(record, **expected): + """ + Compare a record with the values of the kwargs + :param record: + :param expected: + :return: + """ + for k in expected: + real_value = getattr(record, k) + if k is 'name': + real_value = real_value.name + + if real_value != expected[k]: + log.err("Expected %s: %s vs %s" % (k, expected[k], real_value)) + return False + + return True + + +def check_deferred(deferred, success): + completed = [] + + def gimme_x_back(is_success): + def x_back(result): + completed.append((is_success, result)) + + return x_back + + deferred.addCallbacks(gimme_x_back(True), gimme_x_back(False)) + if len(completed) != 1: + return False + + status, result = completed[0] + if status != success: + raise AssertionError("Expected: %r, got %r" % (success, result)) + return False + + return result + + +class TestDockerResolver(object): + def setUp(self): + self.CONFIG = {} + + self.db = create_mock_db2() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + assert_equal(len(rec), 1) + rec = rec[0] + assert_true(check_record(rec, **expected_record)) + return rec + + # + # TEST _a_records + # + def test__a_records_hostname(self): + name, expected_record = 'sneaky-foxes', {'name': + 'sneaky-foxes.docker', 'type': dns.A} + rec = self.harn_expected(name, expected_record) + assert_equal(rec.payload.dottedQuad(), '8.8.8.8') + + def test__a_records_id(self): + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A} + rec = self.harn_expected(name, expected_record) + assert_equal(rec.payload.dottedQuad(), '127.0.0.1') + + @raises(DomainError) + def test__a_records_shutdown(self): + self.resolver._a_records('cidsloths.docker') + + @raises(DomainError) + def test__a_records_invalid(self): + self.resolver._a_records('invalid.docker') + + @raises(DomainError) + def test__a_records_blank_query(self): + self.resolver._a_records("") + + def test__a_records_authoritative(self): + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A, 'auth': True} + self.resolver.config['authoritative'] = True + self.harn_expected(name, expected_record) + + def test__a_records_non_authoritative(self): + name, expected_record = 'cidpandas', {'name': + 'cidpandas.docker', 'type': dns.A, 'auth': False} + self.resolver.config['authoritative'] = False + self.harn_expected(name, expected_record) + + # + # TEST lookupAddress + # + def test_lookupAddress_id(self): + expected_record, expected_authority, expected_additional = ( + {'name': 'cidfoxes.docker', 'type': dns.A}, + tuple(), + tuple() + ) + deferred = self.resolver.lookupAddress('cidfoxes.docker') + + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + # skip this tests as we're now populating + # the authority and additional section + assert_equal(len(response_rr), 1) + + rec = response_rr[0] + assert_true(check_record( + rec, + **expected_record + )) + assert_equal(rec.payload.dottedQuad(), '8.8.8.8') + + def test_lookupAddress_invalid(self): + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + + def test_lookupAddress_invalid_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = False + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal( + result.type, DomainError) # noqa pylint:disable=maybe-no-member + + def test_lookupAddress_invalid_no_nxdomain(self): + self.resolver.config[NO_NXDOMAIN] = True + deferred = self.resolver.lookupAddress('invalid.docker') + + result = check_deferred(deferred, False) + assert_not_equal(result, False) + assert_equal(result.type, DNSQueryTimeoutError) + # noqa pylint:disable=maybe-no-member + + def test_lookupAddress_multi(self): + # search by image + # host -t a impandas.*.docker + # + expected_records = ( + {'name': 'cidpandas.docker'}, + {'name': 'cidpandas0.docker'} + ) + self.resolver.config[NO_NXDOMAIN] = False + + # retrieve hosts by image + deferred = self.resolver.lookupAddress('impandas.*.docker') + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + assert_equal(len(response_rr), len(expected_records)) + + for rec, expected_record in zip(response_rr, expected_records): + assert_true(check_record( + rec, + **expected_record + )) diff --git a/test/test_resolver_multi.py b/test/test_resolver_multi.py new file mode 100644 index 0000000..44cb997 --- /dev/null +++ b/test/test_resolver_multi.py @@ -0,0 +1,76 @@ +#!/usr/bin/python + +""" +Tests for Docker DNS + +Author: Ricky Cook +""" + +# Do not care...... +# noqa pylint:disable=missing-docstring,too-many-public-methods,protected-access,invalid-name + +import unittest + +from twisted.names import dns +from twisted.names.error import DNSQueryTimeoutError, DomainError +from twisted.python import log + +from dockerdns.mappings import DockerMapping +from dockerdns.resolver import DockerResolver, NO_NXDOMAIN +from test.test_events import create_mock_db2, create_mock_db +from test.test_resolver import check_record, in_generator, check_deferred + +from nose.tools import * + + +class TestDockerResolver(object): + def setup(self): + self.CONFIG = {} + + self.db = create_mock_db() + self.mapping = DockerMapping(self.db) + self.resolver = DockerResolver(self.mapping) + + def harn_expected(self, name, expected_record): + rec = self.resolver._a_records(name) + assert_equal(len(rec), 1) + rec = rec[0] + assert_true(check_record(rec, **expected_record)) + return rec + + def harn_lookupAddress_multi(self, image_name, expected_records): + # search by image + # host -t a impandas.*.docker + # + + # retrieve hosts by image + deferred = self.resolver.lookupAddress(image_name) + result = check_deferred(deferred, True) + assert_not_equal(result, False) + + response_rr, authority_rr, additional_rr = result + assert_equal(len(response_rr), len(expected_records)) + + for rec, expected_record in zip(response_rr, expected_records): + assert_true(check_record( + rec, + **expected_record + )) + + def test_lookupAddress_multi(self): + self.resolver.config[NO_NXDOMAIN] = False + expected_records = ( + {'name': 'jboss631.docker'}, + # {'name': 'cidpandas0.docker'} + ) + self.harn_lookupAddress_multi( + 'eap63_tracer:v6.3.1.*.docker', expected_records) + + def test_lookupAddress_multi_notag(self): + self.resolver.config[NO_NXDOMAIN] = False + expected_records = ( + {'name': 'jboss631.docker'}, + # {'name': 'cidpandas0.docker'} + ) + self.harn_lookupAddress_multi( + 'eap63_tracer.*.docker', expected_records) diff --git a/test/test_sftpserver_volumes.py b/test/test_sftpserver_volumes.py new file mode 100644 index 0000000..032f48d --- /dev/null +++ b/test/test_sftpserver_volumes.py @@ -0,0 +1,113 @@ +""" + author: roberto.polli@par-tec.it + + This class tests the SFTP Volume access. + + To access: + - the container jboss63 + - with the volumes "/shared", "/var/log" + + # sftp -P 10022 jboss63@localhost # no password for now + # pwd + / + # ls + /shared + /var + +""" +from dockerdns.sftp import unix +from twisted.python import log +from nose.tools import (assert_true, assert_equal, raises, + assert_false, assert_is_instance) + +MOCK_CONTAINER = "foo" + + +class MockDB(): + """Mocking L{DockerDB}""" + + def get_by_name(self, *a, **kw): + return { + "Volumes": { + "/shared": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/shared1": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90", + "/var/log/sql": "/data/docker/vfs/dir/ecdc183a5d7a2369a8cf1539dfed788dc677b81e0589b3aab7f5854921adbe90" + } + } + +# initialize +db = MockDB() +user = unix.DockerVolumeConchUser(MOCK_CONTAINER, db) +srv = unix.SFTPServerForDockerVolumeConchUser(user) + + +def test_get_homedir_is_rootdir(): + assert user.getHomeDir() == "/" + + +def test_abspath_volumes_to_real_folder(): + for p in ( + '/shared', '/shared/foo.txt', '/shared1', '/var/log/sql', + '/var/log/sql/db.out' + ): + real_path = srv._absPath(p) + log.msg("real path is %r" % real_path) + yield assert_true, real_path.startswith("/data/docker"), \ + "Not starting with /data/docker: %r" % real_path + + +def test_abspath_files_outside_volumes(): + for p in '/var/log/messages /shared2 /shared2/'.split(): + ex = None + try: + real_path = srv._absPath(p) + print("real path is %r" % real_path) + except Exception as ex: + pass + yield assert_is_instance, ex, OSError + + +def test_unixdirectory_1(): + directory = unix.DockerVolumeDirectory(srv, '/') + expected_files = {'shared', 'var', 'shared1'} + assert_equal(set(directory.files), expected_files) + + +def test_unixdir_2(): + test_cases = [ + ('/var', {'log'}), + ('/var/log', {'sql'}) + ] + for dir, expected in test_cases: + directory = unix.DockerVolumeDirectory(srv, dir) + assert_equal, set(directory.files), expected, "%r" % [set(directory.files)] + + +@raises(OSError) +def test_unixdir_3(): + forbidden_dirs = "/root".split() + for dir in forbidden_dirs: + unix.DockerVolumeDirectory(srv, dir) + + +def test_fits(): + cases_true = [ + ("/var", "/var /var/log /var/log/messages".split()), + ("/s", "/s /s/a.out".split()), + ] + for v, paths in cases_true: + for p in paths: + yield harn_fits, assert_true, v, p + + +def test_fits_false(): + cases_true = [ + ('/s', "/s1 /s1/ /s1/a.out".split()) + ] + for v, paths in cases_true: + for p in paths: + yield harn_fits, assert_false, v, p + + +def harn_fits(checker, v, p): + checker(srv._fits(v, p)) diff --git a/test/test_srv.py b/test/test_srv.py new file mode 100644 index 0000000..e9a5867 --- /dev/null +++ b/test/test_srv.py @@ -0,0 +1,91 @@ +""" + Creating srv records + +""" +import twisted +from dockerdns.resolver import DockerResolver +from dockerdns.mappings import DockerMapping +from test import mock_lookup_container, SRV_FMT +from nose import SkipTest +from nose.tools import raises +from nose.twistedtools import deferred as nosedeferred +from test.test_resolver import check_deferred + + +class TestSrv(object): + mapping = DockerMapping(db=None) + mapping.lookup_container = mock_lookup_container + + def check_equals(self, a, b, msg=None): + assert a == b, msg + + def setup(self): + self.resolver = DockerResolver(self.mapping) + + def test_nat_all(self): + host, port = "foo.docker", 8080 + ret = self.mapping.get_nat(host, port) + assert (8080, "tcp", 18080, "0.0.0.0") in ret, "ret: %r" % ret + + def test_nat_ports(self): + expected = [(8080, 18080), + (9999, 19999), (8787, 8787)] + + for pin, pout in expected: + ret = self.mapping.get_nat("foo", pin) + _, _, port, _ = next(ret) + + yield self.check_equals, port, pout, "unexpected value in %r" % ret + + @raises(twisted.names.error.DomainError) + @nosedeferred() + def harn_lookupService_ko(self, n): + """Harness to run lookup in a deferred + """ + return self.resolver.lookupService(n) + + def test_lookupService_ko(self): + expect_fail = 'nondocker.domain noproto.docker noport.container.docker nonint._tcp.container.docker'.split( + ) + for n in expect_fail: + yield self.harn_lookupService_ko, n + + def test_lookupService_ok(self): + ret = self.resolver.lookupService("_8080._tcp.jboss631.docker") + ret = check_deferred(ret, True) + print("resolved: %r" % [ret]) + + def test_mapping(self): + container = self.mapping.lookup_container("foo") + + for local, remote in container['NetworkSettings']['Ports'].items(): + port, proto = local.split("/") + if not remote: + continue + try: + remote = remote[0] + except IndexError: + continue + + print(SRV_FMT.format( + svc=port, + proto=proto, + container=container['Name'][1:], + cclass="IN", + priority=100, + weight=100, + port=remote['HostPort'], + target=remote['HostIp'] if remote[ + 'HostIp'] != '0.0.0.0' else "localhost" + ) + ) + + def test_setup(self): + assert self.resolver + assert self.resolver.mapping + assert self.resolver.mapping.db is None + + @SkipTest + def test_service_image(self): + host = "impandas.docker" + raise NotImplementedError("implement skydns-like") diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..7aa9161 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,118 @@ +""" +Test the traverse_tree function +""" +import docker +import fudge +from dockerdns.utils import traverse_tree + + +theDict = { + 'pandas': { + 'are': 'cuddly', + 'and': 'awesome', + }, + 'foxes': { + 'are': 'sneaky', + 'and': 'orange', + }, + 'badgers': { + 'are': None, + }, +} + + +def harn_basic_check(given, expected, default=None): + ret = traverse_tree(theDict, given, default) + assert ret == expected, "given: %s, expected: %s, actual: %s, default: %s" % (given, expected, ret, default) + + +def test_basic_one(): + for given, expected in [ + ('pandas and', 'awesome'), + ('foxes are', 'sneaky'), + ('nothing', None), + ('pandas bad', None), + ('foxes', theDict['foxes']), + + ]: + yield harn_basic_check, given.split(), expected + + +def test_user_default(): + for given, default in [ + ('nothing', 'Nobody here but us chickens'), + ('pandas bad', 'NO, THAT\'S A DAMN DIRTY LIE'), + ]: + yield harn_basic_check, given.split(), default, default + + +def test_ignore_default(): + for given, expected, default in [ + ('badgers are', None, 'Badgers are none? What?') + ]: + yield harn_basic_check, given.split(), expected, default + + +class MockDockerClient(object): + base_url = 'http://localhost:5000' + version = lambda x: {'ApiVersion': '1.0'} + inspect_container_pandas = { + 'ID': 'cidpandaslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'cuddly-pandas', + }, + 'NetworkSettings': { + 'IPAddress': '127.0.0.1' + }, + } + inspect_container_foxes = { + 'ID': 'cidfoxeslong', + 'Same': 'Value', + 'Config': { + 'Hostname': 'sneaky-foxes', + }, + 'NetworkSettings': { + 'IPAddress': '8.8.8.8' + } + } + inspect_container_sloths = { + 'ID': 'cidslothslong', + 'Config': { + 'Hostname': 'stopped-sloths', + }, + 'NetworkSettings': { + 'IPAddress': '' + } + } + inspect_container_returns = { + 'cidpandas': inspect_container_pandas, + 'cidpandaslong': inspect_container_pandas, + 'cidfoxes': inspect_container_foxes, + 'cidfoxeslong': inspect_container_foxes, + 'cidsloths': inspect_container_sloths, + 'cidslothslong': inspect_container_sloths, + } + containers_return = [ + {'Id': 'cidpandas'}, + {'Id': 'cidfoxes'}, + {'Id': 'cidsloths'}, + ] + + inspect_container_id = None + + def inspect_container(self, cid): + self.inspect_container_id = cid + + try: + return self.inspect_container_returns[cid] + except KeyError: + # Mocks a Docker Client Exception + response = fudge.Fake() + response.has_attr(status_code=404, content='PANDAS!') + + exception = docker.client.APIError('bad', response) + raise exception + + def containers(self, *args, **kwargs): # pylint:disable=unused-argument + return self.containers_return diff --git a/test_requirements.txt b/test_requirements.txt index 731c279..2d6def5 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,5 @@ -r requirements.txt fudge==1.0.3 pep8==1.4.6 -pylint==1.0.0 \ No newline at end of file +pylint==1.0.0 +nose diff --git a/twisted/plugins/dockerdns_plugin.py b/twisted/plugins/dockerdns_plugin.py new file mode 100644 index 0000000..2fa6447 --- /dev/null +++ b/twisted/plugins/dockerdns_plugin.py @@ -0,0 +1,186 @@ +#!/usr/bin/python +""" +A simple TwistD DNS server using custom TLD and Docker as the back end for IP +resolution. + +To look up a container: + - 'A' record query a container NAME that will match + a container with a docker inspect + command with '.d' as the TLD. eg: mysql_server1.d + - 'SRV' record query to _port._srv.container.docker will return + the natted address. + eg. _3306._tcp.mysql_server1.docker returns + _18080._tcp.nice_bohr.docker. 10 IN SRV 100 100 8080 192.168.42.126. + +Code modified from +https://github.com/infoxchange/docker_dns + +Author: Bradley Cicenas +Author: Roberto Polli +""" +from __future__ import print_function, unicode_literals + +from zope.interface import implements +import json + +from twisted.application import service, internet +from twisted.cred.checkers import AllowAnonymousAccess +from twisted.names import dns, server +from twisted.python import log +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.application.service import IServiceMaker +import docker + +from dockerdns.mappings import DockerMapping +from dockerdns.events import DockerDB +from dockerdns.resolver import DockerResolver +from dockerdns.console import ConsoleFactory + + +class Options(usage.Options): + optParameters = [ + ["bind_port", "p", 10053, "The port number to listen on."], + ["bind_interface", "h", "127.0.0.1", "The host address to bind to"], + ["domain", "d", "docker", "The default domain"], + ["config", "c", "dockerdns.json", "Configuration file"], + ["docker_url", "u", 'unix://var/run/docker.sock', "Docker URL"], + ['no_nxdomain', "x", True, + "Return SERVFAIL instead of NXDOMAIN if container not found"], + ["authoritative", "A", True, "Return authoritative replies"], + ["ttl", "", 10, "The default TTL for those entries"], + ["docker-version", "V", '1.15', "Docker API version"], + ["bind_protocols", "B", ['tcp', 'udp'], "Bind protocols"], + ["sftp-bind", "S", "10022", "SFTP port to expose Volume access" + " eg. 10022. To specify an ip you can" + " use eg. 22:interface=127.0.0.1"], + ["sftp-keypath", "", "./", "Path to ssh_host_*key"], + ["sftp-moduli", "", None, "Path to moduli directory"] + + ] + + +class MyServiceMaker(object): + """ + Define a MultiService running: + - dns server for tcp and udp + - http client for retrieving docker events + """ + implements(IServiceMaker, IPlugin) + tapname = "dockerdns" + description = "Run this! It'll make your docker happy." + options = Options + + def sftpVolumeService(self, options, db): + """ + Construct a service for operating a SSH server. + + @param options: An L{Options} instance specifying server options, + including where server keys are stored. + + @param db: A L{DockerDB} instance with container information + + @return: An L{IService} provider which contains the requested + SSH server. + """ + from dockerdns.sftp import factory + from dockerdns.sftp import unix, checkers as docker_checkers + from twisted.cred import portal + from twisted.application import strports + # The factory just sets the ssh keys + t = factory.OpenSSHFactory() + + r = unix.DockerRealm(db) + t.portal = portal.Portal(r, [docker_checkers.PermitChecker()]) + t.dataRoot = options['sftp-keypath'] + t.moduliRoot = options['sftp-moduli'] or options['sftp-keypath'] + + port = options['sftp-bind'] + return strports.service(port, t) + + def makeService(self, options): + """ + Set everything up + """ + try: + with open(options['config']) as fh: + appcfg = json.load(fh) + appcfg = {k: v for k, v in appcfg.items() if k[0] is not '#'} + except IOError as e: + if options['config'] != "dockerdns.json": + raise + log.err("File {config} not found. Using default values".format( + **options)) + appcfg = {} + + # Update config stuff with command line params + options.update(appcfg) + appcfg = options + log.err("config: %r" % appcfg) + # Create docker: by default dict.get returns None on missing keys + # TODO: we could source more than one docker server + docker_client = docker.Client( + appcfg.get('docker_url'), version=appcfg.get('docker-version')) + infos = docker_client.info() + # Test docker connectivity before starting + log.msg("Connecting to docker instance: %r" % infos) + + db = DockerDB(api=docker_client) + # Create our custom mapping and resolver + mapping = DockerMapping(db=db) + resolver = DockerResolver(mapping, config=appcfg) + + # Create twistd stuff to tie in our custom components + factory = server.DNSServerFactory(clients=[resolver]) + factory.noisy = False + + # Protocols to bind + bind_list = [] + if 'tcp' in appcfg['bind_protocols']: + bind_list.append( + (internet.TCPServer, factory)) # noqa pylint:disable=no-member + + if 'udp' in appcfg['bind_protocols']: + proto = dns.DNSDatagramProtocol(factory) + proto.noisy = False + bind_list.append( + (internet.UDPServer, proto)) # noqa pylint:disable=no-member + + # Register the service + ret = service.MultiService() + for (InternetServerKlass, arg) in bind_list: + svc = InternetServerKlass( + int(appcfg['bind_port']), + arg, + interface=appcfg['bind_interface'] + ) + svc.setServiceParent(ret) + + # Add the event Loop + from dockerdns.events import EventFactory + from urlparse import urlparse + if appcfg['docker_url'].startswith("http"): + u = urlparse(appcfg['docker_url']) + efactory = EventFactory(config=appcfg, db=db) + docker_event_monitor = internet.TCPClient( + u.hostname, u.port, efactory) + docker_event_monitor.setServiceParent(ret) + else: + log.err("Cannot load the Event interface: " + "requires an http endpoint, " + "not {docker_url}".format(**appcfg)) + # Add the console + consoleFactory = ConsoleFactory(db) + console = internet.TCPServer(8080, consoleFactory) + console.setServiceParent(ret) + + # sftp Volume + sftp_volumes = self.sftpVolumeService(options, db) + sftp_volumes.setServiceParent(ret) + return ret + + +# +# Create the MultiService +# +serviceMaker = MyServiceMaker()