From e828cc9699eb7ff0339df89227fbcfe3fe122bba Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 10:42:00 +0100
Subject: [PATCH 01/14] Move ConfigError out of __init__.py so config sections
 can use it

prevents circular import
---
 sydent/config/__init__.py  | 4 ----
 sydent/config/_base.py     | 4 ++++
 sydent/replication/peer.py | 3 +--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/sydent/config/__init__.py b/sydent/config/__init__.py
index b21b586e..e0b57955 100644
--- a/sydent/config/__init__.py
+++ b/sydent/config/__init__.py
@@ -153,10 +153,6 @@
 }
 
 
-class ConfigError(Exception):
-    pass
-
-
 class SydentConfig:
     """This is the class in charge of handling Sydent's configuration.
     Handling of each individual section is delegated to other classes
diff --git a/sydent/config/_base.py b/sydent/config/_base.py
index 96a66d72..f59b17ca 100644
--- a/sydent/config/_base.py
+++ b/sydent/config/_base.py
@@ -19,6 +19,10 @@
 CONFIG_PARSER_DICT = Dict[str, Dict[str, str]]
 
 
+class ConfigError(Exception):
+    pass
+
+
 class BaseConfig(ABC):
     @abstractmethod
     def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py
index 037460ee..b9e0a4c6 100644
--- a/sydent/replication/peer.py
+++ b/sydent/replication/peer.py
@@ -26,7 +26,6 @@
 from twisted.web.iweb import IResponse
 from unpaddedbase64 import decode_base64
 
-from sydent.config import ConfigError
 from sydent.db.hashing_metadata import HashingMetadataStore
 from sydent.db.threepid_associations import GlobalAssociationStore
 from sydent.threepid import threePidAssocFromDict
@@ -170,7 +169,7 @@ def __init__(
             try:
                 pubkey_decoded = decode_base64(pubkey)
             except Exception as e:
-                raise ConfigError(
+                raise RuntimeError(
                     "Unable to decode public key for peer %s: %s" % (server_name, e),
                 )
 

From 0f1b6a29c4c2893d1189a1c6423056d5b422bc58 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:06:20 +0100
Subject: [PATCH 02/14] Stop using CONFIG_DEFAULTS, set them when doing
 get(...) instead

---
 README.rst                |  31 ++++++++++++
 sydent/config/__init__.py | 103 +++++++++++---------------------------
 sydent/config/_base.py    |   6 +--
 sydent/config/crypto.py   |  14 ++----
 sydent/config/database.py |  12 +++--
 sydent/config/email.py    |  52 +++++++++++++------
 sydent/config/general.py  |  32 ++++++------
 sydent/config/http.py     |  40 +++++++++------
 sydent/config/sms.py      |  12 ++---
 sydent/http/httpcommon.py |   4 +-
 sydent/terms/terms.py     |   5 +-
 11 files changed, 160 insertions(+), 151 deletions(-)

diff --git a/README.rst b/README.rst
index ce491bf4..1b063ce6 100644
--- a/README.rst
+++ b/README.rst
@@ -190,3 +190,34 @@ Replication
 
 It is possible to configure a mesh of Sydent instances which replicate identity bindings
 between each other. See `<docs/replication.md>`_.
+
+Email obfuscation
+=================
+When a user is invited to a room via their email address, that invite is
+displayed in the room list using an obfuscated version of the user's email
+address.
+
+The amount of obfuscation is controlled by the 
+`email.third_party_invite_username_obfuscate_characters` and
+`email.third_party_invite_domain_obfuscate_characters` config options. The first
+sets the number of characters from the beginning to reveal of the email's username
+portion (left of the '@' sign). The second, the number of characters from the
+beginning to reveal of the email's domain portion (right of the '@' sign).
+
+The '@' sign is always included. 
+
+If the string is longer than a configured limit below, it is truncated to
+that limit with '...' added. For shorter strings, the following rules are
+used:
+
+* If the string has more than 5 characters, it is truncated to 3 characters
+  + '...' (e.g. 'username' would become 'use...')
+
+* If the string has between 2 and 5 characters inclusive, it is truncated
+  to 1 character + '...' (e.g. 'user' would become 'u...')
+
+* If the string is 1 character long, it is converted to just '...'
+  (e.g. 'a' would become '...')
+
+This ensures that a full email address is never shown, even if it is extremely
+short.
diff --git a/sydent/config/__init__.py b/sydent/config/__init__.py
index e0b57955..905c2003 100644
--- a/sydent/config/__init__.py
+++ b/sydent/config/__init__.py
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
 import logging
 import logging.handlers
 import os
-from configparser import DEFAULTSECT, ConfigParser
+import pprint
+from configparser import DEFAULTSECT, ConfigParser, NoSectionError
 from typing import Dict
 
 from twisted.python import log
 
-from sydent.config._base import CONFIG_PARSER_DICT
+from sydent.config._base import CONFIG_PARSER_DICT, ConfigError
 from sydent.config._configparser import SydentConfigParser
 from sydent.config.crypto import CryptoConfig
 from sydent.config.database import DatabaseConfig
@@ -32,6 +32,7 @@
 
 logger = logging.getLogger(__name__)
 
+"""
 CONFIG_DEFAULTS = {
     "general": {
         "server.name": os.environ.get("SYDENT_SERVER_NAME", ""),
@@ -151,6 +152,7 @@
         "ed25519.signingkey": "",
     },
 }
+"""
 
 
 class SydentConfig:
@@ -180,61 +182,37 @@ def __init__(self):
             self.http,
         ]
 
-    def _parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def _parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Run the parse_config method on each of the objects in
         self.config_sections
 
         :param cfg: the configuration to be parsed
-
-        :return: whether or not the config file needs updating. This method CAN
-            return True, but it *shouldn't*. Instead a ConfigError exception
-            should be raised. This is left in for the soon to be deprecated way
-            of generating config files.
         """
-        needs_saving = False
+        # azren TODO: for debugging
+        pprint.pprint(cfg)
         for section in self.config_sections:
-            if section.parse_config(cfg):
-                needs_saving = True
-
-        return needs_saving
-
-    def _parse_from_dict(self, cfg: CONFIG_PARSER_DICT) -> bool:
-        """
-        Parse the configuration from a dict
-
-        :param cfg: the configuration to be parsed
-
-        :return: whether or not the config file needs updating. This method CAN
-            return True, but it *shouldn't*. Instead a ConfigError exception
-            should be raised. This is left in for the soon to be deprecated way
-            of generating config files.
-        """
-        return self._parse_config(cfg)
+            section.parse_config(cfg)
 
-    def _parse_from_sydent_config_parser(self, cfg: SydentConfigParser) -> bool:
+    def _parse_from_sydent_config_parser(self, cfg: SydentConfigParser) -> None:
         """
         Parse the configuration from a SydentConfigParser object
 
         :param cfg: the configuration to be parsed
-
-        :return: whether or not the config file needs updating. This method CAN
-            return True, but it *shouldn't*. Instead a ConfigError exception
-            should be raised. This is left in for the soon to be deprecated way
-            of generating config files.
         """
         config_dict: CONFIG_PARSER_DICT = {}
         for section in cfg.sections():
             config_dict[section] = {}
             # Copy in any values that are in the DEFAULT section
-            # This must be done first as they might be overwritten
+            # This must be done first as they might be overwritten.
+            # This is for legacy support.
             for key, val in cfg.items(DEFAULTSECT):
                 config_dict[section][key] = val
             # Copy in the values set in this section
             for key, val in cfg.items(section):
                 config_dict[section][key] = val
 
-        return self._parse_from_dict(config_dict)
+        self._parse_config(config_dict)
 
     def parse_config_file(
         self, config_file: str, skip_logging_setup: bool = False
@@ -245,15 +223,11 @@ def parse_config_file(
 
         :param config_file: the file to be parsed
         """
-        # If the config file doesn't exist, prepopulate the config object
-        # with the defaults.
-        new_config_file = not os.path.exists(config_file)
+        # If the config file doesn't exist, raise an error
+        if not os.path.exists(config_file):
+            raise ConfigError(f"Unable to find config file: {config_file}")
 
         cfg = SydentConfigParser()
-        for sect, entries in CONFIG_DEFAULTS.items():
-            cfg.add_section(sect)
-            for k, v in entries.items():
-                cfg.set(sect, k, v)
         cfg.read(config_file)
 
         # Logging is configured in cfg, but these options must be parsed first
@@ -261,19 +235,7 @@ def parse_config_file(
         if not skip_logging_setup:
             setup_logging(cfg)
 
-        needs_updating = self._parse_from_sydent_config_parser(cfg)
-
-        # Don't edit config file when starting Sydent unless it's the first run
-        if new_config_file:
-            fp = open(config_file, "w")
-            cfg.write(fp)
-            fp.close()
-            exit(0)
-
-        if needs_updating:
-            # A more specific log message should have been given earlier
-            logger.error("The config file needs updating")
-            exit(1)
+        self._parse_from_sydent_config_parser(cfg)
 
     def parse_config_dict(self, config_dict: Dict) -> None:
         """
@@ -281,18 +243,10 @@ def parse_config_dict(self, config_dict: Dict) -> None:
 
         :param config_dict: the configuration dictionary to be parsed
         """
-        # Build a config dictionary from the defaults merged with the given dictionary
-        config = copy.deepcopy(CONFIG_DEFAULTS)
-        for section, section_dict in config_dict.items():
-            if section not in config:
-                config[section] = {}
-            for option in section_dict.keys():
-                config[section][option] = config_dict[section][option]
-
         # This is only ever called by tests so don't configure logging
         # as tests do this themselves
 
-        self._parse_from_dict(config)
+        self._parse_config(config_dict)
 
 
 def setup_logging(cfg: ConfigParser) -> None:
@@ -304,24 +258,27 @@ def setup_logging(cfg: ConfigParser) -> None:
     log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"
     formatter = logging.Formatter(log_format)
 
-    logPath = cfg.get("general", "log.path")
-    if logPath != "":
+    try:
+        # empty or unset log.path means log to stderr
+        log_path = cfg.get("general", "log.path", fallback=None) or None
+        # unset log.level means use default
+        log_level = cfg.get("general", "log.level", fallback="INFO")
+    except NoSectionError:
+        log_path = None
+        log_level = "INFO"
+
+    if log_path is not None:
         handler: logging.StreamHandler = logging.handlers.TimedRotatingFileHandler(
-            logPath, when="midnight", backupCount=365
+            log_path, when="midnight", backupCount=365
         )
         handler.setFormatter(formatter)
 
-        def sighup(signum, stack):
-            logger.info("Closing log file due to SIGHUP")
-            handler.doRollover()
-            logger.info("Opened new log file due to SIGHUP")
-
     else:
         handler = logging.StreamHandler()
 
     handler.setFormatter(formatter)
     rootLogger = logging.getLogger("")
-    rootLogger.setLevel(cfg.get("general", "log.level"))
+    rootLogger.setLevel(log_level)
     rootLogger.addHandler(handler)
 
     observer = log.PythonLoggingObserver()
diff --git a/sydent/config/_base.py b/sydent/config/_base.py
index f59b17ca..7f985e33 100644
--- a/sydent/config/_base.py
+++ b/sydent/config/_base.py
@@ -25,15 +25,11 @@ class ConfigError(Exception):
 
 class BaseConfig(ABC):
     @abstractmethod
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the a section of the config
 
         :param cfg: the configuration to be parsed
-
-        :return: whether or not cfg has been altered. This method CAN
-            return True, but it *shouldn't* as this leads to altering the
-            config file.
         """
         pass
 
diff --git a/sydent/config/crypto.py b/sydent/config/crypto.py
index 692749b6..e015cb68 100644
--- a/sydent/config/crypto.py
+++ b/sydent/config/crypto.py
@@ -17,31 +17,27 @@
 import nacl
 import signedjson.key
 
-from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig
+from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig, ConfigError
 
 logger = logging.getLogger(__name__)
 
 
 class CryptoConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the crypto section of the config
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("crypto")
+        config = cfg.get("crypto", {})
 
         signing_key_str = config.get("ed25519.signingkey") or None
 
         if signing_key_str is None:
-            logger.warning(
+            raise ConfigError(
                 "'ed25519.signingkey' cannot be blank. Please generate a new"
                 " signing key with the 'generate-key' script."
             )
 
-            self.signing_key = signedjson.key.generate_signing_key("0")
-
-            return True
-
         signing_key_parts = signing_key_str.split(" ")
 
         if len(signing_key_parts) == 1:
@@ -60,5 +56,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
             self.signing_key = signedjson.key.decode_signing_key_base64(
                 signing_key_parts[0], signing_key_parts[1], signing_key_parts[2]
             )
-
-        return False
diff --git a/sydent/config/database.py b/sydent/config/database.py
index 1e1c50a4..d15c27e1 100644
--- a/sydent/config/database.py
+++ b/sydent/config/database.py
@@ -12,18 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig
 
 
 class DatabaseConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the database section of the config
 
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("db")
-
-        self.database_path = config.get("db.file")
+        config = cfg.get("db", {})
 
-        return False
+        self.database_path = config.get(
+            "db.file", os.environ.get("SYDENT_DB_PATH", "sydent.db")
+        )
diff --git a/sydent/config/email.py b/sydent/config/email.py
index b3567132..7bcb88a5 100644
--- a/sydent/config/email.py
+++ b/sydent/config/email.py
@@ -12,55 +12,75 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import socket
 from typing import Optional
 
 from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig
 
+logger = logging.getLogger(__name__)
+
 
 class EmailConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the email section of the config
 
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("email")
+        config = cfg.get("email", {})
 
         # These two options are deprecated
         self.template: Optional[str] = config.get("email.template", None)
+        if self.template is not None:
+            logger.warning(
+                "'email.template' is a deprecated option."
+                " Please use 'templates.path' and 'brand.default' instead."
+            )
 
         self.invite_template = config.get("email.invite_template", None)
+        if self.invite_template is not None:
+            logger.warning(
+                "'email.invite_template' is a deprecated option."
+                " Please use 'templates.path' and 'brand.default' instead."
+            )
 
         # This isn't used anywhere...
         self.validation_subject = config.get("email.subject")
 
+        if self.invite_template is not None:
+            logger.warning(
+                "'email.subject' is no longer a supported option."
+            )
+
         # Interpolation is turned off for these two options
         # This allows them to use %(variable)s substitution without raising errors
-        self.invite_subject = config.get("email.invite.subject")
-        self.invite_subject_space = config.get("email.invite.subject_space")
+        self.invite_subject = config.get(
+            "email.invite.subject", "%(sender_display_name)s has invited you to chat"
+        )
+        self.invite_subject_space = config.get(
+            "email.invite.subject_space",
+            "%(sender_display_name)s has invited you to a space",
+        )
 
-        self.smtp_server = config.get("email.smtphost")
-        self.smtp_port = config.get("email.smtpport")
-        self.smtp_username = config.get("email.smtpusername")
-        self.smtp_password = config.get("email.smtppassword")
-        self.tls_mode = config.get("email.tlsmode")
+        self.smtp_server = config.get("email.smtphost", "localhost")
+        self.smtp_port = int(config.get("email.smtpport", "25"))
+        self.smtp_username = config.get("email.smtpusername", "")
+        self.smtp_password = config.get("email.smtppassword", "")
+        self.tls_mode = config.get("email.tlsmode", "None")
 
         # This is the fully qualified domain name for SMTP HELO/EHLO
         self.host_name = config.get("email.hostname") or socket.getfqdn()
-
-        self.sender = config.get("email.from")
+        self.sender = config.get("email.from", "Sydent <noreply@example.com>")
 
         self.default_web_client_location = config.get(
-            "email.default_web_client_location"
+            "email.default_web_client_location", "https://app.element.io"
         )
 
         self.username_obfuscate_characters = int(
-            config.get("email.third_party_invite_username_obfuscate_characters")
+            config.get("email.third_party_invite_username_obfuscate_characters", "3")
         )
 
         self.domain_obfuscate_characters = int(
-            config.get("email.third_party_invite_domain_obfuscate_characters")
+            config.get("email.third_party_invite_domain_obfuscate_characters", "3")
         )
-
-        return False
diff --git a/sydent/config/general.py b/sydent/config/general.py
index d68b731e..dae4c96b 100644
--- a/sydent/config/general.py
+++ b/sydent/config/general.py
@@ -26,26 +26,26 @@
 
 
 class GeneralConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the 'general' section of the config
 
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("general")
+        config = cfg.get("general", {})
 
         self.server_name = config.get("server.name") or None
         if self.server_name is None:
             self.server_name = os.uname()[1]
             logger.warning(
-                "You have not specified a server name. I have guessed that this server is called '%s'. "
-                "If this is incorrect, you should edit 'general.server.name' in the config file."
+                "'server.name' should not be blank. Please enter a value for it in the config."
+                " For this run, I have guessed that this server is called '%s'."
                 % (self.server_name,)
             )
 
         # Get the possible brands by looking at directories under the
         # templates.path directory.
-        self.templates_path = config.get("templates.path")
+        self.templates_path = config.get("templates.path", "res")
         if os.path.exists(self.templates_path):
             self.valid_brands = {
                 p
@@ -65,13 +65,15 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
             autoescape=True,
         )
 
-        self.default_brand = config.get("brand.default")
+        self.default_brand = config.get("brand.default", "matrix-org")
 
-        self.pidfile = config.get("pidfile.path")
+        self.pidfile = config.get(
+            "pidfile.path", os.environ.get("SYDENT_PID_FILE", "sydent.pid")
+        )
 
-        self.terms_path = config.get("terms.path")
+        self.terms_path = config.get("terms.path") or None
 
-        self.address_lookup_limit = int(config.get("address_lookup_limit"))
+        self.address_lookup_limit = int(config.get("address_lookup_limit", "10000"))
 
         self.prometheus_port = config.get("prometheus_port", None)
         self.prometheus_addr = config.get("prometheus_addr", None)
@@ -86,22 +88,22 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
         self.sentry_enabled = self.sentry_dsn is not None
 
         self.enable_v1_associations = parse_cfg_bool(
-            config.get("enable_v1_associations")
+            config.get("enable_v1_associations", "true")
         )
 
-        self.delete_tokens_on_bind = parse_cfg_bool(config.get("delete_tokens_on_bind"))
+        self.delete_tokens_on_bind = parse_cfg_bool(
+            config.get("delete_tokens_on_bind", "true")
+        )
 
-        ip_blacklist = list_from_comma_sep_string(config.get("ip.blacklist"))
+        ip_blacklist = list_from_comma_sep_string(config.get("ip.blacklist", ""))
         if not ip_blacklist:
             ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST
 
-        ip_whitelist = list_from_comma_sep_string(config.get("ip.whitelist"))
+        ip_whitelist = list_from_comma_sep_string(config.get("ip.whitelist", ""))
 
         self.ip_blacklist = generate_ip_set(ip_blacklist)
         self.ip_whitelist = generate_ip_set(ip_whitelist)
 
-        return False
-
 
 def list_from_comma_sep_string(rawstr: str) -> List[str]:
     """
diff --git a/sydent/config/http.py b/sydent/config/http.py
index ed0aaf2c..572469cc 100644
--- a/sydent/config/http.py
+++ b/sydent/config/http.py
@@ -13,26 +13,34 @@
 # limitations under the License.
 
 
+import logging
+
 from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig, parse_cfg_bool
 
+logger = logging.getLogger(__name__)
+
 
 class HTTPConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the http section of the config
 
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("http")
+        config = cfg.get("http", {})
 
         # This option is deprecated
         self.verify_response_template = config.get("verify_response_template", None)
+        if self.verify_response_template is not None:
+            logger.warning(
+                "'verify_response_template' is a deprecated option."
+                " Please use 'templates.path' and 'brand.default' instead."
+            )
 
-        self.client_bind_address = config.get("clientapi.http.bind_address")
-        self.client_port = int(config.get("clientapi.http.port"))
+        self.client_bind_address = config.get("clientapi.http.bind_address", "::")
+        self.client_port = int(config.get("clientapi.http.port", 8090))
 
-        # internal port is allowed to be set to an empty string in the config
-        internal_api_port = config.get("internalapi.http.port")
+        internal_api_port = config.get("internalapi.http.port") or None
         self.internal_bind_address = config.get("internalapi.http.bind_address", "::1")
 
         if internal_api_port != "":
@@ -41,19 +49,23 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
         else:
             self.internal_api_enabled = False
 
-        self.cert_file = config.get("replication.https.certfile")
-        self.ca_cert_file = config.get("replication.https.cacert")
+        self.cert_file = config.get("replication.https.certfile") or None
+        self.ca_cert_file = config.get("replication.https.cacert") or None
 
-        self.replication_bind_address = config.get("replication.https.bind_address")
-        self.replication_port = int(config.get("replication.https.port"))
+        self.replication_bind_address = config.get(
+            "replication.https.bind_address", "::"
+        )
+        self.replication_port = int(config.get("replication.https.port", 4434))
 
-        self.obey_x_forwarded_for = cfg.getboolean("http", "obey_x_forwarded_for")
+        self.obey_x_forwarded_for = parse_cfg_bool(
+            config.get("obey_x_forwarded_for", "false")
+        )
 
         self.verify_federation_certs = parse_cfg_bool(
-            config.get("federation.verifycerts")
+            config.get("federation.verifycerts", "true")
         )
 
-        self.server_http_url_base = config.get("client_http_base")
+        self.server_http_url_base = config.get("client_http_base", "")
 
         self.base_replication_urls = {}
 
@@ -65,5 +77,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
                 if "base_replication_url" in peer_config.keys():
                     base_url = peer_config.get("base_replication_url")
                     self.base_replication_urls[peer] = base_url
-
-        return False
diff --git a/sydent/config/sms.py b/sydent/config/sms.py
index da427de2..80266512 100644
--- a/sydent/config/sms.py
+++ b/sydent/config/sms.py
@@ -18,20 +18,20 @@
 
 
 class SMSConfig(BaseConfig):
-    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
+    def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         """
         Parse the sms section of the config
 
         :param cfg: the configuration to be parsed
         """
-        config = cfg.get("sms")
+        config = cfg.get("sms", {})
 
-        self.body_template = config.get("bodyTemplate")
+        self.body_template = config.get("bodyTemplate", "Your code is {token}")
 
         # Make sure username and password are bytes otherwise we can't use them with
         # b64encode.
-        self.api_username = config.get("username").encode("UTF-8")
-        self.api_password = config.get("password").encode("UTF-8")
+        self.api_username = config.get("username", "").encode("UTF-8")
+        self.api_password = config.get("password", "").encode("UTF-8")
 
         self.originators: Dict[str, List[Dict[str, str]]] = {}
         self.smsRules = {}
@@ -70,5 +70,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool:
                     )
 
                 self.smsRules[country] = action
-
-        return False
diff --git a/sydent/http/httpcommon.py b/sydent/http/httpcommon.py
index 330f83f6..8d3b8770 100644
--- a/sydent/http/httpcommon.py
+++ b/sydent/http/httpcommon.py
@@ -45,7 +45,7 @@ def makeMyCertificate(self):
         # TODO Move some of this loading into parse_config
         privKeyAndCertFilename = self.sydent.config.http.cert_file
 
-        if privKeyAndCertFilename == "":
+        if privKeyAndCertFilename is None:
             logger.warning(
                 "No HTTPS private key / cert found: not starting replication server "
                 "or doing replication pushes"
@@ -70,7 +70,7 @@ def makeTrustRoot(self):
         # If this option is specified, use a specific root CA cert. This is useful for testing when it's not
         # practical to get the client cert signed by a real root CA but should never be used on a production server.
         caCertFilename = self.sydent.config.http.ca_cert_file
-        if len(caCertFilename) > 0:
+        if caCertFilename is not None:
             try:
                 fp = open(caCertFilename)
                 caCert = twisted.internet.ssl.Certificate.loadPEM(fp.read())
diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py
index cd78187c..cade9dd7 100644
--- a/sydent/terms/terms.py
+++ b/sydent/terms/terms.py
@@ -106,13 +106,12 @@ def get_terms(sydent: "Sydent") -> Optional[Terms]:
     # TODO - move some of this to parse_config
 
     termsPath = sydent.config.general.terms_path
+    if termsPath is None:
+        return Terms(None)
 
     try:
         termsYaml = None
 
-        if termsPath == "":
-            return Terms(None)
-
         with open(termsPath) as fp:
             termsYaml = yaml.safe_load(fp)
         if "master_version" not in termsYaml:

From dce078f6472585900179a522a435e69dc080e6ac Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:07:08 +0100
Subject: [PATCH 03/14] Add script to generate a config file

---
 scripts/generate-config | 75 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100755 scripts/generate-config

diff --git a/scripts/generate-config b/scripts/generate-config
new file mode 100755
index 00000000..538d81c9
--- /dev/null
+++ b/scripts/generate-config
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+# Generates a config file for Sydent to use
+
+
+import argparse
+import os
+from configparser import ConfigParser
+
+import signedjson.key
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--server-name", help="The server domain name.")
+
+    parser.add_argument(
+        "--pid-file",
+        help="The file where the PID of the running Sydent process will be written.",
+    )
+
+    parser.add_argument("--db-path", help="The SQLite Database file for Sydent to use.")
+
+    parser.add_argument(
+        "-o",
+        "--output-file",
+        default="sydent.conf",
+        help="The file to write the configuration to. Default: %(default)s",
+    )
+
+    args = parser.parse_args()
+
+    signing_key = signedjson.key.generate_signing_key(0)
+    sk_str = "%s %s %s" % (
+        signing_key.alg,
+        signing_key.version,
+        signedjson.key.encode_signing_key_base64(signing_key),
+    )
+
+    cfg = ConfigParser()
+
+    # [general]
+    cfg.add_section("general")
+
+    if args.server_name is not None:
+        cfg.set("general", "server.name", args.server_name)
+
+    if args.pid_file is not None:
+        cfg.set("general", "pidfile.path", args.pid_file)
+
+    # [db]
+    cfg.add_section("db")
+
+    if args.db_path is not None:
+        cfg.set("db", "db.file", args.db_path)
+
+    # [http]
+    cfg.add_section("http")
+
+    # [email]
+    cfg.add_section("email")
+
+    # [sms]
+    cfg.add_section("sms")
+
+    # [crypto]
+    cfg.add_section("crypto")
+
+    cfg.set("crypto", "ed25519.signingkey", sk_str)
+
+    if not os.path.exists(args.output_file):
+        with open(args.output_file, "w") as configfile:
+            cfg.write(configfile)
+    else:
+        print(f"ERROR: Cannot overwrite existing file {args.output_file}")

From 7d16cb0e519216fd4f2e48ee5f82a1538af3ef9c Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:07:35 +0100
Subject: [PATCH 04/14] Add documentation on all the config options

---
 docs/Configuration.md    | 174 +++++++++++++++++++++++++++++++++++++++
 docs/example_config.conf |  39 +++++++++
 2 files changed, 213 insertions(+)
 create mode 100644 docs/Configuration.md
 create mode 100644 docs/example_config.conf

diff --git a/docs/Configuration.md b/docs/Configuration.md
new file mode 100644
index 00000000..75800efa
--- /dev/null
+++ b/docs/Configuration.md
@@ -0,0 +1,174 @@
+# Configuration
+
+## Config file structure
+Sydent uses Python's `configparser` libary to parse the config file. 
+This libary's documentation can be found [here](https://docs.python.org/3/library/configparser.html).
+
+As an example:
+```
+[DEFAULT]
+tail.length = 10
+
+[elephant]
+name = Albert
+trunk.length =
+#wings.width =
+
+[mouse]
+name = Terrance
+tail.length = 5
+
+[human]
+```
+
+This config file is then translated to the following Python dict:
+```python
+cfg = {
+    elephant: {
+        name: "Albert",
+        trunk.length: "",
+        tail.length: "10",
+    },
+    mouse: {
+        name: "Terrance",
+        tail.length: "5",
+    },
+    human: {
+        tail.length: "10",
+    }
+}
+```
+
+The important things to note are:  
+1. Commenting out `wings.width` essentially sets it to `None`. But an empty
+value for `trunk.length` sets it to an empty string.
+
+2. Even though the `human` section was empty, it still has a `tail.length`
+value of `"10"` (due to the `DEFAULT` section).  
+
+3. Everything is a string and it's up to the program to decide what type to
+cast things into.
+
+
+## Example config file
+
+An example minimal config file can be found in
+[docs/example_config.conf](https://github.com/matrix-org/sydent/blob/main/docs/example_config.conf).
+
+
+## Writing your own config
+
+Your config file need only contain entries for things you are overwriting from the 
+default. 
+
+The DEFAULT section should be empty and is a deprecated way of configuring SYDENT.
+
+You can generate a new config file by running the [generate-config script](https://github.com/matrix-org/sydent/blob/main/scripts/generate-config)
+
+## Config options
+
+The string `XYZ` in an option name means that there is a family of config options,
+each with a different substitution for `XYZ`.
+
+See [docs/example_config.conf](https://github.com/matrix-org/sydent/blob/main/docs/example_config.conf)
+for an example of this in the [sms] section.
+
+### Definitions of emptiness:
+
+Consider this snippet of config:
+```
+this.option.is.empty = 
+this.option.is.non-empty = petal
+```
+* `this.option.is.empty` has been set to the empty string. This is **EMPTY**.
+* `this.option.was.never.mentioned` was never mentioned. This is **UNSET**.
+* `this.option.is.non-empty` is neither **EMPTY** nor **UNSET**. This is **NON-EMPTY**.
+
+Leaving an option *empty*, overrides the default value with an empty string.  
+Leaving an option *unset* uses the defualt value for that option.
+
+### [general]
+
+Name                    | Description
+---------               | -------
+`address_lookup_limit`  | The maximum number of addresses that someone can query in a single /lookup request. DEFAULTS TO `1000`
+`brand.default`         | The default brand of templates to use. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `matrix-org`
+`delete_tokens_on_bind` | Whether to delete invite tokens after successful binding has taken place. DEFAULTS TO `true`
+`enable_v1_associations`| Whether clients and homeservers can register an association using v1 API endpoints. DEFAULTS TO `true`
+`ip.blacklist`          | A comma-seperated list of CIDR IP address ranges to block outbound requests to. DEFAULTS TO a list of private IP ranges to prevent DNS rebinding attacks. This list can be found in [sydent/util/ip_range.py](https://github.com/matrix-org/sydent/blob/main/sydent/util/ip_range.py).
+`ip.whitelist`          | A comma-seperated of IP address CIDR ranges that should be allowed for outbound requests. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges. This list overrides the blacklist. DEFAULTS TO EMPTY
+`log.level`             | The log level to use. This can be set to any level used by the Python `logging` module. DEFAULTS TO `INFO`
+`log.path`              | The path of the file to write the logs to. Leaving this empty prints logs to stderr. DEFAULTS TO EMPTY
+`pidfile.path`          | The file to save Sydent's process ID (PID) to. DEFAULTS TO the value stored in `SYDENT_PID_FILE` environment variable, or `sydent.pid` if that is unset.
+`prometheus_addr`       | The local IPv4 or IPv6 address for Prometheus to bind to. This must be set to enable the Prometheus client. DEFAULTS TO UNSET
+`prometheus_port`       | The port for Prometheus to bind to. This must be set to enable Prometheus. DEFAULTS TO UNSET
+`server.name`           | The domain name of the server. DEFAULTS TO value returned by `os.uname()[1]`
+`sentry.dsn`            | The Data Source Name (DSN) for Sentry to use. This must be set to enable Sentry. DEFAULTS TO UNSET
+`templates.path`        | The path to the root template directory. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `res`
+`terms.path`            | The path to the file where the terms and conditions are configured, or empty if no terms are being used. DEFAULTS TO EMPTY.
+
+
+### [db]
+
+Name        | Description
+---------   | -------
+`db.file`   | The path to the SQLite database file for Sydent to use. It can be set to `:memory:` to use a temporary database in RAM instead of on disk. DEFAULTS TO the value stored in the `SYDENT_DB_PATH` environment variable or `sydent.db` if that is unset.
+
+
+### [http]
+
+Name                            | Description
+---------                       | -------
+`client_http_base`              | The base url of Sydent. This should be of the form `scheme://base.url.com/here`. DEFAULTS TO EMPTY
+`clientapi.http.bind_address`   | The local IPv4 or IPv6 for the Identity Server API to bind to. DEFAULTS TO `::` (i.e. bind to all)
+`clientapi.http.port`           | The port for the Identity Server API to bind to. DEFAULTS TO `8090`
+`federation.verifycerts`        | Whether or not Sydent should verify the TLS certificates of homeservers it communicates with. DEFAULTS TO `true`
+`internalapi.http.bind_address` | The local IPv4 or IPv6 for the Internal Testing API to bind to. DEFAULTS TO `::1`
+`internalapi.http.port`         | The port for the Internal Testing API to bind to. This must be non-empty to enable the Internal Testing API. Enabling this allows for binding and unbinding between identifiers and matrix IDs without any 
+`obey_x_forwarded_for`          | Whether or not Sydent should pay attention to X-Forwarded-For headers. DEFAULTS TO `false`
+`replication.https.bind_address`| The local IPv4 or IPv6 for the Replication API to bind to. DEFAULTS TO `::` (i.e. bind to all)
+`replication.https.cacert`      | The file path to a root CA certificate. If this is SET then certificates of other Sydent servers signed by this CA will be trusted. This is useful for testing or when it's not practical to get the client cert signed by a real root CA but should never be used on a production server. DEFAULTS TO UNSET
+`replication.https.certfile`    | The file path to a TLS certificate and private key. This file should contain **both** the public certificate and the private key used to generate it. This must be non-empty to enable the Replication API. DEFAULTS TO UNSET
+`replication.https.port`        | The port for the Replication API to bind to. DEFAULTS TO `4434`
+
+
+### [email]
+
+Name                        | Description
+---------                   | -------
+`email.default_web_client_location` | The web client location which will be used in store invites if one is not provided by the homeserver. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. This should be of the form 'scheme://base.url.com/here'. DEFAULTS TO `https://app.element.io`
+`email.from`                | The email address that all emails should appear to have been sent from. This should take the form: `Display Name Here <actual.email@example.com>`. DEFAULTS TO `Sydent <noreply@example.com>`
+`email.hostname`            | The fully qualified domain name (FQDN) to use with HELO/EHLO command when connecting to the SMTP server. DEFAULTS TO result of `socket.getfqdn()`
+`email.invite.subject`      | The subject line of emails that invite someone to a room. This is a string template using `"%(variable)s"` substitution and can use any of the parameters sent to the `/store-invite` API. See See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `%(sender_display_name)s has invited you to chat`
+`email.invite.subject_space`| The subject line of emails that invite someone to a space. This is a string template using `"%(variable)s"` substitution and can use any of the parameters sent to the `/store-invite` API. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `%(sender_display_name)s has invited you to a space`
+`email.smtphost`            | The address of the SMTP server to use. DEFAULTS TO `localhost`
+`email.smtppassword`        | The password to connect to the SMTP server with. DEFAULTS TO EMPTY
+`email.smtpport`            | The port to connect to the SMTP server on. DEFAULTS TO `25`
+`email.smtpusername`        | The username to connect to the SMTP server with. DEFAULTS TO EMPTY
+`email.third_party_invite_username_obfuscate_characters` | The number of characters to from the beginning to reveal of an email's username portion (left of the '@' sign). See the [README](https://github.com/matrix-org/sydent/blob/main/README.rst) for more information on email obfuscation. DEFAULTS TO `3`
+`email.third_party_invite_domain_obfuscate_characters` | The number of characters to from the beginning to reveal of an email's domain portion (right of the '@' sign). See the [README](https://github.com/matrix-org/sydent/blob/main/README.rst) for more information on email obfuscation. DEFAULTS TO `3`
+`email.tlsmode`             | The security mode to use when connecting with the SMTP server. This can take one of the following options: `NONE`, `TLS`, `SSL`, `STARTTLS`. Any other value is equivalent to `NONE`. DEFAULTS TO `NONE`
+
+
+### [sms]
+
+Name                    | Description
+---------               | -------
+`bodyTemplate`          | The template to use for SMS validation texts. The string '{token}' will get replaced with the validation code. DEFAULTS TO `Your code is {token}`
+`username`              | The username to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY
+`password`              | The password to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY
+`originators.XYZ`       | The list of originators to use when sending an SMS to a number with country code `XYZ`. The originator is chosen deterministically from this list so if someone requests multiple codes, they come from a consistent number. The originators must be in form: `long:<number>`, `short:<number>` or `alpha:<text>`, separated by commas. DEFAULTS TO UNSET
+`originators.default`   | The default originaor to use when sending an SMS. This must be of the form `long:<number>`, `short:<number>` or `alpha:<text>`. DEFAULTS TO UNSET
+`smsrule.XYZ`           | Whether or not to allow verification texts to numbers with country code `XYZ`. If this option is set to `reject` then that country is blacklisted, otherwise it is allowed. DEFAULTS TO UNSET
+
+### [crypto]
+
+Name                | Description
+---------           | -------
+`ed25519.signingkey`| The key used to sign JSON sent by this server. A new key can be generated by running the [generate-key script](https://github.com/matrix-org/sydent/blob/main/scripts/generate-key). This must be set to start the server. NO DEFAULT
+
+### [peer.XYZ]
+
+Name                    | Description
+---------               | -------
+`base_replication_url`  | The base url of the peer with name `XYZ`. This should be of the form `https://internal-address.example.com:4434`. See [docs/replication.md](https://github.com/matrix-org/sydent/blob/main/docs/replication.md) for more information. DEFAULTS TO UNSET
diff --git a/docs/example_config.conf b/docs/example_config.conf
new file mode 100644
index 00000000..4ef87f70
--- /dev/null
+++ b/docs/example_config.conf
@@ -0,0 +1,39 @@
+[general]
+server.name = sydent.mydomain.com
+log.path = /home/sydent/sydent.log
+terms.path = /home/sydent/terms.yaml
+templates.path = /home/sydent/templates
+brand.default = example-client
+
+[db]
+db.file = /home/sydent/sydent.db
+
+[http]
+client_http_base = http://sydent.mydomain.com
+
+[email]
+email.from = My Domain <noreply@mydomain.com>
+email.smtphost = relay.emailserver.com
+email.smtpport = 587
+email.smtpusername = myUsername
+email.smtppassword = myPassword
+email.hostname = sydent.mydomain.com
+email.tlsmode = TLS
+email.default_web_client_location = https://app.example-client.com
+
+[sms]
+username = openmarketUserName
+password = openmarketPassword
+
+# Send texts to people with UK (44) numbers from 12345
+originators.44 = short:12345
+originators.default = alpha:mydomain
+
+# Don't sent verification texts to people with UK (44) or French (33) phone numbers
+smsrule.44 = reject
+smsrule.33 = reject
+
+[crypto]
+# Run the generate-key script to produce a random signing key
+# (There is no example key here to prevent people forgetting to generate a new one!)
+ed25519.signingkey = 

From cd2beffe5bb7cb91add25520b95e3e823094056a Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:17:11 +0100
Subject: [PATCH 05/14] Update README with information about generate-config

---
 README.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index 1b063ce6..d2496b75 100644
--- a/README.rst
+++ b/README.rst
@@ -44,18 +44,18 @@ With the virtualenv activated, you can run Sydent using::
 
     python -m sydent.sydent
 
-If this is the first time Sydent is run, then it will generate a configuration file in
-``sydent.conf`` with some defaults and then stop. You must run the generate-key script
-and update the config with this key before Sydent will start.
+Before you run Sydent for the first time, you must run the generate-config script.
 
 You should not write anything in the ``[DEFAULT]`` section. If a
 setting is defined in both the ``[DEFAULT]`` section and another section in the configuration
 file, then the value in the other section is used.
 
 You'll most likely want to change the server name (``server.name``) and specify an email server
-(look for the settings starting with ``email.``).
+(look for the settings starting with ``email.``). 
 
-By default, Sydent will listen on ``0.0.0.0:8090``. This can be changed by changing the values for
+See `<docs/configuration.md>` for more information on how to configure Sydent.
+
+By default, Sydent will listen on ``[::]:8090``. This can be changed by changing the values for
 the configuration settings ``clientapi.http.bind_address`` and ``clientapi.http.port``.
 
 Sydent uses SQLite as its database backend. By default, it will create the database as ``sydent.db``
@@ -109,7 +109,7 @@ However, you then have to pay attention to the file permissions.
 Environment variables
 ---------------------
 
-.. warning:: These variables are only taken into account at first start and are written to the configuration file.
+.. warning:: These variables are only taken into account if they are not overridden by the configuration file.
 
 +--------------------+-----------------+-----------------------+
 | Variable Name      | Sydent default  | Dockerfile default    |

From fa9133a46bf0e55e4c0d97f74822de2661bb86a7 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:19:35 +0100
Subject: [PATCH 06/14] Fix issue that occured during rebase

---
 sydent/config/http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sydent/config/http.py b/sydent/config/http.py
index 572469cc..b742337d 100644
--- a/sydent/config/http.py
+++ b/sydent/config/http.py
@@ -43,7 +43,7 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         internal_api_port = config.get("internalapi.http.port") or None
         self.internal_bind_address = config.get("internalapi.http.bind_address", "::1")
 
-        if internal_api_port != "":
+        if internal_api_port is not None:
             self.internal_api_enabled = True
             self.internal_port = int(internal_api_port)
         else:

From c1fa6e8d58a70616983f204cbfbb86f83e6bb3e7 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:20:08 +0100
Subject: [PATCH 07/14] Run linters

---
 sydent/config/email.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sydent/config/email.py b/sydent/config/email.py
index 7bcb88a5..25e8e923 100644
--- a/sydent/config/email.py
+++ b/sydent/config/email.py
@@ -49,9 +49,7 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
         self.validation_subject = config.get("email.subject")
 
         if self.invite_template is not None:
-            logger.warning(
-                "'email.subject' is no longer a supported option."
-            )
+            logger.warning("'email.subject' is no longer a supported option.")
 
         # Interpolation is turned off for these two options
         # This allows them to use %(variable)s substitution without raising errors

From 1c417e44533175e1a6fe65672a885ff1f8d7b103 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:25:22 +0100
Subject: [PATCH 08/14] Remove commented out CONFIG_DEFAULTS and debugging
 print statement

---
 sydent/config/__init__.py | 125 --------------------------------------
 1 file changed, 125 deletions(-)

diff --git a/sydent/config/__init__.py b/sydent/config/__init__.py
index 905c2003..3eadbcca 100644
--- a/sydent/config/__init__.py
+++ b/sydent/config/__init__.py
@@ -15,7 +15,6 @@
 import logging
 import logging.handlers
 import os
-import pprint
 from configparser import DEFAULTSECT, ConfigParser, NoSectionError
 from typing import Dict
 
@@ -32,128 +31,6 @@
 
 logger = logging.getLogger(__name__)
 
-"""
-CONFIG_DEFAULTS = {
-    "general": {
-        "server.name": os.environ.get("SYDENT_SERVER_NAME", ""),
-        "log.path": "",
-        "log.level": "INFO",
-        "pidfile.path": os.environ.get("SYDENT_PID_FILE", "sydent.pid"),
-        "terms.path": "",
-        "address_lookup_limit": "10000",  # Maximum amount of addresses in a single /lookup request
-        # The root path to use for load templates. This should contain branded
-        # directories. Each directory should contain the following templates:
-        #
-        # * invite_template.eml
-        # * verification_template.eml
-        # * verify_response_template.html
-        "templates.path": "res",
-        # The brand directory to use if no brand hint (or an invalid brand hint)
-        # is provided by the request.
-        "brand.default": "matrix-org",
-        # The following can be added to your local config file to enable prometheus
-        # support.
-        # 'prometheus_port': '8080',  # The port to serve metrics on
-        # 'prometheus_addr': '',  # The address to bind to. Empty string means bind to all.
-        # The following can be added to your local config file to enable sentry support.
-        # 'sentry_dsn': 'https://...'  # The DSN has configured in the sentry instance project.
-        # Whether clients and homeservers can register an association using v1 endpoints.
-        "enable_v1_associations": "true",
-        "delete_tokens_on_bind": "true",
-        # Prevent outgoing requests from being sent to the following blacklisted
-        # IP address CIDR ranges. If this option is not specified or empty then
-        # it defaults to private IP address ranges.
-        #
-        # The blacklist applies to all outbound requests except replication
-        # requests.
-        #
-        # (0.0.0.0 and :: are always blacklisted, whether or not they are
-        # explicitly listed here, since they correspond to unroutable
-        # addresses.)
-        "ip.blacklist": "",
-        # List of IP address CIDR ranges that should be allowed for outbound
-        # requests. This is useful for specifying exceptions to wide-ranging
-        # blacklisted target IP ranges.
-        #
-        # This whitelist overrides `ip.blacklist` and defaults to an empty
-        # list.
-        "ip.whitelist": "",
-    },
-    "db": {
-        "db.file": os.environ.get("SYDENT_DB_PATH", "sydent.db"),
-    },
-    "http": {
-        "clientapi.http.bind_address": "::",
-        "clientapi.http.port": "8090",
-        "internalapi.http.bind_address": "::1",
-        "internalapi.http.port": "",
-        "replication.https.certfile": "",
-        "replication.https.cacert": "",  # This should only be used for testing
-        "replication.https.bind_address": "::",
-        "replication.https.port": "4434",
-        "obey_x_forwarded_for": "False",
-        "federation.verifycerts": "True",
-        # verify_response_template is deprecated, but still used if defined. Define
-        # templates.path and brand.default under general instead.
-        #
-        # 'verify_response_template': 'res/verify_response_page_template',
-        "client_http_base": "",
-    },
-    "email": {
-        # email.template and email.invite_template are deprecated, but still used
-        # if defined. Define templates.path and brand.default under general instead.
-        #
-        # 'email.template': 'res/verification_template.eml',
-        # 'email.invite_template': 'res/invite_template.eml',
-        "email.from": "Sydent Validation <noreply@{hostname}>",
-        "email.subject": "Your Validation Token",
-        "email.invite.subject": "%(sender_display_name)s has invited you to chat",
-        "email.invite.subject_space": "%(sender_display_name)s has invited you to a space",
-        "email.smtphost": "localhost",
-        "email.smtpport": "25",
-        "email.smtpusername": "",
-        "email.smtppassword": "",
-        "email.hostname": "",
-        "email.tlsmode": "0",
-        # The web client location which will be used if it is not provided by
-        # the homeserver.
-        #
-        # This should be the scheme and hostname only, see res/invite_template.eml
-        # for the full URL that gets generated.
-        "email.default_web_client_location": "https://app.element.io",
-        # When a user is invited to a room via their email address, that invite is
-        # displayed in the room list using an obfuscated version of the user's email
-        # address. These config options determine how much of the email address to
-        # obfuscate. Note that the '@' sign is always included.
-        #
-        # If the string is longer than a configured limit below, it is truncated to that limit
-        # with '...' added. Otherwise:
-        #
-        # * If the string is longer than 5 characters, it is truncated to 3 characters + '...'
-        # * If the string is longer than 1 character, it is truncated to 1 character + '...'
-        # * If the string is 1 character long, it is converted to '...'
-        #
-        # This ensures that a full email address is never shown, even if it is extremely
-        # short.
-        #
-        # The number of characters from the beginning to reveal of the email's username
-        # portion (left of the '@' sign)
-        "email.third_party_invite_username_obfuscate_characters": "3",
-        # The number of characters from the beginning to reveal of the email's domain
-        # portion (right of the '@' sign)
-        "email.third_party_invite_domain_obfuscate_characters": "3",
-    },
-    "sms": {
-        "bodyTemplate": "Your code is {token}",
-        "username": "",
-        "password": "",
-    },
-    "crypto": {
-        "ed25519.signingkey": "",
-    },
-}
-"""
-
 
 class SydentConfig:
     """This is the class in charge of handling Sydent's configuration.
@@ -189,8 +66,6 @@ def _parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
 
         :param cfg: the configuration to be parsed
         """
-        # azren TODO: for debugging
-        pprint.pprint(cfg)
         for section in self.config_sections:
             section.parse_config(cfg)
 

From fe104c5f8b98e80b00db1732a3f5df231f4a6605 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 11:28:02 +0100
Subject: [PATCH 09/14] Touchups to documentation

---
 docs/Configuration.md    | 16 ++++++++--------
 docs/example_config.conf |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/Configuration.md b/docs/Configuration.md
index 75800efa..8df53445 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -61,7 +61,7 @@ An example minimal config file can be found in
 Your config file need only contain entries for things you are overwriting from the 
 default. 
 
-The DEFAULT section should be empty and is a deprecated way of configuring SYDENT.
+The DEFAULT section should be empty and is a deprecated way of configuring Sydent.
 
 You can generate a new config file by running the [generate-config script](https://github.com/matrix-org/sydent/blob/main/scripts/generate-config)
 
@@ -101,11 +101,11 @@ Name                    | Description
 `log.path`              | The path of the file to write the logs to. Leaving this empty prints logs to stderr. DEFAULTS TO EMPTY
 `pidfile.path`          | The file to save Sydent's process ID (PID) to. DEFAULTS TO the value stored in `SYDENT_PID_FILE` environment variable, or `sydent.pid` if that is unset.
 `prometheus_addr`       | The local IPv4 or IPv6 address for Prometheus to bind to. This must be set to enable the Prometheus client. DEFAULTS TO UNSET
-`prometheus_port`       | The port for Prometheus to bind to. This must be set to enable Prometheus. DEFAULTS TO UNSET
+`prometheus_port`       | The port for Prometheus to bind to. This must be set to enable the Prometheus client. DEFAULTS TO UNSET
 `server.name`           | The domain name of the server. DEFAULTS TO value returned by `os.uname()[1]`
 `sentry.dsn`            | The Data Source Name (DSN) for Sentry to use. This must be set to enable Sentry. DEFAULTS TO UNSET
 `templates.path`        | The path to the root template directory. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `res`
-`terms.path`            | The path to the file where the terms and conditions are configured, or empty if no terms are being used. DEFAULTS TO EMPTY.
+`terms.path`            | The path to the file where the terms and conditions are configured, or empty if no terms are being used. DEFAULTS TO EMPTY
 
 
 ### [db]
@@ -120,13 +120,13 @@ Name        | Description
 Name                            | Description
 ---------                       | -------
 `client_http_base`              | The base url of Sydent. This should be of the form `scheme://base.url.com/here`. DEFAULTS TO EMPTY
-`clientapi.http.bind_address`   | The local IPv4 or IPv6 for the Identity Server API to bind to. DEFAULTS TO `::` (i.e. bind to all)
+`clientapi.http.bind_address`   | The local IPv4 or IPv6 address for the Identity Server API to bind to. DEFAULTS TO `::` (i.e. bind to all)
 `clientapi.http.port`           | The port for the Identity Server API to bind to. DEFAULTS TO `8090`
 `federation.verifycerts`        | Whether or not Sydent should verify the TLS certificates of homeservers it communicates with. DEFAULTS TO `true`
-`internalapi.http.bind_address` | The local IPv4 or IPv6 for the Internal Testing API to bind to. DEFAULTS TO `::1`
-`internalapi.http.port`         | The port for the Internal Testing API to bind to. This must be non-empty to enable the Internal Testing API. Enabling this allows for binding and unbinding between identifiers and matrix IDs without any 
+`internalapi.http.bind_address` | The local IPv4 or IPv6 address for the Internal Testing API to bind to. DEFAULTS TO `::1`
+`internalapi.http.port`         | The port for the Internal Testing API to bind to. This must be non-empty to enable the Internal Testing API. Enabling this allows for binding and unbinding between identifiers and matrix IDs without any authentication. DEFAULTS TO EMPTY
 `obey_x_forwarded_for`          | Whether or not Sydent should pay attention to X-Forwarded-For headers. DEFAULTS TO `false`
-`replication.https.bind_address`| The local IPv4 or IPv6 for the Replication API to bind to. DEFAULTS TO `::` (i.e. bind to all)
+`replication.https.bind_address`| The local IPv4 or IPv6 address for the Replication API to bind to. DEFAULTS TO `::` (i.e. bind to all)
 `replication.https.cacert`      | The file path to a root CA certificate. If this is SET then certificates of other Sydent servers signed by this CA will be trusted. This is useful for testing or when it's not practical to get the client cert signed by a real root CA but should never be used on a production server. DEFAULTS TO UNSET
 `replication.https.certfile`    | The file path to a TLS certificate and private key. This file should contain **both** the public certificate and the private key used to generate it. This must be non-empty to enable the Replication API. DEFAULTS TO UNSET
 `replication.https.port`        | The port for the Replication API to bind to. DEFAULTS TO `4434`
@@ -158,7 +158,7 @@ Name                    | Description
 `username`              | The username to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY
 `password`              | The password to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY
 `originators.XYZ`       | The list of originators to use when sending an SMS to a number with country code `XYZ`. The originator is chosen deterministically from this list so if someone requests multiple codes, they come from a consistent number. The originators must be in form: `long:<number>`, `short:<number>` or `alpha:<text>`, separated by commas. DEFAULTS TO UNSET
-`originators.default`   | The default originaor to use when sending an SMS. This must be of the form `long:<number>`, `short:<number>` or `alpha:<text>`. DEFAULTS TO UNSET
+`originators.default`   | The default originator to use when sending an SMS. This must be of the form `long:<number>`, `short:<number>` or `alpha:<text>`. DEFAULTS TO UNSET
 `smsrule.XYZ`           | Whether or not to allow verification texts to numbers with country code `XYZ`. If this option is set to `reject` then that country is blacklisted, otherwise it is allowed. DEFAULTS TO UNSET
 
 ### [crypto]
diff --git a/docs/example_config.conf b/docs/example_config.conf
index 4ef87f70..f73874c6 100644
--- a/docs/example_config.conf
+++ b/docs/example_config.conf
@@ -29,8 +29,8 @@ password = openmarketPassword
 originators.44 = short:12345
 originators.default = alpha:mydomain
 
-# Don't sent verification texts to people with UK (44) or French (33) phone numbers
-smsrule.44 = reject
+# Don't sent verification texts to people with US/Canada (1) or French (33) phone numbers
+smsrule.1 = reject
 smsrule.33 = reject
 
 [crypto]

From 6c32b742e469334cefa09c75d28d360f289d83c2 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 12:19:28 +0100
Subject: [PATCH 10/14] Remove capitalisation of configuration.md

---
 docs/{Configuration.md => configuration.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/{Configuration.md => configuration.md} (100%)

diff --git a/docs/Configuration.md b/docs/configuration.md
similarity index 100%
rename from docs/Configuration.md
rename to docs/configuration.md

From 7b8030adef4f10bc95e5ff85d93989fc734e81a9 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 12:53:03 +0100
Subject: [PATCH 11/14] Take notice of environment variables when running
 generate-config

---
 README.rst              |  4 +++-
 scripts/generate-config | 24 +++++++++++++++---------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/README.rst b/README.rst
index d2496b75..9b147573 100644
--- a/README.rst
+++ b/README.rst
@@ -109,7 +109,9 @@ However, you then have to pay attention to the file permissions.
 Environment variables
 ---------------------
 
-.. warning:: These variables are only taken into account if they are not overridden by the configuration file.
+.. warning:: These variables are taken into account when generate-config is run and
+are written to the config file. They are only used by Sydent if they are not
+overridden by the config file.
 
 +--------------------+-----------------+-----------------------+
 | Variable Name      | Sydent default  | Dockerfile default    |
diff --git a/scripts/generate-config b/scripts/generate-config
index 538d81c9..77c61b54 100755
--- a/scripts/generate-config
+++ b/scripts/generate-config
@@ -12,14 +12,19 @@ import signedjson.key
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
-    parser.add_argument("--server-name", help="The server domain name.")
+    parser.add_argument("--server-name", help="The server domain name. The value in SYDENT_SERVER_NAME takes precedence.")
 
     parser.add_argument(
         "--pid-file",
-        help="The file where the PID of the running Sydent process will be written.",
+        default="sydent.pid",
+        help="The file where the PID of the running Sydent process will be written. The value in SYDENT_PID_FILE takes precedence. Default: %(default)s",
     )
 
-    parser.add_argument("--db-path", help="The SQLite Database file for Sydent to use.")
+    parser.add_argument(
+        "--db-path",
+        default="sydent.db",
+        help="The SQLite Database file for Sydent to use. The value in SYDENT_DB_PATH takes precedence. Default: %(default)s",
+    )
 
     parser.add_argument(
         "-o",
@@ -42,17 +47,18 @@ if __name__ == "__main__":
     # [general]
     cfg.add_section("general")
 
-    if args.server_name is not None:
-        cfg.set("general", "server.name", args.server_name)
+    name = os.environ.get("SYDENT_SERVER_NAME", args.server_name)
+    if name is not None:
+        cfg.set("general", "server.name", name)
 
-    if args.pid_file is not None:
-        cfg.set("general", "pidfile.path", args.pid_file)
+    pidfile = os.environ.get("SYDENT_PID_FILE", args.pid_file)
+    cfg.set("general", "pidfile.path", pidfile)
 
     # [db]
     cfg.add_section("db")
 
-    if args.db_path is not None:
-        cfg.set("db", "db.file", args.db_path)
+    dbpath = os.environ.get("SYDENT_DB_PATH", args.db_path)
+    cfg.set("db", "db.file", dbpath)
 
     # [http]
     cfg.add_section("http")

From d7a15852171e4e7c59f114f32faae98e35f0ed21 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 13:28:52 +0100
Subject: [PATCH 12/14] Add generate-config call to dockerfile. And use
 SYDENT_CONF in script

---
 Dockerfile              | 2 +-
 scripts/generate-config | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0322ed0d..de5dac2b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -50,7 +50,7 @@ WORKDIR /sydent
 USER sydent:sydent
 VOLUME ["/data"]
 
-RUN python3 /sydent/scripts/generate-key >> /data/sydent.conf
+RUN python3 /sydent/scripts/generate-config
 
 EXPOSE 8090/tcp
 CMD [ "python", "-m", "sydent.sydent" ]
diff --git a/scripts/generate-config b/scripts/generate-config
index 77c61b54..abc21763 100755
--- a/scripts/generate-config
+++ b/scripts/generate-config
@@ -30,7 +30,7 @@ if __name__ == "__main__":
         "-o",
         "--output-file",
         default="sydent.conf",
-        help="The file to write the configuration to. Default: %(default)s",
+        help="The file to write the configuration to. The value in SYDENT_CONF takes precedence. Default: %(default)s",
     )
 
     args = parser.parse_args()
@@ -74,8 +74,9 @@ if __name__ == "__main__":
 
     cfg.set("crypto", "ed25519.signingkey", sk_str)
 
-    if not os.path.exists(args.output_file):
-        with open(args.output_file, "w") as configfile:
+    outputfile = os.environ.get("SYDENT_CONF", args.output_file)
+    if not os.path.exists(outputfile):
+        with open(outputfile, "w") as configfile:
             cfg.write(configfile)
     else:
-        print(f"ERROR: Cannot overwrite existing file {args.output_file}")
+        print(f"ERROR: Cannot overwrite existing file {outputfile}")

From 7d29f6c059df6f1b863426d60ba441518615e269 Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 16:07:07 +0100
Subject: [PATCH 13/14] Fix mistake added during rebase

---
 sydent/config/email.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sydent/config/email.py b/sydent/config/email.py
index 25e8e923..620a16df 100644
--- a/sydent/config/email.py
+++ b/sydent/config/email.py
@@ -46,7 +46,7 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None:
             )
 
         # This isn't used anywhere...
-        self.validation_subject = config.get("email.subject")
+        self.validation_subject = config.get("email.subject", "Your Validation Token")
 
         if self.invite_template is not None:
             logger.warning("'email.subject' is no longer a supported option.")

From 7d8abb0a2c4bf8d9900ec9ed5aed9211166cfd3d Mon Sep 17 00:00:00 2001
From: Azrenbeth <7782548+Azrenbeth@users.noreply.github.com>
Date: Mon, 20 Sep 2021 16:20:43 +0100
Subject: [PATCH 14/14] Add changelog

---
 changelog.d/404.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/404.misc

diff --git a/changelog.d/404.misc b/changelog.d/404.misc
new file mode 100644
index 00000000..3269a688
--- /dev/null
+++ b/changelog.d/404.misc
@@ -0,0 +1 @@
+Replace the CONFIG_DEFAULTS dict with a generate-config script and documentation on all the config options.
\ No newline at end of file