diff --git a/Dockerfile b/Dockerfile index 0322ed0d..de5dac2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ WORKDIR /sydent USER sydent:sydent VOLUME ["/data"] -RUN python3 /sydent/scripts/generate-key >> /data/sydent.conf +RUN python3 /sydent/scripts/generate-config EXPOSE 8090/tcp CMD [ "python", "-m", "sydent.sydent" ] diff --git a/README.rst b/README.rst index ce491bf4..9b147573 100644 --- a/README.rst +++ b/README.rst @@ -44,18 +44,18 @@ With the virtualenv activated, you can run Sydent using:: python -m sydent.sydent -If this is the first time Sydent is run, then it will generate a configuration file in -``sydent.conf`` with some defaults and then stop. You must run the generate-key script -and update the config with this key before Sydent will start. +Before you run Sydent for the first time, you must run the generate-config script. You should not write anything in the ``[DEFAULT]`` section. If a setting is defined in both the ``[DEFAULT]`` section and another section in the configuration file, then the value in the other section is used. You'll most likely want to change the server name (``server.name``) and specify an email server -(look for the settings starting with ``email.``). +(look for the settings starting with ``email.``). -By default, Sydent will listen on ``0.0.0.0:8090``. This can be changed by changing the values for +See `` for more information on how to configure Sydent. + +By default, Sydent will listen on ``[::]:8090``. This can be changed by changing the values for the configuration settings ``clientapi.http.bind_address`` and ``clientapi.http.port``. Sydent uses SQLite as its database backend. By default, it will create the database as ``sydent.db`` @@ -109,7 +109,9 @@ However, you then have to pay attention to the file permissions. Environment variables --------------------- -.. warning:: These variables are only taken into account at first start and are written to the configuration file. +.. warning:: These variables are taken into account when generate-config is run and +are written to the config file. They are only used by Sydent if they are not +overridden by the config file. +--------------------+-----------------+-----------------------+ | Variable Name | Sydent default | Dockerfile default | @@ -190,3 +192,34 @@ Replication It is possible to configure a mesh of Sydent instances which replicate identity bindings between each other. See ``_. + +Email obfuscation +================= +When a user is invited to a room via their email address, that invite is +displayed in the room list using an obfuscated version of the user's email +address. + +The amount of obfuscation is controlled by the +`email.third_party_invite_username_obfuscate_characters` and +`email.third_party_invite_domain_obfuscate_characters` config options. The first +sets the number of characters from the beginning to reveal of the email's username +portion (left of the '@' sign). The second, the number of characters from the +beginning to reveal of the email's domain portion (right of the '@' sign). + +The '@' sign is always included. + +If the string is longer than a configured limit below, it is truncated to +that limit with '...' added. For shorter strings, the following rules are +used: + +* If the string has more than 5 characters, it is truncated to 3 characters + + '...' (e.g. 'username' would become 'use...') + +* If the string has between 2 and 5 characters inclusive, it is truncated + to 1 character + '...' (e.g. 'user' would become 'u...') + +* If the string is 1 character long, it is converted to just '...' + (e.g. 'a' would become '...') + +This ensures that a full email address is never shown, even if it is extremely +short. diff --git a/changelog.d/404.misc b/changelog.d/404.misc new file mode 100644 index 00000000..3269a688 --- /dev/null +++ b/changelog.d/404.misc @@ -0,0 +1 @@ +Replace the CONFIG_DEFAULTS dict with a generate-config script and documentation on all the config options. \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..8df53445 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,174 @@ +# Configuration + +## Config file structure +Sydent uses Python's `configparser` libary to parse the config file. +This libary's documentation can be found [here](https://docs.python.org/3/library/configparser.html). + +As an example: +``` +[DEFAULT] +tail.length = 10 + +[elephant] +name = Albert +trunk.length = +#wings.width = + +[mouse] +name = Terrance +tail.length = 5 + +[human] +``` + +This config file is then translated to the following Python dict: +```python +cfg = { + elephant: { + name: "Albert", + trunk.length: "", + tail.length: "10", + }, + mouse: { + name: "Terrance", + tail.length: "5", + }, + human: { + tail.length: "10", + } +} +``` + +The important things to note are: +1. Commenting out `wings.width` essentially sets it to `None`. But an empty +value for `trunk.length` sets it to an empty string. + +2. Even though the `human` section was empty, it still has a `tail.length` +value of `"10"` (due to the `DEFAULT` section). + +3. Everything is a string and it's up to the program to decide what type to +cast things into. + + +## Example config file + +An example minimal config file can be found in +[docs/example_config.conf](https://github.com/matrix-org/sydent/blob/main/docs/example_config.conf). + + +## Writing your own config + +Your config file need only contain entries for things you are overwriting from the +default. + +The DEFAULT section should be empty and is a deprecated way of configuring Sydent. + +You can generate a new config file by running the [generate-config script](https://github.com/matrix-org/sydent/blob/main/scripts/generate-config) + +## Config options + +The string `XYZ` in an option name means that there is a family of config options, +each with a different substitution for `XYZ`. + +See [docs/example_config.conf](https://github.com/matrix-org/sydent/blob/main/docs/example_config.conf) +for an example of this in the [sms] section. + +### Definitions of emptiness: + +Consider this snippet of config: +``` +this.option.is.empty = +this.option.is.non-empty = petal +``` +* `this.option.is.empty` has been set to the empty string. This is **EMPTY**. +* `this.option.was.never.mentioned` was never mentioned. This is **UNSET**. +* `this.option.is.non-empty` is neither **EMPTY** nor **UNSET**. This is **NON-EMPTY**. + +Leaving an option *empty*, overrides the default value with an empty string. +Leaving an option *unset* uses the defualt value for that option. + +### [general] + +Name | Description +--------- | ------- +`address_lookup_limit` | The maximum number of addresses that someone can query in a single /lookup request. DEFAULTS TO `1000` +`brand.default` | The default brand of templates to use. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `matrix-org` +`delete_tokens_on_bind` | Whether to delete invite tokens after successful binding has taken place. DEFAULTS TO `true` +`enable_v1_associations`| Whether clients and homeservers can register an association using v1 API endpoints. DEFAULTS TO `true` +`ip.blacklist` | A comma-seperated list of CIDR IP address ranges to block outbound requests to. DEFAULTS TO a list of private IP ranges to prevent DNS rebinding attacks. This list can be found in [sydent/util/ip_range.py](https://github.com/matrix-org/sydent/blob/main/sydent/util/ip_range.py). +`ip.whitelist` | A comma-seperated of IP address CIDR ranges that should be allowed for outbound requests. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges. This list overrides the blacklist. DEFAULTS TO EMPTY +`log.level` | The log level to use. This can be set to any level used by the Python `logging` module. DEFAULTS TO `INFO` +`log.path` | The path of the file to write the logs to. Leaving this empty prints logs to stderr. DEFAULTS TO EMPTY +`pidfile.path` | The file to save Sydent's process ID (PID) to. DEFAULTS TO the value stored in `SYDENT_PID_FILE` environment variable, or `sydent.pid` if that is unset. +`prometheus_addr` | The local IPv4 or IPv6 address for Prometheus to bind to. This must be set to enable the Prometheus client. DEFAULTS TO UNSET +`prometheus_port` | The port for Prometheus to bind to. This must be set to enable the Prometheus client. DEFAULTS TO UNSET +`server.name` | The domain name of the server. DEFAULTS TO value returned by `os.uname()[1]` +`sentry.dsn` | The Data Source Name (DSN) for Sentry to use. This must be set to enable Sentry. DEFAULTS TO UNSET +`templates.path` | The path to the root template directory. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `res` +`terms.path` | The path to the file where the terms and conditions are configured, or empty if no terms are being used. DEFAULTS TO EMPTY + + +### [db] + +Name | Description +--------- | ------- +`db.file` | The path to the SQLite database file for Sydent to use. It can be set to `:memory:` to use a temporary database in RAM instead of on disk. DEFAULTS TO the value stored in the `SYDENT_DB_PATH` environment variable or `sydent.db` if that is unset. + + +### [http] + +Name | Description +--------- | ------- +`client_http_base` | The base url of Sydent. This should be of the form `scheme://base.url.com/here`. DEFAULTS TO EMPTY +`clientapi.http.bind_address` | The local IPv4 or IPv6 address for the Identity Server API to bind to. DEFAULTS TO `::` (i.e. bind to all) +`clientapi.http.port` | The port for the Identity Server API to bind to. DEFAULTS TO `8090` +`federation.verifycerts` | Whether or not Sydent should verify the TLS certificates of homeservers it communicates with. DEFAULTS TO `true` +`internalapi.http.bind_address` | The local IPv4 or IPv6 address for the Internal Testing API to bind to. DEFAULTS TO `::1` +`internalapi.http.port` | The port for the Internal Testing API to bind to. This must be non-empty to enable the Internal Testing API. Enabling this allows for binding and unbinding between identifiers and matrix IDs without any authentication. DEFAULTS TO EMPTY +`obey_x_forwarded_for` | Whether or not Sydent should pay attention to X-Forwarded-For headers. DEFAULTS TO `false` +`replication.https.bind_address`| The local IPv4 or IPv6 address for the Replication API to bind to. DEFAULTS TO `::` (i.e. bind to all) +`replication.https.cacert` | The file path to a root CA certificate. If this is SET then certificates of other Sydent servers signed by this CA will be trusted. This is useful for testing or when it's not practical to get the client cert signed by a real root CA but should never be used on a production server. DEFAULTS TO UNSET +`replication.https.certfile` | The file path to a TLS certificate and private key. This file should contain **both** the public certificate and the private key used to generate it. This must be non-empty to enable the Replication API. DEFAULTS TO UNSET +`replication.https.port` | The port for the Replication API to bind to. DEFAULTS TO `4434` + + +### [email] + +Name | Description +--------- | ------- +`email.default_web_client_location` | The web client location which will be used in store invites if one is not provided by the homeserver. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. This should be of the form 'scheme://base.url.com/here'. DEFAULTS TO `https://app.element.io` +`email.from` | The email address that all emails should appear to have been sent from. This should take the form: `Display Name Here `. DEFAULTS TO `Sydent ` +`email.hostname` | The fully qualified domain name (FQDN) to use with HELO/EHLO command when connecting to the SMTP server. DEFAULTS TO result of `socket.getfqdn()` +`email.invite.subject` | The subject line of emails that invite someone to a room. This is a string template using `"%(variable)s"` substitution and can use any of the parameters sent to the `/store-invite` API. See See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `%(sender_display_name)s has invited you to chat` +`email.invite.subject_space`| The subject line of emails that invite someone to a space. This is a string template using `"%(variable)s"` substitution and can use any of the parameters sent to the `/store-invite` API. See [docs/templates.md](https://github.com/matrix-org/sydent/blob/main/docs/templates.md) for more information. DEFAULTS TO `%(sender_display_name)s has invited you to a space` +`email.smtphost` | The address of the SMTP server to use. DEFAULTS TO `localhost` +`email.smtppassword` | The password to connect to the SMTP server with. DEFAULTS TO EMPTY +`email.smtpport` | The port to connect to the SMTP server on. DEFAULTS TO `25` +`email.smtpusername` | The username to connect to the SMTP server with. DEFAULTS TO EMPTY +`email.third_party_invite_username_obfuscate_characters` | The number of characters to from the beginning to reveal of an email's username portion (left of the '@' sign). See the [README](https://github.com/matrix-org/sydent/blob/main/README.rst) for more information on email obfuscation. DEFAULTS TO `3` +`email.third_party_invite_domain_obfuscate_characters` | The number of characters to from the beginning to reveal of an email's domain portion (right of the '@' sign). See the [README](https://github.com/matrix-org/sydent/blob/main/README.rst) for more information on email obfuscation. DEFAULTS TO `3` +`email.tlsmode` | The security mode to use when connecting with the SMTP server. This can take one of the following options: `NONE`, `TLS`, `SSL`, `STARTTLS`. Any other value is equivalent to `NONE`. DEFAULTS TO `NONE` + + +### [sms] + +Name | Description +--------- | ------- +`bodyTemplate` | The template to use for SMS validation texts. The string '{token}' will get replaced with the validation code. DEFAULTS TO `Your code is {token}` +`username` | The username to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY +`password` | The password to use when connecting to the SMS sender at https://smsc.openmarket.com/sms/v4/mt. DEFAULTS TO EMPTY +`originators.XYZ` | The list of originators to use when sending an SMS to a number with country code `XYZ`. The originator is chosen deterministically from this list so if someone requests multiple codes, they come from a consistent number. The originators must be in form: `long:`, `short:` or `alpha:`, separated by commas. DEFAULTS TO UNSET +`originators.default` | The default originator to use when sending an SMS. This must be of the form `long:`, `short:` or `alpha:`. DEFAULTS TO UNSET +`smsrule.XYZ` | Whether or not to allow verification texts to numbers with country code `XYZ`. If this option is set to `reject` then that country is blacklisted, otherwise it is allowed. DEFAULTS TO UNSET + +### [crypto] + +Name | Description +--------- | ------- +`ed25519.signingkey`| The key used to sign JSON sent by this server. A new key can be generated by running the [generate-key script](https://github.com/matrix-org/sydent/blob/main/scripts/generate-key). This must be set to start the server. NO DEFAULT + +### [peer.XYZ] + +Name | Description +--------- | ------- +`base_replication_url` | The base url of the peer with name `XYZ`. This should be of the form `https://internal-address.example.com:4434`. See [docs/replication.md](https://github.com/matrix-org/sydent/blob/main/docs/replication.md) for more information. DEFAULTS TO UNSET diff --git a/docs/example_config.conf b/docs/example_config.conf new file mode 100644 index 00000000..f73874c6 --- /dev/null +++ b/docs/example_config.conf @@ -0,0 +1,39 @@ +[general] +server.name = sydent.mydomain.com +log.path = /home/sydent/sydent.log +terms.path = /home/sydent/terms.yaml +templates.path = /home/sydent/templates +brand.default = example-client + +[db] +db.file = /home/sydent/sydent.db + +[http] +client_http_base = http://sydent.mydomain.com + +[email] +email.from = My Domain +email.smtphost = relay.emailserver.com +email.smtpport = 587 +email.smtpusername = myUsername +email.smtppassword = myPassword +email.hostname = sydent.mydomain.com +email.tlsmode = TLS +email.default_web_client_location = https://app.example-client.com + +[sms] +username = openmarketUserName +password = openmarketPassword + +# Send texts to people with UK (44) numbers from 12345 +originators.44 = short:12345 +originators.default = alpha:mydomain + +# Don't sent verification texts to people with US/Canada (1) or French (33) phone numbers +smsrule.1 = reject +smsrule.33 = reject + +[crypto] +# Run the generate-key script to produce a random signing key +# (There is no example key here to prevent people forgetting to generate a new one!) +ed25519.signingkey = diff --git a/scripts/generate-config b/scripts/generate-config new file mode 100755 index 00000000..abc21763 --- /dev/null +++ b/scripts/generate-config @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +# Generates a config file for Sydent to use + + +import argparse +import os +from configparser import ConfigParser + +import signedjson.key + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--server-name", help="The server domain name. The value in SYDENT_SERVER_NAME takes precedence.") + + parser.add_argument( + "--pid-file", + default="sydent.pid", + help="The file where the PID of the running Sydent process will be written. The value in SYDENT_PID_FILE takes precedence. Default: %(default)s", + ) + + parser.add_argument( + "--db-path", + default="sydent.db", + help="The SQLite Database file for Sydent to use. The value in SYDENT_DB_PATH takes precedence. Default: %(default)s", + ) + + parser.add_argument( + "-o", + "--output-file", + default="sydent.conf", + help="The file to write the configuration to. The value in SYDENT_CONF takes precedence. Default: %(default)s", + ) + + args = parser.parse_args() + + signing_key = signedjson.key.generate_signing_key(0) + sk_str = "%s %s %s" % ( + signing_key.alg, + signing_key.version, + signedjson.key.encode_signing_key_base64(signing_key), + ) + + cfg = ConfigParser() + + # [general] + cfg.add_section("general") + + name = os.environ.get("SYDENT_SERVER_NAME", args.server_name) + if name is not None: + cfg.set("general", "server.name", name) + + pidfile = os.environ.get("SYDENT_PID_FILE", args.pid_file) + cfg.set("general", "pidfile.path", pidfile) + + # [db] + cfg.add_section("db") + + dbpath = os.environ.get("SYDENT_DB_PATH", args.db_path) + cfg.set("db", "db.file", dbpath) + + # [http] + cfg.add_section("http") + + # [email] + cfg.add_section("email") + + # [sms] + cfg.add_section("sms") + + # [crypto] + cfg.add_section("crypto") + + cfg.set("crypto", "ed25519.signingkey", sk_str) + + outputfile = os.environ.get("SYDENT_CONF", args.output_file) + if not os.path.exists(outputfile): + with open(outputfile, "w") as configfile: + cfg.write(configfile) + else: + print(f"ERROR: Cannot overwrite existing file {outputfile}") diff --git a/sydent/config/__init__.py b/sydent/config/__init__.py index b21b586e..3eadbcca 100644 --- a/sydent/config/__init__.py +++ b/sydent/config/__init__.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import logging import logging.handlers import os -from configparser import DEFAULTSECT, ConfigParser +from configparser import DEFAULTSECT, ConfigParser, NoSectionError from typing import Dict from twisted.python import log -from sydent.config._base import CONFIG_PARSER_DICT +from sydent.config._base import CONFIG_PARSER_DICT, ConfigError from sydent.config._configparser import SydentConfigParser from sydent.config.crypto import CryptoConfig from sydent.config.database import DatabaseConfig @@ -32,130 +31,6 @@ logger = logging.getLogger(__name__) -CONFIG_DEFAULTS = { - "general": { - "server.name": os.environ.get("SYDENT_SERVER_NAME", ""), - "log.path": "", - "log.level": "INFO", - "pidfile.path": os.environ.get("SYDENT_PID_FILE", "sydent.pid"), - "terms.path": "", - "address_lookup_limit": "10000", # Maximum amount of addresses in a single /lookup request - # The root path to use for load templates. This should contain branded - # directories. Each directory should contain the following templates: - # - # * invite_template.eml - # * verification_template.eml - # * verify_response_template.html - "templates.path": "res", - # The brand directory to use if no brand hint (or an invalid brand hint) - # is provided by the request. - "brand.default": "matrix-org", - # The following can be added to your local config file to enable prometheus - # support. - # 'prometheus_port': '8080', # The port to serve metrics on - # 'prometheus_addr': '', # The address to bind to. Empty string means bind to all. - # The following can be added to your local config file to enable sentry support. - # 'sentry_dsn': 'https://...' # The DSN has configured in the sentry instance project. - # Whether clients and homeservers can register an association using v1 endpoints. - "enable_v1_associations": "true", - "delete_tokens_on_bind": "true", - # Prevent outgoing requests from being sent to the following blacklisted - # IP address CIDR ranges. If this option is not specified or empty then - # it defaults to private IP address ranges. - # - # The blacklist applies to all outbound requests except replication - # requests. - # - # (0.0.0.0 and :: are always blacklisted, whether or not they are - # explicitly listed here, since they correspond to unroutable - # addresses.) - "ip.blacklist": "", - # List of IP address CIDR ranges that should be allowed for outbound - # requests. This is useful for specifying exceptions to wide-ranging - # blacklisted target IP ranges. - # - # This whitelist overrides `ip.blacklist` and defaults to an empty - # list. - "ip.whitelist": "", - }, - "db": { - "db.file": os.environ.get("SYDENT_DB_PATH", "sydent.db"), - }, - "http": { - "clientapi.http.bind_address": "::", - "clientapi.http.port": "8090", - "internalapi.http.bind_address": "::1", - "internalapi.http.port": "", - "replication.https.certfile": "", - "replication.https.cacert": "", # This should only be used for testing - "replication.https.bind_address": "::", - "replication.https.port": "4434", - "obey_x_forwarded_for": "False", - "federation.verifycerts": "True", - # verify_response_template is deprecated, but still used if defined. Define - # templates.path and brand.default under general instead. - # - # 'verify_response_template': 'res/verify_response_page_template', - "client_http_base": "", - }, - "email": { - # email.template and email.invite_template are deprecated, but still used - # if defined. Define templates.path and brand.default under general instead. - # - # 'email.template': 'res/verification_template.eml', - # 'email.invite_template': 'res/invite_template.eml', - "email.from": "Sydent Validation ", - "email.subject": "Your Validation Token", - "email.invite.subject": "%(sender_display_name)s has invited you to chat", - "email.invite.subject_space": "%(sender_display_name)s has invited you to a space", - "email.smtphost": "localhost", - "email.smtpport": "25", - "email.smtpusername": "", - "email.smtppassword": "", - "email.hostname": "", - "email.tlsmode": "0", - # The web client location which will be used if it is not provided by - # the homeserver. - # - # This should be the scheme and hostname only, see res/invite_template.eml - # for the full URL that gets generated. - "email.default_web_client_location": "https://app.element.io", - # When a user is invited to a room via their email address, that invite is - # displayed in the room list using an obfuscated version of the user's email - # address. These config options determine how much of the email address to - # obfuscate. Note that the '@' sign is always included. - # - # If the string is longer than a configured limit below, it is truncated to that limit - # with '...' added. Otherwise: - # - # * If the string is longer than 5 characters, it is truncated to 3 characters + '...' - # * If the string is longer than 1 character, it is truncated to 1 character + '...' - # * If the string is 1 character long, it is converted to '...' - # - # This ensures that a full email address is never shown, even if it is extremely - # short. - # - # The number of characters from the beginning to reveal of the email's username - # portion (left of the '@' sign) - "email.third_party_invite_username_obfuscate_characters": "3", - # The number of characters from the beginning to reveal of the email's domain - # portion (right of the '@' sign) - "email.third_party_invite_domain_obfuscate_characters": "3", - }, - "sms": { - "bodyTemplate": "Your code is {token}", - "username": "", - "password": "", - }, - "crypto": { - "ed25519.signingkey": "", - }, -} - - -class ConfigError(Exception): - pass - class SydentConfig: """This is the class in charge of handling Sydent's configuration. @@ -184,61 +59,35 @@ def __init__(self): self.http, ] - def _parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def _parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Run the parse_config method on each of the objects in self.config_sections :param cfg: the configuration to be parsed - - :return: whether or not the config file needs updating. This method CAN - return True, but it *shouldn't*. Instead a ConfigError exception - should be raised. This is left in for the soon to be deprecated way - of generating config files. """ - needs_saving = False for section in self.config_sections: - if section.parse_config(cfg): - needs_saving = True - - return needs_saving - - def _parse_from_dict(self, cfg: CONFIG_PARSER_DICT) -> bool: - """ - Parse the configuration from a dict - - :param cfg: the configuration to be parsed + section.parse_config(cfg) - :return: whether or not the config file needs updating. This method CAN - return True, but it *shouldn't*. Instead a ConfigError exception - should be raised. This is left in for the soon to be deprecated way - of generating config files. - """ - return self._parse_config(cfg) - - def _parse_from_sydent_config_parser(self, cfg: SydentConfigParser) -> bool: + def _parse_from_sydent_config_parser(self, cfg: SydentConfigParser) -> None: """ Parse the configuration from a SydentConfigParser object :param cfg: the configuration to be parsed - - :return: whether or not the config file needs updating. This method CAN - return True, but it *shouldn't*. Instead a ConfigError exception - should be raised. This is left in for the soon to be deprecated way - of generating config files. """ config_dict: CONFIG_PARSER_DICT = {} for section in cfg.sections(): config_dict[section] = {} # Copy in any values that are in the DEFAULT section - # This must be done first as they might be overwritten + # This must be done first as they might be overwritten. + # This is for legacy support. for key, val in cfg.items(DEFAULTSECT): config_dict[section][key] = val # Copy in the values set in this section for key, val in cfg.items(section): config_dict[section][key] = val - return self._parse_from_dict(config_dict) + self._parse_config(config_dict) def parse_config_file( self, config_file: str, skip_logging_setup: bool = False @@ -249,15 +98,11 @@ def parse_config_file( :param config_file: the file to be parsed """ - # If the config file doesn't exist, prepopulate the config object - # with the defaults. - new_config_file = not os.path.exists(config_file) + # If the config file doesn't exist, raise an error + if not os.path.exists(config_file): + raise ConfigError(f"Unable to find config file: {config_file}") cfg = SydentConfigParser() - for sect, entries in CONFIG_DEFAULTS.items(): - cfg.add_section(sect) - for k, v in entries.items(): - cfg.set(sect, k, v) cfg.read(config_file) # Logging is configured in cfg, but these options must be parsed first @@ -265,19 +110,7 @@ def parse_config_file( if not skip_logging_setup: setup_logging(cfg) - needs_updating = self._parse_from_sydent_config_parser(cfg) - - # Don't edit config file when starting Sydent unless it's the first run - if new_config_file: - fp = open(config_file, "w") - cfg.write(fp) - fp.close() - exit(0) - - if needs_updating: - # A more specific log message should have been given earlier - logger.error("The config file needs updating") - exit(1) + self._parse_from_sydent_config_parser(cfg) def parse_config_dict(self, config_dict: Dict) -> None: """ @@ -285,18 +118,10 @@ def parse_config_dict(self, config_dict: Dict) -> None: :param config_dict: the configuration dictionary to be parsed """ - # Build a config dictionary from the defaults merged with the given dictionary - config = copy.deepcopy(CONFIG_DEFAULTS) - for section, section_dict in config_dict.items(): - if section not in config: - config[section] = {} - for option in section_dict.keys(): - config[section][option] = config_dict[section][option] - # This is only ever called by tests so don't configure logging # as tests do this themselves - self._parse_from_dict(config) + self._parse_config(config_dict) def setup_logging(cfg: ConfigParser) -> None: @@ -308,24 +133,27 @@ def setup_logging(cfg: ConfigParser) -> None: log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s" formatter = logging.Formatter(log_format) - logPath = cfg.get("general", "log.path") - if logPath != "": + try: + # empty or unset log.path means log to stderr + log_path = cfg.get("general", "log.path", fallback=None) or None + # unset log.level means use default + log_level = cfg.get("general", "log.level", fallback="INFO") + except NoSectionError: + log_path = None + log_level = "INFO" + + if log_path is not None: handler: logging.StreamHandler = logging.handlers.TimedRotatingFileHandler( - logPath, when="midnight", backupCount=365 + log_path, when="midnight", backupCount=365 ) handler.setFormatter(formatter) - def sighup(signum, stack): - logger.info("Closing log file due to SIGHUP") - handler.doRollover() - logger.info("Opened new log file due to SIGHUP") - else: handler = logging.StreamHandler() handler.setFormatter(formatter) rootLogger = logging.getLogger("") - rootLogger.setLevel(cfg.get("general", "log.level")) + rootLogger.setLevel(log_level) rootLogger.addHandler(handler) observer = log.PythonLoggingObserver() diff --git a/sydent/config/_base.py b/sydent/config/_base.py index 96a66d72..7f985e33 100644 --- a/sydent/config/_base.py +++ b/sydent/config/_base.py @@ -19,17 +19,17 @@ CONFIG_PARSER_DICT = Dict[str, Dict[str, str]] +class ConfigError(Exception): + pass + + class BaseConfig(ABC): @abstractmethod - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the a section of the config :param cfg: the configuration to be parsed - - :return: whether or not cfg has been altered. This method CAN - return True, but it *shouldn't* as this leads to altering the - config file. """ pass diff --git a/sydent/config/crypto.py b/sydent/config/crypto.py index 692749b6..e015cb68 100644 --- a/sydent/config/crypto.py +++ b/sydent/config/crypto.py @@ -17,31 +17,27 @@ import nacl import signedjson.key -from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig +from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig, ConfigError logger = logging.getLogger(__name__) class CryptoConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the crypto section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("crypto") + config = cfg.get("crypto", {}) signing_key_str = config.get("ed25519.signingkey") or None if signing_key_str is None: - logger.warning( + raise ConfigError( "'ed25519.signingkey' cannot be blank. Please generate a new" " signing key with the 'generate-key' script." ) - self.signing_key = signedjson.key.generate_signing_key("0") - - return True - signing_key_parts = signing_key_str.split(" ") if len(signing_key_parts) == 1: @@ -60,5 +56,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: self.signing_key = signedjson.key.decode_signing_key_base64( signing_key_parts[0], signing_key_parts[1], signing_key_parts[2] ) - - return False diff --git a/sydent/config/database.py b/sydent/config/database.py index 1e1c50a4..d15c27e1 100644 --- a/sydent/config/database.py +++ b/sydent/config/database.py @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig class DatabaseConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the database section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("db") - - self.database_path = config.get("db.file") + config = cfg.get("db", {}) - return False + self.database_path = config.get( + "db.file", os.environ.get("SYDENT_DB_PATH", "sydent.db") + ) diff --git a/sydent/config/email.py b/sydent/config/email.py index b3567132..620a16df 100644 --- a/sydent/config/email.py +++ b/sydent/config/email.py @@ -12,55 +12,73 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import socket from typing import Optional from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig +logger = logging.getLogger(__name__) + class EmailConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the email section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("email") + config = cfg.get("email", {}) # These two options are deprecated self.template: Optional[str] = config.get("email.template", None) + if self.template is not None: + logger.warning( + "'email.template' is a deprecated option." + " Please use 'templates.path' and 'brand.default' instead." + ) self.invite_template = config.get("email.invite_template", None) + if self.invite_template is not None: + logger.warning( + "'email.invite_template' is a deprecated option." + " Please use 'templates.path' and 'brand.default' instead." + ) # This isn't used anywhere... - self.validation_subject = config.get("email.subject") + self.validation_subject = config.get("email.subject", "Your Validation Token") + + if self.invite_template is not None: + logger.warning("'email.subject' is no longer a supported option.") # Interpolation is turned off for these two options # This allows them to use %(variable)s substitution without raising errors - self.invite_subject = config.get("email.invite.subject") - self.invite_subject_space = config.get("email.invite.subject_space") + self.invite_subject = config.get( + "email.invite.subject", "%(sender_display_name)s has invited you to chat" + ) + self.invite_subject_space = config.get( + "email.invite.subject_space", + "%(sender_display_name)s has invited you to a space", + ) - self.smtp_server = config.get("email.smtphost") - self.smtp_port = config.get("email.smtpport") - self.smtp_username = config.get("email.smtpusername") - self.smtp_password = config.get("email.smtppassword") - self.tls_mode = config.get("email.tlsmode") + self.smtp_server = config.get("email.smtphost", "localhost") + self.smtp_port = int(config.get("email.smtpport", "25")) + self.smtp_username = config.get("email.smtpusername", "") + self.smtp_password = config.get("email.smtppassword", "") + self.tls_mode = config.get("email.tlsmode", "None") # This is the fully qualified domain name for SMTP HELO/EHLO self.host_name = config.get("email.hostname") or socket.getfqdn() - - self.sender = config.get("email.from") + self.sender = config.get("email.from", "Sydent ") self.default_web_client_location = config.get( - "email.default_web_client_location" + "email.default_web_client_location", "https://app.element.io" ) self.username_obfuscate_characters = int( - config.get("email.third_party_invite_username_obfuscate_characters") + config.get("email.third_party_invite_username_obfuscate_characters", "3") ) self.domain_obfuscate_characters = int( - config.get("email.third_party_invite_domain_obfuscate_characters") + config.get("email.third_party_invite_domain_obfuscate_characters", "3") ) - - return False diff --git a/sydent/config/general.py b/sydent/config/general.py index d68b731e..dae4c96b 100644 --- a/sydent/config/general.py +++ b/sydent/config/general.py @@ -26,26 +26,26 @@ class GeneralConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the 'general' section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("general") + config = cfg.get("general", {}) self.server_name = config.get("server.name") or None if self.server_name is None: self.server_name = os.uname()[1] logger.warning( - "You have not specified a server name. I have guessed that this server is called '%s'. " - "If this is incorrect, you should edit 'general.server.name' in the config file." + "'server.name' should not be blank. Please enter a value for it in the config." + " For this run, I have guessed that this server is called '%s'." % (self.server_name,) ) # Get the possible brands by looking at directories under the # templates.path directory. - self.templates_path = config.get("templates.path") + self.templates_path = config.get("templates.path", "res") if os.path.exists(self.templates_path): self.valid_brands = { p @@ -65,13 +65,15 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: autoescape=True, ) - self.default_brand = config.get("brand.default") + self.default_brand = config.get("brand.default", "matrix-org") - self.pidfile = config.get("pidfile.path") + self.pidfile = config.get( + "pidfile.path", os.environ.get("SYDENT_PID_FILE", "sydent.pid") + ) - self.terms_path = config.get("terms.path") + self.terms_path = config.get("terms.path") or None - self.address_lookup_limit = int(config.get("address_lookup_limit")) + self.address_lookup_limit = int(config.get("address_lookup_limit", "10000")) self.prometheus_port = config.get("prometheus_port", None) self.prometheus_addr = config.get("prometheus_addr", None) @@ -86,22 +88,22 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: self.sentry_enabled = self.sentry_dsn is not None self.enable_v1_associations = parse_cfg_bool( - config.get("enable_v1_associations") + config.get("enable_v1_associations", "true") ) - self.delete_tokens_on_bind = parse_cfg_bool(config.get("delete_tokens_on_bind")) + self.delete_tokens_on_bind = parse_cfg_bool( + config.get("delete_tokens_on_bind", "true") + ) - ip_blacklist = list_from_comma_sep_string(config.get("ip.blacklist")) + ip_blacklist = list_from_comma_sep_string(config.get("ip.blacklist", "")) if not ip_blacklist: ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST - ip_whitelist = list_from_comma_sep_string(config.get("ip.whitelist")) + ip_whitelist = list_from_comma_sep_string(config.get("ip.whitelist", "")) self.ip_blacklist = generate_ip_set(ip_blacklist) self.ip_whitelist = generate_ip_set(ip_whitelist) - return False - def list_from_comma_sep_string(rawstr: str) -> List[str]: """ diff --git a/sydent/config/http.py b/sydent/config/http.py index 8f3948b6..b742337d 100644 --- a/sydent/config/http.py +++ b/sydent/config/http.py @@ -13,47 +13,59 @@ # limitations under the License. +import logging + from sydent.config._base import CONFIG_PARSER_DICT, BaseConfig, parse_cfg_bool +logger = logging.getLogger(__name__) + class HTTPConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the http section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("http") + config = cfg.get("http", {}) # This option is deprecated self.verify_response_template = config.get("verify_response_template", None) + if self.verify_response_template is not None: + logger.warning( + "'verify_response_template' is a deprecated option." + " Please use 'templates.path' and 'brand.default' instead." + ) - self.client_bind_address = config.get("clientapi.http.bind_address") - self.client_port = int(config.get("clientapi.http.port")) + self.client_bind_address = config.get("clientapi.http.bind_address", "::") + self.client_port = int(config.get("clientapi.http.port", 8090)) - # internal port is allowed to be set to an empty string in the config - internal_api_port = config.get("internalapi.http.port") + internal_api_port = config.get("internalapi.http.port") or None self.internal_bind_address = config.get("internalapi.http.bind_address", "::1") - if internal_api_port != "": + if internal_api_port is not None: self.internal_api_enabled = True self.internal_port = int(internal_api_port) else: self.internal_api_enabled = False - self.cert_file = config.get("replication.https.certfile") - self.ca_cert_file = config.get("replication.https.cacert") + self.cert_file = config.get("replication.https.certfile") or None + self.ca_cert_file = config.get("replication.https.cacert") or None - self.replication_bind_address = config.get("replication.https.bind_address") - self.replication_port = int(config.get("replication.https.port")) + self.replication_bind_address = config.get( + "replication.https.bind_address", "::" + ) + self.replication_port = int(config.get("replication.https.port", 4434)) - self.obey_x_forwarded_for = parse_cfg_bool(config.get("obey_x_forwarded_for")) + self.obey_x_forwarded_for = parse_cfg_bool( + config.get("obey_x_forwarded_for", "false") + ) self.verify_federation_certs = parse_cfg_bool( - config.get("federation.verifycerts") + config.get("federation.verifycerts", "true") ) - self.server_http_url_base = config.get("client_http_base") + self.server_http_url_base = config.get("client_http_base", "") self.base_replication_urls = {} @@ -65,5 +77,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: if "base_replication_url" in peer_config.keys(): base_url = peer_config.get("base_replication_url") self.base_replication_urls[peer] = base_url - - return False diff --git a/sydent/config/sms.py b/sydent/config/sms.py index da427de2..80266512 100644 --- a/sydent/config/sms.py +++ b/sydent/config/sms.py @@ -18,20 +18,20 @@ class SMSConfig(BaseConfig): - def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: + def parse_config(self, cfg: CONFIG_PARSER_DICT) -> None: """ Parse the sms section of the config :param cfg: the configuration to be parsed """ - config = cfg.get("sms") + config = cfg.get("sms", {}) - self.body_template = config.get("bodyTemplate") + self.body_template = config.get("bodyTemplate", "Your code is {token}") # Make sure username and password are bytes otherwise we can't use them with # b64encode. - self.api_username = config.get("username").encode("UTF-8") - self.api_password = config.get("password").encode("UTF-8") + self.api_username = config.get("username", "").encode("UTF-8") + self.api_password = config.get("password", "").encode("UTF-8") self.originators: Dict[str, List[Dict[str, str]]] = {} self.smsRules = {} @@ -70,5 +70,3 @@ def parse_config(self, cfg: CONFIG_PARSER_DICT) -> bool: ) self.smsRules[country] = action - - return False diff --git a/sydent/http/httpcommon.py b/sydent/http/httpcommon.py index 330f83f6..8d3b8770 100644 --- a/sydent/http/httpcommon.py +++ b/sydent/http/httpcommon.py @@ -45,7 +45,7 @@ def makeMyCertificate(self): # TODO Move some of this loading into parse_config privKeyAndCertFilename = self.sydent.config.http.cert_file - if privKeyAndCertFilename == "": + if privKeyAndCertFilename is None: logger.warning( "No HTTPS private key / cert found: not starting replication server " "or doing replication pushes" @@ -70,7 +70,7 @@ def makeTrustRoot(self): # If this option is specified, use a specific root CA cert. This is useful for testing when it's not # practical to get the client cert signed by a real root CA but should never be used on a production server. caCertFilename = self.sydent.config.http.ca_cert_file - if len(caCertFilename) > 0: + if caCertFilename is not None: try: fp = open(caCertFilename) caCert = twisted.internet.ssl.Certificate.loadPEM(fp.read()) diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py index 037460ee..b9e0a4c6 100644 --- a/sydent/replication/peer.py +++ b/sydent/replication/peer.py @@ -26,7 +26,6 @@ from twisted.web.iweb import IResponse from unpaddedbase64 import decode_base64 -from sydent.config import ConfigError from sydent.db.hashing_metadata import HashingMetadataStore from sydent.db.threepid_associations import GlobalAssociationStore from sydent.threepid import threePidAssocFromDict @@ -170,7 +169,7 @@ def __init__( try: pubkey_decoded = decode_base64(pubkey) except Exception as e: - raise ConfigError( + raise RuntimeError( "Unable to decode public key for peer %s: %s" % (server_name, e), ) diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index cd78187c..cade9dd7 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -106,13 +106,12 @@ def get_terms(sydent: "Sydent") -> Optional[Terms]: # TODO - move some of this to parse_config termsPath = sydent.config.general.terms_path + if termsPath is None: + return Terms(None) try: termsYaml = None - if termsPath == "": - return Terms(None) - with open(termsPath) as fp: termsYaml = yaml.safe_load(fp) if "master_version" not in termsYaml: