From 14d66029cef01ae82049f3b611adef2ec72e867e Mon Sep 17 00:00:00 2001 From: David Stosik <816901+davidstosik@users.noreply.github.com> Date: Thu, 27 Feb 2025 22:01:05 +0900 Subject: [PATCH 1/2] Prepare an example on how to run MFYNAB in Docker --- README.md | 41 ++++++++++++++ docker_example/.gitignore | 1 + docker_example/Dockerfile | 19 +++++++ docker_example/Gemfile | 6 +++ docker_example/Gemfile.lock | 62 ++++++++++++++++++++++ docker_example/entrypoint.sh | 7 +++ docker_example/mfynab-cron | 10 ++++ lib/mfynab/money_forward/account_status.rb | 2 +- lib/mfynab/money_forward/session.rb | 3 ++ 9 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 docker_example/.gitignore create mode 100644 docker_example/Dockerfile create mode 100644 docker_example/Gemfile create mode 100644 docker_example/Gemfile.lock create mode 100755 docker_example/entrypoint.sh create mode 100644 docker_example/mfynab-cron diff --git a/README.md b/README.md index 38063d0..47cff59 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ MONEYFORWARD_PASSWORD=Passw0rd! YNAB_ACCESS_TOKEN=abunchofcharacters ``` + + Alternatively, you can also use something like [1Password's CLI](https://developer.1password.com/docs/cli/) to completely avoid storing clear secrets: @@ -90,6 +92,44 @@ op run --env-file=.env -- mfynab mfynab-david.yml After checking out the repo, run `bundle install` to install dependencies. Then, run `bin/rake test` to run the tests. +## Running MFYNAB with cron and Docker + +The `docker-example/` directory contains sample files that'll help you schedule MFYNAB inside a Docker container. + +See the comments in each file for more details on how it works. + +First you'll want to bring your MFYNAB configuration file into this directory: + +```sh +cp path_to/config.yml docker_example/ +``` + +Then you can build the Docker image: + +```sh +docker build -t mfynab docker_example/ +``` + +Finally, you can run the Docker image. Note that you need to pass secrets as environment variables: + +```sh +docker run -d \ + --env YNAB_ACCESS_TOKEN=... \ + --env MONEYFORWARD_USERNAME=... \ + --env MONEYFORWARD_PASSWORD='...' \ + --name mfynab mfynab +``` + +You can also use the 1Password CLI ([documented here](#one-password-cli)) for this step: + +```sh +op run --env-file=.env -- sh -c 'docker run -d \ + --env YNAB_ACCESS_TOKEN=$YNAB_ACCESS_TOKEN \ + --env MONEYFORWARD_USERNAME=$MONEYFORWARD_USERNAME \ + --env MONEYFORWARD_PASSWORD="$MONEYFORWARD_PASSWORD" \ + --name mfynab mfynab' +``` + ## Roadmap ### Deploy/Automate @@ -135,3 +175,4 @@ Previous notes: ``` - Passing logger everywhere feels weird. - Prompt user for captcha and other account extra authentication required by Money Forward? +- One might want to run a single Docker instance for multiple users, but the current setup does not allow that easily. We'll want to bring the secret environment variables into the config file, making it possible to assign them to a given "user", and name them accordingly. diff --git a/docker_example/.gitignore b/docker_example/.gitignore new file mode 100644 index 0000000..1d3ed4c --- /dev/null +++ b/docker_example/.gitignore @@ -0,0 +1 @@ +config.yml diff --git a/docker_example/Dockerfile b/docker_example/Dockerfile new file mode 100644 index 0000000..8fbf692 --- /dev/null +++ b/docker_example/Dockerfile @@ -0,0 +1,19 @@ +FROM ruby:3.4 + +# Install cron and chromium +RUN apt-get update && apt-get install -y cron chromium + +WORKDIR /app + +# Copy Gemfile first and install gems +COPY Gemfile Gemfile.lock ./ +RUN bundle install + +# Copy the rest of the application (this two-step process helps with layer caching) +COPY . . + +# Add cron job +RUN crontab /app/mfynab-cron + +# Run the entrypoint script by default +CMD [ "/app/entrypoint.sh" ] diff --git a/docker_example/Gemfile b/docker_example/Gemfile new file mode 100644 index 0000000..7a0c57c --- /dev/null +++ b/docker_example/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# FIXME: point to the latest RubyGems release +gem "mfynab", github: "davidstosik/mfynab", branch: "sto/docker" diff --git a/docker_example/Gemfile.lock b/docker_example/Gemfile.lock new file mode 100644 index 0000000..cd6ac16 --- /dev/null +++ b/docker_example/Gemfile.lock @@ -0,0 +1,62 @@ +GIT + remote: https://github.com/davidstosik/mfynab.git + revision: 73f36f266bbd879983644c8aadc2189b70d22665 + branch: sto/docker + specs: + mfynab (0.1.4) + csv + ferrum (~> 0.15) + nokogiri (~> 1.18) + psych + ynab (~> 3.6) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.5) + csv (3.3.2) + date (3.4.1) + ethon (0.16.0) + ffi (>= 1.15.0) + ferrum (0.16) + addressable (~> 2.5) + base64 (~> 0.2) + concurrent-ruby (~> 1.1) + webrick (~> 1.7) + websocket-driver (~> 0.7) + ffi (1.17.1) + ffi (1.17.1-arm64-darwin) + mini_portile2 (2.8.8) + nokogiri (1.18.3) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.18.3-arm64-darwin) + racc (~> 1.4) + psych (5.2.3) + date + stringio + public_suffix (6.0.1) + racc (1.8.1) + stringio (3.1.5) + typhoeus (1.4.1) + ethon (>= 0.9.0) + webrick (1.9.1) + websocket-driver (0.7.7) + base64 + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + ynab (3.6.0) + typhoeus (~> 1.0, >= 1.0.1) + +PLATFORMS + arm64-darwin-24 + ruby + +DEPENDENCIES + mfynab! + +BUNDLED WITH + 2.6.2 diff --git a/docker_example/entrypoint.sh b/docker_example/entrypoint.sh new file mode 100755 index 0000000..eae6f8c --- /dev/null +++ b/docker_example/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# Save environment variables to a file, so we can forward them to cron jobs +env >> /etc/environment + +# Run cron in the foreground +cron -f diff --git a/docker_example/mfynab-cron b/docker_example/mfynab-cron new file mode 100644 index 0000000..7274733 --- /dev/null +++ b/docker_example/mfynab-cron @@ -0,0 +1,10 @@ +# Pick up environment variables that were forwarded when starting the container +SHELL=/bin/bash +BASH_ENV=/etc/environment + +# See: +# https://www.reddit.com/r/docker/comments/mzqibu/how_to_correctly_run_cron_in_docker/ + +# Run MFYNAB at 6am, 12pm, and 9pm JST every day +# FIXME: I'd like to be able to use UTC+9 times +0 3,9,21 * * * cd /app; bundle exec mfynab config.yml >/proc/1/fd/1 2>/proc/1/fd/2 diff --git a/lib/mfynab/money_forward/account_status.rb b/lib/mfynab/money_forward/account_status.rb index 7c74028..46c9388 100644 --- a/lib/mfynab/money_forward/account_status.rb +++ b/lib/mfynab/money_forward/account_status.rb @@ -67,7 +67,7 @@ def parse_html_table_row(node) end end - FRESHNESS_LIMIT = 24 * 60 * 60 # 1 day + FRESHNESS_LIMIT = 60 * 60 # 1 hour attr_reader :id_hash, :name, :raw_status, :updated_at, :key, :message diff --git a/lib/mfynab/money_forward/session.rb b/lib/mfynab/money_forward/session.rb index d83d375..7ed3b7c 100644 --- a/lib/mfynab/money_forward/session.rb +++ b/lib/mfynab/money_forward/session.rb @@ -79,6 +79,9 @@ def with_ferrum timeout: 30, process_timeout: 20, # 10s was not always enough on CI (FIXME: make configurable per env) headless: !ENV.key?("NO_HEADLESS"), + # FIXME: this was needed to be able to run Chromium headless + # within Docker as root, but I'd rather not rely on it. + browser_options: { "no-sandbox": nil }, ) user_agent = browser.default_user_agent.sub("HeadlessChrome", "Chrome") browser.headers.add({ From f4c7417d6642ac2cad5c235cac9766e3e485d8b2 Mon Sep 17 00:00:00 2001 From: David Stosik <816901+davidstosik@users.noreply.github.com> Date: Tue, 25 Mar 2025 18:07:31 +0900 Subject: [PATCH 2/2] Use RubyGems release of MFYNAB in Docker example Also don't commit Gemfile.lock in that example, since versions should follow with time. --- docker_example/.gitignore | 1 + docker_example/Gemfile | 3 +- docker_example/Gemfile.lock | 62 ------------------------------------- 3 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 docker_example/Gemfile.lock diff --git a/docker_example/.gitignore b/docker_example/.gitignore index 1d3ed4c..389d6ad 100644 --- a/docker_example/.gitignore +++ b/docker_example/.gitignore @@ -1 +1,2 @@ config.yml +Gemfile.lock diff --git a/docker_example/Gemfile b/docker_example/Gemfile index 7a0c57c..6215dbe 100644 --- a/docker_example/Gemfile +++ b/docker_example/Gemfile @@ -2,5 +2,4 @@ source "https://rubygems.org" -# FIXME: point to the latest RubyGems release -gem "mfynab", github: "davidstosik/mfynab", branch: "sto/docker" +gem "mfynab" diff --git a/docker_example/Gemfile.lock b/docker_example/Gemfile.lock deleted file mode 100644 index cd6ac16..0000000 --- a/docker_example/Gemfile.lock +++ /dev/null @@ -1,62 +0,0 @@ -GIT - remote: https://github.com/davidstosik/mfynab.git - revision: 73f36f266bbd879983644c8aadc2189b70d22665 - branch: sto/docker - specs: - mfynab (0.1.4) - csv - ferrum (~> 0.15) - nokogiri (~> 1.18) - psych - ynab (~> 3.6) - -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - base64 (0.2.0) - concurrent-ruby (1.3.5) - csv (3.3.2) - date (3.4.1) - ethon (0.16.0) - ffi (>= 1.15.0) - ferrum (0.16) - addressable (~> 2.5) - base64 (~> 0.2) - concurrent-ruby (~> 1.1) - webrick (~> 1.7) - websocket-driver (~> 0.7) - ffi (1.17.1) - ffi (1.17.1-arm64-darwin) - mini_portile2 (2.8.8) - nokogiri (1.18.3) - mini_portile2 (~> 2.8.2) - racc (~> 1.4) - nokogiri (1.18.3-arm64-darwin) - racc (~> 1.4) - psych (5.2.3) - date - stringio - public_suffix (6.0.1) - racc (1.8.1) - stringio (3.1.5) - typhoeus (1.4.1) - ethon (>= 0.9.0) - webrick (1.9.1) - websocket-driver (0.7.7) - base64 - websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.5) - ynab (3.6.0) - typhoeus (~> 1.0, >= 1.0.1) - -PLATFORMS - arm64-darwin-24 - ruby - -DEPENDENCIES - mfynab! - -BUNDLED WITH - 2.6.2