diff --git a/bin/run_rake_task b/bin/run_rake_task new file mode 100755 index 0000000000..5a7b0cea2e --- /dev/null +++ b/bin/run_rake_task @@ -0,0 +1,187 @@ +#!/bin/sh -e +# +# Loads the same Secrets Manager env as the app start scripts, then runs a rake +# task by name. Used by the scheduled ECS task (EventBridge -> ECS RunTask) that +# runs the periodic DocuSeal integrity checks (e.g. careerplug:check_webhooks), +# so the one-shot task has the same env (DB creds, AIRBRAKE_*, NEWRELIC_*, +# CAREERPLUG_WEBHOOK_*) as the long-running app. +# +# Usage (as the container command override): +# /app/bin/run_rake_task +# +# Example: +# /app/bin/run_rake_task careerplug:check_webhooks +# +# Migrations are intentionally NOT run here: the app service migrates on deploy, +# and a periodic one-shot shouldn't race with it. +# +# POSIX sh only (Alpine /bin/sh is busybox ash, not bash): no bash-isms like &>. + +RAKE_TASK="${1:-}" +if [ -z "$RAKE_TASK" ]; then + echo "ERROR: no rake task specified. Usage: $0 " >&2 + exit 1 +fi + +# Resolve the env-file suffix from RAILS_ENV (.env.staging / .env.production). +ENV_FILE_SUFFIX="${RAILS_ENV:-production}" +ENV_FILE="./.env.${ENV_FILE_SUFFIX}" + +echo "=== CP Docuseal Rake Task Runner ===" +echo "RAILS_ENV: ${ENV_FILE_SUFFIX}" +echo "Task: ${RAKE_TASK}" + +# Enable jemalloc for reduced memory usage and latency. +if [ -z "${LD_PRELOAD+x}" ]; then + LD_PRELOAD=$(find /usr/lib -name libjemalloc.so.2 -print -quit) + export LD_PRELOAD +fi + +check_aws_setup() { + if [ -z "$AWS_REGION" ]; then + echo "ERROR: AWS_REGION environment variable is not set" + exit 1 + fi + + if ! command -v aws >/dev/null 2>&1; then + echo "ERROR: AWS CLI is not installed. Please install it to proceed." + exit 1 + fi +} + +fetch_db_credentials() { + echo "Fetching database credentials from AWS Secrets Manager..." + + if [ -z "$DB_SECRETS_NAME" ]; then + echo "ERROR: DB_SECRETS_NAME environment variable is not set" + exit 1 + fi + + if ! SECRET_JSON=$(aws secretsmanager get-secret-value \ + --region "$AWS_REGION" \ + --secret-id "$DB_SECRETS_NAME" \ + --query SecretString \ + --output text); then + echo "ERROR: Failed to retrieve secrets from AWS Secrets Manager" + exit 1 + fi + + export DB_USERNAME=$(echo "$SECRET_JSON" | jq -r '.username') + export DB_PASSWORD=$(echo "$SECRET_JSON" | jq -r '.password') + + if [ "$DB_USERNAME" = "null" ] || [ "$DB_PASSWORD" = "null" ] || [ -z "$DB_USERNAME" ] || [ -z "$DB_PASSWORD" ]; then + echo "ERROR: Failed to parse database credentials from secrets" + exit 1 + fi + + if [ -f "$ENV_FILE" ]; then + grep -v "^DB_USERNAME=" "$ENV_FILE" > "${ENV_FILE}.tmp" || true + grep -v "^DB_PASSWORD=" "${ENV_FILE}.tmp" > "$ENV_FILE" || true + rm -f "${ENV_FILE}.tmp" + fi + + echo "DB_USERNAME=$DB_USERNAME" >> "$ENV_FILE" + echo "DB_PASSWORD=$DB_PASSWORD" >> "$ENV_FILE" + echo "✓ Database credentials retrieved" +} + +fetch_encryption_key() { + echo "Fetching encryption key from AWS Secrets Manager..." + + ENCRYPTION_SECRET_NAME="cpdocuseal/encryption_key" + + if ! ENCRYPTION_KEY=$(aws secretsmanager get-secret-value \ + --region "$AWS_REGION" \ + --secret-id "$ENCRYPTION_SECRET_NAME" \ + --query SecretString \ + --output text); then + echo "ERROR: Failed to retrieve encryption key from AWS Secrets Manager" + exit 1 + fi + + if [ -z "$ENCRYPTION_KEY" ] || [ "$ENCRYPTION_KEY" = "null" ]; then + echo "ERROR: Failed to retrieve encryption key from AWS Secrets Manager" + exit 1 + fi + + echo -n "$ENCRYPTION_KEY" > config/master.key + chmod 600 config/master.key + echo "✓ Encryption key written to config/master.key" +} + +fetch_env_variables() { + echo "Fetching environment variables from AWS Secrets Manager..." + + if [ -z "$CP_VARIABLES_NAME" ]; then + echo "ERROR: CP_VARIABLES_NAME environment variable is not set" + exit 1 + fi + + if ! SECRET_JSON=$(aws secretsmanager get-secret-value \ + --region "$AWS_REGION" \ + --secret-id "$CP_VARIABLES_NAME" \ + --query SecretString \ + --output text); then + echo "ERROR: Failed to retrieve secrets from AWS Secrets Manager" + exit 1 + fi + + export DB_HOST=$(echo "$SECRET_JSON" | jq -r '.host') + export REDIS_URL=$(echo "$SECRET_JSON" | jq -r '.redis_url') + export S3_ATTACHMENTS_BUCKET=$(echo "$SECRET_JSON" | jq -r '.s3_attachments_bucket') + export AIRBRAKE_ID=$(echo "$SECRET_JSON" | jq -r '.airbrake_id') + export AIRBRAKE_KEY=$(echo "$SECRET_JSON" | jq -r '.airbrake_key') + export NEWRELIC_LICENSE_KEY=$(echo "$SECRET_JSON" | jq -r '.newrelic_license_key') + export NEWRELIC_APP_NAME=$(echo "$SECRET_JSON" | jq -r '.newrelic_app_name') + export NEWRELIC_MONITOR_MODE=$(echo "$SECRET_JSON" | jq -r '.newrelic_monitor_mode') + export ENCRYPTION_SECRET=$(echo "$SECRET_JSON" | jq -r '.ENCRYPTION_SECRET // empty') + export CAREERPLUG_WEBHOOK_SECRET=$(echo "$SECRET_JSON" | jq -r '.CAREERPLUG_WEBHOOK_SECRET // empty') + export CAREERPLUG_WEBHOOK_URL=$(echo "$SECRET_JSON" | jq -r '.CAREERPLUG_WEBHOOK_URL // empty') + + if [ "$DB_HOST" = "null" ] || [ "$REDIS_URL" = "null" ] || [ "$S3_ATTACHMENTS_BUCKET" = "null" ] || [ -z "$DB_HOST" ] || [ -z "$REDIS_URL" ] || [ -z "$S3_ATTACHMENTS_BUCKET" ]; then + echo "ERROR: Failed to parse variables from secrets" + exit 1 + fi + + if [ -f "$ENV_FILE" ]; then + grep -Ev "^(DB_HOST|REDIS_URL|S3_ATTACHMENTS_BUCKET|AIRBRAKE_ID|AIRBRAKE_KEY|NEWRELIC_LICENSE_KEY|NEWRELIC_APP_NAME|NEWRELIC_MONITOR_MODE|ENCRYPTION_SECRET|CAREERPLUG_WEBHOOK_SECRET|CAREERPLUG_WEBHOOK_URL)=" "$ENV_FILE" > "${ENV_FILE}.tmp" || true + mv "${ENV_FILE}.tmp" "$ENV_FILE" + fi + + echo "DB_HOST=$DB_HOST" >> "$ENV_FILE" + echo "REDIS_URL=$REDIS_URL" >> "$ENV_FILE" + echo "S3_ATTACHMENTS_BUCKET=$S3_ATTACHMENTS_BUCKET" >> "$ENV_FILE" + echo "AIRBRAKE_ID=$AIRBRAKE_ID" >> "$ENV_FILE" + echo "AIRBRAKE_KEY=$AIRBRAKE_KEY" >> "$ENV_FILE" + echo "NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY" >> "$ENV_FILE" + echo "NEWRELIC_APP_NAME=$NEWRELIC_APP_NAME" >> "$ENV_FILE" + echo "NEWRELIC_MONITOR_MODE=$NEWRELIC_MONITOR_MODE" >> "$ENV_FILE" + + [ -n "$ENCRYPTION_SECRET" ] && echo "ENCRYPTION_SECRET=$ENCRYPTION_SECRET" >> "$ENV_FILE" + [ -n "$CAREERPLUG_WEBHOOK_SECRET" ] && echo "CAREERPLUG_WEBHOOK_SECRET=$CAREERPLUG_WEBHOOK_SECRET" >> "$ENV_FILE" + [ -n "$CAREERPLUG_WEBHOOK_URL" ] && echo "CAREERPLUG_WEBHOOK_URL=$CAREERPLUG_WEBHOOK_URL" >> "$ENV_FILE" + + echo "✓ Environment variables retrieved" +} + +set_environment() { + if [ -f "$ENV_FILE" ]; then + echo "Setting environment variables from $ENV_FILE" + set -a + . "$ENV_FILE" + set +a + fi +} + +# cd ../../app is relative to the bin/ dir within the container, matching the start scripts. +cd ../../app/ + +set_environment +check_aws_setup +fetch_db_credentials +fetch_encryption_key +fetch_env_variables +set_environment + +echo "=== Startup Complete - Running rake task: ${RAKE_TASK} ===" +exec ./bin/rails "$RAKE_TASK" diff --git a/lib/send_webhook_request.rb b/lib/send_webhook_request.rb index d0f4fcec17..fadd3caa4e 100644 --- a/lib/send_webhook_request.rb +++ b/lib/send_webhook_request.rb @@ -24,6 +24,18 @@ def call(webhook_url, event_type:, data:) raise LocalhostError, "Can't send to localhost." if uri.host.in?(LOCALHOSTS) end + response = post(webhook_url, uri, event_type: event_type, data: data) + rescue Faraday::Error => e + record_delivery(event_type, nil) + Rails.logger.error("Webhook transport error (#{event_type}): #{e.class} #{e.message}") + Airbrake.notify("DocuSeal outbound webhook transport error (#{event_type}): #{e.class}") + nil + else + record_delivery(event_type, response.status.to_i) + response + end + + def post(webhook_url, uri, event_type:, data:) Faraday.post(uri) do |req| req.headers['Content-Type'] = 'application/json' req.headers['User-Agent'] = USER_AGENT @@ -42,7 +54,16 @@ def call(webhook_url, event_type:, data:) req.options.read_timeout = 8 req.options.open_timeout = 8 end - rescue Faraday::Error - nil + end + + # Records delivery outcome as New Relic custom metrics so non-2xx rates and + # transport failures (previously swallowed silently to nil) are visible on a + # dashboard. No-op when the New Relic agent isn't loaded. + def record_delivery(event_type, status) + return unless defined?(::NewRelic::Agent) + + bucket = status&.between?(200, 299) ? 'success' : 'failure' + ::NewRelic::Agent.record_metric("DocuSeal/Webhook/#{event_type}/#{bucket}", 1) + ::NewRelic::Agent.record_metric("DocuSeal/Webhook/total/#{bucket}", 1) end end diff --git a/lib/tasks/careerplug_health.rake b/lib/tasks/careerplug_health.rake new file mode 100644 index 0000000000..32b9e77e7a --- /dev/null +++ b/lib/tasks/careerplug_health.rake @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +namespace :careerplug do + desc 'Alert (via Airbrake) when an ATS-linked account/partnership has no ATS-pointed WebhookUrl' + task check_webhooks: :environment do + WebhookHealthCheck.report! + end +end diff --git a/lib/webhook_health_check.rb b/lib/webhook_health_check.rb new file mode 100644 index 0000000000..99d1ee3171 --- /dev/null +++ b/lib/webhook_health_check.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +# Runtime guardrail that alerts when an ATS-linked account or partnership is +# missing the WebhookUrl that points at the CareerPlug ATS endpoint. +# +# Complements the boot-time env-var guard (config/initializers/careerplug_webhook_config.rb) +# by catching *drift* at runtime: an owner whose ATS WebhookUrl was deleted, never +# created (e.g. created while the env var was temporarily blank), or points at a +# stale URL. Designed to be invoked periodically (rake task / scheduled ECS task). +module WebhookHealthCheck + module_function + + # Returns ATS-linked owners (Accounts + Partnerships) that have no WebhookUrl + # pointing at the configured CAREERPLUG_WEBHOOK_URL. + # + # @return [Array] empty when env is blank (boot guard owns + # that case) or when every ATS-linked owner has the expected webhook. + def missing_ats_webhooks + target_sha1 = sha1_for(ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil)) + return [] if target_sha1.nil? + + accounts = Account.active + .where.not(external_account_id: nil) + .where.not(id: ats_webhook_owner_ids(target_sha1, :account_id)) + + partnerships = Partnership.where.not(id: ats_webhook_owner_ids(target_sha1, :partnership_id)) + + accounts.to_a + partnerships.to_a + end + + # Detects missing ATS webhooks and notifies Airbrake when any are found. + # Airbrake groups notices by message string, so repeated identical findings + # (same count + same owners) are deduplicated into one notice. A new notice + # fires only when the findings change (new missing webhooks appear). + # + # @return [Array] the findings (empty when healthy) + def report! + findings = missing_ats_webhooks + return findings if findings.empty? + + message = format( + 'DocuSeal integrity: %d ATS-linked owner(s) missing an ATS-pointed ' \ + 'WebhookUrl (expected url sha1=%s): %s', + count: findings.size, + sha1: sha1_for(ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil)), + owners: findings.map { |o| "#{o.class.name}##{o.id}" }.first(20).join(', ') + ) + Rails.logger.error(message) + Airbrake.notify(message) + findings + end + + # Cleartext SHA1 of a webhook URL, matching the dedup key set by WebhookUrl#set_sha1. + # Returns nil for a blank URL so callers can short-circuit (missing env is the + # boot guard's responsibility, not ours). + def sha1_for(url) + return nil if url.blank? + + Digest::SHA1.hexdigest(url) + end + + # IDs of owners that already have a WebhookUrl with the target sha1, for the + # given foreign key column. Used as a NOT IN subquery so we never load every + # account/partnership into memory. + def ats_webhook_owner_ids(target_sha1, owner_column) + WebhookUrl.where(sha1: target_sha1) + .where.not(owner_column => nil) + .select(owner_column) + end +end diff --git a/spec/lib/send_webhook_request_spec.rb b/spec/lib/send_webhook_request_spec.rb new file mode 100644 index 0000000000..3d01816121 --- /dev/null +++ b/spec/lib/send_webhook_request_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe SendWebhookRequest do + let(:account) { create(:account) } + let(:webhook_url) { create(:webhook_url, account: account, url: 'https://example.com/hook') } + # Plain double (not verified) because we stub the whole ::NewRelic::Agent constant, + # which may not be loaded depending on env — verifying doubles would fail to resolve it. + let(:nr_agent) { double('NewRelic::Agent') } # rubocop:disable RSpec/VerifiedDoubles + + before do + # The production code calls ::NewRelic::Agent.record_metric. The gem may not be + # loaded in the test env, so stub the whole constant and record_metric on it. + stub_const('NewRelic::Agent', nr_agent) + allow(nr_agent).to receive(:record_metric) + allow(Airbrake).to receive(:notify) + allow(Rails.logger).to receive(:error) + end + + describe '.call' do + context 'when the request succeeds (2xx)' do + before { stub_request(:post, 'https://example.com/hook').to_return(status: 200, body: '{}') } + + it 'records a success metric and does not notify Airbrake' do + described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 }) + + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/success', 1) + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/success', 1) + expect(Airbrake).not_to have_received(:notify) + end + end + + context 'when the request returns a non-2xx status' do + before { stub_request(:post, 'https://example.com/hook').to_return(status: 500) } + + it 'records a failure metric without notifying Airbrake (non-2xx is handled by retry logic)' do + described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 }) + + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/failure', 1) + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/failure', 1) + expect(Airbrake).not_to have_received(:notify) + end + end + + context 'when the request raises a Faraday::Error (transport failure)' do + before do + stub_request(:post, 'https://example.com/hook').to_raise(Faraday::ConnectionFailed.new('boom')) + end + + it 'records a failure metric, logs, notifies Airbrake, and returns nil (retry path intact)' do + result = described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 }) + + expect(result).to be_nil + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/failure', 1) + expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/failure', 1) + expect(Rails.logger).to have_received(:error).with(/Webhook transport error.*form\.completed.*boom/) + expect(Airbrake).to have_received(:notify).with(/DocuSeal outbound webhook transport error \(form.completed\)/) + end + end + end +end diff --git a/spec/lib/webhook_health_check_spec.rb b/spec/lib/webhook_health_check_spec.rb new file mode 100644 index 0000000000..7ef4cdd1d1 --- /dev/null +++ b/spec/lib/webhook_health_check_spec.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe WebhookHealthCheck do + let(:ats_url) { 'https://www.careerplug.com/api/docuseal/events' } + + around do |example| + previous = ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil) + ENV['CAREERPLUG_WEBHOOK_URL'] = ats_url + example.run + ensure + ENV['CAREERPLUG_WEBHOOK_URL'] = previous + end + + describe '.sha1_for' do + it 'returns the SHA1 hexdigest matching WebhookUrl#set_sha1' do + expect(described_class.sha1_for(ats_url)).to eq(Digest::SHA1.hexdigest(ats_url)) + end + + it 'returns nil for a blank url' do + expect(described_class.sha1_for(nil)).to be_nil + expect(described_class.sha1_for('')).to be_nil + end + end + + describe '.missing_ats_webhooks' do + let(:expected_sha1) { Digest::SHA1.hexdigest(ats_url) } + + context 'with an ATS-linked account that has the expected webhook' do + let!(:account) do + create(:account, external_account_id: 101).tap do |a| + create(:webhook_url, account: a, url: ats_url, + events: %w[form.completed], + secret: { 'X-CareerPlug-Secret' => 's' }) + end + end + + it 'is not flagged' do + expect(described_class.missing_ats_webhooks).not_to include(an_object_having_attributes(id: account.id)) + end + end + + context 'with an ATS-linked account missing the expected webhook' do + let!(:account) { create(:account, external_account_id: 102) } + + it 'is flagged' do + expect(described_class.missing_ats_webhooks).to include(an_object_having_attributes(id: account.id)) + end + end + + context 'with an account whose webhook points at a different url' do + let!(:account) { create(:account, external_account_id: 103) } + + before do + create(:webhook_url, account: account, url: 'https://example.com/other', events: %w[form.completed]) + end + + it 'is still flagged (sha1 mismatch)' do + expect(described_class.missing_ats_webhooks).to include(an_object_having_attributes(id: account.id)) + end + end + + it 'ignores accounts with no external_account_id (not ATS-linked)' do + account = create(:account, external_account_id: nil) + expect(described_class.missing_ats_webhooks).not_to include(an_object_having_attributes(id: account.id)) + end + + it 'ignores archived accounts' do + account = create(:account, external_account_id: 104, archived_at: Time.current) + expect(described_class.missing_ats_webhooks).not_to include(an_object_having_attributes(id: account.id)) + end + + context 'with partnerships' do + let!(:with_webhook) do + create(:partnership).tap do |p| + create(:webhook_url, partnership: p, account: nil, url: ats_url, + events: %w[template.preferences_updated]) + end + end + let!(:without_webhook) { create(:partnership) } + + it 'flags partnerships missing the expected webhook but not those with it' do + findings = described_class.missing_ats_webhooks + expect(findings).to include(an_object_having_attributes(id: without_webhook.id)) + expect(findings).not_to include(an_object_having_attributes(id: with_webhook.id)) + end + end + + context 'when CAREERPLUG_WEBHOOK_URL is blank' do + around do |example| + previous = ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil) + ENV['CAREERPLUG_WEBHOOK_URL'] = '' + example.run + ensure + ENV['CAREERPLUG_WEBHOOK_URL'] = previous + end + + it 'returns an empty list (boot guard owns the missing-env case)' do + create(:account, external_account_id: 999) + expect(described_class.missing_ats_webhooks).to eq([]) + end + end + end + + describe '.report!' do + before do + allow(Rails.logger).to receive(:error) + allow(Airbrake).to receive(:notify) + end + + context 'when findings exist' do + let!(:account) { create(:account, external_account_id: 201) } + + it 'logs an error and notifies Airbrake' do + described_class.report! + + expect(Rails.logger).to have_received(:error).with(/missing an ATS-pointed WebhookUrl/) + expect(Airbrake).to have_received(:notify).with(/Account##{account.id}/) + end + + it 'returns the findings' do + expect(described_class.report!.map(&:id)).to include(account.id) + end + end + + context 'when there are no findings' do + before { create(:account, external_account_id: 202).tap { |a| create(:webhook_url, account: a, url: ats_url) } } + + it 'does not log or notify' do + described_class.report! + + expect(Rails.logger).not_to have_received(:error) + expect(Airbrake).not_to have_received(:notify) + end + + it 'returns an empty array' do + expect(described_class.report!).to eq([]) + end + end + end +end