CP-14167 - Add webhook health monitoring for ATS integration

bin/run_rake_task

@@ -0,0 +1,187 @@
+#!/bin/sh -e
+#
+# Loads the same Secrets Manager env as the app start scripts, then runs a rake
+# task by name. Used by the scheduled ECS task (EventBridge -> ECS RunTask) that
+# runs the periodic DocuSeal integrity checks (e.g. careerplug:check_webhooks),
+# so the one-shot task has the same env (DB creds, AIRBRAKE_*, NEWRELIC_*,
+# CAREERPLUG_WEBHOOK_*) as the long-running app.
+#
+# Usage (as the container command override):
+#   /app/bin/run_rake_task <rake task name>
+#
+# Example:
+#   /app/bin/run_rake_task careerplug:check_webhooks
+#
+# Migrations are intentionally NOT run here: the app service migrates on deploy,
+# and a periodic one-shot shouldn't race with it.
+#
+# POSIX sh only (Alpine /bin/sh is busybox ash, not bash): no bash-isms like &>.
+
+RAKE_TASK="${1:-}"
+if [ -z "$RAKE_TASK" ]; then
+    echo "ERROR: no rake task specified. Usage: $0 <rake task name>" >&2
+    exit 1
+fi
+
+# Resolve the env-file suffix from RAILS_ENV (.env.staging / .env.production).
+ENV_FILE_SUFFIX="${RAILS_ENV:-production}"
+ENV_FILE="./.env.${ENV_FILE_SUFFIX}"
+
+echo "=== CP Docuseal Rake Task Runner ==="
+echo "RAILS_ENV: ${ENV_FILE_SUFFIX}"
+echo "Task: ${RAKE_TASK}"
+
+# Enable jemalloc for reduced memory usage and latency.
+if [ -z "${LD_PRELOAD+x}" ]; then
+    LD_PRELOAD=$(find /usr/lib -name libjemalloc.so.2 -print -quit)
+    export LD_PRELOAD
+fi
+
+check_aws_setup() {
+    if [ -z "$AWS_REGION" ]; then
+        echo "ERROR: AWS_REGION environment variable is not set"
+        exit 1
+    fi
+
+    if ! command -v aws >/dev/null 2>&1; then
+        echo "ERROR: AWS CLI is not installed. Please install it to proceed."
+        exit 1
+    fi
+}
+
+fetch_db_credentials() {
+    echo "Fetching database credentials from AWS Secrets Manager..."
+
+    if [ -z "$DB_SECRETS_NAME" ]; then
+        echo "ERROR: DB_SECRETS_NAME environment variable is not set"
+        exit 1
+    fi
+
+    if ! SECRET_JSON=$(aws secretsmanager get-secret-value \
+        --region "$AWS_REGION" \
+        --secret-id "$DB_SECRETS_NAME" \
+        --query SecretString \
+        --output text); then
+        echo "ERROR: Failed to retrieve secrets from AWS Secrets Manager"
+        exit 1
+    fi
+
+    export DB_USERNAME=$(echo "$SECRET_JSON" | jq -r '.username')
+    export DB_PASSWORD=$(echo "$SECRET_JSON" | jq -r '.password')
+
+    if [ "$DB_USERNAME" = "null" ] || [ "$DB_PASSWORD" = "null" ] || [ -z "$DB_USERNAME" ] || [ -z "$DB_PASSWORD" ]; then
+        echo "ERROR: Failed to parse database credentials from secrets"
+        exit 1
+    fi
+
+    if [ -f "$ENV_FILE" ]; then
+        grep -v "^DB_USERNAME=" "$ENV_FILE" > "${ENV_FILE}.tmp" || true
+        grep -v "^DB_PASSWORD=" "${ENV_FILE}.tmp" > "$ENV_FILE" || true
+        rm -f "${ENV_FILE}.tmp"
+    fi
+
+    echo "DB_USERNAME=$DB_USERNAME" >> "$ENV_FILE"
+    echo "DB_PASSWORD=$DB_PASSWORD" >> "$ENV_FILE"
+    echo "✓ Database credentials retrieved"
+}
+
+fetch_encryption_key() {
+    echo "Fetching encryption key from AWS Secrets Manager..."
+
+    ENCRYPTION_SECRET_NAME="cpdocuseal/encryption_key"
+
+    if ! ENCRYPTION_KEY=$(aws secretsmanager get-secret-value \
+        --region "$AWS_REGION" \
+        --secret-id "$ENCRYPTION_SECRET_NAME" \
+        --query SecretString \
+        --output text); then
+        echo "ERROR: Failed to retrieve encryption key from AWS Secrets Manager"
+        exit 1
+    fi
+
+    if [ -z "$ENCRYPTION_KEY" ] || [ "$ENCRYPTION_KEY" = "null" ]; then
+        echo "ERROR: Failed to retrieve encryption key from AWS Secrets Manager"
+        exit 1
+    fi
+
+    echo -n "$ENCRYPTION_KEY" > config/master.key
+    chmod 600 config/master.key
+    echo "✓ Encryption key written to config/master.key"
+}
+
+fetch_env_variables() {
+    echo "Fetching environment variables from AWS Secrets Manager..."
+
+    if [ -z "$CP_VARIABLES_NAME" ]; then
+        echo "ERROR: CP_VARIABLES_NAME environment variable is not set"
+        exit 1
+    fi
+
+    if ! SECRET_JSON=$(aws secretsmanager get-secret-value \
+        --region "$AWS_REGION" \
+        --secret-id "$CP_VARIABLES_NAME" \
+        --query SecretString \
+        --output text); then
+        echo "ERROR: Failed to retrieve secrets from AWS Secrets Manager"
+        exit 1
+    fi
+
+    export DB_HOST=$(echo "$SECRET_JSON" | jq -r '.host')
+    export REDIS_URL=$(echo "$SECRET_JSON" | jq -r '.redis_url')
+    export S3_ATTACHMENTS_BUCKET=$(echo "$SECRET_JSON" | jq -r '.s3_attachments_bucket')
+    export AIRBRAKE_ID=$(echo "$SECRET_JSON" | jq -r '.airbrake_id')
+    export AIRBRAKE_KEY=$(echo "$SECRET_JSON" | jq -r '.airbrake_key')
+    export NEWRELIC_LICENSE_KEY=$(echo "$SECRET_JSON" | jq -r '.newrelic_license_key')
+    export NEWRELIC_APP_NAME=$(echo "$SECRET_JSON" | jq -r '.newrelic_app_name')
+    export NEWRELIC_MONITOR_MODE=$(echo "$SECRET_JSON" | jq -r '.newrelic_monitor_mode')
+    export ENCRYPTION_SECRET=$(echo "$SECRET_JSON" | jq -r '.ENCRYPTION_SECRET // empty')
+    export CAREERPLUG_WEBHOOK_SECRET=$(echo "$SECRET_JSON" | jq -r '.CAREERPLUG_WEBHOOK_SECRET // empty')
+    export CAREERPLUG_WEBHOOK_URL=$(echo "$SECRET_JSON" | jq -r '.CAREERPLUG_WEBHOOK_URL // empty')
+
+    if [ "$DB_HOST" = "null" ] || [ "$REDIS_URL" = "null" ] || [ "$S3_ATTACHMENTS_BUCKET" = "null" ] || [ -z "$DB_HOST" ] || [ -z "$REDIS_URL" ] || [ -z "$S3_ATTACHMENTS_BUCKET" ]; then
+        echo "ERROR: Failed to parse variables from secrets"
+        exit 1
+    fi
+
+    if [ -f "$ENV_FILE" ]; then
+        grep -Ev "^(DB_HOST|REDIS_URL|S3_ATTACHMENTS_BUCKET|AIRBRAKE_ID|AIRBRAKE_KEY|NEWRELIC_LICENSE_KEY|NEWRELIC_APP_NAME|NEWRELIC_MONITOR_MODE|ENCRYPTION_SECRET|CAREERPLUG_WEBHOOK_SECRET|CAREERPLUG_WEBHOOK_URL)=" "$ENV_FILE" > "${ENV_FILE}.tmp" || true
+        mv "${ENV_FILE}.tmp" "$ENV_FILE"
+    fi
+
+    echo "DB_HOST=$DB_HOST" >> "$ENV_FILE"
+    echo "REDIS_URL=$REDIS_URL" >> "$ENV_FILE"
+    echo "S3_ATTACHMENTS_BUCKET=$S3_ATTACHMENTS_BUCKET" >> "$ENV_FILE"
+    echo "AIRBRAKE_ID=$AIRBRAKE_ID" >> "$ENV_FILE"
+    echo "AIRBRAKE_KEY=$AIRBRAKE_KEY" >> "$ENV_FILE"
+    echo "NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY" >> "$ENV_FILE"
+    echo "NEWRELIC_APP_NAME=$NEWRELIC_APP_NAME" >> "$ENV_FILE"
+    echo "NEWRELIC_MONITOR_MODE=$NEWRELIC_MONITOR_MODE" >> "$ENV_FILE"
+
+    [ -n "$ENCRYPTION_SECRET" ] && echo "ENCRYPTION_SECRET=$ENCRYPTION_SECRET" >> "$ENV_FILE"
+    [ -n "$CAREERPLUG_WEBHOOK_SECRET" ] && echo "CAREERPLUG_WEBHOOK_SECRET=$CAREERPLUG_WEBHOOK_SECRET" >> "$ENV_FILE"
+    [ -n "$CAREERPLUG_WEBHOOK_URL" ] && echo "CAREERPLUG_WEBHOOK_URL=$CAREERPLUG_WEBHOOK_URL" >> "$ENV_FILE"
+
+    echo "✓ Environment variables retrieved"
+}
+
+set_environment() {
+    if [ -f "$ENV_FILE" ]; then
+        echo "Setting environment variables from $ENV_FILE"
+        set -a
+        . "$ENV_FILE"
+        set +a
+    fi
+}
+
+# cd ../../app is relative to the bin/ dir within the container, matching the start scripts.
+cd ../../app/
+
+set_environment
+check_aws_setup
+fetch_db_credentials
+fetch_encryption_key
+fetch_env_variables
+set_environment
+
+echo "=== Startup Complete - Running rake task: ${RAKE_TASK} ==="
+exec ./bin/rails "$RAKE_TASK"

lib/send_webhook_request.rb

@@ -24,6 +24,18 @@ def call(webhook_url, event_type:, data:)
       raise LocalhostError, "Can't send to localhost." if uri.host.in?(LOCALHOSTS)
     end
 
+    response = post(webhook_url, uri, event_type: event_type, data: data)
+  rescue Faraday::Error => e
+    record_delivery(event_type, nil)
+    Rails.logger.error("Webhook transport error (#{event_type}): #{e.class} #{e.message}")
+    Airbrake.notify("DocuSeal outbound webhook transport error (#{event_type}): #{e.class}")
+    nil
+  else
+    record_delivery(event_type, response.status.to_i)
+    response
+  end
+
+  def post(webhook_url, uri, event_type:, data:)
     Faraday.post(uri) do |req|
       req.headers['Content-Type'] = 'application/json'
       req.headers['User-Agent'] = USER_AGENT
@@ -42,7 +54,16 @@ def call(webhook_url, event_type:, data:)
       req.options.read_timeout = 8
       req.options.open_timeout = 8
     end
-  rescue Faraday::Error
-    nil
+  end
+
+  # Records delivery outcome as New Relic custom metrics so non-2xx rates and
+  # transport failures (previously swallowed silently to nil) are visible on a
+  # dashboard. No-op when the New Relic agent isn't loaded.
+  def record_delivery(event_type, status)
+    return unless defined?(::NewRelic::Agent)
+
+    bucket = status&.between?(200, 299) ? 'success' : 'failure'
+    ::NewRelic::Agent.record_metric("DocuSeal/Webhook/#{event_type}/#{bucket}", 1)
+    ::NewRelic::Agent.record_metric("DocuSeal/Webhook/total/#{bucket}", 1)
   end
 end

lib/tasks/careerplug_health.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :careerplug do
+  desc 'Alert (via Airbrake) when an ATS-linked account/partnership has no ATS-pointed WebhookUrl'
+  task check_webhooks: :environment do
+    WebhookHealthCheck.report!
+  end
+end

lib/webhook_health_check.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Runtime guardrail that alerts when an ATS-linked account or partnership is
+# missing the WebhookUrl that points at the CareerPlug ATS endpoint.
+#
+# Complements the boot-time env-var guard (config/initializers/careerplug_webhook_config.rb)
+# by catching *drift* at runtime: an owner whose ATS WebhookUrl was deleted, never
+# created (e.g. created while the env var was temporarily blank), or points at a
+# stale URL. Designed to be invoked periodically (rake task / scheduled ECS task).
+module WebhookHealthCheck
+  module_function
+
+  # Returns ATS-linked owners (Accounts + Partnerships) that have no WebhookUrl
+  # pointing at the configured CAREERPLUG_WEBHOOK_URL.
+  #
+  # @return [Array<Account, Partnership>] empty when env is blank (boot guard owns
+  #   that case) or when every ATS-linked owner has the expected webhook.
+  def missing_ats_webhooks
+    target_sha1 = sha1_for(ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil))
+    return [] if target_sha1.nil?
+
+    accounts = Account.active
+                      .where.not(external_account_id: nil)
+                      .where.not(id: ats_webhook_owner_ids(target_sha1, :account_id))
+
+    partnerships = Partnership.where.not(id: ats_webhook_owner_ids(target_sha1, :partnership_id))
+
+    accounts.to_a + partnerships.to_a
+  end
+
+  # Detects missing ATS webhooks and notifies Airbrake when any are found.
+  # Airbrake groups notices by message string, so repeated identical findings
+  # (same count + same owners) are deduplicated into one notice. A new notice
+  # fires only when the findings change (new missing webhooks appear).
+  #
+  # @return [Array<Account, Partnership>] the findings (empty when healthy)
+  def report!
+    findings = missing_ats_webhooks
+    return findings if findings.empty?
+
+    message = format(
+      'DocuSeal integrity: %<count>d ATS-linked owner(s) missing an ATS-pointed ' \
+      'WebhookUrl (expected url sha1=%<sha1>s): %<owners>s',
+      count: findings.size,
+      sha1: sha1_for(ENV.fetch('CAREERPLUG_WEBHOOK_URL', nil)),
+      owners: findings.map { |o| "#{o.class.name}##{o.id}" }.first(20).join(', ')
+    )
+    Rails.logger.error(message)
+    Airbrake.notify(message)
+    findings
+  end
+
+  # Cleartext SHA1 of a webhook URL, matching the dedup key set by WebhookUrl#set_sha1.
+  # Returns nil for a blank URL so callers can short-circuit (missing env is the
+  # boot guard's responsibility, not ours).
+  def sha1_for(url)
+    return nil if url.blank?
+
+    Digest::SHA1.hexdigest(url)
+  end
+
+  # IDs of owners that already have a WebhookUrl with the target sha1, for the
+  # given foreign key column. Used as a NOT IN subquery so we never load every
+  # account/partnership into memory.
+  def ats_webhook_owner_ids(target_sha1, owner_column)
+    WebhookUrl.where(sha1: target_sha1)
+              .where.not(owner_column => nil)
+              .select(owner_column)
+  end
+end

spec/lib/send_webhook_request_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe SendWebhookRequest do
+  let(:account) { create(:account) }
+  let(:webhook_url) { create(:webhook_url, account: account, url: 'https://example.com/hook') }
+  # Plain double (not verified) because we stub the whole ::NewRelic::Agent constant,
+  # which may not be loaded depending on env — verifying doubles would fail to resolve it.
+  let(:nr_agent) { double('NewRelic::Agent') } # rubocop:disable RSpec/VerifiedDoubles
+
+  before do
+    # The production code calls ::NewRelic::Agent.record_metric. The gem may not be
+    # loaded in the test env, so stub the whole constant and record_metric on it.
+    stub_const('NewRelic::Agent', nr_agent)
+    allow(nr_agent).to receive(:record_metric)
+    allow(Airbrake).to receive(:notify)
+    allow(Rails.logger).to receive(:error)
+  end
+
+  describe '.call' do
+    context 'when the request succeeds (2xx)' do
+      before { stub_request(:post, 'https://example.com/hook').to_return(status: 200, body: '{}') }
+
+      it 'records a success metric and does not notify Airbrake' do
+        described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 })
+
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/success', 1)
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/success', 1)
+        expect(Airbrake).not_to have_received(:notify)
+      end
+    end
+
+    context 'when the request returns a non-2xx status' do
+      before { stub_request(:post, 'https://example.com/hook').to_return(status: 500) }
+
+      it 'records a failure metric without notifying Airbrake (non-2xx is handled by retry logic)' do
+        described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 })
+
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/failure', 1)
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/failure', 1)
+        expect(Airbrake).not_to have_received(:notify)
+      end
+    end
+
+    context 'when the request raises a Faraday::Error (transport failure)' do
+      before do
+        stub_request(:post, 'https://example.com/hook').to_raise(Faraday::ConnectionFailed.new('boom'))
+      end
+
+      it 'records a failure metric, logs, notifies Airbrake, and returns nil (retry path intact)' do
+        result = described_class.call(webhook_url, event_type: 'form.completed', data: { foo: 1 })
+
+        expect(result).to be_nil
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/form.completed/failure', 1)
+        expect(nr_agent).to have_received(:record_metric).with('DocuSeal/Webhook/total/failure', 1)
+        expect(Rails.logger).to have_received(:error).with(/Webhook transport error.*form\.completed.*boom/)
+        expect(Airbrake).to have_received(:notify).with(/DocuSeal outbound webhook transport error \(form.completed\)/)
+      end
+    end
+  end
+end