diff --git a/.eslintignore b/.eslintignore index e019f3ca..e9b4f077 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,4 @@ node_modules/ main.js +archive/ diff --git a/.eslintrc b/.eslintrc index f9d396c2..c64703dd 100644 --- a/.eslintrc +++ b/.eslintrc @@ -43,5 +43,14 @@ ] } ] - } + }, + "overrides": [ + { + "files": ["__tests__/**/*.ts", "unit/**/*.ts"], + "rules": { + "no-restricted-imports": "off", + "@typescript-eslint/no-var-requires": "off" + } + } + ] } diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5573a5dd --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +__tests__/** filter=git-crypt diff=git-crypt diff --git a/.github/workflows/e2e-smoke.yml b/.github/workflows/e2e-smoke.yml deleted file mode 100644 index 8d10f839..00000000 --- a/.github/workflows/e2e-smoke.yml +++ /dev/null @@ -1,320 +0,0 @@ -name: E2E Smoke Tests - -on: - push: - branches: [merge-hsm] - workflow_dispatch: - inputs: - branch: - description: 'Branch to test (e.g. origin/main, origin/feature-x)' - required: true - default: 'origin/main' - -env: - VM_NAME: ${{ secrets.GCP_VM_NAME }} - VM_ZONE: ${{ secrets.GCP_VM_ZONE }} - GCP_PROJECT: ${{ secrets.GCP_PROJECT }} - GCP_USE_IAP: "true" - -jobs: - smoke-test: - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Auth to GCP - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Setup gcloud - uses: google-github-actions/setup-gcloud@v2 - - - name: Clone relay-harness - run: | - mkdir -p ~/.ssh - echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > ~/.ssh/harness-key - chmod 600 ~/.ssh/harness-key - cat >> ~/.ssh/config << 'EOF' - Host github.com - IdentityFile ~/.ssh/harness-key - StrictHostKeyChecking accept-new - EOF - - git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness - - - name: Ensure VM is running - run: | - STATUS=$(gcloud compute instances describe "$VM_NAME" \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --format='get(status)' 2>/dev/null || echo "NOT_FOUND") - - if [ "$STATUS" = "NOT_FOUND" ]; then - echo "VM does not exist. Creating via harness infra scripts..." - cd ~/relay-harness - GCP_PROJECT="$GCP_PROJECT" GCP_ZONE="$VM_ZONE" ./infra/gcp-linux-vm.sh create - elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "STOPPED" ]; then - echo "Starting stopped VM..." - gcloud compute instances start "$VM_NAME" \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" - elif [ "$STATUS" = "RUNNING" ]; then - echo "VM already running" - else - echo "Unexpected VM status: $STATUS" - exit 1 - fi - - # Pre-generate SSH key quietly (gcloud's ssh-keygen leaks to stdout) - ssh-keygen -t rsa -b 3072 -f ~/.ssh/google_compute_engine -N "" -q 2>/dev/null || true - # Set connect timeout so SSH attempts fail fast instead of hanging - echo -e "\nHost *\n ConnectTimeout 10" >> ~/.ssh/config - - echo "Waiting for SSH readiness..." - for i in $(seq 1 30); do - if gcloud compute ssh "$VM_NAME" --quiet \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap \ - --command="echo ready" 2>/dev/null; then - echo "VM is ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "Timed out waiting for VM SSH" - exit 1 - fi - sleep 5 - done - - - name: Provision VM credentials - run: | - echo '${{ secrets.GCP_SA_KEY }}' > /tmp/sa-key.json - gcloud compute scp --quiet /tmp/sa-key.json "$VM_NAME":~/ci-sa-key.json \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap 2>/dev/null - rm -f /tmp/sa-key.json - - echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > /tmp/deploy-key - chmod 600 /tmp/deploy-key - gcloud compute scp --quiet /tmp/deploy-key "$VM_NAME":~/ci-deploy-key \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap 2>/dev/null - rm -f /tmp/deploy-key - - # GitHub App private key for posting Check Runs - echo '${{ secrets.E2E_APP_KEY }}' > /tmp/github-app-key.pem - chmod 600 /tmp/github-app-key.pem - gcloud compute ssh "$VM_NAME" --quiet \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap \ - --command="mkdir -p ~/.config/relay-e2e" 2>/dev/null - gcloud compute scp --quiet /tmp/github-app-key.pem "$VM_NAME":~/.config/relay-e2e/github-app-key.pem \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap 2>/dev/null - rm -f /tmp/github-app-key.pem - - # R2 credentials for ci.system3.dev uploads - gcloud compute ssh "$VM_NAME" --quiet \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap \ - --command="cat > ~/.config/relay-e2e/r2-env.sh << 'CREDS' - export CI_R2_ENDPOINT='${{ secrets.CI_R2_ENDPOINT }}' - export CI_R2_ACCESS_KEY_ID='${{ secrets.CI_R2_ACCESS_KEY_ID }}' - export CI_R2_SECRET_ACCESS_KEY='${{ secrets.CI_R2_SECRET_ACCESS_KEY }}' - CREDS - chmod 600 ~/.config/relay-e2e/r2-env.sh" 2>/dev/null - - - name: Run smoke tests on VM - run: | - BRANCH="${{ github.event.inputs.branch || format('origin/{0}', github.ref_name) }}" - - gcloud compute ssh "$VM_NAME" --quiet \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap \ - --command="bash -s -- '$BRANCH'" << 'REMOTE_SCRIPT' - set -eo pipefail - BRANCH="$1" - - # Load R2 credentials - [ -f ~/.config/relay-e2e/r2-env.sh ] && source ~/.config/relay-e2e/r2-env.sh - - # Ensure Obsidian AppImage is installed - export OBSIDIAN_PATH="$HOME/obsidian/Obsidian.AppImage" - if [ ! -f "$OBSIDIAN_PATH" ]; then - echo "Installing Obsidian AppImage..." - mkdir -p ~/obsidian - OBSIDIAN_VERSION="1.8.9" - wget -q -O "$OBSIDIAN_PATH" \ - "https://github.com/obsidianmd/obsidian-releases/releases/download/v${OBSIDIAN_VERSION}/Obsidian-${OBSIDIAN_VERSION}.AppImage" - chmod +x "$OBSIDIAN_PATH" - echo "Installed Obsidian $OBSIDIAN_VERSION" - fi - - # Start virtual display for headless Obsidian - export DISPLAY=:99 - Xvfb :99 -screen 0 1920x1080x24 >/dev/null 2>&1 & - XVFB_PID=$! - sleep 1 - - # Clean stale test reports from previous runs - rm -rf /tmp/test-reports - - # Activate GCP service account for gcloud/GCS uploads - gcloud auth activate-service-account --key-file="$HOME/ci-sa-key.json" 2>/dev/null - - # Set up SSH for private repo access - mkdir -p ~/.ssh - cp ~/ci-deploy-key ~/.ssh/ci-deploy-key - chmod 600 ~/.ssh/ci-deploy-key - if ! grep -q 'ci-deploy-key' ~/.ssh/config 2>/dev/null; then - cat >> ~/.ssh/config << 'SSHCFG' - Host github.com - IdentityFile ~/.ssh/ci-deploy-key - StrictHostKeyChecking accept-new - SSHCFG - fi - - # Clone or update relay-plugin (public) - if [ ! -d ~/relay-plugin ]; then - git clone https://github.com/No-Instructions/Relay.git ~/relay-plugin - fi - cd ~/relay-plugin - git fetch --all --prune - git checkout -- . - git clean -fd - # Detached HEAD checkout — works for origin/branch, tags, and SHAs - git checkout --detach "$BRANCH" - - # Clone or update relay-harness (private) - if [ ! -d ~/relay-harness ]; then - git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness - fi - cd ~/relay-harness && git checkout main && git pull - - # Install playwright test dependencies - cd ~/relay-harness/playwright && npm install --ignore-scripts 2>/dev/null - - # Run tests using the harness test-branch.sh orchestrator - export RELAY_PLUGIN_DIR=~/relay-plugin - cd ~/relay-harness - - TEST_EXIT=0 - ./scripts/test-branch.sh "$BRANCH" --upload --checks || TEST_EXIT=$? - - # Extract results for the GitHub Actions runner - LATEST=$(ls -td /tmp/test-reports/runs/*/* 2>/dev/null | head -1) - if [ -n "$LATEST" ] && [ -f "$LATEST/summary.json" ]; then - cp "$LATEST/summary.json" ~/test-summary.json - - COMMIT=$(basename "$(dirname "$LATEST")") - EXEC_ID=$(basename "$LATEST") - REPORT_URL="https://ci.system3.dev/runs/${COMMIT}/${EXEC_ID}/index.html" - echo "{\"reportUrl\": \"$REPORT_URL\", \"commit\": \"$COMMIT\", \"execId\": \"$EXEC_ID\"}" > ~/test-metadata.json - else - echo '{"summary":{"total":0,"pass":0,"fail":0},"tests":[],"state":"error"}' > ~/test-summary.json - echo '{"reportUrl":"","commit":"","execId":""}' > ~/test-metadata.json - fi - - # Clean up virtual display - kill $XVFB_PID 2>/dev/null || true - - exit $TEST_EXIT - REMOTE_SCRIPT - - - name: Clean up VM credentials - if: always() - run: | - gcloud compute ssh "$VM_NAME" --quiet \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap \ - --command="rm -f ~/ci-sa-key.json ~/ci-deploy-key ~/.ssh/ci-deploy-key ~/.config/relay-e2e/github-app-key.pem ~/.config/relay-e2e/r2-env.sh" 2>/dev/null || true - - - name: Fetch results - if: always() - run: | - gcloud compute scp --quiet "$VM_NAME":~/test-summary.json ./summary.json \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap 2>/dev/null || true - gcloud compute scp --quiet "$VM_NAME":~/test-metadata.json ./metadata.json \ - --zone="$VM_ZONE" --project="$GCP_PROJECT" \ - --tunnel-through-iap 2>/dev/null || true - - - name: Generate job summary - if: always() - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - - let summary, metadata; - try { - summary = JSON.parse(fs.readFileSync('summary.json', 'utf-8')); - } catch { - core.summary.addRaw('## E2E Smoke Test Results\n\n:x: **Failed to retrieve test results from VM**\n'); - await core.summary.write(); - core.setFailed('No test results available'); - return; - } - try { - metadata = JSON.parse(fs.readFileSync('metadata.json', 'utf-8')); - } catch { - metadata = { reportUrl: '', commit: '', execId: '' }; - } - - const s = summary.summary || {}; - const tests = summary.tests || []; - const reportBase = (metadata.reportUrl || '').replace(/\/index\.html$/, ''); - - // Status emoji map - const statusEmoji = { - pass: ':white_check_mark:', - fail: ':x:', - expected_fail: ':warning:', - unexpected_pass: ':sparkles:', - }; - - // Verdict - const totalDur = Math.round((s.duration || 0) / 1000); - const verdictEmoji = (s.fail || 0) > 0 ? ':x:' : ':white_check_mark:'; - const verdictText = (s.fail || 0) > 0 ? 'FAIL' : 'PASS'; - const parts = []; - if (s.pass) parts.push(`${s.pass} pass`); - if (s.fail) parts.push(`${s.fail} fail`); - if (s.expectedFail) parts.push(`${s.expectedFail} expected-fail`); - if (s.unexpectedPass) parts.push(`${s.unexpectedPass} unexpected-pass`); - - let md = `## E2E Smoke Test Results\n\n`; - md += `${verdictEmoji} **${verdictText}** | ${parts.join(', ')} | ${totalDur}s`; - if (metadata.reportUrl) { - md += ` | [Full Report](${metadata.reportUrl})`; - } - md += `\n\n`; - - // Test table - md += `| Test | Status | Assertions | Duration |\n`; - md += `|------|--------|------------|----------|\n`; - for (const t of tests) { - const emoji = statusEmoji[t.status] || ':grey_question:'; - const dur = `${(t.duration / 1000).toFixed(1)}s`; - const assertions = `${t.passedAssertions ?? '?'}/${t.assertions ?? '?'}`; - const testUrl = reportBase ? `${reportBase}/${t.dir}/index.html` : ''; - const name = testUrl ? `[${t.name}](${testUrl})` : t.name; - - let row = `| ${name} | ${emoji} ${t.status} | ${assertions} | ${dur} |`; - md += row + '\n'; - - // Add error detail for failures - if (t.status === 'fail' && t.firstError) { - const errText = t.firstError.split('\n')[0].substring(0, 120); - md += `| | | \`${errText}\` | |\n`; - } - } - - core.summary.addRaw(md); - await core.summary.write(); - - // Set job status - if ((s.total || 0) === 0 || summary.state === 'error') { - core.setFailed('No tests were executed'); - } else if ((s.fail || 0) > 0) { - core.setFailed(`${s.fail} test(s) failed`); - } diff --git a/.github/workflows/e2e-standard.yml b/.github/workflows/e2e-standard.yml new file mode 100644 index 00000000..f93a21f1 --- /dev/null +++ b/.github/workflows/e2e-standard.yml @@ -0,0 +1,672 @@ +name: E2E standard suite + +on: + push: + branches: [merge-hsm] + tags: + - '0.8.0-rc*' + workflow_dispatch: + inputs: + ref: + description: 'Ref to test (e.g. origin/main, origin/feature-x, a tag, or a SHA)' + required: true + default: 'origin/main' + tests: + description: 'Optional comma-separated Level 2 strands to run (e.g. test-basic-open-close,test-share-folder)' + required: false + default: '' + +concurrency: + group: e2e-standard-linux + cancel-in-progress: false + +env: + VM_ZONE: ${{ secrets.GCP_VM_ZONE }} + GCP_PROJECT: ${{ secrets.GCP_PROJECT }} + GCP_USE_IAP: "true" + +jobs: + standard-test: + name: ${{ format('standard-test ({0})', matrix.shard.name) }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 7 + matrix: + shard: + - name: foundation-core + slug: foundation-core + tests: test-basic-open-close,test-folder-lifecycle,test-view-reuse-file-switching + - name: foundation-share + slug: foundation-share + tests: test-share-folder,test-in-note-status + - name: sync-state + slug: sync-state + tests: test-idle-registration,test-active-editor-sync,test-offline-banner,test-binary-free-user + - name: conflicts-differ + slug: conflicts-differ + tests: test-differ-resolve,test-trigger-differ,test-differ-dismissal + - name: conflicts-resolution + slug: conflicts-resolution + tests: test-conflict-dismiss,test-conflict-cancel,test-upgrade-no-lca-conflict + - name: bulk-upload + slug: bulk + tests: test-bulk-upload + - name: restart-heavy + slug: restart + tests: test-bases-checkbox-sync,test-obsidian-restart,test-cold-reopen-external-edit + env: + VM_NAME: ${{ format('{0}-{1}', secrets.GCP_VM_NAME, matrix.shard.slug) }} + SHARD_NAME: ${{ matrix.shard.name }} + SHARD_SLUG: ${{ matrix.shard.slug }} + SHARD_TESTS: ${{ matrix.shard.tests }} + GCP_VM_DISK_SIZE: 20GB + GCP_VM_DISK_TYPE: pd-standard + + steps: + - name: Auth to GCP + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Select tests for shard + id: select-tests + run: | + TARGETED_TESTS="${{ github.event.inputs.tests || '' }}" + SHARD_TESTS="${{ matrix.shard.tests }}" + + select_tests() { + local available="$1" + local wanted="$2" + local filtered="" + local test match + + IFS=',' read -r -a available_tests <<< "$available" + IFS=',' read -r -a wanted_tests <<< "$wanted" + + for test in "${available_tests[@]}"; do + test="${test//[[:space:]]/}" + [ -n "$test" ] || continue + for match in "${wanted_tests[@]}"; do + match="${match//[[:space:]]/}" + [ -n "$match" ] || continue + if [ "$test" = "$match" ]; then + filtered="${filtered:+$filtered,}$test" + break + fi + done + done + + printf '%s' "$filtered" + } + + SELECTED_TESTS="$SHARD_TESTS" + if [ -n "$TARGETED_TESTS" ]; then + SELECTED_TESTS="$(select_tests "$SHARD_TESTS" "$TARGETED_TESTS")" + fi + + if [ -n "$SELECTED_TESTS" ]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "selected_tests=$SELECTED_TESTS" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "selected_tests=" >> "$GITHUB_OUTPUT" + fi + + - name: Clone relay-harness + if: steps.select-tests.outputs.should_run == 'true' + run: | + mkdir -p ~/.ssh + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > ~/.ssh/harness-key + chmod 600 ~/.ssh/harness-key + cat > ~/.ssh/known_hosts << 'EOF' + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= + EOF + chmod 600 ~/.ssh/known_hosts + cat > ~/.ssh/config << 'EOF' + Host github.com + IdentityFile ~/.ssh/harness-key + IdentitiesOnly yes + StrictHostKeyChecking yes + UserKnownHostsFile ~/.ssh/known_hosts + EOF + + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + + - name: Ensure VM is running + if: steps.select-tests.outputs.should_run == 'true' + run: | + STATUS="" + for i in $(seq 1 60); do + STATUS=$(gcloud compute instances describe "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --format='get(status)' 2>/dev/null || echo "NOT_FOUND") + + case "$STATUS" in + PROVISIONING|STAGING|STOPPING|SUSPENDING|REPAIRING) + echo "VM is $STATUS; waiting for a stable state ($i/60) ..." + sleep 10 + ;; + *) + break + ;; + esac + done + + if [ "$STATUS" = "NOT_FOUND" ]; then + echo "VM does not exist. Creating via harness infra scripts..." + cd ~/relay-harness + GCP_PROJECT="$GCP_PROJECT" GCP_ZONE="$VM_ZONE" GCP_VM_NAME="$VM_NAME" \ + GCP_VM_DISK_SIZE="$GCP_VM_DISK_SIZE" GCP_VM_DISK_TYPE="$GCP_VM_DISK_TYPE" \ + ./infra/gcp-linux-vm.sh create + elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "STOPPED" ]; then + echo "Starting stopped VM..." + gcloud compute instances start "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" + elif [ "$STATUS" = "RUNNING" ]; then + echo "VM already running" + else + echo "Unexpected VM status: $STATUS" + exit 1 + fi + + # Pre-generate SSH key quietly (gcloud's ssh-keygen leaks to stdout) + ssh-keygen -t rsa -b 3072 -f ~/.ssh/google_compute_engine -N "" -q 2>/dev/null || true + # Set connect timeout so SSH attempts fail fast instead of hanging + echo -e "\nHost *\n ConnectTimeout 10" >> ~/.ssh/config + + echo "Waiting for SSH readiness..." + for i in $(seq 1 30); do + if gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="echo ready" 2>/dev/null; then + echo "VM is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "Timed out waiting for VM SSH" + exit 1 + fi + sleep 5 + done + + - name: Provision VM credentials + if: steps.select-tests.outputs.should_run == 'true' + run: | + scp_retry() { + local src="$1" + local dest="$2" + local label="$3" + local attempt + + for attempt in $(seq 1 5); do + if gcloud compute scp --quiet "$src" "$VM_NAME":"$dest" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap; then + return 0 + fi + echo "Retrying $label scp ($attempt/5) ..." + sleep 5 + done + + echo "Failed to copy $label to VM after 5 attempts" + return 1 + } + + ssh_retry() { + local label="$1" + local command="$2" + local attempt + + for attempt in $(seq 1 5); do + if gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="$command"; then + return 0 + fi + echo "Retrying $label ssh command ($attempt/5) ..." + sleep 5 + done + + echo "Failed $label ssh command after 5 attempts" + return 1 + } + + echo '${{ secrets.GCP_SA_KEY }}' > /tmp/sa-key.json + scp_retry /tmp/sa-key.json ~/ci-sa-key.json "service-account key" + rm -f /tmp/sa-key.json + + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > /tmp/deploy-key + chmod 600 /tmp/deploy-key + scp_retry /tmp/deploy-key ~/ci-deploy-key "deploy key" + rm -f /tmp/deploy-key + + # GitHub App private key for posting Check Runs + echo '${{ secrets.E2E_APP_KEY }}' > /tmp/github-app-key.pem + chmod 600 /tmp/github-app-key.pem + ssh_retry "relay-e2e config dir creation" "mkdir -p ~/.config/relay-e2e" + scp_retry /tmp/github-app-key.pem ~/.config/relay-e2e/github-app-key.pem "GitHub App key" + rm -f /tmp/github-app-key.pem + + # R2 credentials for ci.system3.dev uploads + ssh_retry "R2 env write" "cat > ~/.config/relay-e2e/r2-env.sh << 'CREDS' + export CI_R2_ENDPOINT='${{ secrets.CI_R2_ENDPOINT }}' + export CI_R2_ACCESS_KEY_ID='${{ secrets.CI_R2_ACCESS_KEY_ID }}' + export CI_R2_SECRET_ACCESS_KEY='${{ secrets.CI_R2_SECRET_ACCESS_KEY }}' + CREDS + chmod 600 ~/.config/relay-e2e/r2-env.sh" + + - name: Run standard suite on VM + if: steps.select-tests.outputs.should_run == 'true' + run: | + INPUT_REF="${{ github.event.inputs.ref || '' }}" + REF_TYPE="${{ github.ref_type }}" + REF_NAME="${{ github.ref_name }}" + + if [ -n "$INPUT_REF" ]; then + REF="$INPUT_REF" + elif [ "$REF_TYPE" = "tag" ]; then + REF="refs/tags/$REF_NAME" + else + REF="origin/$REF_NAME" + fi + + SELECTED_TESTS="${{ steps.select-tests.outputs.selected_tests }}" + SHARD_SLUG_ARG="${{ matrix.shard.slug }}" + + gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="bash -s -- '$REF' '$SELECTED_TESTS' '$SHARD_SLUG_ARG'" 2>/dev/null << 'REMOTE_SCRIPT' + set -eo pipefail + REF="$1" + SELECTED_TESTS="$2" + SHARD_SLUG="$3" + + # Load R2 credentials + [ -f ~/.config/relay-e2e/r2-env.sh ] && source ~/.config/relay-e2e/r2-env.sh + + # Ensure Obsidian AppImage is installed + export OBSIDIAN_PATH="$HOME/obsidian/Obsidian.AppImage" + if [ ! -f "$OBSIDIAN_PATH" ]; then + echo "Installing Obsidian AppImage..." + mkdir -p ~/obsidian + OBSIDIAN_VERSION="1.8.9" + wget -q -O "$OBSIDIAN_PATH" \ + "https://github.com/obsidianmd/obsidian-releases/releases/download/v${OBSIDIAN_VERSION}/Obsidian-${OBSIDIAN_VERSION}.AppImage" + chmod +x "$OBSIDIAN_PATH" + echo "Installed Obsidian $OBSIDIAN_VERSION" + fi + + # Start virtual display for headless Obsidian + export DISPLAY=:99 + Xvfb :99 -screen 0 1920x1080x24 >/dev/null 2>&1 & + XVFB_PID=$! + sleep 1 + + # Clean stale test reports from previous runs + rm -rf /tmp/test-reports + rm -f ~/test-summary.json ~/test-metadata.json + + # Activate GCP service account for gcloud/GCS uploads + gcloud auth activate-service-account --key-file="$HOME/ci-sa-key.json" 2>/dev/null + + # Set up SSH for private repo access + mkdir -p ~/.ssh + cp ~/ci-deploy-key ~/.ssh/ci-deploy-key + chmod 600 ~/.ssh/ci-deploy-key + cat > ~/.ssh/known_hosts << 'EOF' + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= + EOF + chmod 600 ~/.ssh/known_hosts + export GIT_SSH_COMMAND="ssh -i $HOME/.ssh/ci-deploy-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts" + + # Clone or update relay-plugin (public) + if [ ! -d ~/relay-plugin ]; then + git clone https://github.com/No-Instructions/Relay.git ~/relay-plugin + fi + cd ~/relay-plugin + git fetch origin --force --prune \ + '+refs/heads/*:refs/remotes/origin/*' \ + '+refs/tags/*:refs/tags/*' + git checkout -- . + git clean -fd + # Detached HEAD checkout — works for origin/branch, tags, and SHAs + git checkout --detach "$REF" + echo "Relay product checkout:" + git show -s --format=' commit: %H%n subject: %s' + echo " describe: $(git describe --tags --always --long 2>/dev/null || git rev-parse --short HEAD)" + + # Clone or update relay-harness (private) + if [ ! -d ~/relay-harness ]; then + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + fi + cd ~/relay-harness + git fetch origin --prune + git checkout -- . + git clean -fd + git checkout --detach origin/main + + # Clear stale repo-local test instances left by canceled or crashed runs + echo "Clearing stale repo-local slots..." + ./scripts/stop-obsidian.sh >/dev/null 2>&1 || true + + # Install playwright test dependencies + cd ~/relay-harness/playwright && npm install --ignore-scripts 2>/dev/null + + # Run tests using the harness standard-suite orchestrator + export RELAY_PLUGIN_DIR=~/relay-plugin + cd ~/relay-harness + + TEST_EXIT=0 + TEST_CMD=(./scripts/run-e2e-suite.sh "$REF" --upload) + TEST_CMD+=("--tests=$SELECTED_TESTS") + RELAY_RUN_EXEC_SUFFIX="$SHARD_SLUG" "${TEST_CMD[@]}" || TEST_EXIT=$? + + # Extract results for the GitHub Actions runner + LATEST_SUMMARY=$(find /tmp/test-reports/runs -mindepth 3 -maxdepth 3 -type f -name summary.json -print0 2>/dev/null \ + | xargs -0 ls -t 2>/dev/null \ + | head -1) + if [ -n "$LATEST_SUMMARY" ]; then + LATEST="$(dirname "$LATEST_SUMMARY")" + cp "$LATEST_SUMMARY" ~/test-summary.json + + COMMIT=$(basename "$(dirname "$LATEST")") + EXEC_ID=$(basename "$LATEST") + REPORT_URL="https://ci.system3.dev/runs/${COMMIT}/${EXEC_ID}/index.html" + echo "{\"reportUrl\": \"$REPORT_URL\", \"commit\": \"$COMMIT\", \"execId\": \"$EXEC_ID\"}" > ~/test-metadata.json + else + echo '{"summary":{"total":0,"pass":0,"fail":0},"tests":[],"state":"error"}' > ~/test-summary.json + echo '{"reportUrl":"","commit":"","execId":""}' > ~/test-metadata.json + fi + + # Clean up virtual display + kill $XVFB_PID 2>/dev/null || true + + exit $TEST_EXIT + REMOTE_SCRIPT + + - name: Clean up VM credentials + if: always() && steps.select-tests.outputs.should_run == 'true' + run: | + gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="rm -f ~/ci-sa-key.json ~/ci-deploy-key ~/.ssh/ci-deploy-key ~/.config/relay-e2e/github-app-key.pem ~/.config/relay-e2e/r2-env.sh" 2>/dev/null || true + + - name: Fetch results + if: always() && steps.select-tests.outputs.should_run == 'true' + run: | + rm -f summary.json metadata.json + gcloud compute scp --quiet "$VM_NAME":~/test-summary.json ./summary.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap 2>/dev/null || true + gcloud compute scp --quiet "$VM_NAME":~/test-metadata.json ./metadata.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap 2>/dev/null || true + + - name: Upload shard metadata + if: always() && steps.select-tests.outputs.should_run == 'true' + uses: actions/upload-artifact@v4 + with: + name: standard-${{ matrix.shard.slug }}-metadata + path: | + summary.json + metadata.json + if-no-files-found: ignore + + - name: Request shard VM stop + if: always() && steps.select-tests.outputs.should_run == 'true' + run: | + STATUS=$(gcloud compute instances describe "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --format='get(status)' 2>/dev/null || echo "NOT_FOUND") + + if [ "$STATUS" = "RUNNING" ]; then + echo "Requesting asynchronous stop for shard VM..." + gcloud compute instances stop "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --quiet --async 2>/dev/null || true + else + echo "Skipping stop request (status: $STATUS)" + fi + + - name: Generate job summary + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const shardName = process.env.SHARD_NAME || 'shard'; + const shouldRun = process.env.RUN_SHARD === 'true'; + + if (!shouldRun) { + core.summary.addRaw(`## E2E standard suite results (${shardName})\n\n:warning: **SKIPPED** | no matching tests for this shard\n`); + await core.summary.write(); + return; + } + + let summary, metadata; + try { + summary = JSON.parse(fs.readFileSync('summary.json', 'utf-8')); + } catch { + core.summary.addRaw(`## E2E standard suite results (${shardName})\n\n:x: **Failed to retrieve test results from VM**\n`); + await core.summary.write(); + core.setFailed('No test results available'); + return; + } + try { + metadata = JSON.parse(fs.readFileSync('metadata.json', 'utf-8')); + } catch { + metadata = { reportUrl: '', commit: '', execId: '' }; + } + + const s = summary.summary || {}; + const tests = summary.tests || []; + const reportBase = (metadata.reportUrl || '').replace(/\/index\.html$/, ''); + + // Status emoji map + const statusEmoji = { + pass: ':white_check_mark:', + fail: ':x:', + expected_fail: ':warning:', + unexpected_pass: ':sparkles:', + }; + + // Verdict + const totalDur = Math.round((s.duration || 0) / 1000); + const verdictEmoji = (s.fail || 0) > 0 ? ':x:' : ':white_check_mark:'; + const verdictText = (s.fail || 0) > 0 ? 'FAIL' : 'PASS'; + const parts = []; + if (s.pass) parts.push(`${s.pass} pass`); + if (s.fail) parts.push(`${s.fail} fail`); + if (s.expectedFail) parts.push(`${s.expectedFail} expected-fail`); + if (s.unexpectedPass) parts.push(`${s.unexpectedPass} unexpected-pass`); + + let md = `## E2E standard suite results (${shardName})\n\n`; + md += `${verdictEmoji} **${verdictText}** | ${parts.join(', ')} | ${totalDur}s`; + if (metadata.reportUrl) { + md += ` | [Full Report](${metadata.reportUrl})`; + } + md += `\n\n`; + + // Test table + md += `| Test | Status | Assertions | Duration |\n`; + md += `|------|--------|------------|----------|\n`; + for (const t of tests) { + const emoji = statusEmoji[t.status] || ':grey_question:'; + const dur = `${(t.duration / 1000).toFixed(1)}s`; + const assertions = `${t.passedAssertions ?? '?'}/${t.assertions ?? '?'}`; + const testUrl = reportBase ? `${reportBase}/${t.dir}/index.html` : ''; + const name = testUrl ? `[${t.name}](${testUrl})` : t.name; + + let row = `| ${name} | ${emoji} ${t.status} | ${assertions} | ${dur} |`; + md += row + '\n'; + + // Add error detail for failures + if (t.status === 'fail' && t.firstError) { + const errText = t.firstError.split('\n')[0].substring(0, 120); + md += `| | | \`${errText}\` | |\n`; + } + } + + core.summary.addRaw(md); + await core.summary.write(); + + // Set job status + if ((s.total || 0) === 0 || summary.state === 'error') { + core.setFailed('No tests were executed'); + } else if ((s.fail || 0) > 0) { + core.setFailed(`${s.fail} test(s) failed`); + } + env: + RUN_SHARD: ${{ steps.select-tests.outputs.should_run }} + + publish-report-site: + name: publish report site + runs-on: ubuntu-latest + needs: standard-test + if: always() + steps: + - name: Clone relay-harness + run: | + mkdir -p ~/.ssh + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > ~/.ssh/harness-key + chmod 600 ~/.ssh/harness-key + cat > ~/.ssh/known_hosts << 'EOF' + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= + EOF + chmod 600 ~/.ssh/known_hosts + cat > ~/.ssh/config << 'EOF' + Host github.com + IdentityFile ~/.ssh/harness-key + IdentitiesOnly yes + StrictHostKeyChecking yes + UserKnownHostsFile ~/.ssh/known_hosts + EOF + + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + + - name: Download shard metadata + uses: actions/download-artifact@v4 + with: + pattern: standard-*-metadata + path: shard-metadata + + - name: Resolve tested commit + id: resolve-commit + run: | + set -euo pipefail + + COMMIT="$(node <<'NODE' + const fs = require('fs'); + const path = require('path'); + + function walk(dir) { + if (!fs.existsSync(dir)) return []; + const out = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...walk(full)); + else if (entry.name === 'metadata.json') out.push(full); + } + return out; + } + + for (const file of walk('shard-metadata')) { + try { + const metadata = JSON.parse(fs.readFileSync(file, 'utf8')); + if (metadata.commit) { + console.log(metadata.commit); + process.exit(0); + } + } catch { + // Try the next shard metadata file. + } + } + NODE + )" + + if [ -z "$COMMIT" ]; then + INPUT_REF="${{ github.event.inputs.ref || '' }}" + REF_TYPE="${{ github.ref_type }}" + REF_NAME="${{ github.ref_name }}" + + if [ -n "$INPUT_REF" ]; then + REF="$INPUT_REF" + elif [ "$REF_TYPE" = "tag" ]; then + REF="refs/tags/$REF_NAME" + else + REF="origin/$REF_NAME" + fi + + git init /tmp/relay-ref >/dev/null + git -C /tmp/relay-ref remote add origin https://github.com/No-Instructions/Relay.git + git -C /tmp/relay-ref fetch origin --force --prune \ + '+refs/heads/*:refs/remotes/origin/*' \ + '+refs/tags/*:refs/tags/*' + COMMIT="$(git -C /tmp/relay-ref rev-parse "$REF^{commit}")" + fi + + COMMIT_SHORT="$(printf '%s' "$COMMIT" | cut -c1-7)" + echo "commit=$COMMIT_SHORT" >> "$GITHUB_OUTPUT" + echo "Resolved report-site commit: $COMMIT_SHORT" + + - name: Install report-site dependencies + run: npm ci --prefix ~/relay-harness/report-site + + - name: Publish report site + env: + CI_R2_ENDPOINT: ${{ secrets.CI_R2_ENDPOINT }} + CI_R2_ACCESS_KEY_ID: ${{ secrets.CI_R2_ACCESS_KEY_ID }} + CI_R2_SECRET_ACCESS_KEY: ${{ secrets.CI_R2_SECRET_ACCESS_KEY }} + CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_ID: ${{ secrets.CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_ID }} + CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_SECRET: ${{ secrets.CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_SECRET }} + TARGET_COMMIT: ${{ steps.resolve-commit.outputs.commit }} + run: | + set -euo pipefail + cd ~/relay-harness + + COMMIT_SET="$(node --input-type=module <<'NODE' + import { readJson } from './scripts/lib/ci-artifact-store.mjs'; + + const target = process.env.TARGET_COMMIT || ''; + const manifest = await readJson('runs/manifest.json'); + const existing = (manifest?.commitGroups || manifest?.runs || []) + .map(entry => entry.sha || entry.id || entry.runId) + .filter(Boolean); + console.log([...new Set([target, ...existing].filter(Boolean))].join(',')); + NODE + )" + + if [ -z "$COMMIT_SET" ]; then + echo "No commit set resolved for report-site publish" + exit 1 + fi + + READBACK_ARGS=() + if [ -n "${CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_ID:-}" ] && [ -n "${CI_SYSTEM3_DEV_CF_ACCESS_CLIENT_SECRET:-}" ]; then + READBACK_ARGS=(--live-readback --live-readback-base-url=https://ci.system3.dev) + else + echo "Cloudflare Access readback credentials not configured; skipping live readback" + fi + + node scripts/publish-report-site-static.mjs \ + --commit="$COMMIT_SET" \ + --refresh-commit="$TARGET_COMMIT" \ + --materialize-missing-commits \ + --rebuild-manifest-from-commits \ + --upload \ + "${READBACK_ARGS[@]}" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 076ea3ba..637e936f 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -1,15 +1,549 @@ name: Unit Tests + on: push: + branches: [main, merge-hsm] + tags: + - '0.8.0-rc*' + workflow_dispatch: + inputs: + ref: + description: 'Ref to test (e.g. origin/main, origin/feature-x, a tag, or a SHA)' + required: true + default: 'origin/main' + +concurrency: + group: unit-tests-private + cancel-in-progress: false + +env: + VM_ZONE: ${{ secrets.GCP_VM_ZONE }} + GCP_PROJECT: ${{ secrets.GCP_PROJECT }} + GCP_USE_IAP: "true" jobs: - test: + unit-test: + name: private unit tests runs-on: ubuntu-latest + timeout-minutes: 45 + env: + VM_NAME: ${{ format('{0}-unit', secrets.GCP_VM_NAME) }} + GCP_VM_DISK_SIZE: 20GB + GCP_VM_DISK_TYPE: pd-standard + steps: - - uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - run: npm ci - - run: npm test + - name: Auth to GCP + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Clone relay-harness + run: | + mkdir -p ~/.ssh + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > ~/.ssh/harness-key + chmod 600 ~/.ssh/harness-key + cat > ~/.ssh/known_hosts << 'EOF' + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= + EOF + chmod 600 ~/.ssh/known_hosts + cat > ~/.ssh/config << 'EOF' + Host github.com + IdentityFile ~/.ssh/harness-key + IdentitiesOnly yes + StrictHostKeyChecking yes + UserKnownHostsFile ~/.ssh/known_hosts + EOF + + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + + - name: Ensure VM is running + run: | + STATUS="" + for i in $(seq 1 60); do + STATUS=$(gcloud compute instances describe "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --format='get(status)' 2>/dev/null || echo "NOT_FOUND") + + case "$STATUS" in + PROVISIONING|STAGING|STOPPING|SUSPENDING|REPAIRING) + echo "VM is $STATUS; waiting for a stable state ($i/60) ..." + sleep 10 + ;; + *) + break + ;; + esac + done + + if [ "$STATUS" = "NOT_FOUND" ]; then + echo "VM does not exist. Creating via harness infra scripts..." + cd ~/relay-harness + GCP_PROJECT="$GCP_PROJECT" GCP_ZONE="$VM_ZONE" GCP_VM_NAME="$VM_NAME" \ + GCP_VM_DISK_SIZE="$GCP_VM_DISK_SIZE" GCP_VM_DISK_TYPE="$GCP_VM_DISK_TYPE" \ + ./infra/gcp-linux-vm.sh create + elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "STOPPED" ]; then + echo "Starting stopped VM..." + gcloud compute instances start "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" + elif [ "$STATUS" = "RUNNING" ]; then + echo "VM already running" + else + echo "Unexpected VM status: $STATUS" + exit 1 + fi + + ssh-keygen -t rsa -b 3072 -f ~/.ssh/google_compute_engine -N "" -q 2>/dev/null || true + echo -e "\nHost *\n ConnectTimeout 10" >> ~/.ssh/config + + echo "Waiting for SSH readiness..." + for i in $(seq 1 30); do + if gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="echo ready" 2>/dev/null; then + echo "VM is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "Timed out waiting for VM SSH" + exit 1 + fi + sleep 5 + done + + - name: Provision VM credentials + run: | + scp_retry() { + local src="$1" + local dest="$2" + local label="$3" + local attempt + + for attempt in $(seq 1 5); do + if gcloud compute scp --quiet "$src" "$VM_NAME":"$dest" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap; then + return 0 + fi + echo "Retrying $label scp ($attempt/5) ..." + sleep 5 + done + + echo "Failed to copy $label to VM after 5 attempts" + return 1 + } + + ssh_retry() { + local label="$1" + local command="$2" + local attempt + + for attempt in $(seq 1 5); do + if gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="$command"; then + return 0 + fi + echo "Retrying $label ssh command ($attempt/5) ..." + sleep 5 + done + + echo "Failed $label ssh command after 5 attempts" + return 1 + } + + ssh_retry "relay-e2e config dir creation" "mkdir -p ~/.config/relay-e2e" + + echo '${{ secrets.GCP_SA_KEY }}' > /tmp/sa-key.json + scp_retry /tmp/sa-key.json ~/ci-sa-key.json "service-account key" + rm -f /tmp/sa-key.json + + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > /tmp/deploy-key + chmod 600 /tmp/deploy-key + scp_retry /tmp/deploy-key ~/ci-deploy-key "deploy key" + rm -f /tmp/deploy-key + + cat > /tmp/relay-git-crypt-key.b64 << 'KEY' + ${{ secrets.RELAY_GIT_CRYPT_KEY_B64 }} + KEY + if [ ! -s /tmp/relay-git-crypt-key.b64 ]; then + echo "Missing RELAY_GIT_CRYPT_KEY_B64 secret" + exit 1 + fi + base64 -d /tmp/relay-git-crypt-key.b64 > /tmp/relay-git-crypt-key + chmod 600 /tmp/relay-git-crypt-key + scp_retry /tmp/relay-git-crypt-key ~/.config/relay-e2e/relay-git-crypt-key "git-crypt key" + rm -f /tmp/relay-git-crypt-key /tmp/relay-git-crypt-key.b64 + + ssh_retry "R2 env write" "cat > ~/.config/relay-e2e/r2-env.sh << 'CREDS' + export CI_R2_ENDPOINT='${{ secrets.CI_R2_ENDPOINT }}' + export CI_R2_ACCESS_KEY_ID='${{ secrets.CI_R2_ACCESS_KEY_ID }}' + export CI_R2_SECRET_ACCESS_KEY='${{ secrets.CI_R2_SECRET_ACCESS_KEY }}' + CREDS + chmod 600 ~/.config/relay-e2e/r2-env.sh" + + - name: Run unit tests on VM + run: | + INPUT_REF="${{ github.event.inputs.ref || '' }}" + REF_TYPE="${{ github.ref_type }}" + REF_NAME="${{ github.ref_name }}" + + if [ -n "$INPUT_REF" ]; then + REF="$INPUT_REF" + elif [ "$REF_TYPE" = "tag" ]; then + REF="refs/tags/$REF_NAME" + else + REF="origin/$REF_NAME" + fi + + gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="bash -s -- '$REF'" 2>/dev/null << 'REMOTE_SCRIPT' + set -eo pipefail + REF="$1" + + [ -f ~/.config/relay-e2e/r2-env.sh ] && source ~/.config/relay-e2e/r2-env.sh + rm -rf /tmp/test-reports + rm -f ~/test-summary.json ~/test-metadata.json + + if ! command -v git-crypt >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install -y -qq git-crypt + fi + + gcloud auth activate-service-account --key-file="$HOME/ci-sa-key.json" 2>/dev/null + + mkdir -p ~/.ssh + cp ~/ci-deploy-key ~/.ssh/ci-deploy-key + chmod 600 ~/.ssh/ci-deploy-key + cat > ~/.ssh/known_hosts << 'EOF' + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= + EOF + chmod 600 ~/.ssh/known_hosts + export GIT_SSH_COMMAND="ssh -i $HOME/.ssh/ci-deploy-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts" + + if [ ! -d ~/relay-plugin ]; then + git clone https://github.com/No-Instructions/Relay.git ~/relay-plugin + fi + cd ~/relay-plugin + git fetch origin --force --prune \ + '+refs/heads/*:refs/remotes/origin/*' \ + '+refs/tags/0.8.0*:refs/tags/0.8.0*' + git checkout -- . + git clean -fd + git checkout --detach "$REF" + git crypt unlock ~/.config/relay-e2e/relay-git-crypt-key + if ! grep -q "describe\\|test\\|it(" __tests__/TestMinimark.ts; then + echo "git-crypt unlock did not expose the private unit tests" + exit 1 + fi + + COMMIT=$(git rev-parse --short HEAD) + EXEC_ID="unit-$(date -u +%Y%m%dT%H%M%SZ)" + REPORT_DIR="/tmp/test-reports/runs/$COMMIT/$EXEC_ID" + mkdir -p "$REPORT_DIR" + JEST_JSON="$REPORT_DIR/jest-results.json" + JEST_LOG="$REPORT_DIR/jest-output.txt" + + echo "Relay product checkout:" + git show -s --format=' commit: %H%n subject: %s' + echo " describe: $(git describe --tags --always --long 2>/dev/null || git rev-parse --short HEAD)" + + if [ ! -d ~/relay-harness ]; then + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + fi + cd ~/relay-harness + git fetch origin --prune + git checkout -- . + git clean -fd + git checkout --detach origin/main + + cd ~/relay-plugin + npm ci + + TEST_EXIT=0 + npm test -- --silent --json --outputFile="$JEST_JSON" >"$JEST_LOG" 2>&1 || TEST_EXIT=$? + + node - "$JEST_JSON" "$JEST_LOG" "$REPORT_DIR" "$COMMIT" "$EXEC_ID" <<'NODE' + const fs = require('fs'); + const path = require('path'); + + const [jestJsonPath, jestLogPath, reportDir, commit, execId] = process.argv.slice(2); + const raw = JSON.parse(fs.readFileSync(jestJsonPath, 'utf8')); + + const escapeHtml = value => String(value ?? '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + const safeSegment = value => String(value ?? '') + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 40); + const statusFor = status => { + if (status === 'passed') return 'pass'; + if (status === 'pending' || status === 'todo' || status === 'skipped') return 'expected_fail'; + return 'fail'; + }; + + const started = Number(raw.startTime || Date.now()); + const ended = Number(raw.endTime || Date.now()); + const tests = []; + let index = 0; + + for (const suite of raw.testResults || []) { + const suiteName = path.relative(process.cwd(), suite.name || ''); + for (const assertion of suite.assertionResults || []) { + index += 1; + const name = [...(assertion.ancestorTitles || []), assertion.title] + .filter(Boolean) + .join(' > '); + const status = statusFor(assertion.status); + const dir = `unit-${String(index).padStart(4, '0')}`; + const failureMessages = assertion.failureMessages || []; + const duration = Number(assertion.duration || 0); + const test = { + name: name || suiteName || `unit ${index}`, + status, + assertions: 1, + passedAssertions: status === 'pass' ? 1 : 0, + duration, + dir, + suite: suiteName, + firstError: failureMessages[0] || '', + }; + tests.push(test); + + fs.mkdirSync(path.join(reportDir, dir), { recursive: true }); + const passed = status === 'pass'; + fs.writeFileSync(path.join(reportDir, dir, 'results.json'), JSON.stringify({ + testName: test.name, + passed, + status, + startTime: started, + endTime: started + duration, + duration, + steps: [ + { + name: 'Jest assertion', + passed, + error: failureMessages.join('\n\n'), + assertions: [ + { + description: test.name, + expected: 'pass', + actual: status, + passed, + }, + ], + actions: [], + trace: [], + }, + ], + summary: { + totalSteps: 1, + passedSteps: passed ? 1 : 0, + totalAssertions: 1, + passedAssertions: passed ? 1 : 0, + totalActions: 0, + }, + }, null, 2)); + fs.writeFileSync(path.join(reportDir, dir, 'index.html'), ` + + + + ${escapeHtml(test.name)} + + + +

Back to unit report

+

${escapeHtml(test.name)}

+

${escapeHtml(status)}

+

Suite: ${escapeHtml(suiteName)}

+

Duration: ${escapeHtml(duration)}ms

+ ${failureMessages.length ? `

Failure

${escapeHtml(failureMessages.join('\n\n'))}
` : ''} + + + `); + } + } + + const counts = tests.reduce((acc, test) => { + acc[test.status] = (acc[test.status] || 0) + 1; + return acc; + }, {}); + const summary = { + summary: { + total: tests.length, + pass: counts.pass || 0, + fail: counts.fail || 0, + expectedFail: counts.expected_fail || 0, + unexpectedPass: 0, + duration: Math.max(0, ended - started), + }, + tests, + state: (counts.fail || 0) > 0 ? 'failure' : 'success', + }; + + fs.writeFileSync(path.join(reportDir, 'summary.json'), JSON.stringify(summary, null, 2)); + + const rows = tests.map(test => ` + + ${escapeHtml(test.name)} + ${escapeHtml(test.status)} + ${escapeHtml(test.suite)} + ${escapeHtml(test.duration)}ms + + `).join(''); + fs.writeFileSync(path.join(reportDir, 'index.html'), ` + + + + Relay Unit Tests ${escapeHtml(commit)} + + + +

Relay Unit Tests

+

Commit ${escapeHtml(commit)} / ${escapeHtml(execId)}

+
+ ${summary.summary.pass} pass + ${summary.summary.fail} fail + ${summary.summary.expectedFail} expected fail + ${summary.summary.total} total +
+

Jest JSON | Jest output | Summary JSON

+ + + ${rows} +
TestStatusSuiteDuration
+ + + `); + + if (!fs.existsSync(jestLogPath)) { + fs.writeFileSync(jestLogPath, ''); + } + NODE + + if ! node ~/relay-harness/scripts/upload-report.mjs "$REPORT_DIR" "$COMMIT" "$EXEC_ID" >"$REPORT_DIR/upload-output.txt" 2>&1; then + echo "Report upload failed" + exit 1 + fi + + cp "$REPORT_DIR/summary.json" ~/test-summary.json + REPORT_URL="https://ci.system3.dev/runs/${COMMIT}/${EXEC_ID}/index.html" + echo "{\"reportUrl\":\"$REPORT_URL\",\"commit\":\"$COMMIT\",\"execId\":\"$EXEC_ID\"}" > ~/test-metadata.json + + exit $TEST_EXIT + REMOTE_SCRIPT + + - name: Clean up VM credentials + if: always() + run: | + gcloud compute ssh "$VM_NAME" --quiet \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap \ + --command="rm -f ~/ci-sa-key.json ~/ci-deploy-key ~/.ssh/ci-deploy-key ~/.config/relay-e2e/relay-git-crypt-key ~/.config/relay-e2e/r2-env.sh" 2>/dev/null || true + + - name: Fetch results + if: always() + run: | + rm -f summary.json metadata.json + gcloud compute scp --quiet "$VM_NAME":~/test-summary.json ./summary.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap 2>/dev/null || true + gcloud compute scp --quiet "$VM_NAME":~/test-metadata.json ./metadata.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --tunnel-through-iap 2>/dev/null || true + + - name: Request VM stop + if: always() + run: | + STATUS=$(gcloud compute instances describe "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --format='get(status)' 2>/dev/null || echo "NOT_FOUND") + + if [ "$STATUS" = "RUNNING" ]; then + echo "Requesting asynchronous stop for unit-test VM..." + gcloud compute instances stop "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --quiet --async 2>/dev/null || true + else + echo "Skipping stop request (status: $STATUS)" + fi + + - name: Generate job summary + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + let summary, metadata; + try { + summary = JSON.parse(fs.readFileSync('summary.json', 'utf-8')); + } catch { + core.summary.addRaw('## Unit test results\n\n:x: **Failed to retrieve unit test results from VM**\n'); + await core.summary.write(); + core.setFailed('No unit test results available'); + return; + } + try { + metadata = JSON.parse(fs.readFileSync('metadata.json', 'utf-8')); + } catch { + metadata = { reportUrl: '', commit: '', execId: '' }; + } + + const s = summary.summary || {}; + const totalDur = Math.round((s.duration || 0) / 1000); + const fail = s.fail || 0; + const verdictEmoji = fail > 0 ? ':x:' : ':white_check_mark:'; + const verdictText = fail > 0 ? 'FAIL' : 'PASS'; + const parts = []; + parts.push(`${s.pass || 0} pass`); + if (fail) parts.push(`${fail} fail`); + if (s.expectedFail) parts.push(`${s.expectedFail} expected-fail`); + + let md = '## Unit test results\n\n'; + md += `${verdictEmoji} **${verdictText}** | ${parts.join(', ')} | ${s.total || 0} total | ${totalDur}s`; + if (metadata.reportUrl) { + md += ` | [Private Report](${metadata.reportUrl})`; + } + md += '\n'; + + core.summary.addRaw(md); + await core.summary.write(); + + if ((s.total || 0) === 0 || summary.state === 'error') { + core.setFailed('No unit tests were executed'); + } else if (fail > 0) { + core.setFailed(`${fail} unit test(s) failed`); + } diff --git a/.gitignore b/.gitignore index 5ae8f945..2263fb1d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,11 +19,16 @@ data.json # Exclude macOS Finder (System Explorer) View States .DS_Store +# local archive +archive/ +/bug-reports/ + # igit .igit .igitignore # developer harness +.mcp.json AGENTS.md CLAUDE.md AGENTS.md @@ -37,10 +42,12 @@ infra/ scripts/ tickets/ specs/ +arch/ unit/ templates/ test-plans/ test-results/ +merge-queue/ prompts/ vision/ diff --git a/__tests__/Document.test.ts b/__tests__/Document.test.ts new file mode 100644 index 00000000..c0bd6693 Binary files /dev/null and b/__tests__/Document.test.ts differ diff --git a/__tests__/FileLogDetails.test.ts b/__tests__/FileLogDetails.test.ts new file mode 100644 index 00000000..514f7541 Binary files /dev/null and b/__tests__/FileLogDetails.test.ts differ diff --git a/__tests__/HasProvider.test.ts b/__tests__/HasProvider.test.ts new file mode 100644 index 00000000..5e55770d Binary files /dev/null and b/__tests__/HasProvider.test.ts differ diff --git a/__tests__/MetadataRenderer.test.ts b/__tests__/MetadataRenderer.test.ts new file mode 100644 index 00000000..a5f845fb Binary files /dev/null and b/__tests__/MetadataRenderer.test.ts differ diff --git a/__tests__/Observable.test.ts b/__tests__/Observable.test.ts new file mode 100644 index 00000000..a809198f Binary files /dev/null and b/__tests__/Observable.test.ts differ diff --git a/__tests__/ObservableMap.test.ts b/__tests__/ObservableMap.test.ts index 8174debf..4740b6cd 100644 Binary files a/__tests__/ObservableMap.test.ts and b/__tests__/ObservableMap.test.ts differ diff --git a/__tests__/RelayDebugAPI-conflicts.test.ts b/__tests__/RelayDebugAPI-conflicts.test.ts new file mode 100644 index 00000000..af82d96e Binary files /dev/null and b/__tests__/RelayDebugAPI-conflicts.test.ts differ diff --git a/__tests__/RemoteActivityIndex.test.ts b/__tests__/RemoteActivityIndex.test.ts new file mode 100644 index 00000000..0e67ac8a Binary files /dev/null and b/__tests__/RemoteActivityIndex.test.ts differ diff --git a/__tests__/SharedFolder.test.ts b/__tests__/SharedFolder.test.ts index 6d71ba62..0735d759 100644 Binary files a/__tests__/SharedFolder.test.ts and b/__tests__/SharedFolder.test.ts differ diff --git a/__tests__/SyncFile.test.ts b/__tests__/SyncFile.test.ts new file mode 100644 index 00000000..29780d36 Binary files /dev/null and b/__tests__/SyncFile.test.ts differ diff --git a/__tests__/SyncSettings.test.ts b/__tests__/SyncSettings.test.ts new file mode 100644 index 00000000..c6dd87a3 Binary files /dev/null and b/__tests__/SyncSettings.test.ts differ diff --git a/__tests__/TestMinimark.ts b/__tests__/TestMinimark.ts index a09b48f5..7520fd1c 100644 Binary files a/__tests__/TestMinimark.ts and b/__tests__/TestMinimark.ts differ diff --git a/__tests__/TestSettingsStorage.ts b/__tests__/TestSettingsStorage.ts index 0aa47651..2c1e71ee 100644 Binary files a/__tests__/TestSettingsStorage.ts and b/__tests__/TestSettingsStorage.ts differ diff --git a/__tests__/TestSyncStore.ts b/__tests__/TestSyncStore.ts index 9ff4e93e..6e752759 100644 Binary files a/__tests__/TestSyncStore.ts and b/__tests__/TestSyncStore.ts differ diff --git a/__tests__/TestTimeProvider.ts b/__tests__/TestTimeProvider.ts index f506e280..40973faf 100644 Binary files a/__tests__/TestTimeProvider.ts and b/__tests__/TestTimeProvider.ts differ diff --git a/__tests__/TestTokenStore.ts b/__tests__/TestTokenStore.ts index d9dc4553..12e05155 100644 Binary files a/__tests__/TestTokenStore.ts and b/__tests__/TestTokenStore.ts differ diff --git a/__tests__/TextViewPlugin.test.ts b/__tests__/TextViewPlugin.test.ts new file mode 100644 index 00000000..82db3ba3 Binary files /dev/null and b/__tests__/TextViewPlugin.test.ts differ diff --git a/__tests__/ViewHookPlugin.test.ts b/__tests__/ViewHookPlugin.test.ts new file mode 100644 index 00000000..32d4bf19 Binary files /dev/null and b/__tests__/ViewHookPlugin.test.ts differ diff --git a/__tests__/client/provider.test.ts b/__tests__/client/provider.test.ts new file mode 100644 index 00000000..4684721d Binary files /dev/null and b/__tests__/client/provider.test.ts differ diff --git a/__tests__/merge-hsm/E2ERecorded.test.ts b/__tests__/merge-hsm/E2ERecorded.test.ts new file mode 100644 index 00000000..07e66354 Binary files /dev/null and b/__tests__/merge-hsm/E2ERecorded.test.ts differ diff --git a/__tests__/merge-hsm/MergeHSM.test.ts b/__tests__/merge-hsm/MergeHSM.test.ts new file mode 100644 index 00000000..d68dd680 Binary files /dev/null and b/__tests__/merge-hsm/MergeHSM.test.ts differ diff --git a/__tests__/merge-hsm/MergeManager.test.ts b/__tests__/merge-hsm/MergeManager.test.ts new file mode 100644 index 00000000..e31bbd4b Binary files /dev/null and b/__tests__/merge-hsm/MergeManager.test.ts differ diff --git a/__tests__/merge-hsm/cancel-persistence.test.ts b/__tests__/merge-hsm/cancel-persistence.test.ts new file mode 100644 index 00000000..f21a71f6 Binary files /dev/null and b/__tests__/merge-hsm/cancel-persistence.test.ts differ diff --git a/__tests__/merge-hsm/crdt-integrity.test.ts b/__tests__/merge-hsm/crdt-integrity.test.ts new file mode 100644 index 00000000..33b3ca46 Binary files /dev/null and b/__tests__/merge-hsm/crdt-integrity.test.ts differ diff --git a/__tests__/merge-hsm/cross-vault.test.ts b/__tests__/merge-hsm/cross-vault.test.ts new file mode 100644 index 00000000..27356e6c Binary files /dev/null and b/__tests__/merge-hsm/cross-vault.test.ts differ diff --git a/__tests__/merge-hsm/deltaToPositionedChanges.test.ts b/__tests__/merge-hsm/deltaToPositionedChanges.test.ts new file mode 100644 index 00000000..d8e2b056 Binary files /dev/null and b/__tests__/merge-hsm/deltaToPositionedChanges.test.ts differ diff --git a/__tests__/merge-hsm/fork-syncgate.test.ts b/__tests__/merge-hsm/fork-syncgate.test.ts new file mode 100644 index 00000000..49b66a9d Binary files /dev/null and b/__tests__/merge-hsm/fork-syncgate.test.ts differ diff --git a/__tests__/merge-hsm/frontmatter-repair.test.ts b/__tests__/merge-hsm/frontmatter-repair.test.ts new file mode 100644 index 00000000..0b826d41 Binary files /dev/null and b/__tests__/merge-hsm/frontmatter-repair.test.ts differ diff --git a/__tests__/merge-hsm/guid-remap.test.ts b/__tests__/merge-hsm/guid-remap.test.ts new file mode 100644 index 00000000..e06cd916 Binary files /dev/null and b/__tests__/merge-hsm/guid-remap.test.ts differ diff --git a/__tests__/merge-hsm/headless-conflict-resolution.test.ts b/__tests__/merge-hsm/headless-conflict-resolution.test.ts new file mode 100644 index 00000000..d9b8c976 Binary files /dev/null and b/__tests__/merge-hsm/headless-conflict-resolution.test.ts differ diff --git a/__tests__/merge-hsm/hibernation.test.ts b/__tests__/merge-hsm/hibernation.test.ts new file mode 100644 index 00000000..36d02b98 Binary files /dev/null and b/__tests__/merge-hsm/hibernation.test.ts differ diff --git a/__tests__/merge-hsm/invariants.test.ts b/__tests__/merge-hsm/invariants.test.ts new file mode 100644 index 00000000..2cde35b6 Binary files /dev/null and b/__tests__/merge-hsm/invariants.test.ts differ diff --git a/__tests__/merge-hsm/machine-definition.test.ts b/__tests__/merge-hsm/machine-definition.test.ts new file mode 100644 index 00000000..00f89bde Binary files /dev/null and b/__tests__/merge-hsm/machine-definition.test.ts differ diff --git a/__tests__/merge-hsm/machine-edit-rewind.test.ts b/__tests__/merge-hsm/machine-edit-rewind.test.ts new file mode 100644 index 00000000..e886fbbf Binary files /dev/null and b/__tests__/merge-hsm/machine-edit-rewind.test.ts differ diff --git a/__tests__/merge-hsm/machine-visualization.test.ts b/__tests__/merge-hsm/machine-visualization.test.ts new file mode 100644 index 00000000..76a2c1eb Binary files /dev/null and b/__tests__/merge-hsm/machine-visualization.test.ts differ diff --git a/__tests__/merge-hsm/merge-path-adversarial.test.ts b/__tests__/merge-hsm/merge-path-adversarial.test.ts new file mode 100644 index 00000000..e2d6a5bf Binary files /dev/null and b/__tests__/merge-hsm/merge-path-adversarial.test.ts differ diff --git a/__tests__/merge-hsm/network-resilience.test.ts b/__tests__/merge-hsm/network-resilience.test.ts new file mode 100644 index 00000000..e5649258 Binary files /dev/null and b/__tests__/merge-hsm/network-resilience.test.ts differ diff --git a/__tests__/merge-hsm/persistence.test.ts b/__tests__/merge-hsm/persistence.test.ts new file mode 100644 index 00000000..f92510aa Binary files /dev/null and b/__tests__/merge-hsm/persistence.test.ts differ diff --git a/__tests__/merge-hsm/provider-integration-lifecycle.test.ts b/__tests__/merge-hsm/provider-integration-lifecycle.test.ts new file mode 100644 index 00000000..5a0f06d0 Binary files /dev/null and b/__tests__/merge-hsm/provider-integration-lifecycle.test.ts differ diff --git a/__tests__/merge-hsm/provider-sync-guard.test.ts b/__tests__/merge-hsm/provider-sync-guard.test.ts new file mode 100644 index 00000000..71bf427b Binary files /dev/null and b/__tests__/merge-hsm/provider-sync-guard.test.ts differ diff --git a/__tests__/merge-hsm/recalculate-conflict-positions.test.ts b/__tests__/merge-hsm/recalculate-conflict-positions.test.ts new file mode 100644 index 00000000..a164adb7 Binary files /dev/null and b/__tests__/merge-hsm/recalculate-conflict-positions.test.ts differ diff --git a/__tests__/merge-hsm/recording.test.ts b/__tests__/merge-hsm/recording.test.ts new file mode 100644 index 00000000..5ec58f74 Binary files /dev/null and b/__tests__/merge-hsm/recording.test.ts differ diff --git a/__tests__/merge-hsm/replayBufferedEdits.test.ts b/__tests__/merge-hsm/replayBufferedEdits.test.ts new file mode 100644 index 00000000..a83627e4 Binary files /dev/null and b/__tests__/merge-hsm/replayBufferedEdits.test.ts differ diff --git a/__tests__/merge-hsm/state-helpers.test.ts b/__tests__/merge-hsm/state-helpers.test.ts new file mode 100644 index 00000000..67f5e4c1 Binary files /dev/null and b/__tests__/merge-hsm/state-helpers.test.ts differ diff --git a/__tests__/merge-hsm/state-vectors.test.ts b/__tests__/merge-hsm/state-vectors.test.ts new file mode 100644 index 00000000..670911e1 Binary files /dev/null and b/__tests__/merge-hsm/state-vectors.test.ts differ diff --git a/__tests__/merge-hsm/syncbridge.test.ts b/__tests__/merge-hsm/syncbridge.test.ts new file mode 100644 index 00000000..ffc36f0b Binary files /dev/null and b/__tests__/merge-hsm/syncbridge.test.ts differ diff --git a/__tests__/merge-hsm/testing/fixtures.ts b/__tests__/merge-hsm/testing/fixtures.ts new file mode 100644 index 00000000..08ac62ba Binary files /dev/null and b/__tests__/merge-hsm/testing/fixtures.ts differ diff --git a/__tests__/merge-hsm/testing/integration.ts b/__tests__/merge-hsm/testing/integration.ts new file mode 100644 index 00000000..fdf7fbd1 Binary files /dev/null and b/__tests__/merge-hsm/testing/integration.ts differ diff --git a/__tests__/merge-hsm/testing/replay-fixtures.ts b/__tests__/merge-hsm/testing/replay-fixtures.ts new file mode 100644 index 00000000..64bd96ab Binary files /dev/null and b/__tests__/merge-hsm/testing/replay-fixtures.ts differ diff --git a/__tests__/merge-hsm/update-queues.test.ts b/__tests__/merge-hsm/update-queues.test.ts new file mode 100644 index 00000000..a564432a Binary files /dev/null and b/__tests__/merge-hsm/update-queues.test.ts differ diff --git a/__tests__/mocks/MockTimeProvider.ts b/__tests__/mocks/MockTimeProvider.ts index 2cd2d6f4..1b5eb082 100644 Binary files a/__tests__/mocks/MockTimeProvider.ts and b/__tests__/mocks/MockTimeProvider.ts differ diff --git a/__tests__/ui/CanvasData.test.ts b/__tests__/ui/CanvasData.test.ts new file mode 100644 index 00000000..1453f3a2 Binary files /dev/null and b/__tests__/ui/CanvasData.test.ts differ diff --git a/__tests__/ui/FolderPillProgress.test.ts b/__tests__/ui/FolderPillProgress.test.ts new file mode 100644 index 00000000..db031d29 Binary files /dev/null and b/__tests__/ui/FolderPillProgress.test.ts differ diff --git a/__tests__/ui/SyncStatusModel.test.ts b/__tests__/ui/SyncStatusModel.test.ts new file mode 100644 index 00000000..3b74cf8f Binary files /dev/null and b/__tests__/ui/SyncStatusModel.test.ts differ diff --git a/__tests__/ui/UserFacingError.test.ts b/__tests__/ui/UserFacingError.test.ts new file mode 100644 index 00000000..f137ca97 Binary files /dev/null and b/__tests__/ui/UserFacingError.test.ts differ diff --git a/__tests__/ui/timeAgo.test.ts b/__tests__/ui/timeAgo.test.ts new file mode 100644 index 00000000..4304fcf8 Binary files /dev/null and b/__tests__/ui/timeAgo.test.ts differ diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 5fa392cc..3a80197f 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -7,6 +7,7 @@ import { execSync } from "child_process"; import chokidar from "chokidar"; import path from "path"; import fs, { mkdirSync } from "fs"; +import crypto from "crypto"; const banner = `/* THIS IS A GENERATED/BUNDLED FILE BY ESBUILD @@ -14,15 +15,26 @@ if you want to view the source, please visit the github repository of this plugi */ `; -const gitTag = execSync("git describe --tags --always", { +const runBuildMetadataCommand = (command) => execSync(command, { encoding: "utf8", -}).trim(); + stdio: ["ignore", "pipe", "pipe"], +}); + +const getGitTag = () => { + try { + return runBuildMetadataCommand("git describe --tags --always").trim(); + } catch (e) { + return "dev"; + } +}; +const gitTag = getGitTag(); const develop = process.argv[2] === "develop"; const staging = process.argv[2] === "staging"; -const watch = process.argv[2] === "watch" || staging; +const watch = process.argv[2] === "watch" || process.argv.includes("--watch"); const debug = process.argv[2] === "debug" || watch || staging || develop; -const out = process.argv[3] || "."; +const positionalArgs = process.argv.slice(3).filter((a) => !a.startsWith("--")); +const out = positionalArgs[0] || "."; const tld = staging ? "dev" : "md"; const apiUrl = `https://api.system3.${tld}`; @@ -31,11 +43,58 @@ const healthUrl = `${apiUrl}/health?version=${gitTag}`; console.log("git tag:", gitTag); console.log("health URL", healthUrl); +// Fingerprint the working tree: HEAD commit + hash of uncommitted changes. +// Recompute this to check if a build artifact is stale. +const getSourceFingerprint = () => { + try { + const head = runBuildMetadataCommand("git rev-parse --short HEAD").trim(); + const diff = runBuildMetadataCommand("git diff HEAD"); + if (!diff) return head; + const diffHash = crypto.createHash("sha1").update(diff).digest("hex").slice(0, 8); + return `${head}+${diffHash}`; + } catch (e) { + return "unknown"; + } +}; + +const writeBuildStatus = (build, result) => { + const outdir = build.initialOptions.outfile; + // Derive vault name from outdir (e.g. "vaults/live1/.obsidian/..." → "live1") + const match = outdir.match(/vaults\/([^/]+)\//); + const name = match ? match[1] : path.basename(path.dirname(outdir)); + const statusDir = ".build-status"; + fs.mkdirSync(statusDir, { recursive: true }); + const timestamp = new Date().toLocaleTimeString("en-GB", { hour12: false }); + const fingerprint = getSourceFingerprint(); + if (result.errors.length > 0) { + const errors = result.errors.map(e => e.text).join("\n"); + fs.writeFileSync(`${statusDir}/${name}.log`, `FAIL ${timestamp} src:${fingerprint}\n${errors}\n`); + } else { + const stat = fs.statSync(outdir); + const kb = Math.round(stat.size / 1024); + fs.writeFileSync(`${statusDir}/${name}.log`, `OK ${timestamp} main.js ${kb}kb src:${fingerprint} → ${outdir}\n`); + } +}; + const NotifyPlugin = { name: "on-end", setup(build) { build.onEnd((result) => { - if (result.errors.length > 0) execSync(`notify-send "Build Failed"`); + if (result.errors.length > 0) { + execSync(`notify-send "Build Failed"`); + if (staging) { + writeBuildStatus(build, result); + } + } else if (watch) { + const tag = getGitTag(); + const outfile = build.initialOptions.outfile; + const content = fs.readFileSync(outfile, "utf8"); + fs.writeFileSync(outfile, content.replace(/__GIT_TAG__/g, tag)); + console.log(`GIT_TAG: ${tag}`); + if (staging) { + writeBuildStatus(build, result); + } + } }); }, }; @@ -43,6 +102,13 @@ const NotifyPlugin = { const YjsInternalsPlugin = { name: "yjs-internals", setup(build) { + // Keep public Yjs imports and internal helper imports on the same + // module graph so Item/GC instanceof checks agree in the bundle. + build.onResolve({ filter: /^yjs$/ }, args => { + return { + path: path.resolve("node_modules/yjs/src/index.js"), + }; + }); build.onResolve({ filter: /^yjs\/dist\/src\/internals$/ }, args => { return { path: path.resolve("node_modules/yjs/src/internals.js"), @@ -77,7 +143,7 @@ const context = await esbuild.context({ format: "cjs", plugins: [ esbuildSvelte({ - compilerOptions: { css: true }, + compilerOptions: { css: "injected" }, preprocess: sveltePreprocess(), }), YjsInternalsPlugin, @@ -89,7 +155,7 @@ const context = await esbuild.context({ sourcemap: debug ? "inline" : false, define: { BUILD_TYPE: debug ? '"debug"' : '"prod"', - GIT_TAG: `"${gitTag}"`, + GIT_TAG: watch ? '"__GIT_TAG__"' : `"${gitTag}"`, HEALTH_URL: `"${healthUrl}"`, API_URL: `"${apiUrl}"`, AUTH_URL: `"${authUrl}"`, @@ -110,7 +176,7 @@ const copyFile = (src, dest) => { const watchAndMove = (fnames, mapping) => { // only usable on top level directory const watcher = chokidar.watch(fnames, { - ignored: /(^|[\/\\])\../, // ignore dotfiles + ignored: /(^|[\/\\])\./, // ignore dotfiles persistent: true, }); @@ -123,7 +189,7 @@ const watchAndMove = (fnames, mapping) => { const mapping = debug ? { "manifest-beta.json": "manifest.json" } : {}; const manifest = debug ? "manifest-beta.json" : "manifest.json"; -const files = ["styles.css", manifest]; +const files = debug && out === "." ? ["styles.css"] : ["styles.css", manifest]; const updateManifest = (manifest) => { const manifestPath = path.join(out, path.basename("manifest.json")); @@ -155,7 +221,7 @@ if (watch) { } else { await context.rebuild(); move(files, mapping); - if (!develop) { + if (!develop && out !== ".") { updateManifest(); } process.exit(0); diff --git a/jest.config.js b/jest.config.js index 9dd81529..71a8d404 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,22 +1,27 @@ -//module.exports = { -// preset: 'ts-jest', -// testEnvironment: 'node', -//}; -// /** @type {import('ts-jest').JestConfigWithTsJest} */ + +// Resolve yjs paths dynamically so tests work in git worktrees where +// node_modules may live in a parent directory rather than . +const path = require("path"); +const yjsDir = path.dirname(require.resolve("yjs/package.json")); +const yjsIndex = path.join(yjsDir, "src/index.js"); +const yjsInternals = path.join(yjsDir, "src/internals.js"); + module.exports = { - // [...] - preset: "ts-jest/presets/default-esm", // or other ESM presets + preset: "ts-jest/presets/default-esm", moduleNameMapper: { "^(\\.{1,2}/.*)\\.js$": "$1", "^src/(.*)$": "/src/$1", + "^yjs$": yjsIndex, + "^yjs/dist/src/internals$": yjsInternals, }, - testPathIgnorePatterns: ["/__tests__/mocks/"], + testPathIgnorePatterns: ["/__tests__/mocks/", "/__tests__/merge-hsm/testing/", "archive/", ".claude"], globals: { "BUILD_TYPE": "production", }, + transformIgnorePatterns: ["[\\/]node_modules[\\/](?!(yjs|lib0)[\\/])"], transform: { - ".ts": [ + "\\.ts$": [ "ts-jest", { // Note: We shouldn't need to include `isolatedModules` here because it's a deprecated config option in TS 5, @@ -26,5 +31,19 @@ module.exports = { useESM: true, }, ], + "src/.+\\.js$": [ + "ts-jest", + { + isolatedModules: true, + useESM: true, + }, + ], + "node_modules[\\/](yjs|lib0)[\\/].+\\.js$": [ + "ts-jest", + { + isolatedModules: true, + useESM: true, + }, + ], }, }; diff --git a/package-lock.json b/package-lock.json index a694c217..748658ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "cbor-x": "^1.6.0", "diff": "^5.2.0", "diff-match-patch": "^1.0.5", "eventsource": "^2.0.2", @@ -16,13 +17,13 @@ "jose": "^5.3.0", "lucide-svelte": "^0.377.0", "monkey-around": "^3.0.0", + "node-diff3": "^3.2.0", "obsidian-daily-notes-interface": "^0.9.4", "path-browserify": "^1.0.1", "pocketbase": "^0.20.3", "svelte-step-wizard": "^0.0.2", "tslib": "2.4.0", "uuid": "^9.0.1", - "y-indexeddb": "^9.0.9", "y-leveldb": "^0.1.2", "y-protocols": "^1.0.5", "y-websocket": "^1.5.3" @@ -33,6 +34,7 @@ "@types/diff-match-patch": "^1.0.36", "@types/eventsource": "^1.1.15", "@types/jest": "^29.5.12", + "@types/js-yaml": "^4.0.9", "@types/node": "^16.11.6", "@types/path-browserify": "^1.0.2", "@types/uuid": "^9.0.8", @@ -43,6 +45,7 @@ "esbuild": "^0.27.0", "esbuild-svelte": "^0.8.0", "jest": "^29.7.0", + "js-yaml": "^4.1.1", "obsidian": "^1.7.2", "svelte": "^4.2.19", "svelte-preprocess": "^5.1.4", @@ -565,6 +568,84 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@cbor-extract/cbor-extract-darwin-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-darwin-arm64/-/cbor-extract-darwin-arm64-2.2.0.tgz", + "integrity": "sha512-P7swiOAdF7aSi0H+tHtHtr6zrpF3aAq/W9FXx5HektRvLTM2O89xCyXF3pk7pLc7QpaY7AoaE8UowVf9QBdh3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@cbor-extract/cbor-extract-darwin-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-darwin-x64/-/cbor-extract-darwin-x64-2.2.0.tgz", + "integrity": "sha512-1liF6fgowph0JxBbYnAS7ZlqNYLf000Qnj4KjqPNW4GViKrEql2MgZnAsExhY9LSy8dnvA4C0qHEBgPrll0z0w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@cbor-extract/cbor-extract-linux-arm": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-arm/-/cbor-extract-linux-arm-2.2.0.tgz", + "integrity": "sha512-QeBcBXk964zOytiedMPQNZr7sg0TNavZeuUCD6ON4vEOU/25+pLhNN6EDIKJ9VLTKaZ7K7EaAriyYQ1NQ05s/Q==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@cbor-extract/cbor-extract-linux-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-arm64/-/cbor-extract-linux-arm64-2.2.0.tgz", + "integrity": "sha512-rQvhNmDuhjTVXSPFLolmQ47/ydGOFXtbR7+wgkSY0bdOxCFept1hvg59uiLPT2fVDuJFuEy16EImo5tE2x3RsQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@cbor-extract/cbor-extract-linux-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-x64/-/cbor-extract-linux-x64-2.2.0.tgz", + "integrity": "sha512-cWLAWtT3kNLHSvP4RKDzSTX9o0wvQEEAj4SKvhWuOVZxiDAeQazr9A+PSiRILK1VYMLeDml89ohxCnUNQNQNCw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@cbor-extract/cbor-extract-win32-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-win32-x64/-/cbor-extract-win32-x64-2.2.0.tgz", + "integrity": "sha512-l2M+Z8DO2vbvADOBNLbbh9y5ST1RY5sqkWOg/58GkUPBYou/cuNZ68SGQ644f1CvZ8kcOxyZtw06+dxWHIoN/w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@codemirror/state": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.4.1.tgz", @@ -1025,25 +1106,30 @@ } }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, + "license": "MIT", "dependencies": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, + "funding": { + "url": "https://opencollective.com/eslint" + }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", - "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, + "license": "MIT", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } @@ -1053,6 +1139,7 @@ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "ajv": "^6.12.4", @@ -1073,23 +1160,26 @@ } }, "node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", "dev": true, + "license": "Apache-2.0", "peer": true, "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, @@ -1102,6 +1192,7 @@ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", "dev": true, + "license": "Apache-2.0", "peer": true, "engines": { "node": ">=12.22" @@ -1115,7 +1206,9 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", "dev": true, + "license": "BSD-3-Clause", "peer": true }, "node_modules/@istanbuljs/load-nyc-config": { @@ -1553,6 +1646,7 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -1566,6 +1660,7 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } @@ -1575,6 +1670,7 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -1728,11 +1824,19 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@types/node": { "version": "16.18.86", @@ -1753,10 +1857,11 @@ "dev": true }, "node_modules/@types/semver": { - "version": "7.5.8", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", - "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", - "dev": true + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==", + "dev": true, + "license": "MIT" }, "node_modules/@types/stack-utils": { "version": "2.0.3", @@ -1798,6 +1903,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.21.0.tgz", "integrity": "sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.5.1", "@typescript-eslint/scope-manager": "6.21.0", @@ -1833,6 +1939,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.21.0.tgz", "integrity": "sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "@typescript-eslint/scope-manager": "6.21.0", "@typescript-eslint/types": "6.21.0", @@ -1861,6 +1968,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz", "integrity": "sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg==", "dev": true, + "license": "MIT", "dependencies": { "@typescript-eslint/types": "6.21.0", "@typescript-eslint/visitor-keys": "6.21.0" @@ -1878,6 +1986,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.21.0.tgz", "integrity": "sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag==", "dev": true, + "license": "MIT", "dependencies": { "@typescript-eslint/typescript-estree": "6.21.0", "@typescript-eslint/utils": "6.21.0", @@ -1905,6 +2014,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.21.0.tgz", "integrity": "sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg==", "dev": true, + "license": "MIT", "engines": { "node": "^16.0.0 || >=18.0.0" }, @@ -1918,6 +2028,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz", "integrity": "sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "@typescript-eslint/types": "6.21.0", "@typescript-eslint/visitor-keys": "6.21.0", @@ -1942,9 +2053,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", "dev": true, "license": "MIT", "dependencies": { @@ -1956,6 +2067,7 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1971,6 +2083,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.21.0.tgz", "integrity": "sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", @@ -1996,6 +2109,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz", "integrity": "sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==", "dev": true, + "license": "MIT", "dependencies": { "@typescript-eslint/types": "6.21.0", "eslint-visitor-keys": "^3.4.1" @@ -2009,10 +2123,11 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.1.tgz", + "integrity": "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ==", "dev": true, + "license": "ISC", "peer": true }, "node_modules/abstract-leveldown": { @@ -2046,16 +2161,18 @@ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, + "license": "MIT", "peer": true, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", @@ -2136,8 +2253,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "peer": true + "dev": true }, "node_modules/aria-query": { "version": "5.3.0", @@ -2152,6 +2268,7 @@ "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -2324,9 +2441,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "dev": true, "license": "MIT", "dependencies": { @@ -2487,6 +2604,37 @@ } ] }, + "node_modules/cbor-extract": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cbor-extract/-/cbor-extract-2.2.0.tgz", + "integrity": "sha512-Ig1zM66BjLfTXpNgKpvBePq271BPOvu8MR0Jl080yG7Jsl+wAZunfrwiwA+9ruzm/WEdIV5QF/bjDZTqyAIVHA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.1.1" + }, + "bin": { + "download-cbor-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@cbor-extract/cbor-extract-darwin-arm64": "2.2.0", + "@cbor-extract/cbor-extract-darwin-x64": "2.2.0", + "@cbor-extract/cbor-extract-linux-arm": "2.2.0", + "@cbor-extract/cbor-extract-linux-arm64": "2.2.0", + "@cbor-extract/cbor-extract-linux-x64": "2.2.0", + "@cbor-extract/cbor-extract-win32-x64": "2.2.0" + } + }, + "node_modules/cbor-x": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/cbor-x/-/cbor-x-1.6.0.tgz", + "integrity": "sha512-0kareyRwHSkL6ws5VXHEf8uY1liitysCVJjlmhaLG+IXLqhSaOO+t63coaso7yjwEzWZzLy8fJo06gZDVQM9Qg==", + "license": "MIT", + "optionalDependencies": { + "cbor-extract": "^2.2.0" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -2724,6 +2872,7 @@ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/deepmerge": { @@ -2764,6 +2913,16 @@ "node": ">=8" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -2774,9 +2933,10 @@ } }, "node_modules/diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", + "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -2800,6 +2960,7 @@ "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", "dev": true, + "license": "MIT", "dependencies": { "path-type": "^4.0.0" }, @@ -2812,6 +2973,7 @@ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", "dev": true, + "license": "Apache-2.0", "peer": true, "dependencies": { "esutils": "^2.0.2" @@ -2927,10 +3089,11 @@ } }, "node_modules/esbuild-svelte": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/esbuild-svelte/-/esbuild-svelte-0.8.0.tgz", - "integrity": "sha512-uKcPf1kl2UGMjrfHChv4dLxGAvCNhf9s72mHo19ZhKP+LrVOuQkOM/g8GE7MiGpoqjpk8UHqL08uLRbSKXhmhw==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/esbuild-svelte/-/esbuild-svelte-0.8.2.tgz", + "integrity": "sha512-tG97WrhH/OH8wFCmRBk6sRggMVMPNZDktGx1jJcvh9Obvjwm8M1UYoXmliuLL+4fs/wfyVVwM2fvyNYz2e6UPQ==", "dev": true, + "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.19" }, @@ -2939,7 +3102,7 @@ }, "peerDependencies": { "esbuild": ">=0.9.6", - "svelte": ">=3.43.0 <5" + "svelte": ">=3.43.0 <6 || ^5.0.0-next.0" } }, "node_modules/escalade": { @@ -2956,6 +3119,7 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=10" @@ -2965,17 +3129,19 @@ } }, "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -3025,6 +3191,7 @@ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "dependencies": { "esrecurse": "^4.3.0", @@ -3042,6 +3209,7 @@ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -3054,6 +3222,7 @@ "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "dependencies": { "acorn": "^8.9.0", @@ -3080,10 +3249,11 @@ } }, "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", "dev": true, + "license": "BSD-3-Clause", "peer": true, "dependencies": { "estraverse": "^5.1.0" @@ -3097,6 +3267,7 @@ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "dependencies": { "estraverse": "^5.2.0" @@ -3110,6 +3281,7 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "engines": { "node": ">=4.0" @@ -3128,6 +3300,7 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "engines": { "node": ">=0.10.0" @@ -3205,19 +3378,21 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", - "micromatch": "^4.0.4" + "micromatch": "^4.0.8" }, "engines": { "node": ">=8.6.0" @@ -3228,6 +3403,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.1" }, @@ -3246,13 +3422,15 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", "dev": true, + "license": "ISC", "dependencies": { "reusify": "^1.0.4" } @@ -3271,6 +3449,7 @@ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "flat-cache": "^3.0.4" @@ -3296,6 +3475,7 @@ "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "locate-path": "^6.0.0", @@ -3313,6 +3493,7 @@ "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "flatted": "^3.2.9", @@ -3324,10 +3505,11 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, + "license": "ISC", "peer": true }, "node_modules/fs.realpath": { @@ -3423,6 +3605,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, + "license": "ISC", "peer": true, "dependencies": { "is-glob": "^4.0.3" @@ -3436,6 +3619,7 @@ "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "type-fest": "^0.20.2" @@ -3452,6 +3636,7 @@ "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", "dev": true, + "license": "MIT", "dependencies": { "array-union": "^2.1.0", "dir-glob": "^3.0.1", @@ -3477,7 +3662,8 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/gray-matter": { "version": "4.0.3", @@ -3570,10 +3756,11 @@ ] }, "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 4" } @@ -3584,10 +3771,11 @@ "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" }, "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "parent-module": "^1.0.0", @@ -3734,6 +3922,7 @@ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=8" @@ -4416,7 +4605,6 @@ "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "argparse": "^2.0.1" }, @@ -4441,6 +4629,7 @@ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/json-parse-even-better-errors": { @@ -4454,6 +4643,7 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/json-stable-stringify-without-jsonify": { @@ -4461,6 +4651,7 @@ "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/json5": { @@ -4480,6 +4671,7 @@ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "json-buffer": "3.0.1" @@ -4639,6 +4831,7 @@ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "prelude-ls": "^1.2.1", @@ -4683,6 +4876,7 @@ "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "p-locate": "^5.0.0" @@ -4710,6 +4904,7 @@ "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/lru-cache": { @@ -4788,6 +4983,7 @@ "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } @@ -4824,10 +5020,11 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4886,6 +5083,15 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/node-diff3": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/node-diff3/-/node-diff3-3.2.0.tgz", + "integrity": "sha512-vLh2xJFSyniBLYDEDbXKqD32fQ5vAxmYT4hco8t0EHQ4CQ4BDHhshi7kdvDc6Y1MwGSi1Mhl4unUukPbCayZdw==", + "engines": { + "bun": ">=1.3.0", + "node": ">=18" + } + }, "node_modules/node-gyp-build": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", @@ -4896,6 +5102,21 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.1.1.tgz", + "integrity": "sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5002,6 +5223,7 @@ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "deep-is": "^0.1.3", @@ -5035,6 +5257,7 @@ "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "p-limit": "^3.0.2" @@ -5060,6 +5283,7 @@ "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "callsites": "^3.0.0" @@ -5129,6 +5353,7 @@ "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -5151,10 +5376,11 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -5245,6 +5471,7 @@ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">= 0.8.0" @@ -5299,6 +5526,7 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=6" @@ -5338,7 +5566,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/react-is": { "version": "18.2.0", @@ -5423,6 +5652,7 @@ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=4" @@ -5438,10 +5668,11 @@ } }, "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", "dev": true, + "license": "MIT", "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" @@ -5453,6 +5684,7 @@ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, + "license": "ISC", "peer": true, "dependencies": { "glob": "^7.1.3" @@ -5483,6 +5715,7 @@ "url": "https://feross.org/support" } ], + "license": "MIT", "dependencies": { "queue-microtask": "^1.2.2" } @@ -5813,9 +6046,10 @@ } }, "node_modules/svelte": { - "version": "4.2.19", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.19.tgz", - "integrity": "sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==", + "version": "4.2.20", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.20.tgz", + "integrity": "sha512-eeEgGc2DtiUil5ANdtd8vPwt9AgaMdnuUFnPft9F5oMvU/FHu5IHFic+p1dR/UOB7XU2mX2yHW+NcTch4DCh5Q==", + "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.2.1", "@jridgewell/sourcemap-codec": "^1.4.15", @@ -5922,6 +6156,7 @@ "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/tmpl": { @@ -5943,10 +6178,11 @@ } }, "node_modules/ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", "dev": true, + "license": "MIT", "engines": { "node": ">=16" }, @@ -6008,6 +6244,7 @@ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "prelude-ls": "^1.2.1" @@ -6030,6 +6267,7 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, + "license": "(MIT OR CC0-1.0)", "peer": true, "engines": { "node": ">=10" @@ -6086,6 +6324,7 @@ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "dependencies": { "punycode": "^2.1.0" @@ -6157,6 +6396,7 @@ "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=0.10.0" @@ -6215,25 +6455,6 @@ "node": ">=0.4" } }, - "node_modules/y-indexeddb": { - "version": "9.0.12", - "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.12.tgz", - "integrity": "sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==", - "dependencies": { - "lib0": "^0.2.74" - }, - "engines": { - "node": ">=16.0.0", - "npm": ">=8.0.0" - }, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - }, - "peerDependencies": { - "yjs": "^13.0.0" - } - }, "node_modules/y-leveldb": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", @@ -6748,6 +6969,42 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "@cbor-extract/cbor-extract-darwin-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-darwin-arm64/-/cbor-extract-darwin-arm64-2.2.0.tgz", + "integrity": "sha512-P7swiOAdF7aSi0H+tHtHtr6zrpF3aAq/W9FXx5HektRvLTM2O89xCyXF3pk7pLc7QpaY7AoaE8UowVf9QBdh3w==", + "optional": true + }, + "@cbor-extract/cbor-extract-darwin-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-darwin-x64/-/cbor-extract-darwin-x64-2.2.0.tgz", + "integrity": "sha512-1liF6fgowph0JxBbYnAS7ZlqNYLf000Qnj4KjqPNW4GViKrEql2MgZnAsExhY9LSy8dnvA4C0qHEBgPrll0z0w==", + "optional": true + }, + "@cbor-extract/cbor-extract-linux-arm": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-arm/-/cbor-extract-linux-arm-2.2.0.tgz", + "integrity": "sha512-QeBcBXk964zOytiedMPQNZr7sg0TNavZeuUCD6ON4vEOU/25+pLhNN6EDIKJ9VLTKaZ7K7EaAriyYQ1NQ05s/Q==", + "optional": true + }, + "@cbor-extract/cbor-extract-linux-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-arm64/-/cbor-extract-linux-arm64-2.2.0.tgz", + "integrity": "sha512-rQvhNmDuhjTVXSPFLolmQ47/ydGOFXtbR7+wgkSY0bdOxCFept1hvg59uiLPT2fVDuJFuEy16EImo5tE2x3RsQ==", + "optional": true + }, + "@cbor-extract/cbor-extract-linux-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-linux-x64/-/cbor-extract-linux-x64-2.2.0.tgz", + "integrity": "sha512-cWLAWtT3kNLHSvP4RKDzSTX9o0wvQEEAj4SKvhWuOVZxiDAeQazr9A+PSiRILK1VYMLeDml89ohxCnUNQNQNCw==", + "optional": true + }, + "@cbor-extract/cbor-extract-win32-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cbor-extract/cbor-extract-win32-x64/-/cbor-extract-win32-x64-2.2.0.tgz", + "integrity": "sha512-l2M+Z8DO2vbvADOBNLbbh9y5ST1RY5sqkWOg/58GkUPBYou/cuNZ68SGQ644f1CvZ8kcOxyZtw06+dxWHIoN/w==", + "optional": true + }, "@codemirror/state": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.4.1.tgz", @@ -6948,18 +7205,18 @@ "optional": true }, "@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, "requires": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" } }, "@eslint-community/regexpp": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", - "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true }, "@eslint/eslintrc": { @@ -6981,20 +7238,20 @@ } }, "@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, "peer": true }, "@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "dev": true, "peer": true, "requires": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" } @@ -7521,6 +7778,12 @@ "pretty-format": "^29.0.0" } }, + "@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true + }, "@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -7546,9 +7809,9 @@ "dev": true }, "@types/semver": { - "version": "7.5.8", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", - "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==", "dev": true }, "@types/stack-utils": { @@ -7663,9 +7926,9 @@ }, "dependencies": { "brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", "dev": true, "requires": { "balanced-match": "^1.0.0" @@ -7708,9 +7971,9 @@ } }, "@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.1.tgz", + "integrity": "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ==", "dev": true, "peer": true }, @@ -7740,9 +8003,9 @@ "requires": {} }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "peer": true, "requires": { @@ -7798,8 +8061,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "peer": true + "dev": true }, "aria-query": { "version": "5.3.0", @@ -7938,9 +8200,9 @@ "dev": true }, "brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "dev": true, "requires": { "balanced-match": "^1.0.0", @@ -8031,6 +8293,29 @@ "integrity": "sha512-UWM1zlo3cZfkpBysd7AS+z+v007q9G1+fLTUU42rQnY6t2axoogPW/xol6T7juU5EUoOhML4WgBIdG+9yYqAjQ==", "dev": true }, + "cbor-extract": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cbor-extract/-/cbor-extract-2.2.0.tgz", + "integrity": "sha512-Ig1zM66BjLfTXpNgKpvBePq271BPOvu8MR0Jl080yG7Jsl+wAZunfrwiwA+9ruzm/WEdIV5QF/bjDZTqyAIVHA==", + "optional": true, + "requires": { + "@cbor-extract/cbor-extract-darwin-arm64": "2.2.0", + "@cbor-extract/cbor-extract-darwin-x64": "2.2.0", + "@cbor-extract/cbor-extract-linux-arm": "2.2.0", + "@cbor-extract/cbor-extract-linux-arm64": "2.2.0", + "@cbor-extract/cbor-extract-linux-x64": "2.2.0", + "@cbor-extract/cbor-extract-win32-x64": "2.2.0", + "node-gyp-build-optional-packages": "5.1.1" + } + }, + "cbor-x": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/cbor-x/-/cbor-x-1.6.0.tgz", + "integrity": "sha512-0kareyRwHSkL6ws5VXHEf8uY1liitysCVJjlmhaLG+IXLqhSaOO+t63coaso7yjwEzWZzLy8fJo06gZDVQM9Qg==", + "requires": { + "cbor-extract": "^2.2.0" + } + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -8232,6 +8517,12 @@ "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", "dev": true }, + "detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "optional": true + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8239,9 +8530,9 @@ "dev": true }, "diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==" + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", + "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==" }, "diff-match-patch": { "version": "1.0.5", @@ -8360,9 +8651,9 @@ } }, "esbuild-svelte": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/esbuild-svelte/-/esbuild-svelte-0.8.0.tgz", - "integrity": "sha512-uKcPf1kl2UGMjrfHChv4dLxGAvCNhf9s72mHo19ZhKP+LrVOuQkOM/g8GE7MiGpoqjpk8UHqL08uLRbSKXhmhw==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/esbuild-svelte/-/esbuild-svelte-0.8.2.tgz", + "integrity": "sha512-tG97WrhH/OH8wFCmRBk6sRggMVMPNZDktGx1jJcvh9Obvjwm8M1UYoXmliuLL+4fs/wfyVVwM2fvyNYz2e6UPQ==", "dev": true, "requires": { "@jridgewell/trace-mapping": "^0.3.19" @@ -8382,17 +8673,17 @@ "peer": true }, "eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "dev": true, "peer": true, "requires": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -8463,9 +8754,9 @@ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, "esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", "dev": true, "peer": true, "requires": { @@ -8561,16 +8852,16 @@ "peer": true }, "fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", "dev": true, "requires": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", - "micromatch": "^4.0.4" + "micromatch": "^4.0.8" }, "dependencies": { "glob-parent": { @@ -8598,9 +8889,9 @@ "peer": true }, "fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -8658,9 +8949,9 @@ } }, "flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "peer": true }, @@ -8830,9 +9121,9 @@ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true }, "immediate": { @@ -8841,9 +9132,9 @@ "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" }, "import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", "dev": true, "peer": true, "requires": { @@ -9469,7 +9760,6 @@ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, - "peer": true, "requires": { "argparse": "^2.0.1" } @@ -9781,9 +10071,9 @@ "dev": true }, "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -9831,11 +10121,25 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node-diff3": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/node-diff3/-/node-diff3-3.2.0.tgz", + "integrity": "sha512-vLh2xJFSyniBLYDEDbXKqD32fQ5vAxmYT4hco8t0EHQ4CQ4BDHhshi7kdvDc6Y1MwGSi1Mhl4unUukPbCayZdw==" + }, "node-gyp-build": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" }, + "node-gyp-build-optional-packages": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.1.1.tgz", + "integrity": "sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==", + "optional": true, + "requires": { + "detect-libc": "^2.0.1" + } + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -10029,9 +10333,9 @@ "dev": true }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true }, "pirates": { @@ -10226,9 +10530,9 @@ "dev": true }, "reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", "dev": true }, "rimraf": { @@ -10493,9 +10797,9 @@ "dev": true }, "svelte": { - "version": "4.2.19", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.19.tgz", - "integrity": "sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==", + "version": "4.2.20", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.20.tgz", + "integrity": "sha512-eeEgGc2DtiUil5ANdtd8vPwt9AgaMdnuUFnPft9F5oMvU/FHu5IHFic+p1dR/UOB7XU2mX2yHW+NcTch4DCh5Q==", "requires": { "@ampproject/remapping": "^2.2.1", "@jridgewell/sourcemap-codec": "^1.4.15", @@ -10565,9 +10869,9 @@ } }, "ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", "dev": true, "requires": {} }, @@ -10735,14 +11039,6 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, - "y-indexeddb": { - "version": "9.0.12", - "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.12.tgz", - "integrity": "sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==", - "requires": { - "lib0": "^0.2.74" - } - }, "y-leveldb": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", diff --git a/package.json b/package.json index 57336274..0b0588aa 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs develop", "release": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production", "beta": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs debug", - "staging": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs staging", + "staging": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs staging --watch", "version": "node version-bump.mjs && git add manifest.json versions.json", "test": "jest", "lint": "npx eslint . --ext .ts" @@ -22,6 +22,7 @@ "@types/diff-match-patch": "^1.0.36", "@types/eventsource": "^1.1.15", "@types/jest": "^29.5.12", + "@types/js-yaml": "^4.0.9", "@types/node": "^16.11.6", "@types/path-browserify": "^1.0.2", "@types/uuid": "^9.0.8", @@ -32,6 +33,7 @@ "esbuild": "^0.27.0", "esbuild-svelte": "^0.8.0", "jest": "^29.7.0", + "js-yaml": "^4.1.1", "obsidian": "^1.7.2", "svelte": "^4.2.19", "svelte-preprocess": "^5.1.4", @@ -40,6 +42,7 @@ "typescript": "^5.4.5" }, "dependencies": { + "cbor-x": "^1.6.0", "diff": "^5.2.0", "diff-match-patch": "^1.0.5", "eventsource": "^2.0.2", @@ -47,6 +50,7 @@ "jose": "^5.3.0", "lucide-svelte": "^0.377.0", "monkey-around": "^3.0.0", + "node-diff3": "^3.2.0", "obsidian-daily-notes-interface": "^0.9.4", "path-browserify": "^1.0.1", "pocketbase": "^0.20.3", diff --git a/src/AwarenessViewPlugin.ts b/src/AwarenessViewPlugin.ts index 54e5b0ba..f39647d0 100644 --- a/src/AwarenessViewPlugin.ts +++ b/src/AwarenessViewPlugin.ts @@ -1,37 +1,66 @@ import { HasLogging } from "./debug"; -import { MarkdownView } from "obsidian"; -import { Document } from "./Document"; -import { type LiveView } from "./LiveViews"; import UserAwareness from "./components/UserAwareness.svelte"; -import type { RelayUser } from "./Relay"; +import { trackPromise } from "./trackPromise"; +import type { HasProvider } from "./HasProvider"; + +export interface AwarenessHost { + /** The Obsidian view whose containerEl owns the avatar overlay. */ + view: { containerEl: HTMLElement }; + /** + * The provider-backed doc (Document or Canvas). Must expose `_provider`, + * `whenReady()`, `guid`, and optionally `path` for logger tagging. + */ + doc: HasProvider & { whenReady(): Promise; path?: string }; + /** + * Resolves the anchor inside `containerEl` that the avatar element is + * inserted relative to, along with the insertion position. Called once + * during `install()`. Returning `null` skips mounting. + */ + resolveAnchor(containerEl: HTMLElement): { + anchor: HTMLElement; + position: InsertPosition; + } | null; + /** Extra class applied to the container (e.g. layout variant). */ + variantClass?: string; + /** Optional hook to apply inline styles after the container is inserted. */ + configureContainer?: (el: HTMLElement) => void; + /** Lay out the stack vertically (column) instead of horizontally (row). */ + vertical?: boolean; + /** + * Optional accessor for the CM6-backed editor this awareness surface is + * attached to. When present, the popover exposes attribution controls. + */ + getEditor?: () => unknown; +} + export class AwarenessViewPlugin extends HasLogging { - view: LiveView; - doc: Document; + private host: AwarenessHost; private destroyed = false; private awarenessComponent?: UserAwareness; - private targetElement?: HTMLElement; private awarenessElement?: HTMLElement; private relayUsersStore: any; - constructor(view: LiveView, relayUsersStore: any) { + constructor(host: AwarenessHost, relayUsersStore: any) { super(); - this.view = view; - this.doc = view.document; + this.host = host; this.relayUsersStore = relayUsersStore; - this.setLoggers(`[AwarenessView](${this.doc.path})`); + this.setLoggers(`[AwarenessView](${this.host.doc.path ?? this.host.doc.guid})`); this.install(); } private async install() { - if (!this.view || this.destroyed) return; + if (!this.host || this.destroyed) return; - this.log("Installing awareness component for", this.view.view.file?.path); + this.log("Installing awareness component"); // Wrap the title immediately to avoid focus loss later this.wrapTitle(); // Wait for the document to be ready - await this.doc.whenReady(); + await trackPromise( + `awareness:whenReady:${this.host.doc.guid}`, + this.host.doc.whenReady(), + ); if (this.destroyed) return; @@ -40,34 +69,37 @@ export class AwarenessViewPlugin extends HasLogging { } private wrapTitle() { - if (!this.view.view.containerEl || this.destroyed) return; + const containerEl = this.host.view?.containerEl; + if (!containerEl || this.destroyed) return; // Already created if (this.awarenessElement) return; - // Find the target element (inline-title) to position relative to - const inlineTitle = this.view.view.containerEl.querySelector( - ".inline-title", - ) as HTMLElement; - if (!inlineTitle) { - this.warn( - "Could not find inline-title element to position awareness component", - ); + const resolved = this.host.resolveAnchor(containerEl); + if (!resolved) { + this.warn("Could not resolve anchor for awareness component"); return; } // Create container for the awareness component - this.awarenessElement = document.createElement("div"); + this.awarenessElement = containerEl.ownerDocument.createElement("div"); this.awarenessElement.className = "user-awareness-container"; + if (this.host.variantClass) { + this.awarenessElement.classList.add(this.host.variantClass); + } + + resolved.anchor.insertAdjacentElement(resolved.position, this.awarenessElement); - // Insert as sibling after the inline-title (more robust than wrapping) - // Use absolute positioning via CSS to place it next to the title - inlineTitle.insertAdjacentElement("afterend", this.awarenessElement); + this.host.configureContainer?.(this.awarenessElement); - // Make the parent position relative so we can position absolutely - const parent = inlineTitle.parentElement; - if (parent) { - parent.style.position = "relative"; + // The CSS pins the container top-right of its positioning parent. Make + // sure that parent can host absolute children. + const positioningParent = + resolved.position === "afterbegin" || resolved.position === "beforeend" + ? resolved.anchor + : resolved.anchor.parentElement; + if (positioningParent) { + positioningParent.addClass("user-awareness-positioning-parent"); } } @@ -75,7 +107,7 @@ export class AwarenessViewPlugin extends HasLogging { if (!this.awarenessElement || this.destroyed) return; // Get the awareness instance from the provider - const provider = this.doc._provider; + const provider = this.host.doc._provider; if (!provider?.awareness) { this.warn("No awareness provider available"); return; @@ -88,6 +120,8 @@ export class AwarenessViewPlugin extends HasLogging { props: { awareness: provider.awareness, relayUsers: this.relayUsersStore, + vertical: this.host.vertical ?? false, + getEditor: this.host.getEditor, }, }); @@ -115,7 +149,6 @@ export class AwarenessViewPlugin extends HasLogging { this.awarenessElement = undefined; } - this.view = null as any; - this.doc = null as any; + this.host = null as any; } } diff --git a/src/BackgroundSync.ts b/src/BackgroundSync.ts index 921843e3..56b27ceb 100644 --- a/src/BackgroundSync.ts +++ b/src/BackgroundSync.ts @@ -5,17 +5,30 @@ import { S3RN, S3RemoteCanvas, S3RemoteDocument } from "./S3RN"; import { isDocument, type Document } from "./Document"; import { isCanvas } from "./Canvas"; import type { TimeProvider } from "./TimeProvider"; -import { HasLogging, RelayInstances } from "./debug"; -import type { Subscriber, Unsubscriber } from "./observable/Observable"; +import { HasLogging, RelayInstances, metrics } from "./debug"; +import { + Observable, + type Subscriber, + type Unsubscriber, +} from "./observable/Observable"; import { ObservableSet } from "./observable/ObservableSet"; import { ObservableMap } from "./observable/ObservableMap"; import type { SharedFolder, SharedFolders } from "./SharedFolder"; import { compareFilePaths } from "./FolderSort"; import type { ClientToken } from "./client/types"; import { Canvas } from "./Canvas"; -import { areObjectsEqual } from "./areObjectsEqual"; import type { CanvasData } from "./CanvasView"; +import { areCanvasDataEqual } from "./CanvasData"; import { SyncFile, isSyncFile } from "./SyncFile"; +import { isEmptyDoc, snapshotFromDoc } from "./merge-hsm/state-vectors"; +import { + buildFolderSyncSnapshot, + FolderSyncSnapshotSmoother, + type FolderSyncSnapshot, + type FolderSyncWorkItemInput, +} from "./BackgroundSyncProgress"; +import { errorFromUnknown, formatUserFacingError } from "./UserFacingError"; +import { getRelayRequestHeaders } from "./customFetch"; export interface QueueItem { guid: string; @@ -23,17 +36,38 @@ export interface QueueItem { doc: Document | Canvas | SyncFile; status: "pending" | "running" | "completed" | "failed"; sharedFolder: SharedFolder; + userVisible: boolean; + syncIntent?: "sync" | "upload" | "lca-backfill"; + retryAttempts?: number; + nextAttemptAt?: number; +} + +export interface BackgroundSyncFailure { + id: string; + guid: string; + path: string; + kind: "sync" | "download" | "local"; + message: string; + sharedFolder: SharedFolder; } export interface SyncGroup { sharedFolder: SharedFolder; total: number; // Total operations (syncs + downloads) - completed: number; // Total completed operations + completed: number; // Successful operations status: "pending" | "running" | "completed" | "failed"; downloads: number; syncs: number; completedDownloads: number; completedSyncs: number; + failedDownloads: number; + failedSyncs: number; + skippedDownloads: number; + skippedSyncs: number; + userDownloads: number; + completedUserDownloads: number; + failedUserDownloads: number; + skippedUserDownloads: number; } export interface SyncProgress { @@ -56,10 +90,46 @@ export interface GroupProgress { status: "pending" | "running" | "completed" | "failed"; } +interface FolderSyncSnapshotSubscription { + smoother: FolderSyncSnapshotSmoother; + subscribers: Set>; + latestSnapshot: FolderSyncSnapshot | null; + unsubscribers: Unsubscriber[]; + emit: () => void; +} + +class RetryableProviderSyncError extends Error { + constructor(message: string) { + super(message); + this.name = "RetryableProviderSyncError"; + } +} + +const MAX_PROVIDER_SYNC_RETRIES = 5; + +function isRetryableProviderSyncError( + error: unknown, +): error is RetryableProviderSyncError { + return error instanceof RetryableProviderSyncError; +} + +export interface QueueStatus { + syncsQueued: number; + syncsActive: number; + downloadsQueued: number; + downloadsActive: number; + isPaused: boolean; +} + export class BackgroundSync extends HasLogging { public activeSync = new ObservableSet(); public activeDownloads = new ObservableSet(); public syncGroups = new ObservableMap(); + private folderResyncs = new ObservableSet(); + private failures = new ObservableMap( + "BackgroundSync.failures", + ); + private queueStatusChanged = new Observable("BackgroundSync.queueStatus"); private syncQueue: QueueItem[] = []; private downloadQueue: QueueItem[] = []; @@ -68,6 +138,8 @@ export class BackgroundSync extends HasLogging { private isPaused = true; private inProgressSyncs = new Set(); private inProgressDownloads = new Set(); + private cancelledSyncs = new Set(); + private cancelledDownloads = new Set(); private syncCompletionCallbacks = new Map< string, { @@ -75,13 +147,19 @@ export class BackgroundSync extends HasLogging { reject: (error: Error) => void; } >(); + private syncPromises = new Map>(); private downloadCompletionCallbacks = new Map< string, { - resolve: () => void; + resolve: (result?: Uint8Array) => void; reject: (error: Error) => void; } >(); + private downloadPromises = new Map>(); + private folderSyncSnapshotSubscriptions = new Map< + SharedFolder, + FolderSyncSnapshotSubscription + >(); // A map to track items we've already logged to avoid duplicates private loggedItems = new Map(); @@ -100,6 +178,13 @@ export class BackgroundSync extends HasLogging { this.processSyncQueue(); this.processDownloadQueue(); }, 1000); + + // Add polling timer for disk changes (poll all folders) + this.timeProvider.setInterval(() => { + this.sharedFolders.forEach((folder) => { + folder.poll(); + }); + }, 5000); // Poll every 5 seconds } /** @@ -116,6 +201,106 @@ export class BackgroundSync extends HasLogging { return this.downloadQueue; } + cancelDocumentWork(guid: string): void { + let changed = false; + + const queuedSyncs = this.syncQueue.filter((item) => item.guid === guid); + if (queuedSyncs.length > 0) { + for (const item of queuedSyncs) { + this.removeQueuedSyncFromGroup(item); + } + this.syncQueue = this.syncQueue.filter((item) => item.guid !== guid); + changed = true; + } + + const queuedDownloads = this.downloadQueue.filter((item) => item.guid === guid); + if (queuedDownloads.length > 0) { + for (const item of queuedDownloads) { + this.removeQueuedDownloadFromGroup(item); + } + this.downloadQueue = this.downloadQueue.filter((item) => item.guid !== guid); + changed = true; + } + + if (this.activeSync.some((item) => item.guid === guid)) { + this.cancelledSyncs.add(guid); + } else { + this.resolveSyncCancellation(guid); + } + + if (this.activeDownloads.some((item) => item.guid === guid)) { + this.cancelledDownloads.add(guid); + } else { + this.resolveDownloadCancellation(guid); + } + + this.clearFailure(this.failureKey("sync", guid)); + this.clearFailure(this.failureKey("download", guid)); + + if (changed) { + metrics.setBgSyncQueueLength("sync", this.syncQueue.length); + metrics.setBgSyncQueueLength("download", this.downloadQueue.length); + this.queueStatusChanged.notifyListeners(); + } + } + + private removeQueuedSyncFromGroup(item: QueueItem): void { + const group = this.syncGroups.get(item.sharedFolder); + if (!group) return; + + group.total = Math.max(0, group.total - 1); + group.syncs = Math.max(0, group.syncs - 1); + this.updateGroupTerminalStatus(group); + this.syncGroups.set(item.sharedFolder, group); + } + + private removeQueuedDownloadFromGroup(item: QueueItem): void { + const group = this.syncGroups.get(item.sharedFolder); + if (!group) return; + + group.total = Math.max(0, group.total - 1); + group.downloads = Math.max(0, group.downloads - 1); + if (item.userVisible) { + group.userDownloads = Math.max(0, group.userDownloads - 1); + } + this.updateGroupTerminalStatus(group); + this.syncGroups.set(item.sharedFolder, group); + } + + private resolveSyncCancellation(guid: string): void { + const callback = this.syncCompletionCallbacks.get(guid); + if (callback) callback.resolve(); + this.syncCompletionCallbacks.delete(guid); + this.syncPromises.delete(guid); + this.inProgressSyncs.delete(guid); + this.cancelledSyncs.delete(guid); + } + + private resolveDownloadCancellation(guid: string): void { + const callback = this.downloadCompletionCallbacks.get(guid); + if (callback) callback.resolve(undefined); + this.downloadCompletionCallbacks.delete(guid); + this.downloadPromises.delete(guid); + this.inProgressDownloads.delete(guid); + this.cancelledDownloads.delete(guid); + } + + private isSyncCancelled(item: QueueItem): boolean { + return item.doc.destroyed || this.cancelledSyncs.has(item.guid); + } + + private isSyncCancelledForDoc(doc: Document | Canvas | SyncFile): boolean { + return doc.destroyed || this.cancelledSyncs.has(doc.guid); + } + + private isDownloadCancelled(item: QueueItem): boolean { + return item.doc.destroyed || this.cancelledDownloads.has(item.guid); + } + + private shouldSkipDocumentSync(item: Document | Canvas | SyncFile): boolean { + return isDocument(item) && item.hsm?.getSyncStatus().status === "conflict"; + } + getOverallProgress(): SyncProgress { let totalItems = 0; let completedItems = 0; @@ -126,11 +311,11 @@ export class BackgroundSync extends HasLogging { this.syncGroups.forEach((group) => { totalItems += group.total; - completedItems += group.completed; + completedItems += this.groupFinishedTotal(group); syncItems += group.syncs; - completedSyncs += group.completedSyncs; + completedSyncs += this.groupFinishedSyncs(group); downloadItems += group.downloads; - completedDownloads += group.completedDownloads; + completedDownloads += this.groupFinishedDownloads(group); }); const totalPercent = @@ -156,12 +341,15 @@ export class BackgroundSync extends HasLogging { const group = this.syncGroups.get(sharedFolder); if (!group) return null; - const percent = group.total > 0 ? (group.completed / group.total) * 100 : 0; + const percent = + group.total > 0 ? (this.groupFinishedTotal(group) / group.total) * 100 : 0; const syncPercent = - group.syncs > 0 ? (group.completedSyncs / group.syncs) * 100 : 0; + group.syncs > 0 + ? (this.groupFinishedSyncs(group) / group.syncs) * 100 + : 0; const downloadPercent = group.downloads > 0 - ? (group.completedDownloads / group.downloads) * 100 + ? (this.groupFinishedDownloads(group) / group.downloads) * 100 : 0; return { @@ -173,6 +361,330 @@ export class BackgroundSync extends HasLogging { }; } + /** + * Returns download-only progress for a shared folder. + * Used to show only user-visible downloads in folder progress indicators. + */ + getUserVisibleProgress(sharedFolder: SharedFolder): GroupProgress | null { + const group = this.syncGroups.get(sharedFolder); + if (!group) return null; + + const total = group.userDownloads; + const finished = + group.completedUserDownloads + + group.failedUserDownloads + + group.skippedUserDownloads; + const percent = total > 0 ? (finished / total) * 100 : 0; + const status = + total === 0 + ? group.status + : finished === total + ? group.failedUserDownloads > 0 + ? "failed" + : "completed" + : group.status === "failed" + ? "failed" + : "running"; + + return { + percent: Math.round(percent), + syncPercent: 0, + downloadPercent: Math.round(percent), + sharedFolder, + status, + }; + } + + private groupFinishedSyncs(group: SyncGroup): number { + return Math.min( + group.syncs, + group.completedSyncs + group.failedSyncs + group.skippedSyncs, + ); + } + + private groupFinishedDownloads(group: SyncGroup): number { + return Math.min( + group.downloads, + group.completedDownloads + group.failedDownloads + group.skippedDownloads, + ); + } + + private groupFinishedTotal(group: SyncGroup): number { + return Math.min( + group.total, + this.groupFinishedSyncs(group) + this.groupFinishedDownloads(group), + ); + } + + private groupFailureCount(group: SyncGroup): number { + return group.failedSyncs + group.failedDownloads; + } + + private updateGroupTerminalStatus(group: SyncGroup): void { + if (this.groupFinishedTotal(group) >= group.total) { + group.status = this.groupFailureCount(group) > 0 ? "failed" : "completed"; + } else if (this.groupFailureCount(group) > 0) { + group.status = "failed"; + } else if (group.total > 0) { + group.status = "running"; + } else { + group.status = "completed"; + } + } + + private markSyncTerminal( + sharedFolder: SharedFolder, + outcome: "completed" | "failed" | "skipped", + ): void { + const group = this.syncGroups.get(sharedFolder); + if (!group) return; + if (outcome === "completed") { + group.completedSyncs++; + group.completed++; + } else if (outcome === "failed") { + group.failedSyncs++; + } else { + group.skippedSyncs++; + } + this.updateGroupTerminalStatus(group); + this.syncGroups.set(sharedFolder, group); + } + + private markDownloadTerminal( + item: QueueItem, + outcome: "completed" | "failed" | "skipped", + ): void { + const group = this.syncGroups.get(item.sharedFolder); + if (!group) return; + if (outcome === "completed") { + group.completedDownloads++; + group.completed++; + if (item.userVisible) { + group.completedUserDownloads++; + } + } else if (outcome === "failed") { + group.failedDownloads++; + if (item.userVisible) { + group.failedUserDownloads++; + } + } else { + group.skippedDownloads++; + if (item.userVisible) { + group.skippedUserDownloads++; + } + } + this.updateGroupTerminalStatus(group); + this.syncGroups.set(item.sharedFolder, group); + } + + private requeueRetryableSync( + item: QueueItem, + error: RetryableProviderSyncError, + ): boolean { + const retries = (item.retryAttempts ?? 0) + 1; + item.retryAttempts = retries; + if (retries > MAX_PROVIDER_SYNC_RETRIES) { + item.nextAttemptAt = undefined; + this.warn( + `[syncDocWS] provider sync failed after ${MAX_PROVIDER_SYNC_RETRIES} retries for ${item.path}: ${error.message}`, + ); + return false; + } + + const delayMs = Math.min(30_000, 1000 * 2 ** Math.min(retries - 1, 5)); + item.status = "pending"; + item.nextAttemptAt = this.timeProvider.now() + delayMs; + + this.clearFailure(this.failureKey("sync", item.guid)); + if (!this.syncQueue.some((queued) => queued.guid === item.guid)) { + this.syncQueue.push(item); + this.syncQueue.sort(compareFilePaths); + } + this.debug( + `[syncDocWS] retryable provider sync failure for ${item.path}: ${error.message}; retrying in ${delayMs}ms`, + ); + metrics.setBgSyncQueueLength("sync", this.syncQueue.length); + this.queueStatusChanged.notifyListeners(); + return true; + } + + getFolderPillProgress(sharedFolder: SharedFolder): GroupProgress | null { + const group = this.syncGroups.get(sharedFolder); + if (!group) return null; + + const snapshot = this.getFolderSyncSnapshot(sharedFolder); + return { + percent: snapshot.percent, + syncPercent: snapshot.syncPercent, + downloadPercent: snapshot.downloadPercent, + sharedFolder, + status: snapshot.progressStatus, + }; + } + + getFolderSyncSnapshot(sharedFolder: SharedFolder): FolderSyncSnapshot { + const activeDownloads = this.activeDownloads.filter( + (item) => item.sharedFolder === sharedFolder, + ); + const activeSync = this.activeSync.filter( + (item) => item.sharedFolder === sharedFolder, + ); + const queuedDownloads = this.downloadQueue.filter( + (item) => item.sharedFolder === sharedFolder, + ); + const queuedSyncs = this.syncQueue.filter( + (item) => item.sharedFolder === sharedFolder, + ); + const folderResyncActive = this.folderResyncs.has(sharedFolder) ? 1 : 0; + const activeItem = this.activeItemForSnapshot(activeDownloads, activeSync); + const queuedReason = this.queuedReasonForSnapshot( + sharedFolder, + activeDownloads.length + activeSync.length, + queuedDownloads.length + queuedSyncs.length, + ); + + return buildFolderSyncSnapshot({ + group: this.syncGroups.get(sharedFolder) ?? null, + queued: queuedDownloads.length + queuedSyncs.length, + active: activeDownloads.length + activeSync.length + folderResyncActive, + isPaused: this.isPaused, + failureCount: this.getFailures(sharedFolder).length, + canResync: sharedFolder.connected && !sharedFolder.localOnly, + folderActivity: folderResyncActive ? "checking" : null, + activeItem, + queuedReason, + }); + } + + private activeItemForSnapshot( + activeDownloads: QueueItem[], + activeSync: QueueItem[], + ): FolderSyncWorkItemInput | null { + const download = activeDownloads[0]; + if (download) return { kind: "download", path: download.path }; + const sync = activeSync[0]; + if (sync) return { kind: "sync", path: sync.path }; + return null; + } + + private queuedReasonForSnapshot( + sharedFolder: SharedFolder, + active: number, + queued: number, + ): "connection" | "reconnecting" | null { + if (active > 0 || queued === 0) return null; + if (!sharedFolder.connected) { + return sharedFolder.state.status === "connecting" + ? "reconnecting" + : "connection"; + } + return null; + } + + getFailures(sharedFolder: SharedFolder): BackgroundSyncFailure[] { + return this.failures + .values() + .filter((failure) => failure.sharedFolder === sharedFolder) + .sort((a, b) => a.path.localeCompare(b.path)); + } + + clearFailure(id: string): void { + this.failures.delete(id); + } + + clearFailuresForFolder(sharedFolder: SharedFolder): void { + for (const failure of this.failures.values()) { + if (failure.sharedFolder === sharedFolder) { + this.clearFailure(failure.id); + } + } + } + + beginFolderResync(sharedFolder: SharedFolder): Unsubscriber { + this.clearFailuresForFolder(sharedFolder); + this.folderResyncs.add(sharedFolder); + return () => { + this.folderResyncs.delete(sharedFolder); + }; + } + + async refreshLocalFileFailures(sharedFolder: SharedFolder): Promise { + const liveLocalFailureIds = new Set(); + for (const file of sharedFolder.files.values()) { + if (!isCanvas(file)) continue; + const id = this.failureKey("local", file.guid); + liveLocalFailureIds.add(id); + const message = await this.getCanvasLocalStateFailure(file); + if (message) { + this.setFailure({ + id, + guid: file.guid, + path: file.path, + kind: "local", + message, + sharedFolder, + }); + } else { + this.clearFailure(id); + } + } + + for (const failure of this.failures.values()) { + if ( + failure.sharedFolder === sharedFolder && + failure.kind === "local" && + !liveLocalFailureIds.has(failure.id) + ) { + this.clearFailure(failure.id); + } + } + } + + private async getCanvasLocalStateFailure( + canvas: Canvas, + ): Promise { + await canvas.whenSynced(); + let currentFileContents: string; + try { + currentFileContents = await canvas.sharedFolder.read(canvas); + } catch (e) { + return null; + } + if (!currentFileContents) return null; + + let currentFileJson: CanvasData; + try { + currentFileJson = JSON.parse(currentFileContents) as CanvasData; + } catch (e) { + return "Canvas file contains invalid JSON. Open the canvas and repair it before syncing."; + } + + const currentCanvasData = Canvas.exportCanvasData(canvas.ydoc); + if (areCanvasDataEqual(currentCanvasData, currentFileJson)) { + return null; + } + if (await this.repairStaleCanvasText(canvas, currentFileJson)) { + return null; + } + return "Canvas file does not match local sync state. Open the canvas and resolve the local changes before syncing."; + } + + private async repairStaleCanvasText( + canvas: Canvas, + currentFileJson: CanvasData, + ): Promise { + const currentCanvasMapData = Canvas.exportCanvasMapData(canvas.ydoc); + if (!areCanvasDataEqual(currentCanvasMapData, currentFileJson)) { + return false; + } + + await canvas.applyData(currentFileJson); + return areCanvasDataEqual( + Canvas.exportCanvasData(canvas.ydoc), + currentFileJson, + ); + } + getAllGroupsProgress(): GroupProgress[] { const progress: GroupProgress[] = []; this.syncGroups.forEach((group, sharedFolder) => { @@ -188,9 +700,26 @@ export class BackgroundSync extends HasLogging { if (this.isPaused || this.isProcessingSync) return; this.isProcessingSync = true; + // Evict destroyed documents from the queue and clean up their inProgress entries + const destroyed = this.syncQueue.filter((item) => item.doc.destroyed); + for (const item of destroyed) { + this.markSyncTerminal(item.sharedFolder, "skipped"); + this.inProgressSyncs.delete(item.guid); + const callback = this.syncCompletionCallbacks.get(item.guid); + if (callback) callback.reject(new Error("Document destroyed")); + this.syncCompletionCallbacks.delete(item.guid); + this.syncPromises.delete(item.guid); + } + this.syncQueue = this.syncQueue.filter((item) => !item.doc.destroyed); + + metrics.setBgSyncQueueLength("sync", this.syncQueue.length); + // Filter for items with connected folders + const now = this.timeProvider.now(); const connectableItems = this.syncQueue.filter( - (item) => item.sharedFolder.connected, + (item) => + item.sharedFolder.connected && + (item.nextAttemptAt === undefined || item.nextAttemptAt <= now), ); while ( @@ -204,7 +733,10 @@ export class BackgroundSync extends HasLogging { this.syncQueue = this.syncQueue.filter((i) => i.guid !== item.guid); item.status = "running"; + const opStart = performance.now(); this.activeSync.add(item); + metrics.setBgSyncActive("sync", this.activeSync.size); + metrics.setBgSyncQueueLength("sync", this.syncQueue.length); try { const doc = item.doc; @@ -212,6 +744,10 @@ export class BackgroundSync extends HasLogging { if (doc instanceof SyncFile) { syncPromise = this.syncFile(doc); + } else if (item.syncIntent === "upload") { + syncPromise = this.syncDocumentUpload(doc); + } else if (item.syncIntent === "lca-backfill" && isDocument(doc)) { + syncPromise = this.syncDocumentLCABackfill(doc); } else { syncPromise = this.syncDocument(doc); } @@ -219,56 +755,53 @@ export class BackgroundSync extends HasLogging { syncPromise .then(() => { item.status = "completed"; + metrics.incBgSyncOps("sync", "completed"); const callback = this.syncCompletionCallbacks.get(item.guid); if (callback) { callback.resolve(); this.syncCompletionCallbacks.delete(item.guid); + this.syncPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - this.debug( - `[Sync Progress] Before: completed=${group.completed}, total=${group.total}, ` + - `syncs=${group.syncs}, completedSyncs=${group.completedSyncs}`, - ); - - group.completedSyncs++; - group.completed++; - - this.debug( - `[Sync Progress] After: completed=${group.completed}, total=${group.total}, ` + - `syncs=${group.syncs}, completedSyncs=${group.completedSyncs}`, - ); - - if (group.completed === group.total) { - group.status = "completed"; - this.debug("[Sync Progress] Group completed!"); - } - - this.syncGroups.set(item.sharedFolder, group); - } + this.markSyncTerminal(item.sharedFolder, "completed"); }) .catch((error) => { + if (this.isSyncCancelled(item)) { + item.status = "completed"; + this.markSyncTerminal(item.sharedFolder, "skipped"); + this.resolveSyncCancellation(item.guid); + return; + } + + if ( + isRetryableProviderSyncError(error) && + this.requeueRetryableSync(item, error) + ) { + return; + } + item.status = "failed"; + metrics.incBgSyncOps("sync", "failed"); const callback = this.syncCompletionCallbacks.get(item.guid); if (callback) { - callback.reject( - error instanceof Error ? error : new Error(String(error)), - ); + callback.reject(errorFromUnknown(error)); this.syncCompletionCallbacks.delete(item.guid); + this.syncPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - this.error("[Sync Failed]", error); - group.status = "failed"; - this.syncGroups.set(item.sharedFolder, group); - } + this.logError("[Sync Failed]", error); + this.recordFailure("sync", item, error); + this.markSyncTerminal(item.sharedFolder, "failed"); }) .finally(() => { + metrics.observeBgSyncOp("sync", (performance.now() - opStart) / 1000); this.activeSync.delete(item); - this.inProgressSyncs.delete(item.guid); + metrics.setBgSyncActive("sync", this.activeSync.size); + if (!this.syncQueue.some((queued) => queued.guid === item.guid)) { + this.inProgressSyncs.delete(item.guid); + this.cancelledSyncs.delete(item.guid); + } // Unwind the call stack before checking for more work this.timeProvider.setTimeout(() => { @@ -276,24 +809,45 @@ export class BackgroundSync extends HasLogging { }, 0); }); } catch (error) { + if (this.isSyncCancelled(item)) { + item.status = "completed"; + metrics.observeBgSyncOp("sync", (performance.now() - opStart) / 1000); + this.markSyncTerminal(item.sharedFolder, "skipped"); + this.resolveSyncCancellation(item.guid); + this.activeSync.delete(item); + metrics.setBgSyncActive("sync", this.activeSync.size); + this.inProgressSyncs.delete(item.guid); + this.cancelledSyncs.delete(item.guid); + continue; + } + + if ( + isRetryableProviderSyncError(error) && + this.requeueRetryableSync(item, error) + ) { + metrics.observeBgSyncOp("sync", (performance.now() - opStart) / 1000); + this.activeSync.delete(item); + metrics.setBgSyncActive("sync", this.activeSync.size); + continue; + } + item.status = "failed"; + metrics.incBgSyncOps("sync", "failed"); + metrics.observeBgSyncOp("sync", (performance.now() - opStart) / 1000); const callback = this.syncCompletionCallbacks.get(item.guid); if (callback) { - callback.reject( - error instanceof Error ? error : new Error(String(error)), - ); + callback.reject(errorFromUnknown(error)); this.syncCompletionCallbacks.delete(item.guid); + this.syncPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - this.error("[Sync Startup Failed]", error); - group.status = "failed"; - this.syncGroups.set(item.sharedFolder, group); - } + this.logError("[Sync Startup Failed]", error); + this.recordFailure("sync", item, error); + this.markSyncTerminal(item.sharedFolder, "failed"); this.activeSync.delete(item); + metrics.setBgSyncActive("sync", this.activeSync.size); this.inProgressSyncs.delete(item.guid); } } @@ -305,6 +859,20 @@ export class BackgroundSync extends HasLogging { if (this.isPaused || this.isProcessingDownloads) return; this.isProcessingDownloads = true; + // Evict destroyed documents from the queue and clean up their inProgress entries + const destroyedDownloads = this.downloadQueue.filter((item) => item.doc.destroyed); + for (const item of destroyedDownloads) { + this.markDownloadTerminal(item, "skipped"); + this.inProgressDownloads.delete(item.guid); + const callback = this.downloadCompletionCallbacks.get(item.guid); + if (callback) callback.reject(new Error("Document destroyed")); + this.downloadCompletionCallbacks.delete(item.guid); + this.downloadPromises.delete(item.guid); + } + this.downloadQueue = this.downloadQueue.filter((item) => !item.doc.destroyed); + + metrics.setBgSyncQueueLength("download", this.downloadQueue.length); + // Filter for items with connected folders const connectableItems = this.downloadQueue.filter( (item) => item.sharedFolder.connected, @@ -323,7 +891,10 @@ export class BackgroundSync extends HasLogging { ); item.status = "running"; + const opStart = performance.now(); this.activeDownloads.add(item); + metrics.setBgSyncActive("download", this.activeDownloads.size); + metrics.setBgSyncQueueLength("download", this.downloadQueue.length); try { let downloadPromise: Promise; @@ -338,46 +909,47 @@ export class BackgroundSync extends HasLogging { } downloadPromise - .then(() => { + .then((result) => { item.status = "completed"; + metrics.incBgSyncOps("download", "completed"); const callback = this.downloadCompletionCallbacks.get(item.guid); if (callback) { - callback.resolve(); + callback.resolve(result as Uint8Array | undefined); this.downloadCompletionCallbacks.delete(item.guid); + this.downloadPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - group.completedDownloads++; - group.completed++; - if (group.completed === group.total) { - group.status = "completed"; - } - this.syncGroups.set(item.sharedFolder, group); - } + this.markDownloadTerminal(item, "completed"); }) .catch((error) => { + if (this.isDownloadCancelled(item)) { + item.status = "completed"; + this.markDownloadTerminal(item, "skipped"); + this.resolveDownloadCancellation(item.guid); + return; + } + item.status = "failed"; + metrics.incBgSyncOps("download", "failed"); const callback = this.downloadCompletionCallbacks.get(item.guid); if (callback) { - callback.reject( - error instanceof Error ? error : new Error(String(error)), - ); + callback.reject(errorFromUnknown(error)); this.downloadCompletionCallbacks.delete(item.guid); + this.downloadPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - group.status = "failed"; - this.syncGroups.set(item.sharedFolder, group); - } - this.error("[processDownloadQueue]", error); + this.recordFailure("download", item, error); + this.logError("[processDownloadQueue]", error); + this.markDownloadTerminal(item, "failed"); }) .finally(() => { + metrics.observeBgSyncOp("download", (performance.now() - opStart) / 1000); this.activeDownloads.delete(item); + metrics.setBgSyncActive("download", this.activeDownloads.size); this.inProgressDownloads.delete(item.guid); + this.cancelledDownloads.delete(item.guid); // Unwind the call stack before checking for more work this.timeProvider.setTimeout(() => { @@ -385,24 +957,35 @@ export class BackgroundSync extends HasLogging { }, 0); }); } catch (error) { + if (this.isDownloadCancelled(item)) { + item.status = "completed"; + metrics.observeBgSyncOp("download", (performance.now() - opStart) / 1000); + this.markDownloadTerminal(item, "skipped"); + this.resolveDownloadCancellation(item.guid); + this.activeDownloads.delete(item); + metrics.setBgSyncActive("download", this.activeDownloads.size); + this.inProgressDownloads.delete(item.guid); + this.cancelledDownloads.delete(item.guid); + continue; + } + item.status = "failed"; + metrics.incBgSyncOps("download", "failed"); + metrics.observeBgSyncOp("download", (performance.now() - opStart) / 1000); const callback = this.downloadCompletionCallbacks.get(item.guid); if (callback) { - callback.reject( - error instanceof Error ? error : new Error(String(error)), - ); + callback.reject(errorFromUnknown(error)); this.downloadCompletionCallbacks.delete(item.guid); + this.downloadPromises.delete(item.guid); } - const group = this.syncGroups.get(item.sharedFolder); - if (group) { - this.error("[Download Startup Failed]", error); - group.status = "failed"; - this.syncGroups.set(item.sharedFolder, group); - } + this.logError("[Download Startup Failed]", error); + this.recordFailure("download", item, error); + this.markDownloadTerminal(item, "failed"); this.activeDownloads.delete(item); + metrics.setBgSyncActive("download", this.activeDownloads.size); this.inProgressDownloads.delete(item.guid); } } @@ -420,22 +1003,17 @@ export class BackgroundSync extends HasLogging { * @returns A promise that resolves when the sync completes */ async enqueueSync(item: SyncFile | Document | Canvas): Promise { - // Skip if already in progress + if (this.shouldSkipDocumentSync(item)) { + this.clearFailure(this.failureKey("sync", item.guid)); + return Promise.resolve(); + } + + // Skip if already in progress — return the same promise all callers share if (this.inProgressSyncs.has(item.guid)) { this.debug( - `[enqueueSync] Item ${item.guid} already in progress, skipping`, + `[enqueueSync] Item ${item.guid} already in progress, sharing promise`, ); - - // Return existing promise if already processing - const existingCallback = this.syncCompletionCallbacks.get(item.guid); - if (existingCallback) { - return new Promise((resolve, reject) => { - existingCallback.resolve = resolve; - existingCallback.reject = reject; - }); - } - this.processSyncQueue(); - return Promise.resolve(); + return this.syncPromises.get(item.guid) ?? Promise.resolve(); } const sharedFolder = item.sharedFolder; @@ -445,7 +1023,9 @@ export class BackgroundSync extends HasLogging { doc: item, status: "pending", sharedFolder, + userVisible: false, }; + this.clearFailure(this.failureKey("sync", item.guid)); // Get or create the sync group let group = this.syncGroups.get(sharedFolder); @@ -459,10 +1039,19 @@ export class BackgroundSync extends HasLogging { syncs: 0, completedDownloads: 0, completedSyncs: 0, + failedDownloads: 0, + failedSyncs: 0, + skippedDownloads: 0, + skippedSyncs: 0, + userDownloads: 0, + completedUserDownloads: 0, + failedUserDownloads: 0, + skippedUserDownloads: 0, }; } group.total++; group.syncs++; + group.status = "running"; this.syncGroups.set(sharedFolder, group); this.inProgressSyncs.add(item.guid); @@ -473,46 +1062,54 @@ export class BackgroundSync extends HasLogging { reject, }); }); + this.syncPromises.set(item.guid, syncPromise); this.syncQueue.push(queueItem); this.syncQueue.sort(compareFilePaths); + this.queueStatusChanged.notifyListeners(); this.processSyncQueue(); return syncPromise; } /** - * Enqueues a document for download - * - * This method adds a document to the download queue and creates/updates - * the associated sync group to track progress. - * - * @param item The document to download - * @returns A promise that resolves when the download completes + * Enqueue a local-authoritative upload before markUploaded(). For documents, + * this seeds remoteDoc from the enrolled local CRDT before provider sync + * resolves; other file types use their normal sync mechanics. */ - enqueueDownload(item: SyncFile | Document | Canvas): Promise { - // Skip if already in progress - if (this.inProgressDownloads.has(item.guid)) { - this.debug( - `[enqueueDownload] Item ${item.guid} already in progress, skipping`, - ); + async enqueueUpload(item: SyncFile | Document | Canvas): Promise { + if (this.shouldSkipDocumentSync(item)) { + this.clearFailure(this.failureKey("sync", item.guid)); + return Promise.resolve(); + } - // Return existing promise if already processing - const existingCallback = this.downloadCompletionCallbacks.get(item.guid); - if (existingCallback) { - this.processDownloadQueue(); - return new Promise((resolve, reject) => { - existingCallback.resolve = resolve; - existingCallback.reject = reject; - }); + if (this.inProgressSyncs.has(item.guid)) { + const queued = this.syncQueue.find((queued) => queued.guid === item.guid); + if (queued) { + queued.syncIntent = "upload"; + return this.syncPromises.get(item.guid) ?? Promise.resolve(); } - this.processDownloadQueue(); - return Promise.resolve(); + + const active = this.activeSync.find((active) => active.guid === item.guid); + if (active?.syncIntent === "upload") { + return this.syncPromises.get(item.guid) ?? Promise.resolve(); + } + + return this.enqueueUploadAfterCurrentSync(item); } const sharedFolder = item.sharedFolder; + const queueItem: QueueItem = { + guid: item.guid, + path: sharedFolder.getPath(item.path), + doc: item, + status: "pending", + sharedFolder, + userVisible: false, + syncIntent: "upload", + }; + this.clearFailure(this.failureKey("sync", item.guid)); - // Get or create the sync group for this folder let group = this.syncGroups.get(sharedFolder); if (!group) { group = { @@ -524,34 +1121,135 @@ export class BackgroundSync extends HasLogging { syncs: 0, completedDownloads: 0, completedSyncs: 0, + failedDownloads: 0, + failedSyncs: 0, + skippedDownloads: 0, + skippedSyncs: 0, + userDownloads: 0, + completedUserDownloads: 0, + failedUserDownloads: 0, + skippedUserDownloads: 0, }; } - - // Update the counters for individual document download - group.downloads++; group.total++; + group.syncs++; + group.status = "running"; this.syncGroups.set(sharedFolder, group); - // Create the queue item - const queueItem: QueueItem = { - guid: item.guid, - path: sharedFolder.getPath(item.path), - doc: item, - status: "pending", - sharedFolder, - }; + this.inProgressSyncs.add(item.guid); - // Mark as in progress + const syncPromise = new Promise((resolve, reject) => { + this.syncCompletionCallbacks.set(item.guid, { + resolve, + reject, + }); + }); + this.syncPromises.set(item.guid, syncPromise); + + this.syncQueue.push(queueItem); + this.syncQueue.sort(compareFilePaths); + this.queueStatusChanged.notifyListeners(); + this.processSyncQueue(); + + return syncPromise; + } + + private async enqueueUploadAfterCurrentSync( + item: SyncFile | Document | Canvas, + ): Promise { + try { + await (this.syncPromises.get(item.guid) ?? Promise.resolve()); + } catch { + // The upload request is the stronger follow-up operation. Let it run + // even if the weaker sync attempt failed. + } + await new Promise((resolve) => { + this.timeProvider.setTimeout(resolve, 0); + }); + return this.enqueueUpload(item); + } + + /** + * Enqueues a document for download + * + * This method adds a document to the download queue and creates/updates + * the associated sync group to track progress. + * + * @param item The document to download + * @returns A promise that resolves when the download completes + */ + enqueueDownload( + item: SyncFile | Document | Canvas, + userVisible = true, + ): Promise { + // Skip if already in progress — return the same promise all callers share + if (this.inProgressDownloads.has(item.guid)) { + this.debug( + `[enqueueDownload] Item ${item.guid} already in progress, sharing promise`, + ); + return this.downloadPromises.get(item.guid) ?? Promise.resolve(undefined); + } + + const sharedFolder = item.sharedFolder; + + // Get or create the sync group for this folder + let group = this.syncGroups.get(sharedFolder); + if (!group) { + group = { + sharedFolder, + total: 0, + completed: 0, + status: "pending", + downloads: 0, + syncs: 0, + completedDownloads: 0, + completedSyncs: 0, + failedDownloads: 0, + failedSyncs: 0, + skippedDownloads: 0, + skippedSyncs: 0, + userDownloads: 0, + completedUserDownloads: 0, + failedUserDownloads: 0, + skippedUserDownloads: 0, + }; + } + + // Update the counters for individual document download + group.downloads++; + group.total++; + if (userVisible) { + group.userDownloads++; + } + group.status = "running"; + this.syncGroups.set(sharedFolder, group); + + // Create the queue item + const queueItem: QueueItem = { + guid: item.guid, + path: sharedFolder.getPath(item.path), + doc: item, + status: "pending", + sharedFolder, + userVisible, + }; + this.clearFailure(this.failureKey("download", item.guid)); + + // Mark as in progress this.inProgressDownloads.add(item.guid); // Create a promise that will resolve when the download completes - const downloadPromise = new Promise((resolve, reject) => { - this.downloadCompletionCallbacks.set(item.guid, { resolve, reject }); - }); + const downloadPromise = new Promise( + (resolve, reject) => { + this.downloadCompletionCallbacks.set(item.guid, { resolve, reject }); + }, + ); + this.downloadPromises.set(item.guid, downloadPromise); // Add to the queue and start processing this.downloadQueue.push(queueItem); this.downloadQueue.sort(compareFilePaths); + this.queueStatusChanged.notifyListeners(); this.processDownloadQueue(); return downloadPromise; @@ -571,35 +1269,197 @@ export class BackgroundSync extends HasLogging { const docs = [...sharedFolder.files.values()].filter(isDocument); const canvases = [...sharedFolder.files.values()].filter(isCanvas); const syncFiles = [...sharedFolder.files.values()].filter(isSyncFile); - const allItems = [...docs, ...canvases, ...syncFiles]; + const allItems = [...docs, ...canvases, ...syncFiles].filter((item) => + this.shouldEnqueueForSharedFolderSync(item), + ); // Create sync group with properly initialized counters const group: SyncGroup = { sharedFolder, total: allItems.length, completed: 0, - status: "pending", + status: allItems.length > 0 ? "pending" : "completed", downloads: 0, syncs: allItems.length, completedDownloads: 0, completedSyncs: 0, + failedDownloads: 0, + failedSyncs: 0, + skippedDownloads: 0, + skippedSyncs: 0, + userDownloads: 0, + completedUserDownloads: 0, + failedUserDownloads: 0, + skippedUserDownloads: 0, }; // Register the group before enqueueing items this.syncGroups.set(sharedFolder, group); + if (allItems.length === 0) return; // Sort items by path for consistent sync order - const sortedDocs = [...docs, ...canvases, ...syncFiles].sort( - compareFilePaths, - ); + const sortedDocs = allItems.sort(compareFilePaths); + const queueLengthBefore = this.syncQueue.length; for (const doc of sortedDocs) { this.enqueueForGroupSync(doc); } - // Update group status to running + if (this.syncQueue.length > queueLengthBefore) { + this.syncQueue.sort(compareFilePaths); + this.queueStatusChanged.notifyListeners(); + this.processSyncQueue(); + } + + this.updateGroupTerminalStatus(group); + this.syncGroups.set(sharedFolder, group); + } + + enqueueLCABackfill(sharedFolder: SharedFolder): number { + if (!sharedFolder.connected) return 0; + + const docs = [...sharedFolder.files.values()] + .filter(isDocument) + .filter((doc) => this.shouldEnqueueForLCABackfill(doc)) + .filter((doc) => !this.inProgressSyncs.has(doc.guid)); + + if (docs.length === 0) return 0; + + for (const doc of docs.sort(compareFilePaths)) { + void this.enqueueLCABackfillDoc(doc); + } + return docs.length; + } + + private async enqueueLCABackfillDoc(doc: Document): Promise { + if (this.shouldSkipDocumentSync(doc)) { + this.clearFailure(this.failureKey("sync", doc.guid)); + return Promise.resolve(); + } + + if (this.inProgressSyncs.has(doc.guid)) { + this.debug( + `[enqueueLCABackfillDoc] Item ${doc.guid} already in progress, sharing promise`, + ); + return this.syncPromises.get(doc.guid) ?? Promise.resolve(); + } + + const sharedFolder = doc.sharedFolder; + const queueItem: QueueItem = { + guid: doc.guid, + path: sharedFolder.getPath(doc.path), + doc, + status: "pending", + sharedFolder, + userVisible: false, + syncIntent: "lca-backfill", + }; + this.clearFailure(this.failureKey("sync", doc.guid)); + + let group = this.syncGroups.get(sharedFolder); + if (!group) { + group = { + sharedFolder, + total: 0, + completed: 0, + status: "pending", + downloads: 0, + syncs: 0, + completedDownloads: 0, + completedSyncs: 0, + failedDownloads: 0, + failedSyncs: 0, + skippedDownloads: 0, + skippedSyncs: 0, + userDownloads: 0, + completedUserDownloads: 0, + failedUserDownloads: 0, + skippedUserDownloads: 0, + }; + } + group.total++; + group.syncs++; group.status = "running"; this.syncGroups.set(sharedFolder, group); + + this.inProgressSyncs.add(doc.guid); + + const syncPromise = new Promise((resolve, reject) => { + this.syncCompletionCallbacks.set(doc.guid, { + resolve, + reject, + }); + }); + this.syncPromises.set(doc.guid, syncPromise); + + this.syncQueue.push(queueItem); + this.syncQueue.sort(compareFilePaths); + this.queueStatusChanged.notifyListeners(); + this.processSyncQueue(); + + return syncPromise; + } + + enqueueRemoteHeadSyncs( + sharedFolder: SharedFolder, + guids: Iterable, + ): number { + if (!sharedFolder.connected) return 0; + + const advertisedGuids = new Set(guids); + if (advertisedGuids.size === 0) return 0; + + const docs = [...sharedFolder.files.values()] + .filter(isDocument) + .filter((doc) => advertisedGuids.has(doc.guid)) + .filter((doc) => !this.inProgressSyncs.has(doc.guid)) + .filter((doc) => this.shouldEnqueueForRemoteHeadSync(doc)); + + for (const doc of docs.sort(compareFilePaths)) { + void this.enqueueSync(doc); + } + return docs.length; + } + + private shouldEnqueueForSharedFolderSync( + item: Document | Canvas | SyncFile, + ): boolean { + if (isCanvas(item)) { + const mergeManager = item.sharedFolder.mergeManager; + if (!mergeManager) return true; + return !mergeManager.isServerAdvertisedInSync( + item.guid, + snapshotFromDoc(item.ydoc).snapshot, + ); + } + if (!isDocument(item)) return true; + + const hsm = item.hsm; + if (!hsm) return true; + if (hsm.getSyncStatus().status === "conflict") return false; + if (!hsm.state.lca) return true; + if (hsm.getSyncStatus().status !== "synced") return true; + + const mergeManager = item.sharedFolder.mergeManager; + if (!mergeManager) return true; + + return !mergeManager.isServerAdvertisedInSync(item.guid); + } + + private shouldEnqueueForRemoteHeadSync(doc: Document): boolean { + if (this.shouldSkipDocumentSync(doc)) return false; + return doc.sharedFolder.mergeManager?.isServerAdvertisedRemoteAhead(doc.guid) + ?? false; + } + + private shouldEnqueueForLCABackfill(doc: Document): boolean { + const hsm = doc.hsm; + if (!hsm) return false; + if (doc.sharedFolder.isPendingUpload(doc.path)) return false; + if (hsm.isActive()) return false; + if (hsm.state.lca) return false; + if (hsm.hasFork()) return false; + return hsm.getSyncStatus().status === "pending"; } /** @@ -616,22 +1476,17 @@ export class BackgroundSync extends HasLogging { private async enqueueForGroupSync( item: Document | Canvas | SyncFile, ): Promise { - // Skip if already in progress + if (this.shouldSkipDocumentSync(item)) { + this.clearFailure(this.failureKey("sync", item.guid)); + return Promise.resolve(); + } + + // Skip if already in progress — return the same promise all callers share if (this.inProgressSyncs.has(item.guid)) { this.debug( - `[enqueueForGroupSync] Item ${item.guid} already in progress, skipping`, + `[enqueueForGroupSync] Item ${item.guid} already in progress, sharing promise`, ); - - // Return existing promise if already processing - const existingCallback = this.syncCompletionCallbacks.get(item.guid); - if (existingCallback) { - this.processSyncQueue(); - return new Promise((resolve, reject) => { - existingCallback.resolve = resolve; - existingCallback.reject = reject; - }); - } - return Promise.resolve(); + return this.syncPromises.get(item.guid) ?? Promise.resolve(); } const sharedFolder = item.sharedFolder; @@ -641,7 +1496,9 @@ export class BackgroundSync extends HasLogging { doc: item, status: "pending", sharedFolder, + userVisible: false, }; + this.clearFailure(this.failureKey("sync", item.guid)); this.inProgressSyncs.add(item.guid); @@ -651,10 +1508,9 @@ export class BackgroundSync extends HasLogging { reject, }); }); + this.syncPromises.set(item.guid, syncPromise); this.syncQueue.push(queueItem); - this.syncQueue.sort(compareFilePaths); - this.processSyncQueue(); return syncPromise; } @@ -662,6 +1518,7 @@ export class BackgroundSync extends HasLogging { private getAuthHeader(clientToken: ClientToken) { return { Authorization: `Bearer ${clientToken.token}`, + ...getRelayRequestHeaders(), }; } @@ -724,58 +1581,218 @@ export class BackgroundSync extends HasLogging { return response; } - async syncDocumentWebsocket(doc: Document | Canvas): Promise { - // if the local file is synced, then we do the two step process - // check if file is tracking - let currentFileContents = ""; + /** + * Download raw CRDT bytes for a document by guid, without needing a + * Document instance. Used by the SharedFolder guid-remap path, where + * the server's content must be fetched *before* the old Document is + * destroyed — a failure here leaves old state intact and retriable. + * + * Does not participate in the download queue, syncGroups, or + * in-progress tracking. It is a bare HTTP fetch. + * + * Returns undefined if the server has the guid registered but no + * peer has uploaded content yet (empty contents, empty users map). + */ + async downloadByGuid( + sharedFolder: SharedFolder, + guid: string, + path: string, + ): Promise { + const entity = new S3RemoteDocument( + sharedFolder.relayId!, + sharedFolder.guid, + guid, + ); + this.log("[downloadByGuid]", path, S3RN.encode(entity)); - // Handle different document types - let currentTextStr = ""; - let currentCanvasData: CanvasData | null = null; + const clientToken = await sharedFolder.tokenStore.getToken( + S3RN.encode(entity), + path, + () => {}, + ); + const headers = this.getAuthHeader(clientToken); + const baseUrl = this.getBaseUrl(clientToken, entity); + const url = `${baseUrl}/as-update`; - if (isCanvas(doc)) { - // Store the exported canvas data rather than a stringified version - currentCanvasData = Canvas.exportCanvasData(doc.ydoc); - currentTextStr = JSON.stringify(currentCanvasData); - } else if (isDocument(doc)) { - currentTextStr = doc.text; - } - try { - currentFileContents = await doc.sharedFolder.read(doc); - } catch (e) { - // File does not exist - } + const response = await requestUrl({ + url, + method: "GET", + headers, + throw: false, + }); - // Only proceed with update if file matches current ydoc state - let contentsMatch = false; - if (isCanvas(doc) && currentCanvasData) { - // For canvas, use deep object comparison instead of string equality - const currentFileJson = currentFileContents - ? JSON.parse(currentFileContents) - : { nodes: [], edges: [] }; - contentsMatch = areObjectsEqual(currentCanvasData, currentFileJson); - } else { - contentsMatch = currentTextStr === currentFileContents; + if (response.status !== 200) { + this.error( + "[downloadByGuid]", + path, + url, + response.status, + response.text, + ); + throw new Error( + `downloadByGuid: status ${response.status} for ${S3RN.encode(entity)}`, + ); } - if (!contentsMatch && currentFileContents) { + const updateBytes = new Uint8Array(response.arrayBuffer); + + // Peek at the update in a throwaway doc to detect empty-server. + const tmpDoc = new Y.Doc(); + Y.applyUpdate(tmpDoc, updateBytes); + if (isEmptyDoc(tmpDoc)) { this.log( - "file is not tracking local disk. resolve merge conflicts before syncing.", + "[downloadByGuid] server has guid registered but no content", + path, ); + return undefined; + } + return updateBytes; + } + + async syncDocumentWebsocket(doc: Document | Canvas): Promise { + if (doc.destroyed) return false; + this.log(`[syncDocWS] start: ${doc.path} guid=${doc.guid} intent=${doc.intent} connected=${doc.connected}`); + if (this.isSyncCancelledForDoc(doc)) return false; + // if the local file is synced, then we do the two step process + if (isCanvas(doc)) { + // Store the exported canvas data rather than a stringified version + const currentCanvasData = Canvas.exportCanvasData(doc.ydoc); + let canvasContentsMismatch = false; + try { + const currentFileContents = await doc.sharedFolder.read(doc); + + // Only proceed with update if file matches current ydoc state + let contentsMatch = false; + if (isCanvas(doc) && currentCanvasData) { + // For canvas, use deep object comparison instead of string equality + const currentFileJson = currentFileContents + ? JSON.parse(currentFileContents) + : { nodes: [], edges: [] }; + contentsMatch = areCanvasDataEqual(currentCanvasData, currentFileJson); + if ( + !contentsMatch && + await this.repairStaleCanvasText(doc, currentFileJson) + ) { + contentsMatch = true; + } + if (!contentsMatch && currentFileContents) { + canvasContentsMismatch = true; + } + } + } catch (e) { + // File does not exist + } + if (canvasContentsMismatch) { + throw new Error( + "Canvas file does not match local sync state. Open the canvas and resolve the local changes before syncing.", + ); + } + } + const sharedFolder = doc.sharedFolder; + const refreshQueueKey = S3RN.encode(doc.s3rn); + const isActive = doc.userLock || sharedFolder?.mergeManager?.isActive(doc.guid); + const startedDisconnected = doc.intent === "disconnected"; + const hadProviderIntegration = isDocument(doc) && doc.hasProviderIntegration(); + const acquiredIdleIntegration = + isDocument(doc) && !isActive + ? doc.ensureIdleProviderIntegration({ + freshRemoteDoc: !!doc.hsm?.hasFork(), + }) + : false; + const shouldCleanupIdleSession = () => + startedDisconnected && + !(doc.userLock || sharedFolder?.mergeManager?.isActive(doc.guid)); + const cleanupIdleSession = () => { + if (isDocument(doc)) { + if (acquiredIdleIntegration) { + doc.destroyIdleProviderIntegration(); + if (shouldCleanupIdleSession()) { + sharedFolder?.tokenStore.removeFromRefreshQueue(refreshQueueKey); + } + return; + } + if (!shouldCleanupIdleSession()) return; + if (hadProviderIntegration || doc.hasProviderIntegration()) { + return; + } + } + if (!shouldCleanupIdleSession()) return; + doc.disconnect(); + sharedFolder?.tokenStore.removeFromRefreshQueue(refreshQueueKey); + }; + if (doc.destroyed) return false; + const connected = await doc.connect(); + if (!connected) { + if (shouldCleanupIdleSession()) { + cleanupIdleSession(); + } + if (this.isSyncCancelledForDoc(doc)) return false; + throw new RetryableProviderSyncError( + `Provider connection is not ready for ${this.fileName(doc.path)}`, + ); + } + if (this.isSyncCancelledForDoc(doc)) { + if (shouldCleanupIdleSession()) { + cleanupIdleSession(); + } return false; } + // Always wait for provider sync — _providerSynced fast-path resolves + // immediately if already synced. Connected does not imply synced. + // Timeout prevents hanging the sync queue if the connection drops. + const SYNC_TIMEOUT_MS = 10_000; + let timerId: number | undefined; + let cancelTimerId: number | undefined; + let providerSyncFailure: unknown; + const synced = await Promise.race([ + doc.onceProviderSynced().then( + () => true, + (e) => { + providerSyncFailure = e; + return false; + }, + ), + new Promise((resolve) => { + timerId = this.timeProvider.setTimeout( + () => resolve(false), + SYNC_TIMEOUT_MS, + ); + }), + new Promise((resolve) => { + cancelTimerId = this.timeProvider.setInterval(() => { + if (this.isSyncCancelledForDoc(doc)) resolve(false); + }, 100); + }), + ]); + if (timerId !== undefined) this.timeProvider.clearTimeout(timerId); + if (cancelTimerId !== undefined) this.timeProvider.clearInterval(cancelTimerId); + if (!synced) { + if (shouldCleanupIdleSession()) { + cleanupIdleSession(); + } + if (this.isSyncCancelledForDoc(doc)) return false; + if (providerSyncFailure) { + this.warn( + `[syncDocWS] provider sync failed: ${doc.path} guid=${doc.guid}: ${this.errorMessage(providerSyncFailure)}`, + ); + throw new RetryableProviderSyncError( + `Provider sync is not ready for ${this.fileName(doc.path)}: ${this.errorMessage(providerSyncFailure)}`, + ); + } else { + this.warn(`[syncDocWS] provider sync timed out: ${doc.path} guid=${doc.guid}`); + throw new RetryableProviderSyncError( + `Provider sync timed out for ${this.fileName(doc.path)}`, + ); + } + } - const promise = doc.onceProviderSynced(); - const intent = doc.intent; - doc.connect(); - if (intent === "disconnected") { - await promise; + if (isDocument(doc)) { + await this.maybeBootstrapDocumentLCA(doc); } // promise can take some time - if (intent === "disconnected" && !doc.userLock) { - doc.disconnect(); - doc.sharedFolder.tokenStore.removeFromRefreshQueue(S3RN.encode(doc.s3rn)); + if (shouldCleanupIdleSession()) { + cleanupIdleSession(); } return true; } @@ -785,8 +1802,11 @@ export class BackgroundSync extends HasLogging { * @param canvas The canvas to download * @returns A promise that resolves when the download completes */ - enqueueCanvasDownload(canvas: Canvas): Promise { - return this.enqueueDownload(canvas); + enqueueCanvasDownload( + canvas: Canvas, + userVisible = true, + ): Promise { + return this.enqueueDownload(canvas, userVisible); } async getCanvas(canvas: Canvas, retry = 3, wait = 3000) { @@ -802,9 +1822,7 @@ export class BackgroundSync extends HasLogging { } // Only proceed with update if file matches current ydoc state - const contentsMatch = - areObjectsEqual(currentJson.edges, currentFileContents.edges) && - areObjectsEqual(currentJson.nodes, currentFileContents.nodes); + const contentsMatch = areCanvasDataEqual(currentJson, currentFileContents); const hasContents = currentFileContents.nodes.length > 0; const response = await this.downloadItem(canvas); @@ -823,74 +1841,94 @@ export class BackgroundSync extends HasLogging { this.log("[getCanvas] flushed"); } } catch (e) { - this.error(e); + this.logError("[getCanvas] failed", e); throw e; } } - private async getDocument(doc: Document, retry = 3, wait = 3000) { + private async getDocument( + doc: Document, + retry = 3, + wait = 3000, + ): Promise { try { - // Get the current contents before applying the update - const currentText = doc.text; - let currentFileContents = ""; - try { - currentFileContents = await doc.sharedFolder.read(doc); - } catch (e) { - // File doesn't exist - } - - // Only proceed with update if file matches current ydoc state - const contentsMatch = currentText === currentFileContents; - const hasContents = currentFileContents !== ""; - const response = await this.downloadItem(doc); const rawUpdate = response.arrayBuffer; const updateBytes = new Uint8Array(rawUpdate); - // Check for newly created documents without content, and reject them + // Validate: reject uninitialized documents. const newDoc = new Y.Doc(); Y.applyUpdate(newDoc, updateBytes); - const users = newDoc.getMap("users"); - const contents = newDoc.getText("contents").toString(); - if (contents === "") { - if (users.size === 0) { - // Hack for better compat with < 0.4.2. - this.log( - "[getDocument] Server contains uninitialized document. Waiting for peer to upload.", - users.size, - retry, - wait, - ); - if (retry > 0) { - this.timeProvider.setTimeout(() => { - this.getDocument(doc, retry - 1, wait * 2); - }, wait); - } - return; - } + if (isEmptyDoc(newDoc)) { if (doc.text) { this.log( - "[getDocument] local crdt has contents, but remote is empty", + "[getDocument] server CRDT empty, local has content — uploading", ); this.enqueueSync(doc); - return; + return undefined; } + this.log( + "[getDocument] Server contains uninitialized document. Waiting for peer to upload.", + retry, + wait, + ); + if (retry > 0) { + this.timeProvider.setTimeout(() => { + this.getDocument(doc, retry - 1, wait * 2); + }, wait); + } + return undefined; } this.log("[getDocument] applying content from server"); Y.applyUpdate(doc.ydoc, updateBytes); + doc.hsm?.setRemoteDoc(doc.ydoc); + this.notifyDownloadedRemoteHead(doc); + await this.maybeBootstrapDocumentLCA(doc); + return updateBytes; + } catch (e) { + this.logError("[getDocument] failed", e); + throw e; + } + } - if (hasContents && !contentsMatch) { - this.log("Skipping flush - file requires merge conflict resolution."); - return; + private notifyDownloadedRemoteHead(doc: Document): void { + const hsm = doc.hsm; + if (!hsm) return; + + hsm.send({ type: "PROVIDER_SYNCED" }); + } + + private async maybeBootstrapDocumentLCA(doc: Document): Promise { + const hsm = doc.hsm; + if (!hsm || hsm.state.lca || hsm.isActive()) return; + + try { + const mergeManager = doc.sharedFolder.mergeManager; + if (mergeManager?.getHibernationState(doc.guid) === "hibernated") { + mergeManager.wake(doc.guid, doc.ensureRemoteDoc()); + await hsm.awaitPersistenceReady(); } - if (doc.sharedFolder.syncStore.has(doc.path)) { - doc.sharedFolder.flush(doc, doc.text); - this.log("[getDocument] flushed"); + + const diskState = await doc.readDiskContent(); + const repaired = await hsm.bootstrapLCAFromDisk(diskState); + if (!repaired && hsm.getSyncStatus().status === "pending") { + if (!hsm.hasPersistenceUserData()) { + this.debug( + `[bootstrapLCA] deferred for ${doc.path}: awaiting local CRDT enrollment`, + ); + return; + } + this.debug( + `[bootstrapLCA] skipped for ${doc.path}: local CRDT is not enrolled or remote state is not ready`, + ); } } catch (e) { - this.error(e); + this.warn( + `[bootstrapLCA] failed for ${doc.path}: ${this.errorMessage(e)}`, + e, + ); throw e; } } @@ -903,17 +1941,123 @@ export class BackgroundSync extends HasLogging { await file.pull(); } - private async syncDocument(doc: Document | Canvas) { + private async syncDocument(doc: Document | Canvas): Promise { + if (doc.destroyed) { + return; + } + if (this.shouldSkipDocumentSync(doc)) { + return; + } try { - if (isDocument(doc)) { - await this.syncDocumentWebsocket(doc); - } else if (isCanvas(doc)) { - await this.syncDocumentWebsocket(doc); + if (isDocument(doc) || isCanvas(doc)) { + const synced = await this.syncDocumentWebsocket(doc); + if (!synced) { + if (this.isSyncCancelledForDoc(doc)) return; + throw new Error(`Unable to sync ${this.fileName(doc.path)}`); + } } } catch (e) { - console.error(e); + if (!isRetryableProviderSyncError(e)) { + this.logError("[syncDocument] failed", e); + } + throw e; + } + } + + private async syncDocumentUpload(doc: Document | Canvas): Promise { + if (isDocument(doc) && doc.hsm) { + await this.prepareDocumentUpload(doc); + } + await this.syncDocument(doc); + if (isDocument(doc) && doc.hsm) { + this.assertUploadedDocumentHasRemoteContent(doc); + } + } + + private async syncDocumentLCABackfill(doc: Document): Promise { + if (doc.destroyed || this.shouldSkipDocumentSync(doc)) { return; } + + const hsm = doc.hsm; + if (!hsm || hsm.state.lca || hsm.isActive() || hsm.hasFork()) { + return; + } + + let updateBytes: Uint8Array | undefined; + try { + updateBytes = await this.downloadByGuid( + doc.sharedFolder, + doc.guid, + doc.path, + ); + } catch (error) { + throw new RetryableProviderSyncError( + `LCA backfill download failed for ${this.fileName(doc.path)}: ${this.errorMessage(error)}`, + ); + } + if (!updateBytes) { + throw new RetryableProviderSyncError( + `Remote document is empty while backfilling LCA: ${this.fileName(doc.path)}`, + ); + } + + const remoteDoc = new Y.Doc(); + Y.applyUpdate(remoteDoc, updateBytes); + if (isEmptyDoc(remoteDoc)) { + throw new RetryableProviderSyncError( + `Remote document is empty while backfilling LCA: ${this.fileName(doc.path)}`, + ); + } + + hsm.setRemoteDoc(remoteDoc); + const diskState = await doc.readDiskContent(); + const settled = await hsm.bootstrapLCAFromDisk(diskState); + if (!settled && hsm.getSyncStatus().status === "pending") { + throw new RetryableProviderSyncError( + `LCA backfill deferred for ${this.fileName(doc.path)}`, + ); + } + } + + private async prepareDocumentUpload(doc: Document): Promise { + const hsm = doc.hsm; + if (!hsm) return; + if (hsm.hasFork()) { + throw new Error(`Cannot upload ${this.fileName(doc.path)} while a fork exists`); + } + + const remoteDoc = doc.ensureRemoteDoc(); + const mergeManager = doc.sharedFolder.mergeManager; + if (!doc.userLock && !mergeManager?.isActive(doc.guid)) { + mergeManager?.wake(doc.guid, remoteDoc); + } else { + hsm.setRemoteDoc(remoteDoc); + } + await hsm.awaitPersistenceReady(); + + if (hsm.hasFork()) { + throw new Error(`Cannot upload ${this.fileName(doc.path)} while a fork exists`); + } + const localDoc = hsm.getLocalDoc(); + if (!localDoc) { + throw new RetryableProviderSyncError( + `Local document is not ready for upload: ${this.fileName(doc.path)}`, + ); + } + + Y.applyUpdate(remoteDoc, Y.encodeStateAsUpdate(localDoc), hsm); + hsm.setRemoteDoc(remoteDoc); + this.assertUploadedDocumentHasRemoteContent(doc); + } + + private assertUploadedDocumentHasRemoteContent(doc: Document): void { + const remoteDoc = doc.hsm?.getRemoteDoc() ?? doc.remoteDocOrNull; + if (!remoteDoc || isEmptyDoc(remoteDoc)) { + throw new RetryableProviderSyncError( + `Remote document is empty after upload preparation: ${this.fileName(doc.path)}`, + ); + } } subscribeToSync( @@ -966,6 +2110,74 @@ export class BackgroundSync extends HasLogging { }); } + subscribeToFolderSyncSnapshot( + sharedFolder: SharedFolder, + callback: Subscriber, + ): Unsubscriber { + const state = this.getFolderSyncSnapshotSubscription(sharedFolder); + state.subscribers.add(callback); + if (state.latestSnapshot) callback(state.latestSnapshot); + + return () => { + state.subscribers.delete(callback); + if (state.subscribers.size === 0) { + this.disposeFolderSyncSnapshotSubscription(sharedFolder, state); + } + }; + } + + private getFolderSyncSnapshotSubscription( + sharedFolder: SharedFolder, + ): FolderSyncSnapshotSubscription { + const existing = this.folderSyncSnapshotSubscriptions.get(sharedFolder); + if (existing) return existing; + + const state: FolderSyncSnapshotSubscription = { + smoother: null as any, + subscribers: new Set(), + latestSnapshot: null, + unsubscribers: [], + emit: () => {}, + }; + state.smoother = new FolderSyncSnapshotSmoother( + this.timeProvider, + (snapshot) => { + state.latestSnapshot = snapshot; + for (const subscriber of state.subscribers) { + subscriber(snapshot); + } + }, + ); + state.emit = () => { + state.smoother.update(this.getFolderSyncSnapshot(sharedFolder)); + }; + const folderStateKey = { type: "folder-sync-snapshot", sharedFolder }; + state.unsubscribers = [ + this.activeSync.on(state.emit), + this.activeDownloads.on(state.emit), + this.syncGroups.on(state.emit), + this.failures.on(state.emit), + this.folderResyncs.on(state.emit), + this.queueStatusChanged.on(state.emit), + sharedFolder.subscribe(folderStateKey, state.emit), + ]; + this.folderSyncSnapshotSubscriptions.set(sharedFolder, state); + state.emit(); + return state; + } + + private disposeFolderSyncSnapshotSubscription( + sharedFolder: SharedFolder, + state: FolderSyncSnapshotSubscription, + ): void { + if (this.folderSyncSnapshotSubscriptions.get(sharedFolder) !== state) return; + this.folderSyncSnapshotSubscriptions.delete(sharedFolder); + state.unsubscribers.forEach((unsubscribe) => unsubscribe()); + state.smoother.destroy(); + state.subscribers.clear(); + state.latestSnapshot = null; + } + /** * Pauses all sync and download queue processing * @@ -974,6 +2186,7 @@ export class BackgroundSync extends HasLogging { */ pause(): void { this.isPaused = true; + this.queueStatusChanged.notifyListeners(); } /** @@ -985,6 +2198,7 @@ export class BackgroundSync extends HasLogging { resume(): void { this.debug("starting"); this.isPaused = false; + this.queueStatusChanged.notifyListeners(); this.processSyncQueue(); this.processDownloadQueue(); } @@ -995,13 +2209,7 @@ export class BackgroundSync extends HasLogging { * * @returns An object with queue statistics */ - getQueueStatus(): { - syncsQueued: number; - syncsActive: number; - downloadsQueued: number; - downloadsActive: number; - isPaused: boolean; - } { + getQueueStatus(): QueueStatus { return { syncsQueued: this.syncQueue.length, syncsActive: this.activeSync.size, @@ -1011,6 +2219,21 @@ export class BackgroundSync extends HasLogging { }; } + subscribeToQueueStatus(callback: Subscriber): Unsubscriber { + const emit = () => callback(this.getQueueStatus()); + const unsubscribers = [ + this.activeSync.subscribe(emit), + this.activeDownloads.subscribe(emit), + this.syncGroups.subscribe(emit), + this.failures.subscribe(emit), + this.queueStatusChanged.subscribe(emit), + ]; + + return () => { + unsubscribers.forEach((unsubscribe) => unsubscribe()); + }; + } + /** * Destroys this instance and cleans up all resources * @@ -1031,16 +2254,27 @@ export class BackgroundSync extends HasLogging { this.downloadCompletionCallbacks.delete(guid); } + for (const [sharedFolder, state] of [ + ...this.folderSyncSnapshotSubscriptions.entries(), + ]) { + this.disposeFolderSyncSnapshotSubscription(sharedFolder, state); + } + // Destroy observable collections this.activeSync.destroy(); this.activeDownloads.destroy(); + this.folderResyncs.destroy(); this.syncGroups.destroy(); + this.failures.destroy(); + this.queueStatusChanged.destroy(); // Clear queues and tracking this.syncQueue = []; this.downloadQueue = []; this.inProgressSyncs.clear(); this.inProgressDownloads.clear(); + this.syncPromises.clear(); + this.downloadPromises.clear(); this.loggedItems.clear(); // Clean up references @@ -1050,4 +2284,53 @@ export class BackgroundSync extends HasLogging { // Unsubscribe from all subscriptions this.subscriptions.forEach((off) => off()); } + + private recordFailure( + kind: BackgroundSyncFailure["kind"], + item: QueueItem, + error: unknown, + ): void { + const id = this.failureKey(kind, item.guid); + this.setFailure({ + id, + guid: item.guid, + path: item.doc.path, + kind, + message: this.errorMessage(error), + sharedFolder: item.sharedFolder, + }); + } + + private setFailure(failure: BackgroundSyncFailure): void { + const existing = this.failures.get(failure.id); + if ( + existing && + existing.guid === failure.guid && + existing.path === failure.path && + existing.kind === failure.kind && + existing.message === failure.message && + existing.sharedFolder === failure.sharedFolder + ) { + return; + } + this.failures.set(failure.id, failure); + } + + private failureKey(kind: BackgroundSyncFailure["kind"], guid: string): string { + return `${kind}:${guid}`; + } + + private errorMessage(error: unknown): string { + return formatUserFacingError(error); + } + + private logError(context: string, error: unknown): void { + this.error(`${context}: ${this.errorMessage(error)}`, error); + } + + private fileName(path: string): string { + const normalized = path.replace(/\\/g, "/"); + const parts = normalized.split("/").filter(Boolean); + return parts[parts.length - 1] || "file"; + } } diff --git a/src/BackgroundSyncProgress.ts b/src/BackgroundSyncProgress.ts new file mode 100644 index 00000000..5d0664f8 --- /dev/null +++ b/src/BackgroundSyncProgress.ts @@ -0,0 +1,524 @@ +import type { TimeProvider } from "./TimeProvider"; + +export interface FolderPillProgressDecisionInput { + hasQueuedOrActiveWork: boolean; + userDownloads: number; + completedUserDownloads: number; + failedUserDownloads: number; + skippedUserDownloads?: number; +} + +export type FolderSyncVisibleState = + | "synced" + | "syncing" + | "queued" + | "paused" + | "sync-issue"; + +export type FolderSyncAction = "pause" | "resume" | "resync" | null; + +export type FolderSyncProgressStatus = + | "pending" + | "running" + | "completed" + | "failed"; + +export interface FolderSyncGroupInput { + total: number; + completed: number; + status: FolderSyncProgressStatus; + downloads: number; + syncs: number; + completedDownloads: number; + completedSyncs: number; + failedDownloads?: number; + failedSyncs?: number; + skippedDownloads?: number; + skippedSyncs?: number; + userDownloads: number; + completedUserDownloads: number; + failedUserDownloads: number; + skippedUserDownloads?: number; +} + +export interface FolderSyncWorkItemInput { + kind: "sync" | "download"; + path: string; +} + +export interface FolderSyncSnapshotInput { + group?: FolderSyncGroupInput | null; + queued: number; + active: number; + isPaused: boolean; + failureCount: number; + canResync?: boolean; + folderActivity?: "checking" | null; + activeItem?: FolderSyncWorkItemInput | null; + queuedReason?: + | "connection" + | "reconnecting" + | null; +} + +export interface FolderSyncSnapshot { + percent: number; + syncPercent: number; + downloadPercent: number; + showProgress: boolean; + progressStatus: FolderSyncProgressStatus; + visibleState: FolderSyncVisibleState; + label: "Synced" | "Syncing" | "Queued" | "Paused" | "Sync issue"; + latestActivity: string | null; + syncAction: FolderSyncAction; + queued: number; + active: number; + total: number; + failureCount: number; + isPaused: boolean; +} + +export function shouldShowCompletedFolderPillProgress( + input: FolderPillProgressDecisionInput, +): boolean { + return !input.hasQueuedOrActiveWork; +} + +export function shouldUseUserVisibleFolderProgress( + input: FolderPillProgressDecisionInput, +): boolean { + const visibleDownloadsFinished = + input.completedUserDownloads + + input.failedUserDownloads + + (input.skippedUserDownloads ?? 0); + return ( + input.userDownloads > 0 && + visibleDownloadsFinished < input.userDownloads + ); +} + +export function buildFolderSyncSnapshot( + input: FolderSyncSnapshotInput, +): FolderSyncSnapshot { + const progress = computeFolderProgress({ + group: input.group, + queued: input.queued, + active: input.active, + isPaused: input.isPaused, + failureCount: input.failureCount, + }); + const visibleState = deriveVisibleState({ + isPaused: input.isPaused, + active: input.active, + queued: input.queued, + failureCount: input.failureCount, + }); + return { + ...progress, + showProgress: shouldShowProgress(progress.percent, progress.progressStatus), + visibleState, + label: labelForVisibleState(visibleState), + latestActivity: latestActivityForSnapshot(input, visibleState), + syncAction: actionForVisibleState( + visibleState, + input.active + input.queued, + input.canResync ?? true, + ), + queued: input.queued, + active: input.active, + total: progress.total, + failureCount: input.failureCount, + isPaused: input.isPaused, + }; +} + +const PROGRESS_TRANSITION_MS = 1000; + +export class FolderSyncSnapshotSmoother { + private latestSnapshot: FolderSyncSnapshot | null = null; + private displayedPercent = 0; + private targetPercent = 0; + private progressTimer: number | null = null; + private transitionStartedAt: number | null = null; + private transitionEndsAt: number | null = null; + private lastEmittedSnapshot: FolderSyncSnapshot | null = null; + private hasBaseline = false; + private hasProgressActivity = false; + + constructor( + private readonly timeProvider: TimeProvider, + private readonly emit: (snapshot: FolderSyncSnapshot) => void, + ) {} + + update(snapshot: FolderSyncSnapshot): void { + const target = normalizeProgress(snapshot.percent); + this.latestSnapshot = snapshot; + + if (!this.hasBaseline) { + this.hasBaseline = true; + this.displayedPercent = target; + this.targetPercent = target; + this.emitCurrent(false); + return; + } + + if (target < this.displayedPercent) { + this.displayedPercent = target; + this.targetPercent = target; + this.hasProgressActivity = false; + this.transitionStartedAt = null; + this.transitionEndsAt = null; + this.clearTimer(); + this.emitCurrent(false); + return; + } + + if (target !== this.targetPercent) { + this.targetPercent = target; + this.hasProgressActivity = true; + if (this.transitionStartedAt === null) { + this.transitionStartedAt = this.timeProvider.now(); + this.transitionEndsAt = + this.transitionStartedAt + PROGRESS_TRANSITION_MS; + } + this.clearTimer(); + this.scheduleProgressStep(); + return; + } + + this.emitCurrent(); + } + + destroy(): void { + this.clearTimer(); + } + + private scheduleProgressStep(): void { + if (!this.latestSnapshot) return; + if (this.displayedPercent === this.targetPercent) { + this.transitionStartedAt = null; + this.transitionEndsAt = null; + this.emitCurrent(); + return; + } + + const remainingMs = Math.max( + 0, + (this.transitionEndsAt ?? this.timeProvider.now()) - + this.timeProvider.now(), + ); + const remainingSteps = Math.abs(this.targetPercent - this.displayedPercent); + const stepDelayMs = + remainingMs > 0 + ? Math.max(1, Math.ceil(remainingMs / remainingSteps)) + : 1; + + this.progressTimer = this.timeProvider.setTimeout(() => { + this.progressTimer = null; + this.advanceProgress(); + }, stepDelayMs); + } + + private advanceProgress(): void { + if (!this.latestSnapshot) return; + if (this.displayedPercent === this.targetPercent) { + this.scheduleProgressStep(); + return; + } + + this.displayedPercent += + this.displayedPercent < this.targetPercent ? 1 : -1; + this.emitCurrent(); + this.scheduleProgressStep(); + } + + private emitCurrent(forceHide = false): void { + if (!this.latestSnapshot) return; + const isFinalizing = this.isFinalizingProgress(); + const progressStatus = isFinalizing + ? "running" + : this.latestSnapshot.progressStatus; + const visibleState = isFinalizing + ? "syncing" + : this.latestSnapshot.visibleState; + const next = { + ...this.latestSnapshot, + percent: this.displayedPercent, + progressStatus, + visibleState, + label: isFinalizing ? "Syncing" : this.latestSnapshot.label, + latestActivity: isFinalizing + ? (this.latestSnapshot.latestActivity ?? "Finalizing...") + : this.latestSnapshot.latestActivity, + syncAction: isFinalizing ? null : this.latestSnapshot.syncAction, + showProgress: + !forceHide && + this.hasProgressActivity && + shouldShowProgress(this.displayedPercent, progressStatus), + }; + if (shallowFolderSyncSnapshotEqual(this.lastEmittedSnapshot, next)) return; + this.lastEmittedSnapshot = { ...next }; + this.emit(next); + } + + private isFinalizingProgress(): boolean { + if (!this.latestSnapshot) return false; + if (!this.hasProgressActivity) return false; + if (this.displayedPercent === this.targetPercent) return false; + return this.latestSnapshot.visibleState === "synced"; + } + + private clearTimer(): void { + if (this.progressTimer === null) return; + this.timeProvider.clearTimeout(this.progressTimer); + this.progressTimer = null; + } +} + +function normalizeProgress(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(100, Math.round(value))); +} + +function shallowFolderSyncSnapshotEqual( + a: FolderSyncSnapshot | null, + b: FolderSyncSnapshot, +): boolean { + if (a === null) return false; + const aKeys = Object.keys(a) as Array; + const bKeys = Object.keys(b) as Array; + if (aKeys.length !== bKeys.length) return false; + for (const key of aKeys) { + if (!Object.prototype.hasOwnProperty.call(b, key)) return false; + if (!Object.is(a[key], b[key])) return false; + } + return true; +} + +function shouldShowProgress( + percent: number, + status: FolderSyncProgressStatus, +): boolean { + return ( + percent > 0 && + (status === "running" || + status === "failed" || + (status === "completed" && percent < 100)) + ); +} + +function computeFolderProgress(input: { + group?: FolderSyncGroupInput | null; + queued: number; + active: number; + isPaused: boolean; + failureCount: number; +}): Pick< + FolderSyncSnapshot, + "percent" | "syncPercent" | "downloadPercent" | "progressStatus" | "total" +> { + const hasQueuedOrActiveWork = input.queued + input.active > 0; + const queuedWithoutActiveWork = + !input.isPaused && input.active === 0 && input.queued > 0; + const group = input.group; + if (!group) { + return { + percent: hasQueuedOrActiveWork && !queuedWithoutActiveWork ? 0 : 100, + syncPercent: hasQueuedOrActiveWork && !queuedWithoutActiveWork ? 0 : 100, + downloadPercent: + hasQueuedOrActiveWork && !queuedWithoutActiveWork ? 0 : 100, + progressStatus: + input.failureCount > 0 + ? "failed" + : hasQueuedOrActiveWork && !queuedWithoutActiveWork + ? "running" + : "completed", + total: 0, + }; + } + + if (queuedWithoutActiveWork) { + return { + percent: 100, + syncPercent: 100, + downloadPercent: 100, + progressStatus: input.failureCount > 0 ? "failed" : "completed", + total: group.total, + }; + } + + const progressInput: FolderPillProgressDecisionInput = { + hasQueuedOrActiveWork, + userDownloads: group.userDownloads, + completedUserDownloads: group.completedUserDownloads, + failedUserDownloads: group.failedUserDownloads, + skippedUserDownloads: group.skippedUserDownloads, + }; + + if (shouldShowCompletedFolderPillProgress(progressInput)) { + return { + percent: 100, + syncPercent: 100, + downloadPercent: 100, + progressStatus: input.failureCount > 0 ? "failed" : "completed", + total: group.total, + }; + } + + if (shouldUseUserVisibleFolderProgress(progressInput)) { + const total = group.userDownloads; + const finished = + group.completedUserDownloads + + group.failedUserDownloads + + (group.skippedUserDownloads ?? 0); + const percent = total > 0 ? (finished / total) * 100 : 0; + const progressStatus = + finished === total + ? group.failedUserDownloads > 0 + ? "failed" + : "completed" + : group.status === "failed" + ? "failed" + : "running"; + return { + percent: Math.round(percent), + syncPercent: 0, + downloadPercent: Math.round(percent), + progressStatus, + total: group.total, + }; + } + + const terminal = terminalCounts(group); + const percent = group.total > 0 ? (terminal.total / group.total) * 100 : 0; + const syncPercent = + group.syncs > 0 ? (terminal.syncs / group.syncs) * 100 : 0; + const downloadPercent = + group.downloads > 0 + ? (terminal.downloads / group.downloads) * 100 + : 0; + const progressStatus = + terminal.total >= group.total + ? terminal.failed > 0 + ? "failed" + : "completed" + : group.status; + return { + percent: Math.round(percent), + syncPercent: Math.round(syncPercent), + downloadPercent: Math.round(downloadPercent), + progressStatus, + total: group.total, + }; +} + +function terminalCounts(group: FolderSyncGroupInput): { + total: number; + syncs: number; + downloads: number; + failed: number; +} { + const failedSyncs = group.failedSyncs ?? 0; + const failedDownloads = group.failedDownloads ?? 0; + const skippedSyncs = group.skippedSyncs ?? 0; + const skippedDownloads = group.skippedDownloads ?? 0; + const syncs = Math.min( + group.syncs, + group.completedSyncs + failedSyncs + skippedSyncs, + ); + const downloads = Math.min( + group.downloads, + group.completedDownloads + failedDownloads + skippedDownloads, + ); + return { + total: Math.min(group.total, syncs + downloads), + syncs, + downloads, + failed: failedSyncs + failedDownloads, + }; +} + +function deriveVisibleState(input: { + isPaused: boolean; + active: number; + queued: number; + failureCount: number; +}): FolderSyncVisibleState { + if (input.isPaused) return "paused"; + if (input.active > 0) return "syncing"; + if (input.queued > 0) return "queued"; + if (input.failureCount > 0) return "sync-issue"; + return "synced"; +} + +function labelForVisibleState( + visibleState: FolderSyncVisibleState, +): FolderSyncSnapshot["label"] { + switch (visibleState) { + case "syncing": + return "Syncing"; + case "queued": + return "Queued"; + case "paused": + return "Paused"; + case "sync-issue": + return "Sync issue"; + case "synced": + default: + return "Synced"; + } +} + +function actionForVisibleState( + visibleState: FolderSyncVisibleState, + workCount: number, + canResync: boolean, +): FolderSyncAction { + if (visibleState === "syncing") return "pause"; + if (visibleState === "paused" && workCount > 0) return "resume"; + if ( + canResync && + (visibleState === "synced" || visibleState === "sync-issue") + ) { + return "resync"; + } + return null; +} + +function latestActivityForSnapshot( + input: FolderSyncSnapshotInput, + visibleState: FolderSyncVisibleState, +): string | null { + if (visibleState === "syncing" && input.activeItem) { + const filename = filenameFromPath(input.activeItem.path); + return input.activeItem.kind === "download" + ? `Downloading ${filename}` + : `Syncing ${filename}`; + } + if (visibleState === "syncing" && input.folderActivity === "checking") { + return "Checking folder"; + } + if (visibleState === "queued") { + switch (input.queuedReason) { + case "connection": + return "Waiting for connection"; + case "reconnecting": + return "Reconnecting"; + default: + return "Queued"; + } + } + if (visibleState === "paused") return "Sync paused"; + if (visibleState === "sync-issue") { + return input.failureCount === 1 + ? "1 sync issue" + : `${input.failureCount} sync issues`; + } + return null; +} + +function filenameFromPath(path: string): string { + const normalized = path.replace(/\\/g, "/"); + const parts = normalized.split("/").filter(Boolean); + return parts[parts.length - 1] || normalized || "file"; +} diff --git a/src/Canvas.ts b/src/Canvas.ts index 004f62c4..96b374de 100644 --- a/src/Canvas.ts +++ b/src/Canvas.ts @@ -8,6 +8,7 @@ import type { SharedFolder } from "./SharedFolder"; import { getMimeType } from "./mimetypes"; import { IndexeddbPersistence } from "./storage/y-indexeddb"; import { Dependency } from "./promiseUtils"; +import { trackAsyncCleanup } from "./reloadUtils"; import type { Unsubscriber } from "./observable/Observable"; import type { CanvasData, @@ -16,11 +17,50 @@ import type { CanvasView, } from "./CanvasView"; import { areObjectsEqual } from "./areObjectsEqual"; +import { trackPromise } from "./trackPromise"; +import { formatCanvasData } from "./CanvasData"; export function isCanvas(file?: IFile | null): file is Canvas { return file instanceof Canvas; } +function replaceYTextContent(ytext: Y.Text, nextText: string): void { + const currentText = ytext.toString(); + if (currentText === nextText) return; + + let prefixLength = 0; + const maxPrefixLength = Math.min(currentText.length, nextText.length); + while ( + prefixLength < maxPrefixLength && + currentText[prefixLength] === nextText[prefixLength] + ) { + prefixLength++; + } + + let suffixLength = 0; + const maxSuffixLength = maxPrefixLength - prefixLength; + while ( + suffixLength < maxSuffixLength && + currentText[currentText.length - 1 - suffixLength] === + nextText[nextText.length - 1 - suffixLength] + ) { + suffixLength++; + } + + const deleteLength = currentText.length - prefixLength - suffixLength; + if (deleteLength > 0) { + ytext.delete(prefixLength, deleteLength); + } + + const insertedText = nextText.slice( + prefixLength, + nextText.length - suffixLength, + ); + if (insertedText.length > 0) { + ytext.insert(prefixLength, insertedText); + } +} + export class Canvas extends HasProvider implements IFile, HasMimeType { private _parent: SharedFolder; private _persistence: IndexeddbPersistence; @@ -31,6 +71,7 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { _tfile: TFile | null; name: string; userLock: boolean = false; + destroyed: boolean = false; extension: string; basename: string; vault: Vault; @@ -53,6 +94,7 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { ? new S3RemoteCanvas(parent.relayId, parent.guid, guid) : new S3Canvas(parent.guid, guid); super(guid, s3rn, parent.tokenStore, loginManager); + this.timeProvider = parent.timeProvider; this._parent = parent; this.path = path; this.name = "[CRDT] " + path.split("/").pop() || ""; @@ -76,32 +118,42 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { this.setLoggers(`[Canvas](${this.path})`); try { const key = `${this.sharedFolder.appId}-relay-canvas-${this.guid}`; - this._persistence = new IndexeddbPersistence(key, this.ydoc); + this._persistence = new IndexeddbPersistence( + key, + this.ydoc, + null, + null, + this.timeProvider, + ); } catch (e) { this.warn("Unable to open persistence.", this.guid); console.error(e); throw e; } - this.whenSynced().then(() => { - this.updateStats(); - try { - this._persistence.set("path", this.path); - this._persistence.set("relay", this.sharedFolder.relayId || ""); - this._persistence.set("appId", this.sharedFolder.appId); - this._persistence.set("s3rn", S3RN.encode(this.s3rn)); - } catch (e) { - // pass - } - - (async () => { - const serverSynced = await this.getServerSynced(); - if (!serverSynced) { - await this.onceProviderSynced(); - await this.markSynced(); + this.whenSynced() + .then(() => { + this.updateStats(); + try { + this._persistence.set("path", this.path); + this._persistence.set("relay", this.sharedFolder.relayId || ""); + this._persistence.set("appId", this.sharedFolder.appId); + this._persistence.set("s3rn", S3RN.encode(this.s3rn)); + } catch (e) { + // pass } - })(); - }); + + (async () => { + const serverSynced = await this.getServerSynced(); + if (!serverSynced) { + const connected = await this.connect(); + if (!connected) return; + await trackPromise(`canvasSync:${this.guid}`, this.onceProviderSynced()); + await this.markSynced(); + } + })().catch((e) => this.warn("canvas provider sync failed", e)); + }) + .catch((e) => this.warn("canvas persistence sync failed", e)); this._tfile = null; } @@ -137,7 +189,21 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { ...{ text: ytext.toString() || ynode.text }, }); } - return { edges: edges, nodes: nodes }; + return { nodes: nodes, edges: edges }; + } + + static exportCanvasMapData(ydoc: Y.Doc): CanvasData { + const yedges = ydoc.getMap("edges"); + const ynodes = ydoc.getMap("nodes"); + const edges = []; + const nodes = []; + for (const [, yedge] of yedges.entries()) { + edges.push({ ...yedge }); + } + for (const [, ynode] of ynodes.entries()) { + nodes.push({ ...ynode }); + } + return { nodes: nodes, edges: edges }; } async markSynced(): Promise { @@ -182,8 +248,8 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { async awaitingUpdates(): Promise { await this.whenSynced(); await this.getServerSynced(); - if (!this._awaitingUpdates) { - return false; + if (this._awaitingUpdates !== undefined) { + return this._awaitingUpdates; } this._awaitingUpdates = !this.hasLocalDB(); return this._awaitingUpdates; @@ -196,9 +262,9 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { // If this is a brand new shared folder, we want to wait for a connection before we start reserving new guids for local files. this.log("awaiting updates"); this.connect(); - await this.onceConnected(); + await trackPromise(`canvasConnected:${this.guid}`, this.onceConnected()); this.log("connected"); - await this.onceProviderSynced(); + await trackPromise(`canvasReady:${this.guid}`, this.onceProviderSynced()); this.log("synced"); return this; } @@ -208,36 +274,37 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { this.readyPromise || new Dependency(promiseFn, (): [boolean, Canvas] => { return [this.ready, this]; - }); - return this.readyPromise.getPromise(); + }, this.timeProvider); + return trackPromise(`canvas:whenReady:${this.guid}`, this.readyPromise.getPromise()); } whenSynced(): Promise { const promiseFn = async (): Promise => { await this.sharedFolder.whenSynced(); - // Check if already synced first - if (this._persistence.synced && !this.persistenceSynced) { - this.persistenceSynced = true; - return Promise.resolve(); - } - - return new Promise((resolve) => { - if (this.persistenceSynced) { - resolve(); - } - this._persistence.once("synced", () => { - this.persistenceSynced = true; - resolve(); - }); - }); + await this._persistence.whenSynced; + this.persistenceSynced = true; }; this.whenSyncedPromise = this.whenSyncedPromise || new Dependency(promiseFn, (): [boolean, void] => { return [this.persistenceSynced, undefined]; - }); - return this.whenSyncedPromise.getPromise(); + }, this.timeProvider); + return trackPromise(`canvas:whenSynced:${this.guid}`, this.whenSyncedPromise.getPromise()); + } + + /** + * Release lock on this canvas. + * Transitions HSM from active back to idle mode. + * Call this when editor closes. + */ + releaseLock(): void { + this.userLock = false; + + const mergeManager = this.sharedFolder.mergeManager; + if (mergeManager) { + mergeManager.unload(this.guid); + } } public get sharedFolder(): SharedFolder { @@ -283,15 +350,19 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { const deleted_nodes = new Set(); const changed_edges = new Map(); const deleted_edges = new Set(); + const changed_text = new Map(); data.nodes.forEach((node: CanvasNodeData) => { seen.add(node.id); + if (node.type === "text" && typeof node.text === "string") { + const ytext = this.ydoc.getText(node.id); + if (ytext.toString() !== node.text) { + changed_text.set(node.id, node.text); + } + } const ynode = ynodes.get(node.id); if (!ynode) { changed_nodes.set(node.id, node); - if (node.type === "text") { - this.textNode(node); - } } else if (!areObjectsEqual(ynode, node)) { changed_nodes.set(node.id, node); } @@ -320,7 +391,8 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { changed_nodes.size > 0 || deleted_nodes.size > 0 || changed_edges.size > 0 || - deleted_edges.size > 0 + deleted_edges.size > 0 || + changed_text.size > 0 ) { Y.transact( this.ydoc, @@ -328,6 +400,9 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { for (const node of changed_nodes.values()) { this.ynodes.set(node.id, node); } + for (const [node_id, text] of changed_text) { + replaceYTextContent(this.ydoc.getText(node_id), text); + } for (const node_id of deleted_nodes) { this.ynodes.delete(node_id); } @@ -358,7 +433,7 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { public get json(): string { const data = Canvas.exportCanvasData(this.ydoc); - return JSON.stringify(data); + return formatCanvasData(data); } public async cleanup(): Promise {} @@ -370,11 +445,15 @@ export class Canvas extends HasProvider implements IFile, HasMimeType { } destroy() { + this.destroyed = true; this.unsubscribes.forEach((unsubscribe) => { unsubscribe(); }); + if (this._persistence) { + const p = this._persistence.destroy().catch(() => {}); + trackAsyncCleanup(p); + } super.destroy(); - this.ydoc.destroy(); this.whenSyncedPromise?.destroy(); this.whenSyncedPromise = null as any; this.readyPromise?.destroy(); diff --git a/src/CanvasData.ts b/src/CanvasData.ts new file mode 100644 index 00000000..98c7e490 --- /dev/null +++ b/src/CanvasData.ts @@ -0,0 +1,117 @@ +import { areObjectsEqual } from "./areObjectsEqual"; +import type { CanvasData } from "./CanvasView"; + +interface CanvasItem { + id: string; +} + +export function areCanvasDataEqual( + left: CanvasData | null | undefined, + right: CanvasData | null | undefined, +): boolean { + if (!left || !right) return false; + return ( + areCanvasItemsEqual(left.nodes ?? [], right.nodes ?? []) && + areCanvasItemsEqual(left.edges ?? [], right.edges ?? []) + ); +} + +function areCanvasItemsEqual( + left: readonly T[], + right: readonly T[], +): boolean { + if (left.length !== right.length) return false; + + const rightById = new Map(right.map((item) => [item.id, item])); + for (const leftItem of left) { + const rightItem = rightById.get(leftItem.id); + if (!rightItem || !areObjectsEqual(leftItem, rightItem)) { + return false; + } + } + return true; +} + +export function formatCanvasData(data: CanvasData): string { + return formatObsidianJson(data) ?? ""; +} + +function formatObsidianJson(value: unknown): string | undefined { + if (value === undefined) return undefined; + return formatObsidianJsonLines(value).join("\n"); +} + +function formatObsidianJsonLines(value: unknown): string[] { + if (value === undefined) return ["null"]; + if ( + isPrimitiveJsonValue(value) || + value === null || + Object.prototype.toString.call(value) === "[object Date]" + ) { + return [JSON.stringify(value)]; + } + + if (Array.isArray(value)) { + if (value.every(isPrimitiveJsonValue)) { + return [JSON.stringify(value)]; + } + + const lines = ["["]; + const lastIndex = value.length - 1; + for (let index = 0; index <= lastIndex; index++) { + const childLines = formatObsidianJsonLines(value[index]); + const lastChildLineIndex = childLines.length - 1; + for (let lineIndex = 0; lineIndex <= lastChildLineIndex; lineIndex++) { + let line = `\t${childLines[lineIndex]}`; + if (lineIndex === lastChildLineIndex && index !== lastIndex) { + line += ","; + } + lines.push(line); + } + } + lines.push("]"); + return lines; + } + + if (typeof value === "object") { + const record = value as Record; + let primitiveOnly = true; + for (const key in record) { + if ( + Object.prototype.hasOwnProperty.call(record, key) && + !isPrimitiveJsonValue(record[key]) + ) { + primitiveOnly = false; + break; + } + } + if (primitiveOnly) { + return [JSON.stringify(record)]; + } + + const keys = Object.keys(record).filter((key) => record[key] !== undefined); + const lines = ["{"]; + const lastIndex = keys.length - 1; + for (let index = 0; index <= lastIndex; index++) { + const key = keys[index]; + const childLines = formatObsidianJsonLines(record[key]); + childLines[0] = `${JSON.stringify(key)}:${childLines[0]}`; + const lastChildLineIndex = childLines.length - 1; + for (let lineIndex = 0; lineIndex <= lastChildLineIndex; lineIndex++) { + let line = `\t${childLines[lineIndex]}`; + if (lineIndex === lastChildLineIndex && index !== lastIndex) { + line += ","; + } + lines.push(line); + } + } + lines.push("}"); + return lines; + } + + return [""]; +} + +function isPrimitiveJsonValue(value: unknown): boolean { + return typeof value !== "object"; +} diff --git a/src/CanvasPlugin.ts b/src/CanvasPlugin.ts index 83042f55..a4605e22 100644 --- a/src/CanvasPlugin.ts +++ b/src/CanvasPlugin.ts @@ -8,12 +8,18 @@ import type { CanvasView, ObsidianCanvas, } from "src/CanvasView"; -import type { RelayCanvasView, LiveViewManager } from "src/LiveViews"; +import type { + RelayCanvasView, + DocumentViewer, + LiveViewManager, +} from "src/LiveViews"; import { HasLogging } from "src/debug"; import * as Y from "yjs"; import { ViewHookPlugin } from "./plugins/ViewHookPlugin"; -import { flags } from "./flagManager"; +import type { EditorViewRef } from "./merge-hsm/types"; +import { HSMEditorPlugin } from "./merge-hsm/integration/HSMEditorPlugin"; +import type { Document } from "./Document"; export class CanvasPlugin extends HasLogging { view: CanvasView; @@ -38,14 +44,11 @@ export class CanvasPlugin extends HasLogging { this.trackedEmbedViews = new Set(); this.install(); - // Enable embedded view synchronization if enableLiveEmbeds is true - if (flags().enableLiveEmbeds) { - for (const node of this.getEmbedViews()) { - if (!node.file) { - continue; - } - this.connectEmbedView(node); + for (const node of this.getEmbedViews()) { + if (!node.file) { + continue; } + this.connectEmbedView(node); } } @@ -101,27 +104,279 @@ export class CanvasPlugin extends HasLogging { return this.trackedEmbedViews.has(embedView); } + private createEmbedEditorViewRef(embedView: any): EditorViewRef { + return { + getViewData() { + if (typeof embedView?.getViewData === "function") { + return embedView.getViewData(); + } + + const cmDoc = embedView?.editor?.cm?.state?.doc; + if (typeof cmDoc?.toString === "function") { + return cmDoc.toString(); + } + + if (typeof embedView?.text === "string") { + return embedView.text; + } + + if (typeof embedView?.data === "string") { + return embedView.data; + } + + if (typeof embedView?.lastSavedData === "string") { + return embedView.lastSavedData; + } + + return ""; + }, + }; + } + + private syncEmbedViewToDocument( + document: Document, + viewRef: EditorViewRef, + reason: string, + ): boolean { + try { + if (!document.isWritable) { + return false; + } + + const contents = viewRef.getViewData(); + if (document.localText === contents) { + return false; + } + + const hsm = document.hsm; + if (!hsm) { + return false; + } + + const changes = hsm.computeDiffChanges(document.localText, contents); + this.debug( + "syncing canvas embed view to HSM", + document.path, + reason, + ); + hsm.send({ + type: "CM6_CHANGE", + changes, + docText: contents, + userEvent: "set", + }); + return true; + } catch (error: unknown) { + this.error( + `Error syncing canvas embed during ${reason}:`, + error, + ); + return false; + } + } + + private requestNativeEmbedSave( + embedView: any, + state: { saving: boolean }, + ): void { + state.saving = true; + try { + embedView.requestSave(); + } finally { + state.saving = false; + } + } + + private syncDocumentToEmbedView( + document: Document, + embedView: any, + viewRef: EditorViewRef, + state: { saving: boolean; tracking: boolean }, + reason: string, + ): boolean { + if (typeof embedView?.setViewData !== "function") { + return false; + } + + const contents = document.localText; + if (viewRef.getViewData() === contents) { + state.tracking = true; + return false; + } + + this.debug("syncing canvas embed HSM to view", document.path, reason); + state.saving = true; + try { + embedView.setViewData(contents, false); + } finally { + state.saving = false; + } + this.requestNativeEmbedSave(embedView, state); + state.tracking = true; + return true; + } + private connectEmbedView(embedView: any): void { if (!embedView.file) { return; } + // Only markdown embeds have CM6 editors that need ViewHookPlugin + HSM. + // Canvas embeds render as canvas views, and media (images, SVG, PDF) + // are SyncFiles — neither uses a text editor. + const path: string = embedView.file.path; + if (!path.endsWith(".md")) { + return; + } + this.trackedEmbedViews.add(embedView); this.unsubscribes.push( (() => { + const document = this.relayCanvas.sharedFolder.proxy.getDoc(embedView.file.path); + const viewRef = this.createEmbedEditorViewRef(embedView); + const syncEmbedViewToDocument = this.syncEmbedViewToDocument.bind(this); + const syncDocumentToEmbedView = this.syncDocumentToEmbedView.bind(this); + const logError = this.error.bind(this); const plugin = new ViewHookPlugin( embedView, - this.relayCanvas.sharedFolder.proxy.getDoc(embedView.file.path), + document, ); - plugin.initialize().catch((error) => { - this.error( - "Error initializing ViewHookPlugin for canvas embed:", - error, - ); + const state = { saving: false, tracking: false }; + let observedYText: Y.Text | null = null; + let ytextObserver: + | ((event: Y.YTextEvent, tr: Y.Transaction) => void) + | null = null; + const requestSaveUnsubscribe = getPatcher().patch(embedView, { + requestSave: (old: any) => { + return function (this: any) { + if (!state.saving && !this?.__relaySaving) { + try { + syncEmbedViewToDocument( + document, + viewRef, + "requestSave", + ); + state.tracking = true; + } catch (error: unknown) { + logError( + "Error syncing canvas embed during requestSave:", + error, + ); + } + } + this?.app?.metadataCache?.trigger?.("resolve", this.file); + return old.call(this); + }; + }, }); + const viewer: DocumentViewer = + embedView.leaf ?? Symbol(`canvas-embed:${embedView.file.path}`); + let cancelled = false; + let lockAcquired = false; + + document + .whenReady() + .then(async () => { + if (cancelled) { + return; + } + + try { + const initialContents = viewRef.getViewData(); + if (!document.hsm?.isActive() && initialContents.length > 0) { + // Canvas embeds do not reliably pass through the normal + // TextFileView load hooks before ACQUIRE_LOCK. Seed the + // HSM with the current embed buffer so active entry does + // not reconcile against an empty localDoc. + document.hsm?.send({ + type: "OBSIDIAN_SET_VIEW_DATA", + data: initialContents, + clear: true, + }); + } + this.connectionManager.acquireDocumentLock( + document, + viewRef, + viewer, + ); + lockAcquired = true; + } catch (error: unknown) { + this.error( + "Error acquiring lock for canvas embed:", + error, + ); + return; + } + + const hsm = document.hsm; + if (hsm?.awaitState) { + await hsm.awaitState((state) => state.startsWith("active.")); + if (cancelled) { + return; + } + } + + const localDoc = document.localDoc; + if (localDoc) { + observedYText = localDoc.getText("contents"); + ytextObserver = (_event: Y.YTextEvent, tr: Y.Transaction) => { + if (cancelled || document.destroyed) { + return; + } + if (tr.origin === document || tr.origin === document.hsm) { + return; + } + syncDocumentToEmbedView( + document, + embedView, + viewRef, + state, + "localDoc.observe", + ); + }; + observedYText.observe(ytextObserver); + } + + syncDocumentToEmbedView( + document, + embedView, + viewRef, + state, + "initial-sync", + ); + + const cm = (embedView.editor as any)?.cm; + const hsmEditorPlugin = cm?.plugin?.(HSMEditorPlugin); + hsmEditorPlugin?.initializeIfReady(); + + plugin.initialize().catch((error) => { + this.error( + "Error initializing ViewHookPlugin for canvas embed:", + error, + ); + }); + }) + .catch((error: unknown) => { + this.error( + "Error waiting for canvas embed readiness:", + error, + ); + }); + return () => { + cancelled = true; this.trackedEmbedViews.delete(embedView); + requestSaveUnsubscribe(); + if (observedYText && ytextObserver) { + observedYText.unobserve(ytextObserver); + } plugin.destroy(); + if (lockAcquired) { + this.connectionManager.releaseDocumentLock( + document, + viewer, + ); + } }; })(), ); @@ -218,12 +473,10 @@ export class CanvasPlugin extends HasLogging { this.observeNode((node as CanvasNode).getData()); // Check if this is a newly created embed node that needs ViewHookPlugin - if (flags().enableLiveEmbeds) { - //@ts-ignore - const embedView = node.child; - if (embedView?.file && !this.isEmbedAlreadyTracked(embedView)) { - this.connectEmbedView(embedView); - } + //@ts-ignore + const embedView = node.child; + if (embedView?.file && !this.isEmbedAlreadyTracked(embedView)) { + this.connectEmbedView(embedView); } } this.canvas.markMoved(node); diff --git a/src/ConnectionPool.ts b/src/ConnectionPool.ts index c1c5b220..62025b86 100644 --- a/src/ConnectionPool.ts +++ b/src/ConnectionPool.ts @@ -110,7 +110,7 @@ export class ConnectionPool extends HasLogging { } private processQueue(): void { - const currentTime = this.timeProvider.getTime(); + const currentTime = this.timeProvider.now(); //this.logConnectionStatus(); this.enforceConnectionLimits(); @@ -147,7 +147,7 @@ export class ConnectionPool extends HasLogging { this.connections.set(request.uuid, { uuid: request.uuid, disconnect: request.disconnect, - leaseExpiryTime: this.timeProvider.getTime() + request.lease_s * 1000, + leaseExpiryTime: this.timeProvider.now() + request.lease_s * 1000, }); this.log(`Created queued temporary connection for ${request.uuid}`); @@ -211,7 +211,7 @@ export class ConnectionPool extends HasLogging { this.connections.set(uuid, { uuid, disconnect, - leaseExpiryTime: this.timeProvider.getTime() + lease_s * 1000, + leaseExpiryTime: this.timeProvider.now() + lease_s * 1000, }); this.log(`Created temporary connection for ${uuid}`); return true; diff --git a/src/DeviceManager.ts b/src/DeviceManager.ts new file mode 100644 index 00000000..ad780284 --- /dev/null +++ b/src/DeviceManager.ts @@ -0,0 +1,176 @@ +"use strict"; + +import { Platform } from "obsidian"; +import type { LoginManager } from "./LoginManager"; +import { Observable } from "./observable/Observable"; + +const DEVICE_ID_KEY = "relay-device-id"; + +/** + * Generate a PocketBase-compatible ID. + * Format: 15 characters, lowercase alphanumeric only. + */ +function generatePocketBaseId(): string { + const chars = "abcdefghijklmnopqrstuvwxyz0123456789"; + const array = new Uint8Array(15); + crypto.getRandomValues(array); + return Array.from(array, (byte) => chars[byte % chars.length]).join(""); +} + +/** + * Get platform string for device registration. + */ +function getPlatform(): string { + if (Platform.isIosApp) return "Phone (iOS)"; + if (Platform.isAndroidApp) return "Phone (Android)"; + if (Platform.isMacOS) return "Desktop (macOS)"; + if (Platform.isWin) return "Desktop (Windows)"; + if (Platform.isLinux) return "Desktop (Linux)"; + if (Platform.isMobile) return "Mobile"; + return "Desktop"; +} + +export class DeviceManager extends Observable { + private deviceId: string | null = null; + private registered = false; + + constructor( + private appId: string, + private loginManager: LoginManager, + ) { + super("DeviceManager"); + } + + /** + * Get or create the device ID from localStorage. + */ + getDeviceId(): string { + if (this.deviceId) return this.deviceId; + + let id = localStorage.getItem(DEVICE_ID_KEY); + if (!id) { + id = generatePocketBaseId(); + localStorage.setItem(DEVICE_ID_KEY, id); + this.log("Generated new device ID:", id); + } + this.deviceId = id; + return id; + } + + /** + * Get platform string. + */ + getPlatform(): string { + return getPlatform(); + } + + /** + * Register device and vault with PocketBase. + * Creates records if they don't exist, updates if they do. + */ + async register(): Promise { + if (this.registered) { + this.debug("Already registered this session"); + return; + } + + if (!this.loginManager.loggedIn) { + this.debug("Not logged in, skipping registration"); + return; + } + + const deviceId = this.getDeviceId(); + const platform = this.getPlatform(); + const userId = this.loginManager.user?.id; + + if (!userId) { + this.warn("No user ID available"); + return; + } + + try { + // Register device + await this.registerDevice(deviceId, platform, userId); + + // Register vault + await this.registerVault(this.appId, deviceId, userId); + + this.registered = true; + this.log("Device and vault registered successfully"); + } catch (error) { + this.error("Failed to register device/vault:", error); + } + } + + private async registerDevice( + deviceId: string, + platform: string, + userId: string, + ): Promise { + const pb = this.loginManager.pb; + + try { + // Try to create new device record + await pb.collection("devices").create({ + id: deviceId, + name: platform, + platform: platform, + user: userId, + }); + this.log("Created new device record:", deviceId); + } catch (e: any) { + // Record may already exist, try to update + if (e.status === 400 || e.status === 409) { + try { + await pb.collection("devices").update(deviceId, { + platform: platform, + user: userId, + }); + this.log("Updated existing device record:", deviceId); + } catch (updateError) { + this.error("Failed to update device:", updateError); + throw updateError; + } + } else { + throw e; + } + } + } + + private async registerVault( + vaultId: string, + deviceId: string, + userId: string, + ): Promise { + const pb = this.loginManager.pb; + + try { + // Try to create new vault record + await pb.collection("vaults").create({ + id: vaultId, + device: deviceId, + user: userId, + }); + this.log("Created new vault record:", vaultId); + } catch (e: any) { + // Record may already exist, try to update + if (e.status === 400 || e.status === 409) { + try { + await pb.collection("vaults").update(vaultId, { + device: deviceId, + }); + this.log("Updated existing vault record:", vaultId); + } catch (updateError) { + this.error("Failed to update vault:", updateError); + throw updateError; + } + } else { + throw e; + } + } + } + + override destroy(): void { + super.destroy(); + } +} diff --git a/src/DiskBuffer.ts b/src/DiskBuffer.ts index ca074b6c..7f1a3b60 100644 --- a/src/DiskBuffer.ts +++ b/src/DiskBuffer.ts @@ -48,73 +48,3 @@ export class DiskBuffer implements TFile { return this.path.substring(0, this.path.lastIndexOf("/")); } } -export class DiskBufferStore { - private dbName = "RelayDiskBuffer"; - private storeName = "diskBuffers"; - private dbPromise: Promise | null = null; - - private async getDB(): Promise { - if (!this.dbPromise) { - this.dbPromise = this.openDB(); - } - return this.dbPromise; - } - - private async openDB(): Promise { - return new Promise((resolve, reject) => { - const request = indexedDB.open(this.dbName, 1); - request.onerror = () => reject(request.error); - request.onsuccess = () => resolve(request.result); - request.onupgradeneeded = (event) => { - const db = (event.target as IDBOpenDBRequest).result; - db.createObjectStore(this.storeName, { keyPath: "guid" }); - }; - }); - } - - async saveDiskBuffer(guid: string, contents: string): Promise { - const db = await this.getDB(); - return new Promise((resolve, reject) => { - try { - const transaction = db.transaction(this.storeName, "readwrite"); - const store = transaction.objectStore(this.storeName); - const request = store.put({ guid, contents }); - request.onerror = () => reject(request.error); - request.onsuccess = () => resolve(); - } catch (e) { - reject(e); - } - }); - } - - async loadDiskBuffer(guid: string): Promise { - const db = await this.getDB(); - return new Promise((resolve, reject) => { - try { - const transaction = db.transaction(this.storeName, "readonly"); - const store = transaction.objectStore(this.storeName); - const request = store.get(guid); - request.onerror = () => reject(request.error); - request.onsuccess = () => - resolve(request.result ? request.result.contents : null); - } catch (e) { - reject(e); - } - }); - } - - async removeDiskBuffer(guid: string): Promise { - const db = await this.getDB(); - return new Promise((resolve, reject) => { - try { - const transaction = db.transaction(this.storeName, "readwrite"); - const store = transaction.objectStore(this.storeName); - const request = store.delete(guid); - request.onerror = () => reject(request.error); - request.onsuccess = () => resolve(); - } catch (e) { - reject(e); - } - }); - } -} diff --git a/src/Document.ts b/src/Document.ts index 4099617b..7ed23113 100644 --- a/src/Document.ts +++ b/src/Document.ts @@ -1,6 +1,5 @@ "use strict"; import { IndexeddbPersistence } from "./storage/y-indexeddb"; -import * as idb from "lib0/indexeddb"; import * as Y from "yjs"; import { HasProvider } from "./HasProvider"; import { LoginManager } from "./LoginManager"; @@ -8,14 +7,22 @@ import { S3Document, S3Folder, S3RN, S3RemoteDocument } from "./S3RN"; import { SharedFolder } from "./SharedFolder"; import type { TFile, Vault, TFolder } from "obsidian"; import { debounce } from "obsidian"; -import { DiskBuffer, DiskBufferStore } from "./DiskBuffer"; import type { Unsubscriber } from "./observable/Observable"; import { Dependency } from "./promiseUtils"; -import { flags, withFlag } from "./flagManager"; +import { withFlag } from "./flagManager"; import { flag } from "./flags"; import type { HasMimeType, IFile } from "./IFile"; import { getMimeType } from "./mimetypes"; -import { diffMatchPatch } from "./y-diffMatchPatch"; +import type { MergeHSM } from "./merge-hsm/MergeHSM"; +import type { EditorViewRef } from "./merge-hsm/types"; +import { + ProviderIntegration, + type YjsProvider, +} from "./merge-hsm/integration/ProviderIntegration"; +import { reconnectProvider } from "./merge-hsm/integration/ProviderLifecycle"; +import { generateHash } from "./hashing"; +import { trackAsyncCleanup } from "./reloadUtils"; +import { trackPromise } from "./trackPromise"; export function isDocument(file?: IFile): file is Document { return file instanceof Document; @@ -23,7 +30,7 @@ export function isDocument(file?: IFile): file is Document { export class Document extends HasProvider implements IFile, HasMimeType { private _parent: SharedFolder; - private _persistence: IndexeddbPersistence; + private _persistence: IndexeddbPersistence | null = null; whenSyncedPromise: Dependency | null = null; persistenceSynced: boolean = false; _awaitingUpdates?: boolean; @@ -40,11 +47,37 @@ export class Document extends HasProvider implements IFile, HasMimeType { mtime: number; size: number; }; - _diskBuffer?: DiskBuffer; - _diskBufferStore?: DiskBufferStore; + destroyed = false; unsubscribes: Unsubscriber[] = []; pendingOps: ((data: string) => string)[] = []; + /** + * MergeHSM instance for this document. + * Created in the constructor and cleared on destroy(). + */ + private _hsm: MergeHSM | null; + + /** + * ProviderIntegration instance for bridging HSM with the provider. + * Created when lock is acquired, destroyed when released. + */ + private _providerIntegration: ProviderIntegration | null = null; + private _idleProviderIntegrationRefs = 0; + private _activeProviderIntegration = false; + + private recordProviderSyncedRemoteHead = (snapshot: Uint8Array): void => { + this.sharedFolder.mergeManager?.seedServerAdvertisedSnapshotFromBytes( + this.guid, + snapshot, + ); + }; + + /** + * Flag to track when we're in the middle of our own save operation. + * Used to distinguish our writes from external modifications. + */ + private _isSaving: boolean = false; + constructor( path: string, guid: string, @@ -55,6 +88,7 @@ export class Document extends HasProvider implements IFile, HasMimeType { ? new S3RemoteDocument(parent.relayId, parent.guid, guid) : new S3Document(parent.guid, guid); super(guid, s3rn, parent.tokenStore, loginManager); + this.timeProvider = parent.timeProvider; this._parent = parent; this.path = path; this.name = "[CRDT] " + path.split("/").pop() || ""; @@ -67,8 +101,51 @@ export class Document extends HasProvider implements IFile, HasMimeType { mtime: Date.now(), size: 0, }; - this._diskBufferStore = this.sharedFolder.diskBufferStore; + // Initialize HSM immediately so it's always available for filtering disk changes. + // The HSM starts in loading state and transitions to idle once persistence loads. + // Document owns the HSM - use ensureHSM() which uses MergeManager as a factory. + const mergeManager = this.sharedFolder?.mergeManager; + if (!mergeManager) { + throw new Error("no merge manager"); + } + + // Create HSM using factory + this._hsm = mergeManager.createHSM({ + guid: this.guid, + getPath: () => this.path, + remoteDoc: this.isRemoteDocLoaded ? this.ydoc : null, + getDiskContent: () => this.readDiskContent(), + getCurrentDiskMetadata: () => + this.sharedFolder.getCurrentDiskMetadata(this), + isFolderConnected: () => this.sharedFolder.connected, + getPersistenceMetadata: () => ({ + path: this.path, + relay: this.sharedFolder.relayId || "", + appId: this.sharedFolder.appId, + s3rn: this.s3rn ? S3RN.encode(this.s3rn) : "", + }), + }); + + // Subscribe to effects + this.unsubscribes.push( + this._hsm.subscribe((effect) => { + this.handleEffect(effect); + }), + ); + + // Subscribe to state changes for sync status updates + this.unsubscribes.push( + this._hsm.onStateChange(() => { + const syncStatus = this._hsm?.getSyncStatus(); + if (!syncStatus) { + return; + } + mergeManager.updateSyncStatus(this.guid, syncStatus); + }), + ); + // Notify MergeManager for hibernation tracking + mergeManager.notifyHSMCreated(this.guid); this.unsubscribes.push( this._parent.subscribe(this.path, (state) => { if (state.intent === "disconnected") { @@ -78,46 +155,41 @@ export class Document extends HasProvider implements IFile, HasMimeType { ); this.setLoggers(`[SharedDoc](${this.path})`); - try { - const key = `${this.sharedFolder.appId}-relay-doc-${this.guid}`; - this._persistence = new IndexeddbPersistence(key, this.ydoc); - } catch (e) { - this.warn("Unable to open persistence.", this.guid); - console.error(e); - throw e; - } - - this.whenSynced().then(() => { - const statsObserver = (event: Y.YTextEvent) => { - const origin = event.transaction.origin; - if (event.changes.keys.size === 0) return; - if (origin == this) return; - this.updateStats(); - }; - this.ytext.observe(statsObserver); - this.unsubscribes.push(() => { - this.ytext?.unobserve(statsObserver); - }); - this.updateStats(); - try { - this._persistence.set("path", this.path); - this._persistence.set("relay", this.sharedFolder.relayId || ""); - this._persistence.set("appId", this.sharedFolder.appId); - this._persistence.set("s3rn", S3RN.encode(this.s3rn)); - } catch (e) { - // pass - } - (async () => { - const serverSynced = await this.getServerSynced(); - if (!serverSynced) { - await this.onceProviderSynced(); - await this.markSynced(); - } - })(); - }); + // need to port this to the HSM + // this.whenSynced().then(() => { + // const statsObserver = (event: Y.YTextEvent) => { + // const origin = event.transaction.origin; + // if (event.changes.keys.size === 0) return; + // if (origin == this) return; + // this.updateStats(); + // }; + // this.ytext.observe(statsObserver); + // this.unsubscribes.push(() => { + // this.ytext?.unobserve(statsObserver); + // }); + // this.updateStats(); + // try { + // this._persistence!.set("path", this.path); + // this._persistence!.set("relay", this.sharedFolder.relayId || ""); + // this._persistence!.set("appId", this.sharedFolder.appId); + // this._persistence!.set("s3rn", S3RN.encode(this.s3rn)); + // } catch (e) { + // // pass + // } + + // (async () => { + // const serverSynced = await this.getServerSynced(); + // if (!serverSynced) { + // await this.onceProviderSynced(); + // await this.markSynced(); + // } + // })(); + // }); withFlag(flag.enableDeltaLogging, () => { + // Only attach observer when remoteDoc is loaded (avoid triggering lazy creation) + if (!this.isRemoteDocLoaded) return; const logObserver = (event: Y.YTextEvent) => { let log = ""; log += `Transaction origin: ${event.transaction.origin} ${event.transaction.origin?.constructor?.name}\n`; @@ -144,10 +216,11 @@ export class Document extends HasProvider implements IFile, HasMimeType { this.updateStats(); } - async process(fn: (data: string) => string) { - if (this.tfile && flags().enableAutomaticDiffResolution) { - this.pendingOps.push(fn); + process(fn: (data: string) => string): boolean { + if (this._hsm) { + this._hsm.registerMachineEdit(fn); } + return false; } public get parent(): TFolder | null { @@ -157,6 +230,148 @@ export class Document extends HasProvider implements IFile, HasMimeType { public get sharedFolder(): SharedFolder { return this._parent; } + + /** + * Get the MergeHSM instance for this document. + * Returns null if HSM active mode is not enabled or lock not acquired. + */ + public get hsm(): MergeHSM | null { + return this._hsm; + } + + /** + * Create the remote YDoc/provider if needed. + * Seed updates are accepted only when server-advertised snapshot metadata + * proves they cannot introduce local-only CRDT state. + */ + ensureRemoteDoc(): Y.Doc { + const isNew = !this.isRemoteDocLoaded; + const doc = super.ensureRemoteDoc(); + if (isNew) { + this.seedRemoteDocFromServerAdvertisedSnapshot(doc); + } + return doc; + } + + private seedRemoteDocFromServerAdvertisedSnapshot(remoteDoc: Y.Doc): void { + const localDoc = this._hsm?.getLocalDoc(); + if (!localDoc) return; + + const seedUpdate = + this.sharedFolder.mergeManager?.getRemoteDocSeedUpdateFromLocalDoc( + this.guid, + localDoc, + ) ?? null; + if (!seedUpdate) { + return; + } + + Y.applyUpdate(remoteDoc, seedUpdate, this._provider); + } + + /** + * Acquire lock on this document for active editing. + * Transitions HSM from idle to active mode. + * + * Editor content flows in via CM6_CHANGE events (from HSMEditorPlugin) — + * not passed here, because callers can't reliably observe post-setViewData + * content at this moment (Obsidian's view-reuse window produces stale reads). + */ + acquireLock(editorViewRef: EditorViewRef): MergeHSM { + const mergeManager = this.sharedFolder.mergeManager; + if (!mergeManager) { + throw new Error("no merge manager"); + } + const hsm = this._hsm; + if (!hsm) { + throw new Error("no hsm"); + } + + // Idempotent fast path: if this document is already active and has an + // integration bridge, keep that lock and simply ensure connectivity. + if (mergeManager.isActive(this.guid) && this._providerIntegration) { + this._activeProviderIntegration = true; + this.connect(); + return hsm; + } + + // Ensure remoteDoc and provider exist before entering active mode. + // This wakes the document from hibernation if needed. + const remoteDoc = this.ensureRemoteDoc(); + hsm.setRemoteDoc(remoteDoc); + + hsm.send({ + type: "ACQUIRE_LOCK", + editorViewRef, + }); + mergeManager.markActive(this.guid); + + // Create ProviderIntegration BEFORE awaiting so it can deliver + // PROVIDER_SYNCED during the entering phase (needed for empty-IDB flow). + if (!this._providerIntegration) { + this._providerIntegration = new ProviderIntegration( + hsm, + remoteDoc, + this._provider! as YjsProvider, + { onSyncedRemoteHead: this.recordProviderSyncedRemoteHead }, + ); + } + this._activeProviderIntegration = true; + + // Ensure provider is connected. After idle-mode fork + // reconciliation, destroyIdleProviderIntegration disconnects + // the provider. Without reconnecting, SYNC_TO_REMOTE updates + // from conflict resolution are buffered but never sent. + // connect() is a no-op if already connected. + this.connect(); + + return hsm; + } + + /** + * Release lock on this document. + * Transitions HSM from active back to idle mode. + * Call this when editor closes. + */ + releaseLock(): void { + this._activeProviderIntegration = false; + this.destroyProviderIntegrationIfUnused(false); + + // Guard: sharedFolder may be null if document was orphaned (file moved out of folder) + const mergeManager = this.sharedFolder?.mergeManager; + if (mergeManager) { + // MergeManager.unload() sends RELEASE_LOCK and runs async IDB cleanup. + const p = mergeManager.unload(this.guid); + trackAsyncCleanup(p); + } + } + + /** + * Get the HSM sync status for this document. + * Returns the status if HSM is available, or null otherwise. + * This can be used instead of checkStale() when HSM is enabled. + */ + getHSMSyncStatus(): import("./merge-hsm/types").SyncStatus | null { + const mergeManager = this.sharedFolder?.mergeManager; + if (!mergeManager) { + return null; + } + return mergeManager.syncStatus.get(this.guid) ?? null; + } + + /** + * Check if the document has a conflict according to HSM. + * Returns true if HSM indicates a conflict, false if synced/pending, + * or null if HSM is not available. + */ + hasHSMConflict(): boolean | null { + const status = this.getHSMSyncStatus(); + if (!status) { + return null; + } + return status.status === "conflict"; + } + public get tfile(): TFile | null { if (!this._tfile) { this._tfile = this.getTFile(); @@ -179,136 +394,131 @@ export class Document extends HasProvider implements IFile, HasMimeType { return this.ytext.toString(); } - public async diskBuffer(read = false): Promise { - if (read || this._diskBuffer === undefined) { - let fileContents: string; - try { - const storedContents = await this._parent.diskBufferStore - .loadDiskBuffer(this.guid) - .catch((e) => { - return null; - }); - if (storedContents !== null && storedContents !== "") { - fileContents = storedContents; - } else { - fileContents = await this._parent.read(this); - } - return this.setDiskBuffer(fileContents.replace(/\r\n/g, "\n")); - } catch (e) { - console.error(e); - throw e; - } + // =========================================================================== + // HSM-aware accessors (localDoc only - no fallback to remoteDoc) + // =========================================================================== + + /** + * Get the HSM's localDoc when available (active mode only). + * Returns null when HSM is not in active mode or not available. + * + * IMPORTANT: All editor operations should use localDoc, not ydoc (remoteDoc). + * Writing to ydoc directly causes corruption. + */ + public get localDoc(): Y.Doc | null { + return this._hsm?.getLocalDoc() ?? null; + } + + /** + * Get the Y.Text from HSM's localDoc. + * @throws Error if HSM is not in active mode (no localDoc available) + */ + public get localYText(): Y.Text { + const doc = this.localDoc; + if (!doc) { + throw new Error( + `Document ${this.path}: Cannot access localYText - HSM not in active mode.`, + ); } - return this._diskBuffer; - } - - setDiskBuffer(contents: string): TFile { - if (this._diskBuffer) { - this._diskBuffer.contents = contents; - } else { - this._diskBuffer = new DiskBuffer( - this._parent.vault, - "local disk", - contents, + return doc.getText("contents"); + } + + /** + * Get text content from HSM's localDoc. + * @throws Error if HSM is not in active mode (no localDoc available) + */ + public get localText(): string { + return this.localYText.toString(); + } + + /** + * Get the YDoc that should be used for write operations. + * Returns localDoc when in active mode, throws when HSM not in active mode. + * + * IMPORTANT: Writing to ydoc (remoteDoc) directly causes corruption. + * All write operations must go through this method or the HSM. + * + * @throws Error if HSM is not in active mode (no localDoc available) + */ + public getWritableDoc(): Y.Doc { + const localDoc = this.localDoc; + if (!localDoc) { + throw new Error( + `Document ${this.path}: Cannot write - HSM not in active mode. ` + + `Writing to ydoc (remoteDoc) directly causes corruption.`, ); } - this._parent.diskBufferStore - .saveDiskBuffer(this.guid, contents) - .catch((e) => {}); - return this._diskBuffer; + return localDoc; } - async clearDiskBuffer(): Promise { - if (this._diskBuffer) { - this._diskBuffer.contents = ""; - this._diskBuffer = undefined; - } - await this._parent.diskBufferStore - .removeDiskBuffer(this.guid) - .catch((e) => {}); + /** + * Check if the document is in a writable state (HSM active mode). + */ + public get isWritable(): boolean { + return this.localDoc !== null; } - public async checkStale(): Promise { - await this.whenSynced(); - const diskBuffer = await this.diskBuffer(true); - const contents = (diskBuffer as DiskBuffer).contents; - const response = await this.sharedFolder.backgroundSync.downloadItem(this); - const updateBytes = new Uint8Array(response.arrayBuffer); - - Y.applyUpdate(this.ydoc, updateBytes); - const stale = this.text !== contents; - - const og = this.text; - let text = og; - - const applied: ((data: string) => string)[] = []; - for (const fn of this.pendingOps) { - text = fn(text); - applied.push(fn); - - if (text == contents) { - this.clearDiskBuffer(); - if (og == this.text) { - diffMatchPatch(this.ydoc, text, this); - } else { - if (flags().enableDeltaLogging) { - this.warn( - "diffMatchPatch solution is stale an cannot be applied", - text, - this.text, - ); - } else { - this.log("diffMatchPatch solution is stale an cannot be applied"); - } - return true; - } - this.pendingOps = []; - return true; - } + async connect(): Promise { + if (this.destroyed) { + return false; } - this.pendingOps = []; - if (!stale) { - this.clearDiskBuffer(); + + const sharedFolder = this._parent; + if (!sharedFolder || sharedFolder.destroyed) { + return false; } - return stale; - } - async connect(): Promise { - if (this.sharedFolder.s3rn instanceof S3Folder) { + if (sharedFolder.s3rn instanceof S3Folder) { // Local only return false; } else if (this.s3rn instanceof S3Document) { // convert to remote document - if (this.sharedFolder.relayId) { + if (sharedFolder.relayId) { this.s3rn = new S3RemoteDocument( - this.sharedFolder.relayId, - this.sharedFolder.guid, + sharedFolder.relayId, + sharedFolder.guid, this.guid, ); } else { - this.s3rn = new S3Document(this.sharedFolder.guid, this.guid); + this.s3rn = new S3Document(sharedFolder.guid, this.guid); } } - return ( - this.sharedFolder.shouldConnect && - this.sharedFolder.connect().then((connected) => { - return super.connect(); - }) - ); + + if (!sharedFolder.shouldConnect) { + return false; + } + + const folderConnected = await sharedFolder.connect().catch(() => false); + if ( + !folderConnected || + this.destroyed || + !this._parent || + this._parent.destroyed + ) { + return false; + } + + return super.connect(); } public get ready(): boolean { - return this._persistence.isReady(this.synced); + return this.persistenceSynced && this._awaitingUpdates === false; } hasLocalDB(): boolean { - return this._persistence.hasServerSync || this._persistence.hasUserData(); + return this._hsm?.hasPersistenceUserData() ?? false; } async awaitingUpdates(): Promise { await this.whenSynced(); await this.getServerSynced(); - if (!this._awaitingUpdates) { + if (this._awaitingUpdates !== undefined) { + return this._awaitingUpdates; + } + // If folder has synced with server (or is authoritative, which sets serverSynced), we don't need to wait + const folderServerSynced = await this.sharedFolder.getServerSynced(); + if (folderServerSynced) { + this._awaitingUpdates = false; return false; } this._awaitingUpdates = !this.hasLocalDB(); @@ -317,16 +527,20 @@ export class Document extends HasProvider implements IFile, HasMimeType { async whenReady(): Promise { const promiseFn = async (): Promise => { + await this.whenSynced(); const awaitingUpdates = await this.awaitingUpdates(); if (awaitingUpdates) { // If this is a brand new shared folder, we want to wait for a connection before we start reserving new guids for local files. this.log("awaiting updates"); this.connect(); - await this.onceConnected(); + await trackPromise(`connected:${this.guid}`, this.onceConnected()); this.log("connected"); - await this.onceProviderSynced(); + await trackPromise( + `providerSync:${this.guid}`, + this.onceProviderSynced(), + ); this.log("synced"); - return this; + this._awaitingUpdates = false; } return this; }; @@ -334,33 +548,29 @@ export class Document extends HasProvider implements IFile, HasMimeType { this.readyPromise || new Dependency(promiseFn, (): [boolean, Document] => { return [this.ready, this]; - }); - return this.readyPromise.getPromise(); + }, this.timeProvider); + return trackPromise( + `doc:whenReady:${this.guid}`, + this.readyPromise.getPromise(), + ); } whenSynced(): Promise { const promiseFn = async (): Promise => { await this.sharedFolder.whenSynced(); - - return new Promise((resolve) => { - if (this.persistenceSynced) { - resolve(); - return; - } - - this._persistence.once("synced", () => { - this.persistenceSynced = true; - resolve(); - }); - }); + await this._hsm?.awaitPersistenceReady(); + this.persistenceSynced = true; }; this.whenSyncedPromise = this.whenSyncedPromise || new Dependency(promiseFn, (): [boolean, void] => { return [this.persistenceSynced, undefined]; - }); - return this.whenSyncedPromise.getPromise(); + }, this.timeProvider); + return trackPromise( + `doc:whenSynced:${this.guid}`, + this.whenSyncedPromise.getPromise(), + ); } async hasKnownPeers(): Promise { @@ -372,7 +582,7 @@ export class Document extends HasProvider implements IFile, HasMimeType { return getMimeType(this.path); } - save() { + async save() { if (!this.tfile) { return; } @@ -380,26 +590,44 @@ export class Document extends HasProvider implements IFile, HasMimeType { this.warn("skipping save for pending delete", this.path); return; } - this.vault.modify(this.tfile, this.text); - this.warn("file saved", this.path); - } - requestSave = debounce(this.save, 2000); - - async markOrigin(origin: "local" | "remote"): Promise { - await this._persistence.setOrigin(origin); + // Mark that we're saving to distinguish from external modifications + this._isSaving = true; + try { + // Use localDoc content when in HSM active mode; ydoc (remoteDoc) is stale there. + const contents = this.localDoc ? this.localText : this.text; + await this.vault.modify(this.tfile, contents); + this.warn("file saved", this.path); + + // Notify HSM of save completion with new mtime and hash. + // Use optional chaining so async save tails don't emit after teardown. + if (this.tfile) { + const mtime = this.tfile.stat.mtime; + const encoder = new TextEncoder(); + const hash = await generateHash(encoder.encode(contents).buffer); + this._hsm?.send({ type: "SAVE_COMPLETE", mtime, hash }); + } + } finally { + this._isSaving = false; + } } - async getOrigin(): Promise<"local" | "remote" | undefined> { - return this._persistence.getOrigin(); + /** + * Check if the document is currently being saved by us. + * Used to distinguish our writes from external modifications. + */ + get isSaving(): boolean { + return this._isSaving; } + requestSave = debounce(this.save, 2000); + async markSynced(): Promise { - await this._persistence.markServerSynced(); + await this._hsm?.markPersistenceServerSynced(); } async getServerSynced(): Promise { - return this._persistence.getServerSynced(); + return (await this._hsm?.getPersistenceServerSynced()) ?? false; } static checkExtension(vpath: string): boolean { @@ -407,21 +635,30 @@ export class Document extends HasProvider implements IFile, HasMimeType { } destroy() { + this.destroyed = true; (this.requestSave as unknown as { cancel?: () => void }).cancel?.(); this.unsubscribes.forEach((unsubscribe) => { unsubscribe(); }); - super.destroy(); - this.ydoc.destroy(); - if (this._diskBuffer) { - this._diskBuffer.contents = ""; - this._diskBuffer = undefined; + + // Release HSM lock if held + this.releaseLock(); + + // The HSM's cleanup invoke closes per-document IDB asynchronously. + // Track it so close failures are logged. + if (this._hsm) { + const p = this._hsm.awaitAsync('cleanup').catch(() => {}); + trackAsyncCleanup(p, `doc:cleanup:${this.guid}`); } - this._diskBufferStore = null as any; + + super.destroy(); + // Note: super.destroy() calls destroyRemoteDoc() which handles ydoc cleanup. + // Do NOT call this.ydoc.destroy() here — it would trigger lazy creation. this.whenSyncedPromise?.destroy(); this.whenSyncedPromise = null as any; this.readyPromise?.destroy(); this.readyPromise = null as any; + this._hsm = null; this._parent = null as any; } @@ -430,24 +667,234 @@ export class Document extends HasProvider implements IFile, HasMimeType { } public async cleanup(): Promise { - this._diskBufferStore?.removeDiskBuffer(this.guid); + this.sharedFolder?.mergeManager?.notifyHSMDestroyed(this.guid); } // Helper method to update file stats private updateStats(): void { this.stat.mtime = Date.now(); - this.stat.size = this.text.length; + // Only access text if remoteDoc is loaded (avoid triggering lazy creation) + if (this.isRemoteDocLoaded) { + this.stat.size = this.text.length; + } } - // Additional methods that might be useful - public async write(content: string): Promise { - this.ytext.delete(0, this.ytext.length); - this.ytext.insert(0, content); - this.updateStats(); + // =========================================================================== + // HSM Effect Handling + // =========================================================================== + + /** + * Read current disk content for the HSM. + * Used as diskLoader callback when creating HSM. + */ + async readDiskContent(): Promise<{ + content: string; + hash: string; + mtime: number; + }> { + const tfile = this.tfile; + if (!tfile) { + throw new Error( + `[Document] Cannot read disk content for ${this.path}: TFile not found`, + ); + } + const content = await this.vault.read(tfile); + const encoder = new TextEncoder(); + const hash = await generateHash(encoder.encode(content).buffer); + return { content, hash, mtime: tfile.stat.mtime }; + } + + /** + * Handle effects emitted by the HSM. + * Called by HSM subscriber in ensureHSM(). + */ + async handleEffect( + effect: import("./merge-hsm/types").MergeEffect, + ): Promise { + switch (effect.type) { + case "WRITE_DISK": + await this.handleWriteDisk(effect.contents, effect.mtime); + break; + case "PERSIST_STATE": + await this.handlePersistState(effect.state); + break; + // Other effects (DISPATCH_CM6, STATUS_CHANGED, etc.) are handled elsewhere + } } - public async append(content: string): Promise { - this.ytext.insert(this.ytext.length, content); - this.updateStats(); + private async handleWriteDisk( + contents: string, + mtime?: number, + ): Promise { + const tfile = this.tfile; + if (!tfile) { + this.warn("[handleEffect:WRITE_DISK] TFile not found, cannot write"); + return; + } + if (this.sharedFolder.isPendingDelete(this.path)) { + this.warn( + "[handleEffect:WRITE_DISK] Skipping write for pending delete", + this.path, + ); + return; + } + + this._isSaving = true; + try { + const options = mtime !== undefined ? { mtime } : undefined; + await this.vault.modify(tfile, contents, options); + this.debug?.("[handleEffect:WRITE_DISK] Wrote to disk", this.path); + + // Notify HSM of save completion with new mtime and hash. + // Use optional chaining so async write tails don't emit after teardown. + const encoder = new TextEncoder(); + const hash = await generateHash(encoder.encode(contents).buffer); + this._hsm?.send({ + type: "SAVE_COMPLETE", + mtime: tfile.stat.mtime, + hash, + }); + } finally { + this._isSaving = false; + } + } + + private async handlePersistState( + _state: import("./merge-hsm/types").PersistedMergeState, + ): Promise { + // MergeManager.handleHSMEffect handles both LCA cache updates + // and IDB persistence via onEffect. No action needed here — + // Document's subscriber exists for other effect types only. + } + + /** + * Connect the provider for idle-mode fork reconciliation. + * Creates a temporary ProviderIntegration so the HSM receives + * CONNECTED/PROVIDER_SYNCED events and SYNC_TO_REMOTE effects + * flow through the live WebSocket. + * + * Cleanup: call destroyIdleProviderIntegration() or releaseLock() + * when the provider is no longer needed (e.g. on hibernate). + */ + async connectForForkReconcile(): Promise { + const hsm = this._hsm; + if (!hsm) return; + if (!this.sharedFolder.shouldConnect) return; + + const acquiredIntegration = this.ensureIdleProviderIntegration({ + freshRemoteDoc: hsm.hasFork(), + }); + let unsubscribeState: (() => void) | null = null; + const cleanupIfDone = () => { + if (hsm.matches("idle.localAhead")) return; + unsubscribeState?.(); + unsubscribeState = null; + if (!hsm.isActive()) { + this.destroyIdleProviderIntegration(); + } + }; + unsubscribeState = hsm.onStateChange(cleanupIfDone); + this.unsubscribes.push(() => unsubscribeState?.()); + const connected = await this.connect(); + if (!connected) { + unsubscribeState?.(); + unsubscribeState = null; + if (acquiredIntegration && !hsm.isActive()) { + this.destroyIdleProviderIntegration(); + } + return; + } + + // Tear down when transitioning to another idle state (fork resolved + // or diverged). The transition may already have happened while connect() + // was awaiting the provider, so check once after connect resolves too. + cleanupIfDone(); + } + + /** + * Tear down idle-mode provider integration (created by connectForForkReconcile). + * Called during hibernation to clean up the WebSocket connection. + */ + destroyIdleProviderIntegration(): void { + if ((this._idleProviderIntegrationRefs ?? 0) > 0) { + this._idleProviderIntegrationRefs--; + } + if (!this.userLock) { + this.destroyProviderIntegrationIfUnused(true); + } + } + + /** + * Ensure the HSM is attached to a live remoteDoc/provider bridge while the + * document stays in idle mode. + * + * Returns true if this call acquired an idle integration lease, false + * if the document is not HSM-backed. + */ + ensureIdleProviderIntegration(options?: { freshRemoteDoc?: boolean }): boolean { + const hsm = this._hsm; + if (!hsm) return false; + this._idleProviderIntegrationRefs = + (this._idleProviderIntegrationRefs ?? 0) + 1; + + const freshRemoteDoc = options?.freshRemoteDoc ?? false; + if (freshRemoteDoc) { + const result = reconnectProvider({ + hsm, + integration: this._providerIntegration, + createFreshRemoteDoc: () => this.ensureRemoteDoc(), + destroyCurrentRemoteDoc: () => this.destroyRemoteDoc(), + createAndConnectProvider: (_remoteDoc) => { + void this.connect(); + return this._provider as YjsProvider; + }, + providerIntegrationOptions: { + onSyncedRemoteHead: this.recordProviderSyncedRemoteHead, + }, + }); + this._providerIntegration = result.integration; + return true; + } + + if (this._providerIntegration) { + return true; + } + + const remoteDoc = this.ensureRemoteDoc(); + hsm.setRemoteDoc(remoteDoc); + if (!this._provider) { + this._idleProviderIntegrationRefs--; + return false; + } + + this._providerIntegration = new ProviderIntegration( + hsm, + remoteDoc, + this._provider as YjsProvider, + { onSyncedRemoteHead: this.recordProviderSyncedRemoteHead }, + ); + return true; + } + + private destroyProviderIntegrationIfUnused(disconnect: boolean): void { + if ( + this._providerIntegration && + !this._activeProviderIntegration && + (this._idleProviderIntegrationRefs ?? 0) === 0 + ) { + this._providerIntegration.destroy(); + this._providerIntegration = null; + if (disconnect) { + this.disconnect(); + } + } + } + + /** + * Whether this document has an active provider integration + * (either from acquireLock or connectForForkReconcile). + */ + hasProviderIntegration(): boolean { + return this._providerIntegration !== null; } } diff --git a/src/EndpointManager.ts b/src/EndpointManager.ts index 89f43763..88b714e5 100644 --- a/src/EndpointManager.ts +++ b/src/EndpointManager.ts @@ -236,7 +236,7 @@ export class EndpointManager { // Wrap validation in a timeout promise const validationPromise = this.performTenantValidation(activeTenant.tenantUrl); const timeoutPromise = new Promise((_, reject) => - setTimeout(() => reject(new Error(`Validation timed out after ${timeoutMs}ms`)), timeoutMs) + window.setTimeout(() => reject(new Error(`Validation timed out after ${timeoutMs}ms`)), timeoutMs) ); const result = await Promise.race([validationPromise, timeoutPromise]); @@ -826,7 +826,7 @@ export class EndpointManager { // Wrap validation in a timeout promise const validationPromise = this.performTestValidation(tenantUrl); const timeoutPromise = new Promise((_, reject) => - setTimeout(() => reject(new Error(`Validation timed out after ${timeoutMs}ms`)), timeoutMs) + window.setTimeout(() => reject(new Error(`Validation timed out after ${timeoutMs}ms`)), timeoutMs) ); const result = await Promise.race([validationPromise, timeoutPromise]); diff --git a/src/FileLogDetails.ts b/src/FileLogDetails.ts new file mode 100644 index 00000000..8e37b4bf --- /dev/null +++ b/src/FileLogDetails.ts @@ -0,0 +1,31 @@ +import type { IFile } from "./IFile"; + +export function formatDuplicateGuidLog( + existing: IFile, + incoming: IFile, +): string { + const guid = incoming.guid || existing.guid || ""; + return [ + "duplicate guid", + `guid=${quoteLogValue(guid)}`, + `existing=${formatFileLogDetails(existing)}`, + `incoming=${formatFileLogDetails(incoming)}`, + ].join(" "); +} + +function formatFileLogDetails(file: IFile): string { + return [ + fileType(file), + `(guid=${quoteLogValue(file.guid)}, `, + `path=${quoteLogValue(file.path)})`, + ].join(""); +} + +function fileType(file: IFile): string { + const type = file.constructor?.name; + return type && type !== "Object" ? type : "IFile"; +} + +function quoteLogValue(value: string): string { + return value ? JSON.stringify(value) : ""; +} diff --git a/src/Frontmatter.ts b/src/Frontmatter.ts index 631787dd..30cfcf0d 100644 --- a/src/Frontmatter.ts +++ b/src/Frontmatter.ts @@ -9,29 +9,28 @@ export function updateFrontMatter( newEntry: Frontmatter, ): string { const parsed = matter(markdownString); + const data = parsed.data as Frontmatter; for (const key in newEntry) { - parsed.data[key] = newEntry[key]; + data[key] = newEntry[key]; } - const result = matter.stringify(parsed.content, parsed.data); + const result = matter.stringify(parsed.content, data); return result.slice(0, -1); // remove trailing \n } export function hasKey(markdownString: string, keyMatch: string) { const parsed = matter(markdownString); - return parsed.data.hasOwnProperty(keyMatch); + const data = parsed.data as Frontmatter; + return Object.prototype.hasOwnProperty.call(data, keyMatch); } export function removeKey(markdownString: string, keyMatch: string) { const parsed = matter(markdownString); + const data = parsed.data as Frontmatter; - for (const key in parsed.data) { - if (key === keyMatch) { - parsed.data.remove(key); - } - } + delete data[keyMatch]; - const result = matter.stringify(parsed.content, parsed.data); + const result = matter.stringify(parsed.content, data); return result.slice(0, -1); // remove trailing \n } diff --git a/src/HasProvider.ts b/src/HasProvider.ts index 23126912..7126aba9 100644 --- a/src/HasProvider.ts +++ b/src/HasProvider.ts @@ -13,7 +13,7 @@ import { LiveTokenStore } from "./LiveTokenStore"; import type { ClientToken } from "./client/types"; import { S3RN, type S3RNType } from "./S3RN"; import { encodeClientToken } from "./client/types"; -import { flags } from "./flagManager"; +import type { TimeProvider } from "./TimeProvider"; export interface Subscription { on: () => void; @@ -23,7 +23,8 @@ export interface Subscription { function makeProvider( clientToken: ClientToken, ydoc: Y.Doc, - user?: User, + user: User | undefined, + timeProvider: TimeProvider, ): YSweetProvider { const params = { token: clientToken.token, @@ -37,6 +38,8 @@ function makeProvider( params: params, disableBc: true, maxConnectionErrors: 3, + readOnly: clientToken.authorization === "read-only", + timeProvider, }, ); @@ -51,16 +54,31 @@ function makeProvider( return provider; } +/** Disconnected state returned when no provider exists */ +const DISCONNECTED_STATE: ConnectionState = { + status: "disconnected", +} as ConnectionState; + type Listener = (state: ConnectionState) => void; export class HasProvider extends HasLogging { - _provider: YSweetProvider; + _provider: YSweetProvider | null = null; path?: string; - ydoc: Y.Doc; + private _ydoc: Y.Doc | null = null; clientToken: ClientToken; - private _offConnectionError: () => void; - private _offState: () => void; + private _deferredDisconnectTimer: number | null = null; + private _deferredDisconnectStatusListener: + | ((state: ConnectionState) => void) + | null = null; + private _providerSyncAbortHandlers = new Set<(reason: Error) => void>(); + // Track whether the current provider connection has completed sync. + // This must reset on disconnect so reconnect flows do not treat a + // stale connection as ready. + _providerSynced: boolean = false; + private _offConnectionError: (() => void) | null = null; + private _offState: (() => void) | null = null; listeners: Map; + timeProvider!: TimeProvider; constructor( public guid: string, @@ -71,28 +89,61 @@ export class HasProvider extends HasLogging { super(); this.listeners = new Map(); this.loginManager = loginManager; - const user = this.loginManager?.user; - this.ydoc = new Y.Doc(); - - if (flags().enableDocumentHistory) { - this.ydoc.gc = false; - } this.tokenStore = tokenStore; this.clientToken = this.tokenStore.getTokenSync(S3RN.encode(this.s3rn)) || ({ token: "", url: "", docId: "-", expiryTime: 0 } as ClientToken); + } + + /** + * Get the remote YDoc. Lazily creates it on first access. + * Most callers should use this property for backward compatibility. + */ + public get ydoc(): Y.Doc { + if (!this._ydoc) { + this.ensureRemoteDoc(); + } + return this._ydoc!; + } + + /** + * Get the remote YDoc without creating it. + * Returns null if the remoteDoc has not been created yet. + */ + public get remoteDocOrNull(): Y.Doc | null { + return this._ydoc; + } + + /** + * Check if the remote YDoc and provider are currently loaded. + */ + public get isRemoteDocLoaded(): boolean { + return this._ydoc !== null; + } + + /** + * Create the remote YDoc and provider if they don't exist. + * Returns the YDoc for convenience. + */ + ensureRemoteDoc(): Y.Doc { + if (this._ydoc) { + return this._ydoc; + } - this._provider = makeProvider(this.clientToken, this.ydoc, user); + const user = this.loginManager?.user; + this._ydoc = new Y.Doc(); + + this._provider = makeProvider( + this.clientToken, + this._ydoc, + user, + this.timeProvider, + ); const connectionErrorSub = this.providerConnectionErrorSubscription( (event) => { - this.log(`[${this.path}] disconnection event`, event); - const shouldConnect = this._provider.canReconnect(); - this.disconnect(); - if (shouldConnect) { - this.connect(); - } + this.log(`[${this.path}] connection error`, event); }, ); connectionErrorSub.on(); @@ -100,11 +151,43 @@ export class HasProvider extends HasLogging { const stateSub = this.providerStateSubscription( (state: ConnectionState) => { + if (state.status !== "connected") { + this._providerSynced = false; + } this.notifyListeners(); }, ); stateSub.on(); this._offState = stateSub.off; + + return this._ydoc; + } + + /** + * Destroy the remote YDoc and provider, freeing memory. + * The document can be re-created later via ensureRemoteDoc(). + */ + destroyRemoteDoc(): void { + this.abortProviderSyncWaiters( + new Error("Provider was destroyed before sync completed"), + ); + if (this._offConnectionError) { + this._offConnectionError(); + this._offConnectionError = null; + } + if (this._offState) { + this._offState(); + this._offState = null; + } + if (this._provider) { + this._provider.destroy(); + this._provider = null; + } + if (this._ydoc) { + this._ydoc.destroy(); + this._ydoc = null; + } + this._providerSynced = false; } public get s3rn(): S3RNType { @@ -113,7 +196,9 @@ export class HasProvider extends HasLogging { public set s3rn(value: S3RNType) { this._s3rn = value; - this.refreshProvider(this.clientToken); + if (this._provider) { + this.refreshProvider(this.clientToken); + } } public get debuggerUrl(): string { @@ -151,7 +236,7 @@ export class HasProvider extends HasLogging { } providerActive() { - if (this.clientToken) { + if (this.clientToken && this._provider) { const tokenIsSet = this._provider.hasUrl(this.clientToken.url); const expired = Date.now() > (this.clientToken?.expiryTime || 0); return tokenIsSet && !expired; @@ -164,13 +249,15 @@ export class HasProvider extends HasLogging { this.clientToken = clientToken; if (!this._provider) { - throw new Error("missing provider!"); + // No provider yet - token will be used when ensureRemoteDoc() is called + return; } const result = this._provider.refreshToken( clientToken.url, clientToken.docId, clientToken.token, + clientToken.authorization === "read-only", ); if (result.urlChanged) { @@ -190,32 +277,114 @@ export class HasProvider extends HasLogging { if (this.connected) { return Promise.resolve(true); } + // Ensure remoteDoc exists before connecting + this.ensureRemoteDoc(); return this.getProviderToken() .then((clientToken) => { this.refreshProvider(clientToken); // XXX is this still needed? - this._provider.connect(); + this._provider!.connect(); this.notifyListeners(); return true; }) .catch((e) => { + this.abortProviderSyncWaiters( + new Error("Provider connection failed before sync completed"), + ); return false; }); } public get state(): ConnectionState { + if (!this._provider) { + return DISCONNECTED_STATE; + } return this._provider.connectionState; } get intent(): ConnectionIntent { + if (!this._provider) { + return "disconnected" as ConnectionIntent; + } return this._provider.intent; } public get synced(): boolean { - return this._provider.synced; + return this._providerSynced; + } + + private clearDeferredDisconnect(): void { + if (this._deferredDisconnectTimer !== null) { + this.timeProvider.clearTimeout(this._deferredDisconnectTimer); + this._deferredDisconnectTimer = null; + } + if (this._provider && this._deferredDisconnectStatusListener) { + this._provider.off("status", this._deferredDisconnectStatusListener); + } + this._deferredDisconnectStatusListener = null; + } + + deferDisconnectForPendingMessages(timeoutMs: number = 2000): boolean { + const provider = this._provider; + if (!provider || provider._pendingMessages.length === 0) { + return false; + } + + this.clearDeferredDisconnect(); + + const finishDisconnect = () => { + if (this._provider !== provider) { + this.clearDeferredDisconnect(); + return; + } + this.disconnect(); + }; + + const queueDisconnect = () => { + // YSweetProvider emits "status: connected" before its onopen + // handler flushes buffered sync frames. Defer one task so the + // pending messages are actually sent before we close the socket. + this.timeProvider.setTimeout(finishDisconnect, 0); + }; + + this._deferredDisconnectStatusListener = (state: ConnectionState) => { + if (this._provider !== provider) { + this.clearDeferredDisconnect(); + return; + } + if (state.status === "connected") { + this.clearDeferredDisconnect(); + queueDisconnect(); + } + }; + provider.on("status", this._deferredDisconnectStatusListener); + + this._deferredDisconnectTimer = this.timeProvider.setTimeout(() => { + if (this._provider !== provider) { + this.clearDeferredDisconnect(); + return; + } + this.disconnect(); + }, timeoutMs); + + // Keep the in-flight connection attempt alive. If the socket was + // dropped during a brief disconnect window, reconnect so the buffered + // sync frames can flush on open. + if (provider.connectionState.status !== "connected") { + void this.connect(); + } + + return true; } disconnect() { - this._provider.disconnect(); + this.clearDeferredDisconnect(); + this.abortProviderSyncWaiters( + new Error("Provider disconnected before sync completed"), + ); + this._providerSynced = false; + if (this._provider) { + this._provider.disconnect(); + } this.tokenStore.removeFromRefreshQueue(this.guid); this.notifyListeners(); } @@ -232,28 +401,100 @@ export class HasProvider extends HasLogging { } onceConnected(): Promise { + this.ensureRemoteDoc(); + if (this.state.status === "connected") { + return Promise.resolve(); + } + const provider = this._provider!; return new Promise((resolve) => { const resolveOnConnect = (state: ConnectionState) => { if (state.status === "connected") { + provider.off("status", resolveOnConnect); resolve(); } }; - // provider observers are manually cleared in destroy() - this._provider.on("status", resolveOnConnect); + provider.on("status", resolveOnConnect); }); } onceProviderSynced(): Promise { - if (this._provider.synced) { + if (this._providerSynced) { return Promise.resolve(); } - return new Promise((resolve) => { - this._provider.once("synced", () => { + this.ensureRemoteDoc(); + const provider = this._provider!; + if (provider.synced) { + this._providerSynced = true; + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = () => { + provider.off("synced", handleSynced); + provider.off("status", handleStatus); + provider.off("connection-error", handleConnectionError); + this._providerSyncAbortHandlers.delete(abort); + }; + const finish = () => { + if (settled) return; + settled = true; + cleanup(); + this._providerSynced = true; resolve(); - }); + }; + const fail = (reason: Error) => { + if (settled) return; + settled = true; + cleanup(); + reject(reason); + }; + const abort = (reason: Error) => { + fail(reason); + }; + const checkTerminalState = () => { + if (this._provider !== provider) { + fail(new Error("Provider was replaced before sync completed")); + return; + } + if (this._providerSynced || provider.synced) { + finish(); + return; + } + const state = provider.connectionState; + if ( + state.status === "disconnected" && + state.intent === "connected" && + !provider.canReconnect() + ) { + fail(new Error("Provider retries were exhausted before sync completed")); + return; + } + }; + const handleSynced = (synced: boolean) => { + if (!synced) return; + finish(); + }; + const handleStatus = () => { + checkTerminalState(); + }; + const handleConnectionError = () => { + checkTerminalState(); + }; + provider.on("synced", handleSynced); + provider.on("status", handleStatus); + provider.on("connection-error", handleConnectionError); + this._providerSyncAbortHandlers.add(abort); + checkTerminalState(); }); } + private abortProviderSyncWaiters(reason: Error): void { + for (const abort of Array.from(this._providerSyncAbortHandlers)) { + abort(reason); + } + this._providerSyncAbortHandlers.clear(); + } + reset() { this.disconnect(); this.clientToken = { @@ -269,10 +510,10 @@ export class HasProvider extends HasLogging { f: (event: Event) => void, ): Subscription { const on = () => { - this._provider.on("connection-error", f); + this._provider?.on("connection-error", f); }; const off = () => { - this._provider.off("connection-error", f); + this._provider?.off("connection-error", f); }; return { on, off } as Subscription; } @@ -281,24 +522,16 @@ export class HasProvider extends HasLogging { f: (state: ConnectionState) => void, ): Subscription { const on = () => { - this._provider.on("status", f); + this._provider?.on("status", f); }; const off = () => { - this._provider.off("status", f); + this._provider?.off("status", f); }; return { on, off } as Subscription; } destroy() { - if (this._offConnectionError) { - this._offConnectionError(); - } - if (this._offState) { - this._offState(); - } - if (this._provider) { - this._provider.destroy(); - } + this.destroyRemoteDoc(); this.loginManager = null as any; } } diff --git a/src/LiveTokenStore.ts b/src/LiveTokenStore.ts index 4054b546..d69c7ca0 100644 --- a/src/LiveTokenStore.ts +++ b/src/LiveTokenStore.ts @@ -14,9 +14,7 @@ import { S3RemoteFile, S3RemoteCanvas, } from "./S3RN"; -import { customFetch } from "./customFetch"; - -declare const GIT_TAG: string; +import { customFetch, getRelayRequestHeaders } from "./customFetch"; function getJwtExpiryFromClientToken(clientToken: ClientToken): number { // lol this is so fake @@ -25,15 +23,17 @@ function getJwtExpiryFromClientToken(clientToken: ClientToken): number { function withLoginManager( loginManager: LoginManager, + deviceId: string, // eslint-disable-next-line @typescript-eslint/no-explicit-any fn: (...args: any[]) => void, ) { // eslint-disable-next-line @typescript-eslint/no-explicit-any - return (...args: any[]) => fn(loginManager, ...args); + return (...args: any[]) => fn(loginManager, deviceId, ...args); } async function refresh( loginManager: LoginManager, + deviceId: string, documentId: string, onSuccess: (clientToken: ClientToken) => void, onError: (err: Error) => void, @@ -42,31 +42,31 @@ async function refresh( const error = curryLog("[TokenStore][Refresh]", "error"); debug(`${documentId}`); const entity: S3RNType = S3RN.decode(documentId); - let payload: string; + let payloadObj: Record; if (entity instanceof S3RemoteDocument) { - payload = JSON.stringify({ + payloadObj = { docId: entity.documentId, relay: entity.relayId, folder: entity.folderId, - }); + }; } else if (entity instanceof S3RemoteCanvas) { - payload = JSON.stringify({ + payloadObj = { docId: entity.canvasId, relay: entity.relayId, folder: entity.folderId, - }); + }; } else if (entity instanceof S3RemoteFolder) { - payload = JSON.stringify({ + payloadObj = { docId: entity.folderId, relay: entity.relayId, folder: entity.folderId, - }); + }; } else if (entity instanceof S3RemoteFile) { - payload = JSON.stringify({ + payloadObj = { docId: entity.fileId, relay: entity.relayId, folder: entity.folderId, - }); + }; } else { onError(new Error("No remote to connect to")); return; @@ -75,9 +75,13 @@ async function refresh( onError(Error("Not logged in")); return; } + if (deviceId) { + payloadObj.device = deviceId; + } + const payload = JSON.stringify(payloadObj); const headers = { Authorization: `Bearer ${loginManager.user?.token}`, - "Relay-Version": GIT_TAG, + ...getRelayRequestHeaders(), "Content-Type": "application/json", }; try { @@ -103,16 +107,20 @@ async function refresh( } export class LiveTokenStore extends TokenStore { + private deviceId: string; + constructor( private loginManager: LoginManager, timeProvider: TimeProvider, vaultName: string, + deviceId: string, maxConnections = 5, + refreshJitterSeed?: string, ) { super( { log: curryLog("[LiveTokenStore]", "debug"), - refresh: withLoginManager(loginManager, refresh), + refresh: withLoginManager(loginManager, deviceId, refresh), getJwtExpiry: getJwtExpiryFromClientToken, getStorage: function () { return new LocalStorage>( @@ -122,9 +130,13 @@ export class LiveTokenStore extends TokenStore { getTimeProvider: () => { return timeProvider; }, + refreshJitterSeed: refreshJitterSeed + ? `${refreshJitterSeed}:${deviceId}` + : undefined, }, maxConnections, ); + this.deviceId = deviceId; } private async getFileTokenFromNetwork( @@ -180,14 +192,18 @@ export class LiveTokenStore extends TokenStore { const entity: S3RNType = S3RN.decode(documentId); let payload: string; if (entity instanceof S3RemoteFile) { - payload = JSON.stringify({ + const payloadObj: Record = { docId: entity.fileId, relay: entity.relayId, folder: entity.folderId, hash: fileHash, contentType, contentLength, - }); + }; + if (this.deviceId) { + payloadObj.device = this.deviceId; + } + payload = JSON.stringify(payloadObj); } else { throw new Error(`No remote to connect to for ${documentId}`); } @@ -196,7 +212,7 @@ export class LiveTokenStore extends TokenStore { } const headers = { Authorization: `Bearer ${this.loginManager.user?.token}`, - "Relay-Version": GIT_TAG, + ...getRelayRequestHeaders(), "Content-Type": "application/json", }; const apiUrl = this.loginManager.getEndpointManager().getApiUrl(); diff --git a/src/LiveViews.ts b/src/LiveViews.ts index 5ce09795..a59bdbb9 100644 --- a/src/LiveViews.ts +++ b/src/LiveViews.ts @@ -1,31 +1,36 @@ import type { Extension } from "@codemirror/state"; -import { StateField, EditorState, Compartment } from "@codemirror/state"; import { EditorView } from "@codemirror/view"; import { App, MarkdownView, - Platform, - requireApiVersion, TFile, TextFileView, Workspace, + editorInfoField, moment, + type WorkspaceLeaf, type CachedMetadata, } from "obsidian"; import ViewActions from "src/components/ViewActions.svelte"; import * as Y from "yjs"; import { Document } from "./Document"; +import type { EditorViewRef } from "./merge-hsm/types"; import type { ConnectionState } from "./HasProvider"; import { LoginManager } from "./LoginManager"; import NetworkStatus from "./NetworkStatus"; import { SharedFolder, SharedFolders } from "./SharedFolder"; -import { curryLog, HasLogging, RelayInstances } from "./debug"; +import { curryLog, HasLogging, RelayInstances, metrics } from "./debug"; import { Banner } from "./ui/Banner"; -import { LiveEdit } from "./y-codemirror.next/LiveEditPlugin"; +import { HSMEditorPlugin } from "./merge-hsm/integration/HSMEditorPlugin"; import { yRemoteSelections, yRemoteSelectionsTheme, } from "./y-codemirror.next/RemoteSelections"; +import { + attributionFilterField, + userAttributionPlugin, + userAttributionTheme, +} from "./y-codemirror.next/UserAttributionPlugin"; import { InvalidLinkPlugin } from "./markdownView/InvalidLinkExtension"; import * as Differ from "./differ/differencesView"; import type { CanvasView } from "./CanvasView"; @@ -35,6 +40,26 @@ import { LiveNode } from "./y-codemirror.next/LiveNodePlugin"; import { flags } from "./flagManager"; import { AwarenessViewPlugin } from "./AwarenessViewPlugin"; import { TextFileViewPlugin } from "./TextViewPlugin"; +import { ViewHookPlugin } from "./plugins/ViewHookPlugin"; +import { DiskBuffer } from "./DiskBuffer"; +import { trackPromise } from "./trackPromise"; + +/** + * Access the LiveViewManager singleton via the Obsidian plugin registry. + * Replaces ConnectionManagerStateField — no CM6 state field needed since + * the plugin is a singleton reachable from any EditorView. + */ +export function getConnectionManager( + editor: EditorView, +): LiveViewManager | null { + const fileInfo = editor.state.field(editorInfoField, false); + return ( + (fileInfo as any)?.app?.plugins?.plugins?.["system3-relay"]?._liveViews ?? + null + ); +} + +export type DocumentViewer = WorkspaceLeaf | symbol; const BACKGROUND_CONNECTIONS = 3; @@ -53,10 +78,7 @@ function iterateTextFileViews( workspace: Workspace, fn: (leaf: TextFileView) => void, ) { - const ALLOWED_TEXT_FILE_VIEWS = ["markdown"]; - if (flags().enableKanbanView) { - ALLOWED_TEXT_FILE_VIEWS.push("kanban"); - } + const ALLOWED_TEXT_FILE_VIEWS = ["markdown", "kanban"]; const allLeaves: any[] = []; workspace.iterateAllLeaves((leaf) => { @@ -72,7 +94,7 @@ function iterateTextFileViews( if (leaf.view instanceof TextFileView) { const viewType = leaf.view.getViewType(); if (viewType === "canvas") return; - if (ALLOWED_TEXT_FILE_VIEWS.contains(viewType)) { + if (ALLOWED_TEXT_FILE_VIEWS.includes(viewType)) { fn(leaf.view); } } @@ -99,6 +121,7 @@ export interface S3View { view: TextFileView | CanvasView; release: () => void; attach: () => Promise; + // eslint-disable-next-line -- Relay document model, not DOM global. document: Document | Canvas | null; destroy: () => void; canConnect: boolean; @@ -109,6 +132,7 @@ export class LoggedOutView implements S3View { view: TextFileView | CanvasView; login: () => Promise; banner?: Banner; + // eslint-disable-next-line -- Relay document model, not DOM global. document = null; canConnect = false; @@ -124,65 +148,24 @@ export class LoggedOutView implements S3View { this.login = login; } - setLoginIcon(): void { - const viewHeaderElement = - this.view.containerEl.querySelector(".view-header"); - const viewHeaderLeftElement = - this.view.containerEl.querySelector(".view-header-left"); - - if (viewHeaderElement && viewHeaderLeftElement) { - this.clearLoginButton(); - - // Create login button element - const loginButton = document.createElement("button"); - loginButton.className = "view-header-left system3-login-button"; - loginButton.textContent = "Login to enable Live edits"; - loginButton.setAttribute("aria-label", "Login to enable Live edits"); - loginButton.setAttribute("tabindex", "0"); - - // Add click handler - loginButton.addEventListener("click", async () => { - await this.login(); - }); - - // Insert after view-header-left - viewHeaderLeftElement.insertAdjacentElement("afterend", loginButton); - } - } - - clearLoginButton() { - const existingButton = this.view.containerEl.querySelector(".system3-login-button"); - if (existingButton) { - existingButton.remove(); - } - } - attach(): Promise { - // Use header button approach on mobile for Obsidian >=1.11.0 to avoid banner positioning issues - if (Platform.isMobile && requireApiVersion("1.11.0")) { - this.setLoginIcon(); - } else { - this.banner = new Banner( - this.view, - "Login to enable Live edits", - async () => { - return await this.login(); - }, - ); - } + this.banner = new Banner( + this.view, + { short: "Login to Relay", long: "Login to enable Live edits" }, + async () => { + return await this.login(); + }, + ); return Promise.resolve(this); } release() { this.banner?.destroy(); - this.clearLoginButton(); } destroy() { - this.release(); this.banner?.destroy(); this.banner = undefined; - this.clearLoginButton(); this.view = null as any; } } @@ -214,12 +197,14 @@ export class RelayCanvasView implements S3View { shouldConnect: boolean; canConnect: boolean; plugin?: CanvasPlugin; + // eslint-disable-next-line -- Relay document model, not DOM global. document: Canvas; private _viewActions?: ViewActions; private offConnectionStatusSubscription?: () => void; private _parent: LiveViewManager; private _banner?: Banner; + private _awarenessPlugin?: AwarenessViewPlugin; tracking: boolean; constructor( @@ -256,11 +241,15 @@ export class RelayCanvasView implements S3View { } } + toggleLocalOnly() { + this.toggleConnection(); + } + offlineBanner(): () => void { if (this.shouldConnect) { const banner = new Banner( this.view, - "You're offline -- click to reconnect", + { short: "Offline", long: "You're offline -- click to reconnect" }, async () => { this._parent.networkStatus.checkStatus(); this.connect(); @@ -291,6 +280,8 @@ export class RelayCanvasView implements S3View { view: this, state: this.canvas.state, remote: this.canvas.sharedFolder.remote, + tracking: this.tracking, + enableDraftMode: flags().enableDraftMode, }, }); this.offConnectionStatusSubscription = this.canvas.subscribe( @@ -300,6 +291,8 @@ export class RelayCanvasView implements S3View { view: this, state: state, remote: this.canvas.sharedFolder.remote, + tracking: this.tracking, + enableDraftMode: flags().enableDraftMode, }); }, ); @@ -308,6 +301,8 @@ export class RelayCanvasView implements S3View { view: this, state: this.canvas.state, remote: this.canvas.sharedFolder.remote, + tracking: this.tracking, + enableDraftMode: flags().enableDraftMode, }); } } @@ -340,9 +335,44 @@ export class RelayCanvasView implements S3View { this.plugin = new CanvasPlugin(this._parent, this); } + if (!this._awarenessPlugin) { + const viewEl = this.view.containerEl; + this._awarenessPlugin = new AwarenessViewPlugin( + { + view: this.view, + doc: this.canvas, + resolveAnchor: (containerEl) => { + const viewContent = containerEl.querySelector( + ".view-content", + ) as HTMLElement | null; + return viewContent + ? { anchor: viewContent, position: "afterbegin" } + : null; + }, + vertical: true, + configureContainer: (el) => { + const controls = viewEl.querySelector( + ".canvas-controls", + ) as HTMLElement | null; + const gap = 12; + const top = controls + ? controls.offsetTop + controls.offsetHeight + gap + : gap; + el.style.top = `${top}px`; + const isMobile = + viewEl.ownerDocument.body.classList.contains("is-mobile"); + el.style.right = isMobile ? "12px" : "6px"; + }, + }, + this._parent.sharedFolders.manager.users, + ); + } + return new Promise((resolve) => { - return this.canvas - .whenReady() + return trackPromise( + `canvasView:whenReady:${this.canvas.guid}`, + this.canvas.whenReady(), + ) .then((doc) => { if ( this._parent.networkStatus.online && @@ -374,6 +404,8 @@ export class RelayCanvasView implements S3View { this.plugin?.destroy(); this.plugin = undefined; + this._awarenessPlugin?.destroy(); + this._awarenessPlugin = undefined; this._viewActions?.$destroy(); this._viewActions = undefined; this._banner?.destroy(); @@ -383,7 +415,7 @@ export class RelayCanvasView implements S3View { this.offConnectionStatusSubscription = undefined; } this.canvas.disconnect(); - this.canvas.userLock = false; + this.canvas.releaseLock(); } destroy() { @@ -403,10 +435,12 @@ export class LiveView implements S3View { view: ViewType; + // eslint-disable-next-line -- Relay document model, not DOM global. document: Document; shouldConnect: boolean; canConnect: boolean; private _plugin?: TextFileViewPlugin; + private _viewHookPlugin?: ViewHookPlugin; private _viewActions?: ViewActions; private offConnectionStatusSubscription?: () => void; @@ -414,6 +448,10 @@ export class LiveView private _banner?: Banner; _tracking: boolean; private _awarenessPlugin?: AwarenessViewPlugin; + private _hsmStateUnsubscribe?: () => void; + private _hasLock = false; + private _released = false; + private readonly _fallbackViewer = Symbol("live-view-viewer"); constructor( connectionManager: LiveViewManager, @@ -449,14 +487,30 @@ export class LiveView } } + toggleLocalOnly() { + const hsm = this.document.hsm; + if (hsm) { + const wasLocalOnly = hsm.isLocalOnly; + hsm.setLocalOnly(!wasLocalOnly); + if (wasLocalOnly && !this.document.connected) { + this.document.connect(); + } + this.attach(); + } + } + public get tracking() { + if (this.document?.hsm) { + return this.document.hsm.state.statePath === "active.tracking"; + } return this._tracking; } public set tracking(value: boolean) { const old = this._tracking; this._tracking = value; - if (this._tracking !== old) { + // Only call attach for non-HSM mode (fallback for views without HSM) + if (this._tracking !== old && !this.document?.hsm) { this.attach(); } } @@ -475,93 +529,110 @@ export class LiveView } } - setMergeButton(): void { - const viewHeaderElement = - this.view.containerEl.querySelector(".view-header"); - const viewHeaderLeftElement = - this.view.containerEl.querySelector(".view-header-left"); - - if (viewHeaderElement && viewHeaderLeftElement) { - this.clearMergeButton(); - - // Create merge button element - const mergeButton = document.createElement("button"); - mergeButton.className = "view-header-left system3-merge-button"; - mergeButton.textContent = "Merge conflict"; - mergeButton.setAttribute("aria-label", "Merge conflict -- click to resolve"); - mergeButton.setAttribute("tabindex", "0"); - - // Add click handler - mergeButton.addEventListener("click", async () => { - const diskBuffer = await this.document.diskBuffer(); - const stale = await this.document.checkStale(); - if (!stale) { - this.clearMergeButton(); - return; - } - this._parent.openDiffView({ - file1: this.document, - file2: diskBuffer, - showMergeOption: true, - onResolve: async () => { - this.document.clearDiskBuffer(); - this.clearMergeButton(); - // Force view to sync to CRDT state after differ resolution - if ( - this._plugin && - typeof this._plugin.syncViewToCRDT === "function" - ) { - await this._plugin.syncViewToCRDT(); + mergeBanner(): () => void { + this._banner = new Banner( + this.view, + { short: "Merge conflict", long: "Merge conflict -- click to resolve" }, + async () => { + // HSM-aware conflict resolution path + const hsm = this.document.hsm; + if (hsm) { + const conflictData = hsm.getConflictData({ fresh: true }); + const localDoc = hsm.getLocalDoc(); + if ( + conflictData && + localDoc && + hsm.state.statePath.includes("conflict") + ) { + this.log("[mergeBanner] Opening diff view for conflict resolution"); + + // Check if there are inline conflict regions (new flow) + const hasInlineConflicts = + conflictData.conflictRegions && + conflictData.conflictRegions.length > 0; + + if (hasInlineConflicts) { + // With inline conflicts, clicking banner opens diff view as alternative + this.log( + "[mergeBanner] Inline conflicts present, opening diff view as alternative", + ); } - }, - }); - }); - - // Insert after view-header-left - viewHeaderLeftElement.insertAdjacentElement("afterend", mergeButton); - } - } - clearMergeButton() { - const existingButton = this.view.containerEl.querySelector(".system3-merge-button"); - if (existingButton) { - existingButton.remove(); - } - } + // Use the conflict payload sides directly so labels and hunk actions + // stay aligned with what the HSM declared as ours/theirs. + const oursContent = conflictData.ours; + const theirsContent = conflictData.theirs; + const currentLocalContent = localDoc.getText("contents").toString(); + if (currentLocalContent !== oursContent) { + this.log( + `[mergeBanner] conflict side drift detected: localDoc=${currentLocalContent.length}, conflict.ours=${oursContent.length}`, + ); + } - mergeBanner(): () => void { - // Use header button approach on mobile for Obsidian >=1.11.0 to avoid banner positioning issues - if (Platform.isMobile && requireApiVersion("1.11.0")) { - this.setMergeButton(); - } else { - this._banner = new Banner( - this.view, - "Merge conflict -- click to resolve", - async () => { - const diskBuffer = await this.document.diskBuffer(); - const stale = await this.document.checkStale(); - if (!stale) { - return true; + this.log( + `[mergeBanner] ours: ${oursContent.length} chars, theirs: ${theirsContent.length} chars`, + ); + + const oursLabel = conflictData.oursLabel ?? "Editor"; + const theirsLabel = conflictData.theirsLabel ?? "Disk"; + const showRemoteOnTop = + oursLabel.toLowerCase().includes("local") + && ( + theirsLabel.toLowerCase().includes("remote") + || theirsLabel.toLowerCase().includes("peer") + ); + const topContent = showRemoteOnTop ? theirsContent : oursContent; + const bottomContent = showRemoteOnTop ? oursContent : theirsContent; + const topLabel = showRemoteOnTop ? theirsLabel : oursLabel; + const bottomLabel = showRemoteOnTop ? oursLabel : theirsLabel; + + // Create DiskBuffer wrappers (differ expects TFile-like objects). + // file1 is always shown on top/left in the differ. + const topFile = new DiskBuffer( + this._parent.app.vault, + this.document.path + ` (${topLabel})`, + topContent, + ); + const bottomFile = new DiskBuffer( + this._parent.app.vault, + this.document.path + ` (${bottomLabel})`, + bottomContent, + ); + + // Transition HSM to resolving state + hsm.send({ type: "OPEN_DIFF_VIEW" }); + + // Open diff view: file1 is rendered on top/left. + this._parent.openDiffView({ + file1: topFile, + file2: bottomFile, + showMergeOption: true, + oursLabel: topLabel, + theirsLabel: bottomLabel, + sourceVaultPath: this.document.tfile?.path, + onResolve: async () => { + this.log("[mergeBanner] HSM conflict resolved via diff view"); + + // The differ modifies file1 in-place via its contents. + // Get the resolved content and apply it to HSM's localDoc. + const resolvedContent = topFile.contents; + + hsm.send({ type: "RESOLVE", contents: resolvedContent }); + + this._banner?.destroy(); + this._banner = undefined; + }, + onCancel: () => { + this.log("[mergeBanner] Diff view closed without resolving"); + hsm.send({ type: "CANCEL" }); + }, + }); + return false; // Don't destroy banner yet - wait for resolution } - this._parent.openDiffView({ - file1: this.document, - file2: diskBuffer, - showMergeOption: true, - onResolve: async () => { - this.document.clearDiskBuffer(); - // Force view to sync to CRDT state after differ resolution - if ( - this._plugin && - typeof this._plugin.syncViewToCRDT === "function" - ) { - await this._plugin.syncViewToCRDT(); - } - }, - }); - return true; - }, - ); - } + } + return false; + }, + ); return () => {}; } @@ -569,7 +640,7 @@ export class LiveView if (this.shouldConnect) { const banner = new Banner( this.view, - "You're offline -- click to reconnect", + { short: "Offline", long: "You're offline -- click to reconnect" }, async () => { this._parent.networkStatus.checkStatus(); this.connect(); @@ -600,6 +671,12 @@ export class LiveView view: this, state: this.document.state, remote: this.document.sharedFolder.remote, + tracking: this.tracking, + localOnly: this.document.hsm?.isLocalOnly ?? false, + enableDraftMode: flags().enableDraftMode, + folderConnected: this.document.sharedFolder.connected, + pendingOutbound: this.document.hsm?.pendingOutbound ?? 0, + pendingInbound: this.document.hsm?.pendingInbound ?? 0, }, }); this.offConnectionStatusSubscription = this.document.subscribe( @@ -609,14 +686,54 @@ export class LiveView view: this, state: state, remote: this.document.sharedFolder.remote, + tracking: this.tracking, + localOnly: this.document.hsm?.isLocalOnly ?? false, + enableDraftMode: flags().enableDraftMode, + folderConnected: this.document.sharedFolder.connected, + pendingOutbound: this.document.hsm?.pendingOutbound ?? 0, + pendingInbound: this.document.hsm?.pendingInbound ?? 0, }); }, ); } + // Subscribe to HSM state changes to update tracking icon and conflict banner + const hsm = this.document.hsm; + if (hsm && !this._hsmStateUnsubscribe) { + this._hsmStateUnsubscribe = hsm.stateChanges.subscribe((state) => { + if (!this.document.sharedFolder) return; + this._viewActions?.$set({ + tracking: state.statePath === "active.tracking", + localOnly: this.document.hsm?.isLocalOnly ?? false, + enableDraftMode: flags().enableDraftMode, + folderConnected: this.document.sharedFolder.connected, + pendingOutbound: this.document.hsm?.pendingOutbound ?? 0, + pendingInbound: this.document.hsm?.pendingInbound ?? 0, + }); + const isConflict = state.statePath.includes("conflict"); + if (isConflict && !this._banner) { + this.log( + "[LiveView] HSM entered conflict state, showing merge banner", + ); + this.mergeBanner(); + } else if (!isConflict && this._banner) { + this.log( + "[LiveView] HSM exited conflict state, hiding merge banner", + ); + this._banner.destroy(); + this._banner = undefined; + } + }); + } this._viewActions.$set({ view: this, state: this.document.state, remote: this.document.sharedFolder.remote, + tracking: this.tracking, + localOnly: this.document.hsm?.isLocalOnly ?? false, + enableDraftMode: flags().enableDraftMode, + folderConnected: this.document.sharedFolder.connected, + pendingOutbound: this.document.hsm?.pendingOutbound ?? 0, + pendingInbound: this.document.hsm?.pendingInbound ?? 0, }); } } @@ -641,25 +758,66 @@ export class LiveView this.view instanceof MarkdownView && this.view.getMode() === "preview" ) { + this.log("[LiveView.checkStale] skipping - preview mode"); return false; } - const stale = await this.document.checkStale(); - if (stale && this.document._diskBuffer?.contents) { + + // Use HSM conflict detection + const hsmConflict = this.document.hasHSMConflict(); + this.log(`[LiveView.checkStale] HSM conflict detection: ${hsmConflict}`); + if (hsmConflict === true) { + this.log( + "[LiveView.checkStale] HSM reports conflict, showing merge banner", + ); this.mergeBanner(); + return true; } else { this._banner?.destroy(); this._banner = undefined; + return false; + } + } + + private initializeEditorIntegration(): void { + if (!(this.view instanceof MarkdownView)) { + return; + } + const cm = (this.view.editor as any)?.cm as EditorView | undefined; + if (!cm) { + return; } - return stale; + const plugin = cm.plugin(HSMEditorPlugin); + plugin?.initializeIfReady(); } attach(): Promise { + this._released = false; + // can be called multiple times, whereas release is only ever called once - this.document.userLock = true; + // Acquire a lock synchronously. Subsequent attach calls for the same view + // are idempotent until release(). + if (!this._hasLock) { + this._parent.acquireDocumentLock( + this.document, + this.view as unknown as EditorViewRef, + this.view.leaf ?? this._fallbackViewer, + ); + this._hasLock = true; + } + this.initializeEditorIntegration(); // Add CSS class to indicate this view should have live editing if (this.view instanceof MarkdownView) { this.view.containerEl.addClass("relay-live-editor"); + + // Initialize ViewHookPlugin to capture non-CM6 edit paths + // (preview mode checkbox toggles, frontmatter saves via Properties panel) + if (!this._viewHookPlugin) { + this._viewHookPlugin = new ViewHookPlugin(this.view, this.document); + this._viewHookPlugin.initialize().catch((error) => { + this.error("Error initializing ViewHookPlugin:", error); + }); + } } if (!(this.view instanceof MarkdownView)) { @@ -675,22 +833,33 @@ export class LiveView this.setConnectionDot(); - // Initialize awareness plugin if not already created and feature flag is enabled - if ( - isLiveMd(this) && - !this._awarenessPlugin && - flags().enablePresenceAvatars - ) { + // Initialize awareness plugin if not already created. + if (isLiveMd(this) && !this._awarenessPlugin) { this._awarenessPlugin = new AwarenessViewPlugin( - this, + { + view: this.view, + doc: this.document, + resolveAnchor: (containerEl) => { + const inlineTitle = containerEl.querySelector( + ".inline-title", + ) as HTMLElement | null; + return inlineTitle + ? { anchor: inlineTitle, position: "afterend" } + : null; + }, + getEditor: () => this.view.editor, + }, this._parent.sharedFolders.manager.users, ); } return new Promise((resolve) => { - return this.document - .whenReady() + return trackPromise( + `liveView:whenReady:${this.document.guid}`, + this.document.whenReady(), + ) .then((doc) => { + this.initializeEditorIntegration(); if ( this._parent.networkStatus.online && this.document.sharedFolder.shouldConnect && @@ -715,6 +884,10 @@ export class LiveView release() { // Called when a view is released from management + if (!this.document || !this.view || this._released) { + return; + } + this._released = true; // Remove the live editor class if (this.view instanceof MarkdownView) { @@ -725,23 +898,42 @@ export class LiveView this._viewActions = undefined; this._banner?.destroy(); this._banner = undefined; - this.clearMergeButton(); if (this.offConnectionStatusSubscription) { this.offConnectionStatusSubscription(); this.offConnectionStatusSubscription = undefined; } + // Clean up HSM state subscription + if (this._hsmStateUnsubscribe) { + this._hsmStateUnsubscribe(); + this._hsmStateUnsubscribe = undefined; + } this._awarenessPlugin?.destroy(); this._awarenessPlugin = undefined; + this._viewHookPlugin?.destroy(); + this._viewHookPlugin = undefined; this._plugin?.destroy(); this._plugin = undefined; - this.document.disconnect(); - this.document.userLock = false; + const sharedFolder = this.document.destroyed ? null : this.document.sharedFolder; + const preservePendingUpload = + !!sharedFolder && sharedFolder.isPendingUpload(this.document.path); + let stillLocked = this.document.userLock; + if (this._hasLock) { + stillLocked = this._parent.releaseDocumentLock( + this.document, + this.view.leaf ?? this._fallbackViewer, + ); + this._hasLock = false; + } + if (!preservePendingUpload && !stillLocked) { + if (!this.document.deferDisconnectForPendingMessages()) { + this.document.disconnect(); + } + } } destroy() { this.release(); this.clearViewActions(); - this.clearMergeButton(); (this.view.leaf as any).rebuildView?.(); this._parent = null as any; this.view = null as any; @@ -755,7 +947,6 @@ export class LiveViewManager { workspace: Workspace; views: S3View[]; private _activePromise?: Promise | null; - _compartment: Compartment; private loginManager: LoginManager; private offListeners: (() => void)[] = []; private folderListeners: Map void> = new Map(); @@ -767,6 +958,7 @@ export class LiveViewManager { extensions: Extension[]; networkStatus: NetworkStatus; refreshQueue: (() => Promise)[]; + private documentViewers: Map>; log: (message: string, ...args: unknown[]) => void; warn: (message: string, ...args: unknown[]) => void; @@ -784,7 +976,7 @@ export class LiveViewManager { this.loginManager = loginManager; this.networkStatus = networkStatus; this.refreshQueue = []; - this._compartment = new Compartment(); + this.documentViewers = new Map(); this.log = curryLog("[LiveViews]", "log"); this.warn = curryLog("[LiveViews]", "warn"); @@ -809,8 +1001,10 @@ export class LiveViewManager { const folderSub = (folder: SharedFolder) => { if (!folder.ready) { (async () => { - folder - .whenReady() + trackPromise( + `liveViews:folderReady:${folder.guid}`, + folder.whenReady(), + ) .then(() => { this.refresh("[Shared Folder Ready]"); }) @@ -848,16 +1042,6 @@ export class LiveViewManager { RelayInstances.set(this, "LiveViewManager"); } - reconfigure(editorView: EditorView) { - editorView.dispatch({ - effects: this._compartment.reconfigure([ - ConnectionManagerStateField.init(() => { - return this; - }), - ]), - }); - } - onMeta(tfile: TFile, cb: (data: string, cache: CachedMetadata) => void) { this.metadataListeners.set(tfile, cb); } @@ -889,9 +1073,156 @@ export class LiveViewManager { return this.views.some((view) => view.document === doc); } + private isDocumentOpenInWorkspace(document: Document): boolean { + const sharedFolder = document.sharedFolder; + if (!sharedFolder) return false; + const fullPath = sharedFolder.getPath(document.path); + let open = false; + this.workspace.iterateAllLeaves((leaf) => { + if (open) return; + if ((leaf.view as any)?.file?.path === fullPath) { + open = true; + } + }); + return open; + } + + acquireDocumentLock( + document: Document, + editorViewRef: EditorViewRef, + viewer: DocumentViewer, + ): void { + let viewers = this.documentViewers.get(document.guid); + if (!viewers) { + viewers = new Set(); + this.documentViewers.set(document.guid, viewers); + } + + if (viewers.has(viewer)) { + document.userLock = true; + return; + } + + const wasEmpty = viewers.size === 0; + viewers.add(viewer); + document.userLock = true; + + if (wasEmpty) { + document.acquireLock(editorViewRef); + } + } + + releaseDocumentLock( + document: Document, + viewer: DocumentViewer, + ): boolean { + const viewers = this.documentViewers.get(document.guid); + if (!viewers) { + if (this.isDocumentOpenInWorkspace(document)) { + document.userLock = true; + return true; + } + document.userLock = false; + document.releaseLock(); + return false; + } + + viewers.delete(viewer); + if (viewers.size > 0) { + document.userLock = true; + return true; + } + + this.documentViewers.delete(document.guid); + if (this.isDocumentOpenInWorkspace(document)) { + document.userLock = true; + return true; + } + document.userLock = false; + document.releaseLock(); + return false; + } + + /** + * Notify MergeManagers which documents have open editors. + * Groups views by their shared folder and calls setActiveDocuments() on each. + * This transitions HSMs from 'loading' to the appropriate mode (idle or active). + * + * Per spec (Gap 8): LiveViews sends bulk update to MergeManager indicating which + * documents have open editors. MergeManager fans out SET_MODE_ACTIVE to those HSMs, + * and SET_MODE_IDLE to all others. + */ + private async updateMergeManagerActiveDocuments( + views: S3View[], + ): Promise { + // Group document GUIDs by their shared folder + const folderToGuids = new Map>(); + + for (const view of views) { + const doc = view.document; + if (!doc) continue; + + const folder = doc.sharedFolder; + if (!folder?.mergeManager) continue; + + if (!folderToGuids.has(folder)) { + folderToGuids.set(folder, new Set()); + } + folderToGuids.get(folder)!.add(doc.guid); + } + + // Also discover embedded markdown files inside canvas views. + // Canvas nodes with type='file' have a child TextFileView whose file + // may belong to a shared folder. Their HSMs need active mode too. + for (const view of views) { + if (!(view instanceof RelayCanvasView)) continue; + const canvas = view.view.canvas; + if (!canvas?.nodes) continue; + + for (const [, node] of canvas.nodes) { + const nodeData = node.getData?.(); + // @ts-ignore — child is not typed on CanvasNode, only on CanvasNodeData + const child = (node as any).child ?? nodeData?.child; + if (!child?.file) continue; + + const filePath: string = child.file.path; + if (!filePath.endsWith(".md")) continue; + + const folder = this.sharedFolders.lookup(filePath); + if (!folder?.mergeManager || !folder.ready) continue; + + const embeddedDoc = folder.proxy.getDoc(filePath); + if (!embeddedDoc) continue; + + if (!folderToGuids.has(folder)) { + folderToGuids.set(folder, new Set()); + } + folderToGuids.get(folder)!.add(embeddedDoc.guid); + } + } + + // Call setActiveDocuments on each folder's MergeManager + for (const [folder, guids] of folderToGuids) { + const allGuids = Array.from(folder.files.keys()); + folder.mergeManager.setActiveDocuments(guids, allGuids); + } + + // Also notify folders with no active views (all HSMs should be idle) + for (const folder of this.sharedFolders.items()) { + if (!folderToGuids.has(folder) && folder.mergeManager) { + const allGuids = Array.from(folder.files.keys()); + folder.mergeManager.setActiveDocuments(new Set(), allGuids); + } + } + } + private releaseViews(views: S3View[]) { views.forEach((view) => { - view.release(); + try { + view.release(); + } catch (e) { + this.warn("[LiveViews] error releasing stale view", e); + } }); } @@ -952,7 +1283,9 @@ export class LiveViewManager { if (folders.size === 0) { return []; } - const readyFolders = [...folders].map((folder) => folder.whenReady()); + const readyFolders = [...folders].map((folder) => + trackPromise(`liveViews:findFolders:${folder.guid}`, folder.whenReady()), + ); return Promise.all(readyFolders); } @@ -991,21 +1324,21 @@ export class LiveViewManager { } const folder = this.sharedFolders.lookup(viewFilePath); if (folder && canvasView.file) { - const canvas = folder.getFile(canvasView.file); - if (isCanvas(canvas)) { - if (!this.loginManager.loggedIn) { - const view = new LoggedOutView(this, canvasView, () => { - return this.loginManager.openLoginPage(); - }); - views.push(view); - } else if (folder.ready) { + if (!this.loginManager.loggedIn) { + const view = new LoggedOutView(this, canvasView, () => { + return this.loginManager.openLoginPage(); + }); + views.push(view); + } else if (folder.ready) { + const canvas = folder.getFile(canvasView.file); + if (isCanvas(canvas)) { const view = new RelayCanvasView(this, canvasView, canvas); views.push(view); } else { - this.log(`Folder not ready, skipping views. folder=${folder.path}`); + this.log(`Skipping canvas view connection for ${viewFilePath}`); } } else { - this.log(`Skipping canvas view connection for ${viewFilePath}`); + this.log(`Folder not ready, skipping views. folder=${folder.path}`); } } }); @@ -1088,10 +1421,38 @@ export class LiveViewManager { ); } + private getExpectedViewPath(document: S3View["document"]): string | null { + if (!document?.path) { + return null; + } + const sharedFolder = document.sharedFolder; + if (!sharedFolder) { + return null; + } + return sharedFolder.getPath(document.path); + } + private deduplicate(views: S3View[]): [S3View[], S3View[]] { const stale: S3View[] = []; const matching: S3View[] = []; this.views.forEach((oldView) => { + const viewPath = oldView.view?.file?.path; + const expectedViewPath = this.getExpectedViewPath(oldView.document); + if ( + oldView.document?.path && + viewPath && + expectedViewPath && + viewPath !== expectedViewPath + ) { + this.warn("[LiveViews] stale view path mismatch", { + viewPath, + expectedViewPath, + documentPath: oldView.document.path, + viewType: oldView.view.getViewType?.(), + }); + stale.push(oldView); + return; + } const found = views.find((newView) => { if ( oldView.document == newView.document && @@ -1143,11 +1504,12 @@ export class LiveViewManager { return false; } const activeDocumentFolders = this.findFolders(); + + // Notify MergeManagers which documents have open editors (Gap 8: mode determination) + // This transitions HSMs from 'loading' to the appropriate mode before attach() calls acquireLock() + await this.updateMergeManagerActiveDocuments(views); + if (activeDocumentFolders.length === 0 && views.length === 0) { - if (this.extensions.length !== 0) { - log("Unexpected plugins loaded."); - this.wipe(); - } logViews("Releasing Views", this.views); this.releaseViews(this.views); this.views = []; @@ -1184,7 +1546,9 @@ export class LiveViewManager { log("loading plugins"); this.load(); const now = moment.utc(); - log(`refresh completed in ${now.diff(queuedAt)}ms`, ctx); + const durationMs = now.diff(queuedAt); + log(`refresh completed in ${durationMs}ms`, ctx); + metrics.observeLiveviewsRefresh(durationMs / 1000); return true; } @@ -1195,6 +1559,7 @@ export class LiveViewManager { this.refreshQueue.push(() => { return this._refreshViews(context, queuedAt); }); + metrics.setLiveviewsQueueDepth(this.refreshQueue.length); if (this._activePromise !== null) { return false; } @@ -1219,22 +1584,18 @@ export class LiveViewManager { } load() { - this.wipe(); - if (this.views.length > 0) { - this.extensions.push([ - this._compartment.of( - ConnectionManagerStateField.init(() => { - return this; - }), - ), - LiveEdit, - LiveNode, - yRemoteSelectionsTheme, - yRemoteSelections, - InvalidLinkPlugin, - ]); - this.workspace.updateOptions(); - } + if (this.extensions.length > 0) return; // already registered + this.extensions.push([ + HSMEditorPlugin, + LiveNode, + yRemoteSelectionsTheme, + yRemoteSelections, + attributionFilterField, + userAttributionTheme, + userAttributionPlugin, + InvalidLinkPlugin, + ]); + this.workspace.updateOptions(); } public destroy() { @@ -1247,6 +1608,8 @@ export class LiveViewManager { this.folderListeners.forEach((off) => off()); this.folderListeners.clear(); this.folderListeners = null as any; + this.documentViewers.clear(); + this.documentViewers = null as any; this.views.forEach((view) => view.destroy()); this.views = []; this.wipe(); @@ -1259,14 +1622,3 @@ export class LiveViewManager { this.workspace = null as any; } } - -export const ConnectionManagerStateField = StateField.define< - LiveViewManager | undefined ->({ - create(state: EditorState) { - return undefined; - }, - update(currentManager, transaction) { - return currentManager; - }, -}); diff --git a/src/LoginManager.ts b/src/LoginManager.ts index 6b4e5837..131b3ebf 100644 --- a/src/LoginManager.ts +++ b/src/LoginManager.ts @@ -11,14 +11,12 @@ import PocketBase, { import { RelayInstances, curryLog } from "./debug"; import { Observable } from "./observable/Observable"; -declare const GIT_TAG: string; - -import { customFetch } from "./customFetch"; +import { customFetch, getRelayRequestHeaders } from "./customFetch"; import { LocalAuthStore } from "./pocketbase/LocalAuthStore"; import type { TimeProvider } from "./TimeProvider"; import { FeatureFlagManager } from "./flagManager"; import type { NamespacedSettings } from "./SettingsStorage"; -import { type EndpointManager, type EndpointSettings } from "./EndpointManager"; +import { type EndpointManager } from "./EndpointManager"; interface GoogleUser { email: string; @@ -174,7 +172,7 @@ export class LoginManager extends Observable { constructor( vaultName: string, openSettings: () => Promise, - timeProvider: TimeProvider, + private timeProvider: TimeProvider, private beforeLogin: () => void, public loginSettings: NamespacedSettings, endpointManager: EndpointManager, @@ -190,9 +188,11 @@ export class LoginManager extends Observable { this.logout(); } options.fetch = customFetch; - options.headers = Object.assign({}, options.headers, { - "Relay-Version": GIT_TAG, - }); + options.headers = Object.assign( + {}, + options.headers, + getRelayRequestHeaders(), + ); return { url, options }; }; this.refreshToken(); @@ -282,7 +282,7 @@ export class LoginManager extends Observable { async checkRelayHost(relay_guid: string): Promise { const headers = { Authorization: `Bearer ${this.pb.authStore.token}`, - "Relay-Version": GIT_TAG, + ...getRelayRequestHeaders(), }; return requestUrl({ url: `${this.endpointManager.getApiUrl()}/relay/${relay_guid}/check-host`, @@ -294,7 +294,7 @@ export class LoginManager extends Observable { getFlags() { const headers = { Authorization: `Bearer ${this.pb.authStore.token}`, - "Relay-Version": GIT_TAG, + ...getRelayRequestHeaders(), }; requestUrl({ url: `${this.endpointManager.getApiUrl()}/flags`, @@ -315,6 +315,7 @@ export class LoginManager extends Observable { whoami() { const headers = { Authorization: `Bearer ${this.pb.authStore.token}`, + ...getRelayRequestHeaders(), }; requestUrl({ url: `${this.endpointManager.getApiUrl()}/whoami`, @@ -351,6 +352,10 @@ export class LoginManager extends Observable { const result = await this.endpointManager.validateAndSetEndpoints(timeoutMs); if (result.success && this.endpointManager.hasValidatedEndpoints()) { + // Clean up old PocketBase instance before creating new one + this.pb.cancelAllRequests(); + this.pb.realtime.unsubscribe(); + // Recreate PocketBase instance with new auth URL const pbLog = curryLog("[Pocketbase]", "debug"); this.pb = new PocketBase(this.endpointManager.getAuthUrl(), this.authStore); @@ -360,9 +365,11 @@ export class LoginManager extends Observable { this.logout(); } options.fetch = customFetch; - options.headers = Object.assign({}, options.headers, { - "Relay-Version": GIT_TAG, - }); + options.headers = Object.assign( + {}, + options.headers, + getRelayRequestHeaders(), + ); return { url, options }; }; this.log("Updated PocketBase instance with validated endpoints"); @@ -389,6 +396,7 @@ export class LoginManager extends Observable { logout() { this.pb.cancelAllRequests(); + this.pb.realtime.unsubscribe(); this.pb.authStore.clear(); this.user = undefined; this.notifyListeners(); @@ -521,10 +529,10 @@ export class LoginManager extends Observable { let counter = 0; const interval = 1000; return new Promise((resolve, reject) => { - const timer = setInterval(() => { + const timer = this.timeProvider.setInterval(() => { counter += 1; if (counter >= 30) { - clearInterval(timer); + this.timeProvider.clearInterval(timer); return reject( new Error( `Auth timeout: Timed out after ${ @@ -538,7 +546,7 @@ export class LoginManager extends Observable { .getOne(provider.info.state.slice(0, 15)) .then((response) => { if (response) { - clearInterval(timer); + this.timeProvider.clearInterval(timer); return resolve(provider.login(response.code)); } }) @@ -549,10 +557,18 @@ export class LoginManager extends Observable { async login(provider: string): Promise { this.beforeLogin(); - const authData = await this.pb.collection("users").authWithOAuth2({ - provider: provider, - }); - return this.setup(authData, provider); + try { + const authData = await this.pb.collection("users").authWithOAuth2({ + provider: provider, + }); + return this.setup(authData, provider); + } catch (e) { + // Clean up realtime subscription to prevent reconnection loops + // authWithOAuth2 internally subscribes to @oauth2 via SSE, and if it fails, + // PocketBase's realtime client will keep trying to reconnect indefinitely + this.pb.realtime.unsubscribe(); + throw e; + } } async openLoginPage() { @@ -563,7 +579,6 @@ export class LoginManager extends Observable { this.off(isLoggedIn); resolve(true); } - resolve(false); }; this.on(isLoggedIn); }); diff --git a/src/NetworkStatus.ts b/src/NetworkStatus.ts index 36fdd600..9b066d8b 100644 --- a/src/NetworkStatus.ts +++ b/src/NetworkStatus.ts @@ -1,8 +1,7 @@ import { requestUrl } from "obsidian"; import { curryLog } from "./debug"; import type { TimeProvider } from "./TimeProvider"; - -declare const GIT_TAG: string; +import { getRelayRequestHeaders } from "./customFetch"; interface ServiceStatus { status: string; @@ -51,7 +50,7 @@ class NetworkStatus { public stop() { if (this.timer) { - clearInterval(this.timer); + this.timeProvider.clearInterval(this.timer); } } @@ -77,7 +76,7 @@ class NetworkStatus { return requestUrl({ url: this.url, method: "GET", - headers: { "Relay-Version": GIT_TAG }, + headers: getRelayRequestHeaders(), }) .then((response) => { if (response.status === 200) { diff --git a/src/Relay.ts b/src/Relay.ts index f1791eaf..45bc1650 100644 --- a/src/Relay.ts +++ b/src/Relay.ts @@ -20,7 +20,7 @@ interface HasPermissionParents { permissionParents: [string, string][]; } interface Serializable { - toDict: () => any; + toDict: () => Record; } export function hasPermissionParents(item: HasPermissionParents) { diff --git a/src/RelayDebugAPI.ts b/src/RelayDebugAPI.ts new file mode 100644 index 00000000..bafad5af --- /dev/null +++ b/src/RelayDebugAPI.ts @@ -0,0 +1,1469 @@ +/** + * RelayDebugAPI — Plugin-level debug surface exposed as `window.__relayDebug`. + * + * Aggregates per-folder recording bridges and provides CDP-accessible + * utilities for E2E tests, live debugging, and diagnostics. + * + * Lifecycle: created in plugin onload(), destroyed in onunload(). + */ + +import * as Y from 'yjs'; +import { diff_match_patch } from 'diff-match-patch'; +import { IndexeddbPersistence } from './storage/y-indexeddb'; +import type { TimeProvider } from './TimeProvider'; +import type { E2ERecordingBridge, E2ERecordingState } from './merge-hsm/recording'; +import type { ConflictInfoSnapshot } from './merge-hsm/conflict'; +import { getHSMBootId, getHSMBootEntries, getRecentEntries, getSessionLogs } from './debug'; +import type { SessionLogOptions } from './debug'; +import { getRecentPromises } from './trackPromise'; + +export type { ConflictHunkInfo, ConflictInfoSnapshot } from './merge-hsm/conflict'; + +// ============================================================================= +// Types +// ============================================================================= + +export interface DocumentContentSnapshot { + path: string; + guid: string; + folder: string; + local: { content: string; stateVector: string } | null; + remote: { content: string; stateVector: string } | null; + idb: { content: string; stateVector: string } | null; + disk: { content: string; mtime: number } | null; + server: { content: string; stateVector: string; updateSize: number } | null; +} + +export interface HsmStateTransition { + ts: number; + seq: number; + event: string; + from: string; + to: string; +} + +export interface HsmSyncGate { + providerConnected: boolean; + providerSynced: boolean; + localOnly: boolean; + pendingInbound: number; + pendingOutbound: number; +} + +export interface IdbContentSnapshot { + path: string; + guid: string; + folder: string; + dbName: string; + metadata: Record; + updatesCount: number; + idbContent: string | null; + idbLength: number; + diskContent: string | null; + diskLength: number | null; + match: boolean; +} + +export interface IdbHistoryEntry { + key: IDBValidKey; + origin: any; + timestamp: number | null; + time: string | null; + insertionsBytes: number; + deletionsBytes: number; +} + +export interface IdbHistorySnapshot { + path: string; + guid: string; + folder: string; + dbName: string; + historyCount: number; + inMemoryCount: number | null; + entries: IdbHistoryEntry[]; + note?: string; +} + +export interface ForkSnapshot { + base: string | null; + baseLength: number; + origin: string | null; + created: number | null; + createdTime: string | null; + captureMark: any; + localStateVectorBytes: number; + remoteStateVectorBytes: number; +} + +export interface IdbForkSnapshot { + path: string; + guid: string; + folder: string; + statePath: string; + hasFork: boolean; + inMemoryFork: ForkSnapshot | null; + persistedFork: ForkSnapshot | { error: string } | null; + persistedMeta: { + lastStatePath: string | null; + persistedAt: number | null; + persistedAtTime: string | null; + hasForkInPersistedState: boolean; + } | null; +} + +/** + * Rich snapshot of an HSM's state, covering every layer the test harness + * routinely inspects: state path + sync gate (from the machine), LCA meta + * and content (from the HSM), localDoc length/content/frontmatter (from + * the in-memory Y.Doc), disk content + mtime (via the vault adapter), and + * recent HSM transitions (via the disk log). Replaces the ad-hoc 120-line + * eval blob that used to live in the Python CLI. + */ +export interface HsmStateSnapshot { + path: string; + guid: string; + folder: string; + statePath: string; + syncGate: HsmSyncGate | null; + hasLCA: boolean; + lcaHash: string | null; + lcaContentLength: number | null; + lcaContent: string | null; + hasConflict: boolean; + conflictData: any | null; + localDocLength: number; + idbContent: string | null; + diskMtime: number | null; + diskContent: string | null; + stateVectorsEqual: boolean | null; + diskMatchesIdb: boolean; + idbMatchesLca: boolean; + frontmatterMap: Record | null; + recentTransitions: HsmStateTransition[]; +} + +// ============================================================================= +// Global interface exposed via CDP +// ============================================================================= + +/** + * A stable reference to an editor leaf. Resolved by matching windowId+leafId; + * operations that require the leaf to still be showing the same file verify + * `handle.path` against the leaf's current file. + */ +export interface EditorHandle { + windowId: string; + leafId: string; + path: string; +} + +export interface OpenEditorResult { + handle: EditorHandle; + viewType: string | null; + mode: string | null; +} + +export interface EditorInfo { + handle: EditorHandle; + /** The leaf's current file path. Differs from handle.path if the leaf drifted. Null if the leaf is gone. */ + currentPath: string | null; + viewType: string | null; + mode: string | null; + active: boolean; +} + +export type SetEditorContentResult = + | { success: true; changeCount: number } + | { success: false; error: string }; + +export interface RelayDebugGlobal { + /** Open PATH in an editor leaf. Pass `{ newLeaf: true }` to force a new tab. */ + openEditor: (path: string, opts?: { newLeaf?: boolean }) => Promise; + /** Close the exact leaf identified by HANDLE. No-op if already gone. */ + closeEditor: (handle: EditorHandle) => Promise; + /** Read the editor text from the exact leaf. Throws if the leaf drifted. */ + getEditorContent: (handle: EditorHandle) => Promise; + /** Inspect a handle without mutating focus or throwing on drift. */ + getEditorInfo: (handle: EditorHandle) => EditorInfo; + /** Enumerate every open markdown editor leaf with its handle and state. */ + listEditors: () => EditorInfo[]; + /** Start recording all HSM activity */ + startRecording: (name?: string) => E2ERecordingState; + /** Stop recording and return lightweight summary JSON */ + stopRecording: () => string; + /** Get current recording state */ + getState: () => E2ERecordingState; + /** Check if recording is active */ + isRecording: () => boolean; + /** Get list of active document GUIDs */ + getActiveDocuments: () => string[]; + /** Get the current boot ID (for disk recording) */ + getBootId: () => string | null; + /** Get entries from current boot (reads disk file, filters by boot ID) */ + getBootEntries: () => Promise; + /** Get last N entries for a specific document (buffer + disk, newest files first) */ + getRecentEntries: (guid: string, limit?: number) => Promise; + /** Read Y.Doc text content from IndexedDB without waking the HSM */ + readIdbContent: (guid: string, appId: string) => Promise<{ content: string; stateVector: Uint8Array } | null>; + /** Get plugin log entries from the current session, with optional level/pattern filtering */ + getSessionLogs: (options?: SessionLogOptions) => Promise; + /** Get a snapshot of all content views for a document */ + getDocumentContent: (path: string) => Promise; + /** Set the editor text via minimal CM6 transactions. Throws if the leaf drifted. */ + setEditorContent: (handle: EditorHandle, content: string) => Promise; + /** Look up a document by vault-level path including the shared-folder prefix (e.g. "/private/foo.md"). Returns document, HSM, folder, and GUID. */ + lookupDocument: (path: string) => { doc: any; hsm: any; guid: string; folder: any; filePath: string } | null; + /** Look up a shared folder by path (e.g. "private"). Returns the SharedFolder or null. */ + lookupFolder: (path: string) => any | null; + /** Folder-scoped sync rows from MergeManager.syncStatus keyed by guid. */ + getFolderSyncStatus: (folderGuid: string) => { guid: string; path: string; status: string }[]; + /** Folder-scoped subset of sync rows where status === "error". */ + getFolderSyncErrors: (folderGuid: string) => { guid: string; path: string; status: string }[]; + /** Folder-scoped subset of sync rows where status === "conflict". */ + getFolderConflicts: (folderGuid: string) => { guid: string; path: string }[]; + /** All files currently in conflict state across every shared folder. */ + listAllConflicts: () => { folderGuid: string; folderPath: string; guid: string; path: string }[]; + /** Get a rich HSM state snapshot for the test harness — state path, LCA, disk, IDB, SV, frontmatter, recent transitions. */ + getHsmStateSnapshot: (path: string) => Promise; + /** Snapshot the per-doc IndexedDB: updates count, custom metadata, IDB content, disk content, match flag. */ + getIdbContent: (path: string) => Promise; + /** Snapshot the OpCapture history store for a document. */ + getIdbHistory: (path: string) => Promise; + /** Snapshot in-memory and persisted fork state for a document. */ + getIdbFork: (path: string) => Promise; + /** + * Wait for an HSM to reach a state path that starts with `statePrefix`, + * subject to a timeout. Resolves with the final state path on success. + * Thin bridge over `MergeHSM.awaitState` — event-driven, no polling. + */ + awaitHsmState: (path: string, statePrefix: string, timeoutMs: number) => Promise; + /** + * Focused conflict snapshot: base/ours/theirs plus labels so callers + * can pick the right side by semantic name without pulling the whole + * HsmStateSnapshot. Throws if the document is not found. + */ + getConflictInfo: (path: string) => Promise; + /** + * Resolve the conflict with the chosen final content. Active conflicts use + * the normal HSM event path; idle.diverged conflicts resolve headlessly + * without opening editors or views. + */ + resolveConflict: (path: string, contents: string) => Promise; + /** + * Dispatch a `RESOLVE_HUNK` event for a single conflict hunk. + * + * `hunkId` is matched against `ConflictHunkInfo.id`; throws on + * ambiguous (collision) or missing. Numeric array indices are not + * accepted at this boundary because digit-only hash prefixes are valid ids. + * + * `resolution` picks the side to apply: + * - "ours" → oursContent + * - "theirs" → theirsContent + * - "both" → oursContent + "\n" + theirsContent + * - "neither" → remove the hunk entirely + * + * The HSM mutates localDoc in place at the hunk's positioned region, + * marks the hunk resolved, and once every hunk is resolved commits + * the final content. idle.diverged conflicts resolve headlessly + * without opening editors or views. + */ + resolveHunk: ( + path: string, + hunkId: string, + resolution: 'ours' | 'theirs' | 'both' | 'neither', + ) => Promise; + /** + * Dispatch an `OPEN_DIFF_VIEW` event — the state-machine-level + * equivalent of the user clicking the conflict banner. Transitions + * `active.conflict.bannerShown` → `active.conflict.resolving`. This + * only drives the HSM; it does not open a diff view leaf in the UI. + * Returns the state path after dispatch. + */ + openDiffView: (path: string) => Promise; + /** + * Dispatch a `CANCEL` event — the state-machine-level equivalent of + * the user closing the diff view without resolving. Transitions + * `active.conflict.resolving` → `active.conflict.bannerShown`. + * Returns the state path after dispatch. + */ + cancelDiffView: (path: string) => Promise; + /** + * Clear the HSM's LCA in place. Low-level internal-state mutation — + * reproduces the no-LCA state that arises after upgrading from a + * plugin version without LCA tracking. On reopen the HSM enters + * `isRecoveryMode` and routes to two-way merge. + */ + clearLca: (path: string) => Promise; + + // -- Promise tracking -- + getPendingPromises: () => { label: string; ageMs: number; owner?: string }[]; + getRecentPromises: () => { label: string; created: number; settledAt: number; state: "fulfilled" | "rejected"; owner?: string }[]; + + // -- Relay server CRUD -- + createRelay: (name: string) => Promise<{ guid: string; name: string }>; + renameRelay: (guid: string, newName: string) => Promise<{ guid: string; name: string }>; + deleteRelay: (guid: string) => Promise; +} + +// ============================================================================= +// RelayDebugAPI +// ============================================================================= + +export class RelayDebugAPI { + private bridges = new Map(); + private activeRecordingName: string | null = null; + private plugin: any; + private destroyed = false; + + constructor(plugin?: any) { + this.plugin = plugin; + this.installGlobal(); + } + + private debugWindow(): Window { + return window; + } + + private debugGlobal(): any { + return (this.debugWindow() as any).__relayDebug; + } + + /** + * Register a per-folder recording bridge. + * Returns a cleanup function to call when the folder is destroyed. + */ + registerBridge(folderPath: string, bridge: E2ERecordingBridge): () => void { + if (this.destroyed) { + return () => { + bridge.dispose(); + }; + } + this.bridges.set(folderPath, bridge); + + // Auto-start recording if one is currently active + if (this.activeRecordingName !== null) { + try { + bridge.startRecording(this.activeRecordingName); + } catch { /* already recording */ } + } + + this.installGlobal(); + + return () => { + bridge.dispose(); + this.bridges.delete(folderPath); + if (!this.destroyed) { + this.installGlobal(); + } + }; + } + + /** + * Install the `window.__relayDebug` global. + */ + private installGlobal(): void { + if (this.destroyed) { + if (this.debugGlobal()?.__owner === this) { + delete (this.debugWindow() as any).__relayDebug; + } + return; + } + + const api: RelayDebugGlobal = { + startRecording: (name) => { + this.activeRecordingName = name ?? 'E2E Recording'; + const results: E2ERecordingState[] = []; + for (const bridge of this.bridges.values()) { + try { results.push(bridge.startRecording(name)); } + catch { /* already recording */ } + } + return { + recording: results.some(r => r.recording), + name: name ?? null, + id: results[0]?.id ?? null, + startedAt: results[0]?.startedAt ?? null, + documentCount: results.reduce((sum, r) => sum + r.documentCount, 0), + totalEntries: results.reduce((sum, r) => sum + r.totalEntries, 0), + }; + }, + + stopRecording: () => { + this.activeRecordingName = null; + const recordings: string[] = []; + for (const bridge of this.bridges.values()) { + try { recordings.push(bridge.stopRecording()); } + catch { /* not recording */ } + } + const combined = recordings.flatMap(r => { + try { return JSON.parse(r); } catch { return []; } + }); + return JSON.stringify(combined, null, 2); + }, + + getState: () => { + let totalDocs = 0; + let totalEntries = 0; + let recording = false; + let name: string | null = null; + let id: string | null = null; + let startedAt: string | null = null; + + for (const bridge of this.bridges.values()) { + const state = bridge.getState(); + if (state.recording) { + recording = true; + name = name ?? state.name; + id = id ?? state.id; + startedAt = startedAt ?? state.startedAt; + } + totalDocs += state.documentCount; + totalEntries += state.totalEntries; + } + + return { recording, name, id, startedAt, documentCount: totalDocs, totalEntries }; + }, + + isRecording: () => { + for (const bridge of this.bridges.values()) { + if (bridge.isRecording()) return true; + } + return false; + }, + + getActiveDocuments: () => { + const docs: string[] = []; + for (const bridge of this.bridges.values()) { + docs.push(...bridge.getActiveDocuments()); + } + return docs; + }, + + getBootId: () => getHSMBootId(), + getBootEntries: () => getHSMBootEntries(), + getRecentEntries: (guid, limit) => getRecentEntries(guid, limit), + readIdbContent: readIdbContent, + getSessionLogs: (options) => getSessionLogs(options), + openEditor: (path, opts) => this.openEditor(path, opts), + closeEditor: (handle) => this.closeEditor(handle), + getEditorContent: (handle) => this.getEditorContent(handle), + getEditorInfo: (handle) => this.getEditorInfo(handle), + listEditors: () => this.listEditors(), + getDocumentContent: async (path) => this.getDocumentContent(path), + getHsmStateSnapshot: async (path) => this.getHsmStateSnapshot(path), + getIdbContent: async (path) => this.getIdbContent(path), + getIdbHistory: async (path) => this.getIdbHistory(path), + getIdbFork: async (path) => this.getIdbFork(path), + awaitHsmState: async (path, statePrefix, timeoutMs) => + this.awaitHsmState(path, statePrefix, timeoutMs), + getConflictInfo: async (path) => this.getConflictInfo(path), + resolveConflict: async (path, contents) => this.resolveConflict(path, contents), + resolveHunk: async (path, hunkId, resolution) => + this.resolveHunk(path, hunkId, resolution), + openDiffView: async (path) => this.sendConflictEvent(path, { type: 'OPEN_DIFF_VIEW' }), + cancelDiffView: async (path) => this.sendConflictEvent(path, { type: 'CANCEL' }), + clearLca: async (path) => this.clearLca(path), + getPendingPromises: () => this.plugin?.promises?.getPending() ?? [], + getRecentPromises: () => getRecentPromises(), + + createRelay: async (name) => { + if (!this.plugin.relayManager) throw new Error('RelayManager not available'); + const relay = await this.plugin.relayManager.createRelay(name); + return { guid: relay.guid, name: relay.name }; + }, + renameRelay: async (guid, newName) => { + if (!this.plugin.relayManager) throw new Error('RelayManager not available'); + const relay = this.findRelayByGuid(guid); + if (!relay) throw new Error(`Relay not found: ${guid}`); + relay.name = newName; + await this.plugin.relayManager.updateRelay(relay); + return { guid: relay.guid, name: relay.name }; + }, + deleteRelay: async (guid) => { + if (!this.plugin.relayManager) throw new Error('RelayManager not available'); + const relay = this.findRelayByGuid(guid); + if (!relay) throw new Error(`Relay not found: ${guid}`); + return await this.plugin.relayManager.destroyRelay(relay); + }, + + setEditorContent: (handle, content) => this.setEditorContent(handle, content), + + lookupFolder: (path: string) => { + if (!this.plugin?.sharedFolders?._set) return null; + for (const folder of this.plugin.sharedFolders._set.values()) { + if ((folder as any).path === path) return folder; + } + // Also try matching as a prefix (e.g. "private" matches folder at path "private") + for (const folder of this.plugin.sharedFolders._set.values()) { + if (path.startsWith((folder as any).path + '/')) return folder; + } + return null; + }, + getFolderSyncStatus: (folderGuid: string) => this.getFolderSyncStatus(folderGuid), + getFolderSyncErrors: (folderGuid: string) => this.getFolderSyncErrors(folderGuid), + getFolderConflicts: (folderGuid: string) => this.getFolderConflicts(folderGuid), + listAllConflicts: () => this.listAllConflicts(), + + lookupDocument: (path: string) => { + const sharedFolders = this.plugin?.sharedFolders; + if (!sharedFolders) return null; + if (!path.startsWith('/')) { + for (const folder of (sharedFolders as any)._set.values()) { + const doc = folder.mergeManager?._getDocument(path); + const hsm = doc?._hsm; + if (hsm) return { doc, hsm, guid: path, folder, filePath: hsm.path || path }; + } + throw new Error(`Document paths must start with '/' (got: ${JSON.stringify(path)})`); + } + const vaultPath = path.slice(1); + const folder: any = sharedFolders.lookup(vaultPath); + if (!folder) { + const available = Array.from((sharedFolders as any)._set.values()) + .map((f: any) => '/' + f.path + '/') + .join(', ') || '(none)'; + throw new Error( + `Document path must be a vault-level path under a shared folder ` + + `(got: ${JSON.stringify(path)}; shared folders: ${available})` + ); + } + const vpath = folder.getVirtualPath(vaultPath); + const guid = folder.syncStore?.get(vpath); + if (!guid) return null; + const doc = folder.mergeManager?._getDocument(guid); + const hsm = doc?._hsm; + if (!hsm) return null; + return { doc, hsm, guid, folder, filePath: hsm.path || vpath }; + }, + + }; + + (this.debugWindow() as any).__relayDebug = { + __owner: this, + ...api, + registerBridge: (folderPath: string, bridge: E2ERecordingBridge) => this.registerBridge(folderPath, bridge), + }; + } + + /** + * Locate the leaf identified by HANDLE.windowId + HANDLE.leafId. Does NOT + * verify the path — callers that require path match call resolveAndVerify. + */ + private findLeaf(handle: EditorHandle): any | null { + let found: any = null; + this.plugin?.app?.workspace?.iterateAllLeaves?.((leaf: any) => { + if (found) return; + const ids = this.leafIds(leaf); + if (ids.windowId === handle.windowId && ids.leafId === handle.leafId) { + found = leaf; + } + }); + return found; + } + + /** + * Resolve the exact leaf for HANDLE and verify it still shows handle.path. + * Throws a precise error on every failure mode the caller cares about. + */ + private resolveAndVerify(handle: EditorHandle): any { + const leaf = this.findLeaf(handle); + if (!leaf) { + throw new Error(`leaf not found: windowId=${handle.windowId} leafId=${handle.leafId}`); + } + const currentPath = leaf.view?.file?.path ?? null; + if (currentPath !== handle.path) { + throw new Error(`leaf drifted to ${currentPath ?? ''} (expected ${handle.path})`); + } + return leaf; + } + + /** + * Stable IDs for a leaf. Uses Obsidian's internal leaf.id and derives a + * windowId from the leaf's root (main window vs popout). + */ + private leafIds(leaf: any): { windowId: string; leafId: string } { + const leafId: string = leaf?.id ?? ''; + const root = leaf?.getRoot?.(); + const rootId: string | undefined = root?.id; + const mainRoot = this.plugin?.app?.workspace?.rootSplit; + let windowId: string; + if (!root || root === mainRoot) { + windowId = 'main'; + } else if (rootId) { + windowId = `popout:${rootId}`; + } else { + // Fallback: identify by the window containing the leaf's DOM. + const ownerWin = leaf?.view?.containerEl?.ownerDocument?.defaultView; + windowId = ownerWin && ownerWin !== window ? 'popout:unknown' : 'main'; + } + return { windowId, leafId }; + } + + private leafViewInfo(leaf: any): { viewType: string | null; mode: string | null; currentPath: string | null } { + const view = leaf?.view; + return { + viewType: view?.getViewType?.() ?? null, + mode: view?.getMode?.() ?? null, + currentPath: view?.file?.path ?? null, + }; + } + + private findLeavesByPath(path: string): any[] { + const matches: any[] = []; + this.plugin?.app?.workspace?.iterateAllLeaves?.((leaf: any) => { + if (leaf?.view?.file?.path === path) { + matches.push(leaf); + } + }); + return matches; + } + + private async openEditor( + path: string, + opts?: { newLeaf?: boolean }, + ): Promise { + const file = this.plugin?.app?.vault?.getAbstractFileByPath(path); + if (!file) { + throw new Error(`File not found: ${path}`); + } + const leaf = this.plugin.app.workspace.getLeaf(opts?.newLeaf ? 'tab' : false); + await leaf.openFile(file); + this.plugin.app.workspace.setActiveLeaf?.(leaf, { focus: true }); + + // Markdown views default to preview; flip to source so the editor is live. + // setViewState is used instead of view.setMode because setMode expects a + // mode instance (from view.modes), not a string — passing a string leaves + // view.currentMode as the string and corrupts the view. + const view = leaf.view; + if (view?.getViewType?.() === 'markdown' && view.getMode?.() !== 'source') { + if (typeof leaf.setViewState === 'function') { + const state = leaf.getViewState?.() ?? { type: 'markdown', state: {} }; + await leaf.setViewState({ + ...state, + state: { ...(state.state || {}), file: path, mode: 'source' }, + }, { focus: true }); + } + } + + // Let Obsidian finish any async view replacement caused by mode switches. + await new Promise((resolve) => window.setTimeout(resolve, 0)); + + const activeLeaf = this.plugin.app.workspace.activeLeaf; + const candidates = this.findLeavesByPath(path); + const resolvedLeaf = ( + (activeLeaf?.view?.file?.path === path ? activeLeaf : null) + ?? candidates.find((candidate) => candidate === leaf) + ?? candidates[0] + ?? leaf + ); + + const ids = this.leafIds(resolvedLeaf); + const info = this.leafViewInfo(resolvedLeaf); + return { + handle: { windowId: ids.windowId, leafId: ids.leafId, path }, + viewType: info.viewType, + mode: info.mode, + }; + } + + private getEditorInfo(handle: EditorHandle): EditorInfo { + const leaf = this.findLeaf(handle); + if (!leaf) { + return { + handle, + currentPath: null, + viewType: null, + mode: null, + active: false, + }; + } + const info = this.leafViewInfo(leaf); + const active = this.plugin?.app?.workspace?.activeLeaf === leaf; + return { + handle, + currentPath: info.currentPath, + viewType: info.viewType, + mode: info.mode, + active, + }; + } + + private listEditors(): EditorInfo[] { + const out: EditorInfo[] = []; + const activeLeaf = this.plugin?.app?.workspace?.activeLeaf; + this.plugin?.app?.workspace?.iterateAllLeaves?.((leaf: any) => { + const info = this.leafViewInfo(leaf); + // Only markdown leaves have an editor; other view types can't be targeted + // by editor commands, so listing them would just add noise. + if (info.viewType !== 'markdown' || !info.currentPath) return; + const ids = this.leafIds(leaf); + out.push({ + handle: { windowId: ids.windowId, leafId: ids.leafId, path: info.currentPath }, + currentPath: info.currentPath, + viewType: info.viewType, + mode: info.mode, + active: leaf === activeLeaf, + }); + }); + return out; + } + + private async getEditorContent(handle: EditorHandle): Promise { + const leaf = this.resolveAndVerify(handle); + const editor = leaf.view?.editor; + if (!editor) { + throw new Error(`leaf has no editor: ${handle.path}`); + } + return editor.getValue(); + } + + private async setEditorContent( + handle: EditorHandle, + content: string, + ): Promise { + const leaf = this.resolveAndVerify(handle); + const editor = leaf.view?.editor; + const cm = editor?.cm; + if (!cm) return { success: false, error: 'leaf has no CM6 EditorView' }; + + const before = cm.state.doc.toString(); + if (before === content) return { success: true, changeCount: 0 }; + + const dmp = new diff_match_patch(); + const diffs = dmp.diff_main(before, content); + dmp.diff_cleanupSemantic(diffs); + + const changes: { from: number; to: number; insert: string }[] = []; + let pos = 0; + for (const [op, text] of diffs) { + if (op === 0) { + pos += text.length; + } else if (op === -1) { + changes.push({ from: pos, to: pos + text.length, insert: '' }); + pos += text.length; + } else if (op === 1) { + changes.push({ from: pos, to: pos, insert: text }); + } + } + + // Merge adjacent delete+insert into replacements + const merged: typeof changes = []; + let i = 0; + while (i < changes.length) { + const cur = changes[i]; + if (i + 1 < changes.length && cur.insert === '' && + changes[i + 1].from === cur.to && changes[i + 1].to === changes[i + 1].from) { + merged.push({ from: cur.from, to: cur.to, insert: changes[i + 1].insert }); + i += 2; + } else { + merged.push(cur); + i++; + } + } + + // Dispatch without ySyncAnnotation so HSM treats this as a user edit + cm.dispatch({ changes: merged }); + return { success: true, changeCount: merged.length }; + } + + private async closeEditor(handle: EditorHandle): Promise { + const leaf = this.findLeaf(handle); + if (leaf && leaf?.view?.file?.path === handle.path) { + leaf.detach?.(); + return; + } + + // If the original leaf was rebuilt and its id drifted, close by path. + const matches = this.findLeavesByPath(handle.path); + if (matches.length === 0) return; + for (const match of matches) { + match.detach?.(); + } + } + + /** + * Encode a Uint8Array as a hex string for JSON serialization. + */ + private toHex(bytes: Uint8Array): string { + return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join(''); + } + + /** + * Get a snapshot of all content views (local, remote, IDB, disk, server) for a document. + */ + private async getDocumentContent(path: string): Promise { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`Document not found: ${path}`); + const { doc, folder, guid, filePath } = lookup; + + const result: DocumentContentSnapshot = { + path: this.toVaultPath(folder, filePath), + guid, + folder: folder.path || folder.name, + local: null, + remote: null, + idb: null, + disk: null, + server: null, + }; + + // Local doc + try { + const localDoc = doc.localDoc; + if (localDoc) { + result.local = { + content: localDoc.getText('contents').toString(), + stateVector: this.toHex(Y.encodeStateVector(localDoc)), + }; + } + } catch { /* localDoc not available */ } + + // Remote doc (ydoc) + try { + const remoteDoc = doc.ydoc; + if (remoteDoc) { + result.remote = { + content: remoteDoc.getText('contents').toString(), + stateVector: this.toHex(Y.encodeStateVector(remoteDoc)), + }; + } + } catch { /* remoteDoc not available */ } + + // IDB + try { + const idbResult = await readIdbContent(guid, folder.appId, this.plugin?.timeProvider); + if (idbResult) { + result.idb = { + content: idbResult.content, + stateVector: this.toHex(idbResult.stateVector), + }; + } + } catch { /* IDB not available */ } + + // Disk + try { + const adapter = this.plugin.app.vault.adapter; + const vaultRelativePath = folder.getPath(filePath); + const content = await adapter.read(vaultRelativePath); + const stat = await adapter.stat(vaultRelativePath); + result.disk = { + content, + mtime: stat?.mtime ?? 0, + }; + } catch { /* disk read failed */ } + + // Server + try { + const response = await folder.backgroundSync.downloadItem(doc); + const rawUpdate = new Uint8Array(response.arrayBuffer); + const tempDoc = new Y.Doc(); + Y.applyUpdate(tempDoc, rawUpdate); + result.server = { + content: tempDoc.getText('contents').toString(), + stateVector: this.toHex(Y.encodeStateVector(tempDoc)), + updateSize: rawUpdate.byteLength, + }; + tempDoc.destroy(); + } catch { /* server download failed */ } + + return result; + } + + /** + * Build the HsmStateSnapshot for a document. Factored here so the CLI + * and the Python RelayClient can both reach the same shape via a + * single `__relayDebug.getHsmStateSnapshot(path)` call. + */ + private async getHsmStateSnapshot(path: string): Promise { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) { + throw new Error(`HSM not found: ${path}`); + } + const { doc, hsm, guid, folder, filePath } = lookup; + + const hsmAny = hsm as any; + + // Disk — prefer the vault adapter so we see exactly what the HSM sees. + const vaultPath = (folder as any).path + filePath; + let diskContent: string | null = null; + try { + diskContent = await this.plugin.app.vault.adapter.read(vaultPath); + } catch { + diskContent = null; + } + + // IDB — prefer the in-memory localDoc so we don't open a parallel + // IndexeddbPersistence when the HSM is warm. + let idbContent: string | null = null; + let idbStateVector: Uint8Array | null = null; + if (hsmAny.localDoc) { + idbContent = hsmAny.localDoc.getText('contents').toString(); + idbStateVector = hsmAny._localStateVector || null; + } else { + try { + const result = await readIdbContent( + guid, + hsmAny._persistenceMetadata?.appId, + this.plugin?.timeProvider, + ); + if (result) { + idbContent = result.content; + idbStateVector = result.stateVector; + } + } catch { /* noop */ } + } + + // SV equality — only meaningful if both sides exist. + let stateVectorsEqual: boolean | null = null; + try { + const remoteStateVector: Uint8Array | null = + hsmAny._remoteStateVector || null; + if (idbStateVector && remoteStateVector) { + const localArr = Array.from(idbStateVector); + const remoteArr = Array.from(remoteStateVector); + stateVectorsEqual = JSON.stringify(localArr) === JSON.stringify(remoteArr); + } + } catch { /* noop */ } + + // Recent transitions from the HSM disk log. + let recentTransitions: HsmStateTransition[] = []; + try { + const entries = await getRecentEntries(guid, 10); + recentTransitions = entries.map((raw: any) => ({ + ts: raw.ts, + seq: raw.seq, + event: typeof raw.event === 'object' ? raw.event.type : raw.event, + from: raw.from, + to: raw.to, + })); + } catch { /* noop */ } + + // Frontmatter Y.Map snapshot. + let frontmatterMap: Record | null = null; + if (hsmAny.localDoc) { + try { + const ymap = hsmAny.localDoc.getMap('frontmatter'); + if (ymap.size > 0) { + frontmatterMap = {}; + for (const [k, v] of ymap.entries()) { + try { frontmatterMap[k] = JSON.parse(v as string); } + catch { frontmatterMap[k] = v; } + } + } + } catch { /* noop */ } + } + + // Capture volatile in-memory HSM fields together after the async reads + // above. Initial enrollment can complete while disk/IDB/log probes await. + const lca = hsmAny._lca; + const hasValidLCA = !!(lca && lca.contents !== undefined && lca.meta?.hash); + const lcaContent: string | null = hasValidLCA ? lca.contents : null; + const localDoc = hsmAny.localDoc; + const statePath = hsmAny._statePath || 'unknown'; + const disk = hsmAny._disk; + const syncGateRaw = hsmAny._syncGate; + const syncGate: HsmSyncGate | null = syncGateRaw ? { + providerConnected: !!syncGateRaw.providerConnected, + providerSynced: !!syncGateRaw.providerSynced, + localOnly: !!syncGateRaw.localOnly, + pendingInbound: syncGateRaw.pendingInbound ?? 0, + pendingOutbound: syncGateRaw.pendingOutbound ?? 0, + } : null; + const diskMatchesIdb = + diskContent !== null && idbContent !== null && diskContent === idbContent; + const idbMatchesLca = + idbContent !== null && lcaContent !== null && idbContent === lcaContent; + + void doc; // referenced for future expansion; silences lint + + return { + path: this.toVaultPath(folder, filePath), + guid, + folder: (folder as any).name, + statePath, + syncGate, + hasLCA: hasValidLCA, + lcaHash: lca?.meta?.hash || null, + lcaContentLength: lca?.contents?.length ?? null, + lcaContent, + hasConflict: !!hsmAny.getConflictData?.(), + conflictData: hsmAny.getConflictData?.() || null, + localDocLength: localDoc + ? (localDoc.getText?.('contents')?.toString()?.length ?? 0) + : 0, + idbContent, + diskMtime: disk?.mtime || null, + diskContent, + stateVectorsEqual, + diskMatchesIdb, + idbMatchesLca, + frontmatterMap, + recentTransitions, + }; + } + + /** + * Wait for an HSM to reach a state path that starts with `statePrefix`, + * racing against a timeout. Thin bridge over `MergeHSM.awaitState`, + * which is event-driven (subscribes to `stateChanges` and resolves + * as soon as the predicate matches) — no polling, no per-tick + * Python↔JS round-trips. + * + * Resolves with the final state path on success. Rejects with a + * timeout error that includes the current state path for debugging. + * + * Use from the Python library to compose "open file and wait for + * active" or "close and wait for idle" flows without baking the + * wait into the action primitives themselves. + */ + private async awaitHsmState( + path: string, + statePrefix: string, + timeoutMs: number, + ): Promise { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`HSM not found: ${path}`); + const hsm = lookup.hsm as any; + + const matcher = (s: string) => s.startsWith(statePrefix); + if (matcher(hsm._statePath)) return hsm._statePath; + + let timer: number | null = null; + try { + await Promise.race([ + hsm.awaitState(matcher), + new Promise((_, reject) => { + timer = window.setTimeout(() => { + reject( + new Error( + `awaitHsmState timeout after ${timeoutMs}ms waiting for ` + + `${path} to reach state starting with "${statePrefix}" ` + + `(current: ${hsm._statePath})`, + ), + ); + }, timeoutMs); + }), + ]); + } finally { + if (timer !== null) window.clearTimeout(timer); + } + return hsm._statePath; + } + + /** + * Conflict APIs translate vault paths into merge-layer targets. State, + * hunk lookup, waking, and mutation behavior live below this boundary. + */ + private resolveConflictTarget(path: string): { manager: any; guid: string; folder: any; filePath: string } { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`HSM not found: ${path}`); + const manager = lookup.folder?.mergeManager; + if (!manager) throw new Error(`Merge manager not found: ${path}`); + return { + manager, + guid: lookup.guid, + folder: lookup.folder, + filePath: lookup.filePath, + }; + } + + private async getConflictInfo(path: string): Promise { + const { manager, guid, folder, filePath } = this.resolveConflictTarget(path); + if (typeof manager.getConflictInfo !== 'function') { + throw new Error(`Conflict info is not available: ${path}`); + } + const info = await manager.getConflictInfo(guid); + return { + ...info, + path: this.toVaultPath(folder, filePath), + }; + } + + private async resolveConflict(path: string, contents: string): Promise { + const { manager, guid } = this.resolveConflictTarget(path); + if (typeof manager.resolveConflict !== 'function') { + throw new Error(`Conflict resolution is not available: ${path}`); + } + return manager.resolveConflict(guid, contents); + } + + /** + * Clear the HSM's LCA in place. Low-level internal-state mutation + * that reproduces the no-LCA state after upgrading from a plugin + * version without LCA tracking. + */ + private findRelayByGuid(guid: string) { + for (const r of this.plugin.relayManager.relays._map.values()) { + if (r.guid === guid) return r; + } + return null; + } + + private getFolderByGuid(folderGuid: string): any | null { + if (!this.plugin?.sharedFolders?._set) return null; + for (const folder of this.plugin.sharedFolders._set.values()) { + if ((folder as any).guid === folderGuid) return folder; + } + return null; + } + + /** + * Canonical vault-path form: leading-slash, includes the shared-folder + * prefix (e.g. `/private/foo.md`). All debug-API outputs emit paths in + * this shape so CLI output can round-trip through any path-accepting + * command. + */ + private toVaultPath(folder: any, vpath: string): string { + return '/' + folder.getPath(vpath); + } + + private getFolderSyncStatus(folderGuid: string): { guid: string; path: string; status: string }[] { + const folder = this.getFolderByGuid(folderGuid); + const mm = folder?.mergeManager; + if (!mm?.syncStatus) return []; + + const rows: { guid: string; path: string; status: string }[] = []; + for (const [guid, syncStatus] of mm.syncStatus.entries()) { + const doc = mm._getDocument?.(guid); + const vpath = doc?.path; + rows.push({ + guid, + path: vpath ? this.toVaultPath(folder, vpath) : guid, + status: syncStatus?.status ?? 'unknown', + }); + } + rows.sort((a, b) => a.path.localeCompare(b.path)); + return rows; + } + + private getFolderSyncErrors(folderGuid: string): { guid: string; path: string; status: string }[] { + return this.getFolderSyncStatus(folderGuid).filter((row) => row.status === 'error'); + } + + private getFolderConflicts(folderGuid: string): { guid: string; path: string }[] { + return this.getFolderSyncStatus(folderGuid) + .filter((row) => row.status === 'conflict') + .map(({ guid, path }) => ({ guid, path })); + } + + private listAllConflicts(): { folderGuid: string; folderPath: string; guid: string; path: string }[] { + if (!this.plugin?.sharedFolders?._set) return []; + const out: { folderGuid: string; folderPath: string; guid: string; path: string }[] = []; + for (const folder of this.plugin.sharedFolders._set.values() as Iterable) { + const folderGuid = folder.guid; + const folderPath = folder.path ?? ''; + for (const row of this.getFolderConflicts(folderGuid)) { + out.push({ folderGuid, folderPath, guid: row.guid, path: row.path }); + } + } + return out; + } + + private async clearLca(path: string): Promise { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`HSM not found: ${path}`); + (lookup.hsm as any)._lca = null; + } + + /** + * Dispatch a simple parameter-less conflict event (OPEN_DIFF_VIEW, + * CANCEL) to an HSM and return the resulting state path. Centralizes + * the lookup + send boilerplate for single-event primitives. + */ + private sendConflictEvent(path: string, event: { type: string }): string { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`HSM not found: ${path}`); + const hsm = lookup.hsm as any; + hsm.send(event); + return hsm._statePath || 'unknown'; + } + + private async resolveHunk( + path: string, + hunkId: string, + resolution: 'ours' | 'theirs' | 'both' | 'neither', + ): Promise { + const { manager, guid } = this.resolveConflictTarget(path); + if (typeof manager.resolveConflictHunk !== 'function') { + throw new Error(`Conflict hunk resolution is not available: ${path}`); + } + return manager.resolveConflictHunk(guid, hunkId, resolution); + } + + /** + * Shared helper: resolve a vault path to a lookup + dbName, so the + * getIdb* methods don't each duplicate the prelude. Throws if the + * document can't be found or has no persistence metadata. + */ + private resolveIdbTarget(path: string): { + hsm: any; guid: string; folder: any; filePath: string; dbName: string; hsmDbName: string; + } { + const lookup = this.debugGlobal()?.lookupDocument?.(path); + if (!lookup) throw new Error(`HSM not found: ${path}`); + const { hsm, guid, folder, filePath } = lookup; + const appId = (hsm as any)._persistenceMetadata?.appId; + if (!appId) throw new Error('No appId in persistence metadata'); + return { + hsm, + guid, + folder, + filePath, + dbName: `${appId}-relay-doc-${guid}`, + hsmDbName: `${appId}-relay-hsm`, + }; + } + + /** + * Open an IndexedDB database by name and return the handle. Promise + * rejects if the open request errors. + */ + private openDb(dbName: string): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(dbName); + request.onerror = () => reject(new Error(`Failed to open DB: ${dbName}`)); + request.onsuccess = () => resolve(request.result); + }); + } + + /** + * Await an IDBRequest as a Promise. + */ + private awaitRequest(request: IDBRequest, label: string): Promise { + return new Promise((resolve, reject) => { + request.onerror = () => reject(new Error(`Failed: ${label}`)); + request.onsuccess = () => resolve(request.result); + }); + } + + /** + * Snapshot the per-doc IndexedDB + compare against disk. Replaces the + * ~100-line inline JS blob that used to live in cmd_relay_idb_content. + */ + private async getIdbContent(path: string): Promise { + const { hsm, guid, folder, filePath, dbName } = this.resolveIdbTarget(path); + + const db = await this.openDb(dbName); + try { + const tx = db.transaction(['updates', 'custom'], 'readonly'); + const updates = await this.awaitRequest( + tx.objectStore('updates').getAll(), + 'read updates', + ); + const customKeys = await this.awaitRequest( + tx.objectStore('custom').getAllKeys(), + 'read custom keys', + ); + const customValues = await this.awaitRequest( + tx.objectStore('custom').getAll(), + 'read custom values', + ); + const metadata: Record = {}; + for (let i = 0; i < customKeys.length; i++) { + metadata[String(customKeys[i])] = customValues[i]; + } + + // Prefer the in-memory localDoc text (matches the HSM's view). + // When hibernated, fall back to opening IndexeddbPersistence via + // readIdbContent. + let idbContent: string | null = null; + if ((hsm as any).localDoc) { + idbContent = (hsm as any).localDoc.getText('contents').toString(); + } else { + try { + const result = await readIdbContent(guid, (hsm as any)._persistenceMetadata?.appId, this.plugin?.timeProvider); + if (result) idbContent = result.content; + } catch { /* noop */ } + } + + // Read disk for comparison. + const vaultPath = (folder as any).path + filePath; + let diskContent: string | null = null; + try { + diskContent = await this.plugin.app.vault.adapter.read(vaultPath); + } catch (e: any) { + diskContent = `[Error reading disk: ${e.message}]`; + } + + return { + path: this.toVaultPath(folder, filePath), + guid, + folder: (folder as any).name, + dbName, + metadata, + updatesCount: updates.length, + idbContent, + idbLength: idbContent?.length ?? 0, + diskContent, + diskLength: diskContent?.length ?? null, + match: diskContent === idbContent, + }; + } finally { + db.close(); + } + } + + /** + * Snapshot the OpCapture history store for a document. Replaces the + * ~90-line inline JS blob that used to live in cmd_relay_idb_history. + */ + private async getIdbHistory(path: string): Promise { + const { hsm, guid, folder, filePath, dbName } = this.resolveIdbTarget(path); + + const db = await this.openDb(dbName); + try { + if (!db.objectStoreNames.contains('history')) { + return { + path: this.toVaultPath(folder, filePath), + guid, + folder: (folder as any).name, + dbName, + historyCount: 0, + inMemoryCount: null, + entries: [], + note: 'No history store (DB version < 2)', + }; + } + + const tx = db.transaction(['history'], 'readonly'); + const store = tx.objectStore('history'); + const keys = await this.awaitRequest(store.getAllKeys(), 'read history keys'); + const values = await this.awaitRequest(store.getAll(), 'read history values'); + + const entries: IdbHistoryEntry[] = keys.map((key, i) => { + const v = values[i] as any; + return { + key, + origin: v.origin ?? null, + timestamp: v.timestamp ?? null, + time: v.timestamp ? new Date(v.timestamp).toISOString() : null, + insertionsBytes: v.insertions?.byteLength ?? 0, + deletionsBytes: v.deletions?.byteLength ?? 0, + }; + }); + + const persistence = (hsm as any)._persistenceMetadata?.persistence; + const inMemoryCount = persistence?.opCapture?.entries?.length ?? null; + + return { + path: this.toVaultPath(folder, filePath), + guid, + folder: (folder as any).name, + dbName, + historyCount: entries.length, + inMemoryCount, + entries, + }; + } finally { + db.close(); + } + } + + /** + * Snapshot in-memory + persisted fork state for a document. Replaces + * the ~90-line inline JS blob that used to live in cmd_relay_idb_fork. + */ + private async getIdbFork(path: string): Promise { + const { hsm, guid, folder, filePath, hsmDbName } = this.resolveIdbTarget(path); + + const toSnapshot = (f: any): ForkSnapshot => ({ + base: f.base ?? null, + baseLength: f.base?.length ?? 0, + origin: f.origin ?? null, + created: f.created ?? null, + createdTime: f.created ? new Date(f.created).toISOString() : null, + captureMark: f.captureMark ?? null, + localStateVectorBytes: f.localStateVector?.byteLength ?? 0, + remoteStateVectorBytes: f.remoteStateVector?.byteLength ?? 0, + }); + + const inMemoryFork = (hsm as any)._fork; + const inMemory: ForkSnapshot | null = inMemoryFork ? toSnapshot(inMemoryFork) : null; + + // Read persisted fork from the shared HSM store. Swallow errors so + // a broken IDB doesn't hide the in-memory snapshot the caller wants. + let persistedFork: ForkSnapshot | { error: string } | null = null; + let persistedMeta: IdbForkSnapshot['persistedMeta'] = null; + try { + const db = await this.openDb(hsmDbName); + try { + if (db.objectStoreNames.contains('states')) { + const tx = db.transaction(['states'], 'readonly'); + const state = await this.awaitRequest( + tx.objectStore('states').get(guid), + 'read persisted state', + ) as any; + if (state?.fork) { + persistedFork = toSnapshot(state.fork); + } + if (state) { + persistedMeta = { + lastStatePath: state.lastStatePath ?? null, + persistedAt: state.persistedAt ?? null, + persistedAtTime: state.persistedAt ? new Date(state.persistedAt).toISOString() : null, + hasForkInPersistedState: !!state.fork, + }; + } + } + } finally { + db.close(); + } + } catch (e: any) { + persistedFork = { error: e.message }; + } + + return { + path: this.toVaultPath(folder, filePath), + guid, + folder: (folder as any).name, + statePath: (hsm as any)._statePath || 'unknown', + hasFork: inMemoryFork != null, + inMemoryFork: inMemory, + persistedFork, + persistedMeta, + }; + } + + /** + * Remove globals and dispose all bridges. + * Call in plugin onunload(). + */ + destroy(): void { + if (this.destroyed) { + return; + } + this.destroyed = true; + for (const bridge of this.bridges.values()) { + bridge.dispose(); + } + this.bridges.clear(); + this.activeRecordingName = null; + this.plugin = null; + + if (this.debugGlobal()?.__owner === this) { + delete (this.debugWindow() as any).__relayDebug; + } + } +} + +// ============================================================================= +// IDB Utility +// ============================================================================= + +async function readIdbContent( + guid: string, + appId: string, + timeProvider?: TimeProvider, +): Promise<{ content: string; stateVector: Uint8Array } | null> { + if (!timeProvider) return null; + const dbName = `${appId}-relay-doc-${guid}`; + const tempDoc = new Y.Doc(); + try { + const persistence = new IndexeddbPersistence(dbName, tempDoc, null, null, timeProvider); + await persistence.whenSynced; + const content = tempDoc.getText('contents').toString(); + const stateVector = Y.encodeStateVector(tempDoc); + await persistence.destroy(); + return { content, stateVector }; + } catch { + tempDoc.destroy(); + return null; + } +} diff --git a/src/RelayManager.ts b/src/RelayManager.ts index 40448411..37a691a3 100644 --- a/src/RelayManager.ts +++ b/src/RelayManager.ts @@ -138,6 +138,28 @@ interface Collection { delete(id: string): void; } +/** + * Marker interface for records that maintain a PocketBase realtime + * per-record subscription in addition to the top-level collection stream. + * RelayManager.resubscribeRecords() scans every Store collection for tagged + * items on reconnect and re-installs their subscriptions. + */ +interface HasRecordSubscription { + id: string; + collectionName: string; + offRecordSubscription?: Unsubscriber; +} + +function hasRecordSubscription(x: unknown): x is HasRecordSubscription { + return ( + typeof x === "object" && + x !== null && + typeof (x as HasRecordSubscription).id === "string" && + typeof (x as HasRecordSubscription).collectionName === "string" && + "offRecordSubscription" in x + ); +} + class RoleCollection implements Collection { collectionName: string = "roles"; roles: ObservableMap; @@ -196,8 +218,9 @@ class Auto implements hasRoot, hasPermissionParents { class StorageQuotaAuto extends Observable - implements StorageQuota + implements StorageQuota, HasRecordSubscription { + public readonly collectionName = "storage_quotas"; public offRecordSubscription?: Unsubscriber; constructor( @@ -1461,6 +1484,7 @@ export class RelayManager extends HasLogging { store?: Store; policyManager?: IPolicyManager; _offLoginManager: Unsubscriber; + private _isSubscribed = false; private pb: PocketBase | null; destroyed = false; @@ -1641,6 +1665,10 @@ export class RelayManager extends HasLogging { this.store?.clear(); this.user = undefined; this.store = undefined; + // LoginManager.logout() calls pb.realtime.unsubscribe() which wipes + // the client-side subscription map; clear our flag so the next login + // re-registers. + this._isSubscribed = false; } async rotateKey(relayInvitation: RelayInvitation): Promise { @@ -1766,6 +1794,15 @@ export class RelayManager extends HasLogging { { name: "subscriptions", expand: ["user", "relay"] }, ]; + // Idempotent. PocketBase's subscribe() stacks listener closures onto + // a per-topic array with no dedup, so calling this twice would fire + // our _handleEvent twice per server event. offline(), logout(), and + // destroy() clear the flag so the next call re-registers. + if (this._isSubscribed) { + return; + } + this._isSubscribed = true; + for (const collection of collections) { this.pb .collection(collection.name) @@ -1776,6 +1813,58 @@ export class RelayManager extends HasLogging { } } + /** + * Tear down every PocketBase realtime subscription and close the SSE + * stream. Called from the network offline handler to stop PB's infinite + * reconnect loop while the browser reports no connectivity. + * + * pb.realtime.unsubscribe() wipes PB's entire subscriptions map, + * including per-record subscriptions held by tagged records in the + * store. online() rebuilds both layers. + */ + offline(): void { + if (!this.pb) return; + this.pb.realtime.unsubscribe(); + this._isSubscribed = false; + } + + /** + * Rebuild all PocketBase realtime state after coming back online: + * re-register the top-level collection streams, reinstall per-record + * subscriptions on tagged records already in the store, then refresh + * the store contents via a full list fetch. + */ + async online(): Promise { + await this.subscribe(); + await this.resubscribeRecords(); + await this.update(); + } + + /** + * Scan every Store collection for records implementing + * HasRecordSubscription and (re)install their per-record realtime + * subscriptions. Safe to call repeatedly; each record's existing + * subscription handle is released before a fresh one is installed. + */ + private async resubscribeRecords(): Promise { + if (!this.store?.collections) return; + for (const collection of this.store.collections.values()) { + for (const item of collection.items()) { + if (!hasRecordSubscription(item)) continue; + try { + await item.offRecordSubscription?.(); + } catch (e) { + this.debug("stale record unsubscribe failed", e); + } + item.offRecordSubscription = await this.subscribeRecord( + item.collectionName, + item.id, + [], + ); + } + } + } + async subscribeRecord( collectionName: string, recordId: string, @@ -1876,14 +1965,14 @@ export class RelayManager extends HasLogging { expand: "relay,user", }), ]; - promises.forEach(async (promise) => { + await Promise.all(promises.map(async (promise) => { const result = await promise; for (const record of result) { if (!this.destroyed && this.store) { this.store.ingest(record); } } - }); + })); } async acceptInvitation(shareKey: string): Promise { @@ -2041,7 +2130,7 @@ export class RelayManager extends HasLogging { } else { this.warn("No role found to leave relay"); } - this.store?.cascade("relay", relay.id); + this.store?.cascade("relays", relay.id); } async kick(relay_role: RelayRole) { @@ -2205,6 +2294,7 @@ export class RelayManager extends HasLogging { this._offLoginManager = null as any; this.pb?.cancelAllRequests(); this.pb?.realtime?.unsubscribe(); + this._isSubscribed = false; this.loginManager = null as any; this.store?.destroy(); this.pb = null as any; diff --git a/src/RemoteActivityIndex.ts b/src/RemoteActivityIndex.ts new file mode 100644 index 00000000..df5f2181 --- /dev/null +++ b/src/RemoteActivityIndex.ts @@ -0,0 +1,154 @@ +"use strict"; + +export interface RemoteActivityEntry { + guid: string; + timestamp: number; + userId?: string; +} + +export const REMOTE_ACTIVITY_MAX_ENTRIES = 100; +export const REMOTE_ACTIVITY_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +const FUTURE_SKEW_MS = 60 * 1000; +const SECONDS_THRESHOLD = 10_000_000_000; + +export class RemoteActivityIndex { + private readonly byGuid = new Map(); + private readonly ordered: RemoteActivityEntry[] = []; + + constructor( + private readonly capacity: number = REMOTE_ACTIVITY_MAX_ENTRIES, + ) {} + + upsert(entry: RemoteActivityEntry): boolean { + if (!entry.guid || !Number.isFinite(entry.timestamp) || entry.timestamp <= 0) { + return false; + } + + const existing = this.byGuid.get(entry.guid); + if (existing && entry.timestamp < existing.timestamp) { + return false; + } + + const next: RemoteActivityEntry = { + guid: entry.guid, + timestamp: entry.timestamp, + userId: entry.userId ?? existing?.userId, + }; + + if ( + existing && + existing.timestamp === next.timestamp && + existing.userId === next.userId + ) { + return false; + } + + if (existing) { + this.removeFromOrdered(entry.guid); + } + + this.byGuid.set(entry.guid, next); + this.insertOrdered(next); + this.trim(); + return true; + } + + get(guid: string): RemoteActivityEntry | undefined { + const entry = this.byGuid.get(guid); + return entry ? { ...entry } : undefined; + } + + remove(guid: string): boolean { + if (!this.byGuid.delete(guid)) { + return false; + } + this.removeFromOrdered(guid); + return true; + } + + entries(limit: number = this.capacity): RemoteActivityEntry[] { + return this.ordered.slice(0, limit).map((entry) => ({ ...entry })); + } + + serialize(): RemoteActivityEntry[] { + return this.entries(this.capacity); + } + + hydrate(entries: readonly RemoteActivityEntry[]): void { + this.byGuid.clear(); + this.ordered.length = 0; + for (const entry of entries) { + this.upsert(entry); + } + } + + pruneOlderThan(cutoff: number): boolean { + let changed = false; + for (let index = this.ordered.length - 1; index >= 0; index--) { + const entry = this.ordered[index]; + if (entry.timestamp >= cutoff) { + break; + } + this.byGuid.delete(entry.guid); + this.ordered.pop(); + changed = true; + } + return changed; + } + + private insertOrdered(entry: RemoteActivityEntry): void { + let low = 0; + let high = this.ordered.length; + while (low < high) { + const mid = Math.floor((low + high) / 2); + if (compareNewestFirst(entry, this.ordered[mid]) < 0) { + high = mid; + } else { + low = mid + 1; + } + } + this.ordered.splice(low, 0, entry); + } + + private removeFromOrdered(guid: string): void { + const index = this.ordered.findIndex((entry) => entry.guid === guid); + if (index >= 0) { + this.ordered.splice(index, 1); + } + } + + private trim(): void { + while (this.ordered.length > this.capacity) { + const removed = this.ordered.pop(); + if (removed) { + this.byGuid.delete(removed.guid); + } + } + } +} + +export function normalizeRemoteActivityTimestamp( + timestamp: unknown, + now: number, +): number | null { + if (typeof timestamp !== "number" || !Number.isFinite(timestamp) || timestamp <= 0) { + return null; + } + + const millis = timestamp < SECONDS_THRESHOLD ? timestamp * 1000 : timestamp; + if (millis > now + FUTURE_SKEW_MS) { + return null; + } + return millis; +} + +function compareNewestFirst( + a: RemoteActivityEntry, + b: RemoteActivityEntry, +): number { + if (a.timestamp !== b.timestamp) { + return b.timestamp - a.timestamp; + } + return a.guid.localeCompare(b.guid); +} diff --git a/src/SettingsStorage.ts b/src/SettingsStorage.ts index 5d34051f..874576d2 100644 --- a/src/SettingsStorage.ts +++ b/src/SettingsStorage.ts @@ -113,6 +113,7 @@ */ import { Observable, type Unsubscriber } from "./observable/Observable"; +import { PostOffice } from "./observable/Postie"; export type PathSegment = string | number; export type Path = PathSegment[]; @@ -248,6 +249,15 @@ export class Settings extends Observable { listener(this.data); } } + + override destroy() { + if (this.destroyed) return; + super.destroy(); + this.data = null as any; + this._loaded = false; + (this as any).storage = null; + (this as any).defaults = null; + } } export class NamespacedSettings< @@ -696,6 +706,7 @@ export class NamespacedSettings< run(this.get()); return () => { this._listeners.delete(run); + PostOffice.peekInstance()?.cancel(run); }; } diff --git a/src/ShareLinkPlugin.ts b/src/ShareLinkPlugin.ts deleted file mode 100644 index a3bf204d..00000000 --- a/src/ShareLinkPlugin.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Annotation, ChangeSet } from "@codemirror/state"; -import { EditorView, ViewPlugin } from "@codemirror/view"; -import type { PluginValue } from "@codemirror/view"; -import { TextFileView } from "obsidian"; -import { LiveView, LiveViewManager } from "./LiveViews"; -import { connectionManagerFacet } from "./y-codemirror.next/LiveEditPlugin"; -import { hasKey, updateFrontMatter } from "./Frontmatter"; -import { diffChars } from "diff"; - -export const shareLinkAnnotation = Annotation.define(); - -function diffToChangeSet(originalText: string, newText: string): ChangeSet { - const changes: { from: number; to?: number; insert?: string }[] = []; - const diffResult = diffChars(originalText, newText); - - let index = 0; - for (const part of diffResult) { - if (!part.count) { - continue; - } - if (part.added) { - changes.push({ from: index, insert: part.value }); - } else if (part.removed) { - changes.push({ from: index, to: index + part.count }); - index += part.value.length; - } else { - index += part.count; - } - } - const reduced: { from: number; to?: number; insert?: string }[] = []; - let lastChange: { from: number; to?: number; insert?: string } | null = null; - changes.forEach((_change) => { - if ( - lastChange && - lastChange.to === _change.from && - !lastChange.insert && - _change.insert - ) { - reduced.push({ - from: lastChange.from, - to: lastChange.to, - insert: _change.insert, - }); - lastChange = null; - } else if (lastChange) { - reduced.push(lastChange); - lastChange = _change; - } else { - lastChange = _change; - } - }); - return ChangeSet.of(changes, originalText.length); -} - -export class ShareLinkPluginValue implements PluginValue { - editor: EditorView; - view?: LiveView; - connectionManager: LiveViewManager; - - constructor(editor: EditorView) { - this.editor = editor; - this.connectionManager = this.editor.state.facet(connectionManagerFacet); - this.view = this.connectionManager.findView(editor); - this.editor = editor; - if (this.view) { - this.view.document?.whenSynced().then(async () => { - const hasKnownPeers = await this.view?.document?.hasKnownPeers(); - if (this.view?.document?.text || !hasKnownPeers) { - this.updateFrontMatter(); - } - }); - } - } - - updateFrontMatter() { - if (!(this.view instanceof LiveView)) { - return; - } - if (!this.view || !this.view.shouldConnect) { - return; - } - if (this.view.document.text != this.editor.state.doc.toString()) { - return; - } - const text = this.editor.state.doc.toString(); - const shareLink = `https://ydoc.live/${this.view.document.guid}`; - const withShareLink = updateFrontMatter(text, { - shareLink: shareLink, - }); - if (!(text || this.view.document.text)) { - // document is empty - this.editor.dispatch({ - changes: { from: 0, insert: withShareLink }, - annotations: [shareLinkAnnotation.of(this)], - }); - } else if (!text.startsWith("---")) { - // frontmatter is missing - this.editor.dispatch({ - changes: { - from: 0, - insert: `---\nshareLink: '${shareLink}'\n---\n`, - }, - annotations: [shareLinkAnnotation.of(this)], - }); - } else if (!hasKey(text, "shareLink")) { - // frontmatter exists, but the key is missing - this.editor.dispatch({ - changes: { from: 3, insert: `\nshareLink: '${shareLink}'\n` }, - annotations: [shareLinkAnnotation.of(this)], - }); - } else { - // frontmatter exists, and the key is present - const changeSet = diffToChangeSet(text, withShareLink); - if (changeSet.empty) { - return; - } - this.editor.dispatch({ - changes: changeSet, - annotations: [shareLinkAnnotation.of(this)], - }); - } - } -} - -export const ShareLinkPlugin = ViewPlugin.fromClass(ShareLinkPluginValue); diff --git a/src/SharedFolder.ts b/src/SharedFolder.ts index 63811255..afcf22a1 100644 --- a/src/SharedFolder.ts +++ b/src/SharedFolder.ts @@ -1,31 +1,37 @@ "use strict"; +import { uuidv4 } from "lib0/random"; import { FileManager, + type MetadataCache, TAbstractFile, TFile, TFolder, Vault, debounce, + getFrontMatterInfo, normalizePath, + parseYaml, + stringifyYaml, } from "obsidian"; -import { IndexeddbPersistence } from "./storage/y-indexeddb"; -import * as idb from "lib0/indexeddb"; +import { + IndexeddbPersistence, +} from "./storage/y-indexeddb"; import { dirname, join, sep } from "path-browserify"; import { HasProvider, type ConnectionIntent } from "./HasProvider"; +import type { EventMessage } from "./client/provider"; import { Document } from "./Document"; import { ObservableSet } from "./observable/ObservableSet"; import { LoginManager } from "./LoginManager"; import { LiveTokenStore } from "./LiveTokenStore"; import { SharedPromise, Dependency, withTimeoutWarning } from "./promiseUtils"; -import { S3Folder, S3RN, S3RemoteFolder } from "./S3RN"; +import { S3Folder, S3RN, S3RemoteFolder, S3RemoteDocument } from "./S3RN"; import type { RemoteSharedFolder } from "./Relay"; import { RelayManager } from "./RelayManager"; import type { Unsubscriber } from "svelte/store"; -import { DiskBufferStore } from "./DiskBuffer"; import { BackgroundSync } from "./BackgroundSync"; import type { NamespacedSettings } from "./SettingsStorage"; -import { RelayInstances } from "./debug"; +import { RelayInstances, metrics } from "./debug"; import { LocalStorage } from "./LocalStorage"; import { SyncFolder, isSyncFolder } from "./SyncFolder"; import { isDocument } from "./Document"; @@ -37,38 +43,62 @@ import { makeFileMeta, makeFolderMeta, isSyncFileMeta, + isDocumentMeta, isCanvasMeta, type FileMeta, type Meta, type SyncFileType, } from "./SyncTypes"; import type { IFile } from "./IFile"; +import { formatDuplicateGuidLog } from "./FileLogDetails"; import { createPathProxy } from "./pathProxy"; import { ContentAddressedStore } from "./CAS"; import { SyncSettingsManager, type SyncFlags } from "./SyncSettings"; import { ContentAddressedFileStore, SyncFile, isSyncFile } from "./SyncFile"; import { Canvas, isCanvas } from "./Canvas"; import { flags } from "./flagManager"; +import { MergeManager } from "./merge-hsm/MergeManager"; +import { + E2ERecordingBridge, + type HSMLogEntry, +} from "./merge-hsm/recording"; +import { recordHSMEntry } from "./debug"; +import { trackAsyncCleanup } from "./reloadUtils"; +import { generateHash } from "./hashing"; +import { + HSMStore, +} from "./merge-hsm/persistence"; +import { trackPromise } from "./trackPromise"; +import { + RemoteActivityIndex, + REMOTE_ACTIVITY_RETENTION_MS, + type RemoteActivityEntry, + normalizeRemoteActivityTimestamp, +} from "./RemoteActivityIndex"; import { expandDesiredRemotePaths } from "./syncPathUtils"; +import type { TimeProvider } from "./TimeProvider"; +import * as Y from "yjs"; export interface SharedFolderSettings { guid: string; path: string; relay?: string; connect?: boolean; + localOnly?: boolean; sync?: SyncFlags; + remoteActivity?: RemoteActivityEntry[]; } interface Operation { op: "create" | "rename" | "delete" | "update" | "upgrade" | "noop"; path: string; - promise: Promise | Promise; + promise: Promise; } interface Create extends Operation { op: "create"; path: string; - promise: Promise; + promise: Promise; } interface Rename extends Operation { @@ -117,7 +147,7 @@ class Files extends ObservableSet { add(item: IFile, update = true): ObservableSet { const existing = this.find((file) => file.guid === item.guid); if (existing && existing !== item) { - this.error("duplicate guid", existing, item); + this.error(formatDuplicateGuidLog(existing, item)); this._set.delete(existing); } this._set.add(item); @@ -135,6 +165,7 @@ export class SharedFolder extends HasProvider { relayId?: string; _remote?: RemoteSharedFolder; _shouldConnect: boolean; + private _localOnly: boolean; destroyed: boolean = false; public vault: Vault; syncStore: SyncStore; @@ -153,11 +184,32 @@ export class SharedFolder extends HasProvider { private pendingDeletes: Set = new Set(); private enabledSyncTypes: Set = new Set(); + private _persistence: IndexeddbPersistence; - diskBufferStore: DiskBufferStore; proxy: SharedFolder; + private revokeProxy: (() => void) | null = null; cas: ContentAddressedStore; syncSettingsManager: SyncSettingsManager; + mergeManager: MergeManager; + private recordingBridge: E2ERecordingBridge; + private _pendingKeyframeUpdates: Map = new Map(); + private _pendingRemaps: Set = new Set(); + private _pendingDownloads: Set = new Set(); + private _pendingDownloadPromises: Map> = + new Map(); + private readonly remoteActivityIndex = new RemoteActivityIndex(); + private readonly remoteActivitySubscribers = new Set<() => void>(); + private onFolderYDocUpdate = (_update: Uint8Array, origin: unknown): void => { + // Folder metadata updates can arrive before SyncStore observers are active, + // or with origins that bypass SyncStore-level callbacks. Reconcile directly + // from Y.Doc updates so file materialization is not missed. + if (this.destroyed || origin === this) { + return; + } + trackPromise(`folder:ydocUpdateSync:${this.guid}`, this.syncFileTree()).catch( + (e) => this.error("syncFileTree on ydoc update failed", e), + ); + }; constructor( public appId: string, @@ -165,20 +217,24 @@ export class SharedFolder extends HasProvider { path: string, loginManager: LoginManager, vault: Vault, + private metadataCache: MetadataCache | undefined, fileManager: FileManager, tokenStore: LiveTokenStore, relayManager: RelayManager, private hashStore: ContentAddressedFileStore, public backgroundSync: BackgroundSync, private _settings: NamespacedSettings, + private _hsmStore: HSMStore, + timeProvider: TimeProvider, relayId?: string, - awaitingUpdates: boolean = true, + authoritative: boolean = false, ) { const s3rn = relayId ? new S3RemoteFolder(relayId, guid) : new S3Folder(guid); super(guid, s3rn, tokenStore, loginManager); + this.timeProvider = timeProvider; this.path = path; this.setLoggers(`[SharedFile](${this.path})`); this.fileManager = fileManager; @@ -200,10 +256,14 @@ export class SharedFolder extends HasProvider { }); this.relayManager = relayManager; this.relayId = relayId; - this.diskBufferStore = new DiskBufferStore(); this._shouldConnect = this.settings.connect ?? true; + this._localOnly = this.settings.localOnly ?? false; + this.remoteActivityIndex.hydrate(this.settings.remoteActivity ?? []); + if (this.pruneRemoteActivity()) { + this.persistRemoteActivity(); + } - this.authoritative = !awaitingUpdates; + this.authoritative = authoritative; this.syncSettingsManager = this._settings.getChild< Record, @@ -217,7 +277,12 @@ export class SharedFolder extends HasProvider { this.syncSettingsManager, ); this.syncStore.on(async () => { - await this.syncFileTree(this.syncStore); + await this.syncFileTree(); + }); + const subscribedYdoc = this.ydoc; + subscribedYdoc.on("update", this.onFolderYDocUpdate); + this.unsubscribes.push(() => { + subscribedYdoc.off("update", this.onFolderYDocUpdate); }); this.unsubscribes.push( @@ -253,55 +318,677 @@ export class SharedFolder extends HasProvider { }), ); - this.proxy = createPathProxy(this, this.path, (globalPath: string) => { + const { proxy, revoke } = createPathProxy(this, this.path, (globalPath: string) => { return this.getVirtualPath(globalPath); }); + this.proxy = proxy; + this.revokeProxy = revoke; try { - this._persistence = new IndexeddbPersistence(this.guid, this.ydoc); + const folderDbName = `${this.appId}-relay-folder-${this.guid}`; + const migrateFrom = flags().enableFolderIdbMigration ? this.guid : null; + this._persistence = new IndexeddbPersistence( + folderDbName, + this.ydoc, + null, + migrateFrom, + this.timeProvider, + ); } catch (e) { this.warn("Unable to open persistence.", this.guid); console.error(e); throw e; } + // If folder is authoritative (local-only, not awaiting server updates), + // mark it as server synced so it's considered "ready" even after reload + if (this.authoritative) { + this._persistence.markServerSynced(); + } + if (loginManager.loggedIn) { this.connect(); } this.cas = new ContentAddressedStore(this); - this.whenReady().then(() => { - if (!this.destroyed) { + // Create MergeManager for this SharedFolder (per-folder instance) + this.mergeManager = new MergeManager({ + folderGuid: this.guid, + getVaultId: (guid: string) => `${this.appId}-relay-doc-${guid}`, + getDocument: (guid: string) => { + const file = this.files.get(guid); + if (!file || !isDocument(file)) return undefined; + return file; + }, + timeProvider: this.timeProvider, + createPersistence: (vaultId, doc, captureOpts) => + new IndexeddbPersistence(vaultId, doc, captureOpts, null, this.timeProvider), + getDiskState: async (docPath: string) => { + // docPath is SharedFolder-relative (e.g., "/note.md") + const vaultPath = this.getPath(docPath); + const tfile = this.vault.getAbstractFileByPath(vaultPath); + if (!(tfile instanceof TFile)) return null; + const contents = await this.vault.read(tfile); + const encoder = new TextEncoder(); + const hash = await generateHash(encoder.encode(contents).buffer); + return { contents, mtime: tfile.stat.mtime, hash }; + }, + loadAllStates: async () => { + try { + return await this._hsmStore.getAllStateMeta(); + } catch { + return []; + } + }, + loadState: async (guid: string) => { + try { + return await this._hsmStore.loadState(guid); + } catch { + return null; + } + }, + onEffect: async (guid, effect) => { + this.debug?.(`[MergeManager] Effect for ${guid}:`, effect.type); + if (effect.type === "PERSIST_STATE") { + // Persisted fork/LCA state writes run in the background; track + // failures so persistence errors are visible. + const p = this._hsmStore + .saveState(guid, effect.state) + .catch((err) => { + this.error( + `[MergeManager] saveState failed for ${guid}:`, + err, + ); + }); + trackAsyncCleanup(p); + } else if (effect.type === "SYNC_TO_REMOTE") { + // When a file is closed, ProviderIntegration is destroyed so no one + // listens for these effects. Handle them at the SharedFolder level. + await this.handleIdleSyncToRemote(guid, effect.update); + } + }, + getPersistenceMetadata: (guid: string, path: string) => { + const s3rn = this.relayId + ? new S3RemoteDocument(this.relayId, this.guid, guid) + : null; + return { + path, + relay: this.relayId || "", + appId: this.appId, + s3rn: s3rn ? S3RN.encode(s3rn) : "", + }; + }, + yaml: { parse: parseYaml, stringify: stringifyYaml, getFrontMatterInfo }, + }); + + // Create per-folder recording bridge and register with the debug API. + this.recordingBridge = new E2ERecordingBridge({ + onEntry: flags().enableHSMRecording + ? (entry: HSMLogEntry) => recordHSMEntry(entry) + : undefined, + getFullPath: (guid: string) => { + const file = this.files.get(guid); + if (!file || !isDocument(file)) return undefined; + return join(this.path, file.path); + }, + }); + const debugAPI = (window as any).__relayDebug; + if (debugAPI?.registerBridge) { + const unregister = debugAPI.registerBridge(this.path, this.recordingBridge); + this.unsubscribes.push(unregister); + } + this.mergeManager.setOnTransition((guid, path, info) => { + this.recordingBridge.recordTransition(guid, path, info); + }); + + // Wire folder-level event subscriptions for idle mode remote updates + this.setupEventSubscriptions(); + + trackPromise(`folder:whenReady:${this.guid}`, this.whenReady()) + .then(async () => { + if (this.destroyed) return; + await this.mergeManager.initialize(); + if (this.destroyed) return; + this.syncFileTree(); + }) + .catch((e) => this.error("folder ready failed", e)); + + trackPromise(`folder:whenSynced:${this.guid}`, this.whenSynced()) + .then(async () => { + // Load persisted HSM metadata before sync startup can create + // Documents. Document construction immediately creates HSMs, + // and cold-start needs this cache to decide whether a doc can + // remain hibernated without opening y-indexeddb. + await this.mergeManager.initialize(); + if (this.destroyed) return; + + this.syncStore.start(); + // Wait until syncStore is observing the committed file metadata before + // creating docs from local disk. On reload, addLocalDocs() can otherwise + // reserve placeholder GUIDs for already-shared files and build HSMs that + // miss their persisted fork/LCA state. + // + // Remote folder metadata can also land before SyncStore observers are + // installed, so replay both local doc discovery and file-tree sync after + // start() to avoid missing the first batch of remote entries. this.enabledSyncTypes = new Set( this.syncStore.typeRegistry.getEnabledFileSyncTypes(), ); this.addLocalDocs(); - this.syncFileTree(this.syncStore); + await this.syncFileTree(); + try { + this._persistence.set("path", this.path); + this._persistence.set("relay", this.relayId || ""); + this._persistence.set("appId", this.appId); + this._persistence.set("s3rn", S3RN.encode(this.s3rn)); + } catch (e) { + // pass + } + }) + .catch((e) => this.error("folder persistence sync failed", e)); + + const isAuthoritative = this.authoritative; + const canAwaitProviderSync = + this.s3rn instanceof S3RemoteFolder && + this.shouldConnect && + this.loginManager.loggedIn && + this.remote !== undefined; + (async () => { + const serverSynced = await this.getServerSynced(); + if (!serverSynced) { + if (isAuthoritative) { + await this.markSynced(); + } else if (canAwaitProviderSync) { + await trackPromise(`folderSync:${this.guid}`, this.onceProviderSynced()); + await this.markSynced(); + } + } else if (!isAuthoritative && canAwaitProviderSync) { + // Even when IDB already has serverSync, we still need the + // provider to sync so _providerSynced is set. Without this, + // the folder's `synced` getter stays false and downstream + // flows (syncFileTree downloads) can fail. + await trackPromise(`folderProviderSync:${this.guid}`, this.onceProviderSynced()); } + })().catch((e) => this.warn("folder provider sync failed", e)); + + RelayInstances.set(this, this.path); + } + + private setupEventSubscriptions() { + if (!this._provider || !this.mergeManager) return; + + this._provider.subscribeToEvents( + ["document.updated"], + (event: EventMessage) => { + this.handleDocumentUpdateEvent(event); + }, + ); + + // On reconnect, query server head metadata for locally committed docs. + // The folder index and live events discover remote paths; subdoc index + // queries only refresh known subdocument heads. + const provider = this._provider; + provider.getSubdocQueryDocIds = () => { + if (!flags().enableSelectiveSubdocQuery || !this.relayId) return []; + return this.syncStore + .getCommittedSubdocGuids() + .map((guid) => this.serverDocIdForGuid(guid)); + }; + provider.onSubdocIndex = (serverIndex) => { + const remoteActivity: RemoteActivityEntry[] = []; + const advertisedGuids: string[] = []; + const now = this.currentTime(); + for (const [docId, entry] of Object.entries(serverIndex)) { + const guid = this.guidFromServerDocId(docId) ?? docId; + advertisedGuids.push(guid); + this.mergeManager?.seedServerAdvertisedHeadFromBytes( + guid, + entry, + ); + if (entry.lastSeen !== undefined) { + const timestamp = normalizeRemoteActivityTimestamp( + entry.lastSeen, + now, + ); + if (timestamp !== null) { + remoteActivity.push({ guid, timestamp }); + } + } + } + this.recordRemoteActivities(remoteActivity); + this.syncFileTree() + .then(() => { + const queued = this.backgroundSync.enqueueRemoteHeadSyncs( + this, + advertisedGuids, + ); + if (queued > 0) { + this.debug(`[subdoc-index] queued ${queued} remote-head syncs`); + } + }) + .catch((e) => this.error("subdoc index sync sweep failed", e)); + }; + this.unsubscribes.push(() => { + provider.onSubdocIndex = null; + provider.getSubdocQueryDocIds = null; }); + } - this.whenSynced().then(async () => { - this.syncStore.start(); - try { - this._persistence.set("path", this.path); - this._persistence.set("relay", this.relayId || ""); - this._persistence.set("appId", this.appId); - this._persistence.set("s3rn", S3RN.encode(this.s3rn)); - } catch (e) { - // pass + private serverDocIdForGuid(guid: string): string { + return `${this.relayId}-${guid}`; + } + + private guidFromServerDocId(docId: string): string | null { + const uuidPattern = + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; + const match = docId.match( + new RegExp(`^${uuidPattern}-(${uuidPattern})$`, "i"), + ); + return match?.[1] ?? null; + } + + private handleDocumentUpdateEvent(event: EventMessage) { + if (!this.mergeManager) return; + + const docId = event.doc_id; + if (!docId) return; + + + // Extract the guid from the doc_id + // The doc_id format is "{relayId}-{guid}" where both are UUIDs + const guid = this.guidFromServerDocId(docId); + if (!guid) return; + metrics.recordDocumentUpdateEvent("received", this.guid); + + if (!event.user || !this.isLocalUserId(event.user)) { + const timestamp = normalizeRemoteActivityTimestamp( + event.timestamp, + this.currentTime(), + ); + if (timestamp !== null) { + this.recordRemoteActivities([ + { guid, timestamp, userId: event.user }, + ]); + } + } + + if (!this.files.has(guid)) { + this.retryDeferredDownloadForGuid(guid); + this.retryDeferredRemapForGuid(guid); + return; + } + + const file = this.files.get(guid); + if (!file || !isDocument(file)) return; + + // Active documents: ProviderIntegration handles sync via y-protocols + if (this.mergeManager.isActive(guid)) { + return; + } + + if (!event.update) return; + + // Normalize update bytes (CBOR decoding may return Buffer or plain object) + const update = + event.update instanceof Uint8Array + ? event.update + : new Uint8Array(event.update); + + // If a keyframe fetch is in progress, buffer the update + const buf = this._pendingKeyframeUpdates.get(guid); + if (buf) { + metrics.recordDocumentUpdateEvent("catchup", this.guid); + buf.push(update); + return; + } + + const classification = this.mergeManager.classifyUpdate(guid, update); + switch (classification) { + case 'apply': + this.mergeManager.handleRemoteUpdate(guid, update); + metrics.recordDocumentUpdateEvent("applied", this.guid); + this.mergeManager.advanceAppliedRemoteUpdate(guid, update); + break; + case 'stale': + break; // already covered by the applied remote baseline + case 'gap': + metrics.recordDocumentUpdateEvent("catchup", this.guid); + this._fetchKeyframeAndDeliver(file, guid, [update]); + break; + } + } + + private findCommittedPathByGuid(guid: string): string | null { + let match: string | null = null; + this.syncStore.forEach((meta, path) => { + if (!match && meta.id === guid) { + match = path; } }); + return match; + } - (async () => { - const serverSynced = await this.getServerSynced(); - if (!serverSynced) { - await this.onceProviderSynced(); - await this.markSynced(); + private retryDeferredRemapForGuid(guid: string): void { + const path = this.findCommittedPathByGuid(guid); + if (!path || this._pendingRemaps.has(path)) return; + + const localGuid = this.syncStore.get(path); + if (!localGuid || localGuid === guid) return; + + const localFile = this.files.get(localGuid); + const committedMeta = this.syncStore.getCommittedMeta(path); + if (!localFile || !isDocument(localFile) || !isDocumentMeta(committedMeta)) { + return; + } + if (committedMeta.id !== guid) return; + + this._pendingRemaps.add(path); + this.executeRemap({ + path, + fromGuid: localGuid, + toGuid: guid, + }).catch((e) => { + this.warn(`[${path}] remap retry from update event failed`, e); + }).finally(() => { + this._pendingRemaps.delete(path); + }); + } + + private retryDeferredDownloadForGuid(guid: string): void { + const path = this.findCommittedPathByGuid(guid); + if (!path || this._pendingDownloads.has(path)) return; + + const committedMeta = this.syncStore.getCommittedMeta(path); + if (!isDocumentMeta(committedMeta) || committedMeta.id !== guid) { + return; + } + + const localGuid = this.syncStore.get(path); + if (!localGuid || localGuid !== guid || this.files.has(guid)) { + return; + } + + this._pendingDownloads.add(path); + this.downloadDoc(path, true) + .catch((e) => { + this.warn(`[${path}] deferred download retry failed`, e); + }) + .finally(() => { + this._pendingDownloads.delete(path); + }); + } + + /** + * Fetch an HTTP keyframe, then deliver it and the buffered updates. + */ + private _fetchKeyframeAndDeliver( + file: Document, + guid: string, + pending: Uint8Array[], + ): void { + this._pendingKeyframeUpdates.set(guid, pending); + this.backgroundSync.enqueueDownload(file, false).then((keyframe) => { + const buf = this._pendingKeyframeUpdates.get(guid); + this._pendingKeyframeUpdates.delete(guid); + if (!buf || buf.length === 0) return; + + if (keyframe) { + this.mergeManager.handleRemoteUpdate(guid, keyframe); + this.mergeManager.seedAppliedRemoteUpdate(guid, keyframe); } - })(); - RelayInstances.set(this, this.path); + for (const u of buf) { + const c = this.mergeManager.classifyUpdate(guid, u); + if (c === 'apply') { + this.mergeManager.handleRemoteUpdate(guid, u); + this.mergeManager.advanceAppliedRemoteUpdate(guid, u); + } + // 'stale' → drop (subsumed by keyframe) + // 'gap' shouldn't happen after a keyframe, but if it does + // the update is dropped — the keyframe is the best we have + } + }); + } + + /** + * Handle SYNC_TO_REMOTE effect in idle mode. + * + * When a document is in idle mode (file closed), the HSM may still need + * to sync local disk changes to the remote server. This happens when: + * 1. External process modifies the file on disk + * 2. HSM detects the change via polling + * 3. HSM performs idle auto-merge (disk → local CRDT) + * 4. HSM emits SYNC_TO_REMOTE effect + * + * Without this handler, the effect is dropped because ProviderIntegration + * is destroyed when the file is closed. + */ + private async handleIdleSyncToRemote( + guid: string, + update: Uint8Array, + ): Promise { + const file = this.files.get(guid); + if (!file || !isDocument(file)) { + this.warn( + `[handleIdleSyncToRemote] Document not found for guid: ${guid}`, + ); + return; + } + + // Skip if the editor has the file open — active mode syncs via ProviderIntegration. + if (file.userLock) { + this.debug?.( + `[handleIdleSyncToRemote] Document ${guid} has user lock, skipping`, + ); + return; + } + + try { + // Apply update to the document's remoteDoc (which is file.ydoc). + // This intentionally triggers lazy creation (wake from hibernation). + const remoteDoc = file.ensureRemoteDoc(); + Y.applyUpdate(remoteDoc, update, "local"); + + // Also update the HSM's remoteDoc reference so it stays in sync + if (file.hsm) { + file.hsm.setRemoteDoc(remoteDoc); + } + + // The per-document provider is not connected in idle mode, so we + // must explicitly sync via backgroundSync to push the update to + // the server. + await this.backgroundSync.enqueueSync(file); + this.log(`[handleIdleSyncToRemote] Synced idle mode update for ${guid}`); + } catch (e) { + this.warn( + `[handleIdleSyncToRemote] Failed to sync update for ${guid}:`, + e, + ); + } + } + + /** + * Handle WRITE_DISK effect in idle mode. + * + * When a document is in idle mode and receives remote updates, the HSM + * may need to write merged content to disk. This happens when: + * 1. Remote update arrives (from server) + * 2. HSM performs idle auto-merge (remote → local CRDT) + * 3. HSM emits WRITE_DISK effect to update the file on disk + * + * Without this handler, the effect is dropped. + */ + private async handleIdleWriteDisk( + guid: string, + contents: string, + ): Promise { + try { + // Look up document by guid to get current path (handles renames) + const file = this.files.get(guid); + if (!file || !isDocument(file)) { + this.warn(`[handleIdleWriteDisk] Document not found for guid: ${guid}`); + return; + } + + const vaultPath = this.getPath(file.path); + let tfile = this.vault.getAbstractFileByPath(vaultPath); + + if (tfile instanceof TFile) { + await this.vault.modify(tfile, contents); + } else { + // File doesn't exist on disk yet (new remote file) — create it + const normalized = normalizePath(vaultPath); + // Ensure parent folders exist + const parentPath = normalized.substring(0, normalized.lastIndexOf("/")); + if (parentPath && !this.vault.getAbstractFileByPath(parentPath)) { + await this.vault.createFolder(parentPath); + } + tfile = await this.vault.create(normalized, contents); + } + + this.log(`[handleIdleWriteDisk] Wrote merged content to ${vaultPath}`); + + } catch (e) { + this.warn(`[handleIdleWriteDisk] Failed to write for guid ${guid}:`, e); + } + } + + /** + * Poll for disk changes on all documents in this SharedFolder. + * Only sends DISK_CHANGED if the disk state actually differs from HSM's knowledge. + * Works for all documents regardless of hibernation state. + * + * @param guids - Optional set of GUIDs to poll. If not provided, polls all documents. + */ + async poll(guids?: string[]): Promise { + const targetGuids = guids ?? Array.from(this.files.keys()); + + for (const guid of targetGuids) { + const file = this.files.get(guid); + if (!file || !isDocument(file)) continue; + + const hsm = file.hsm; + if (!hsm) continue; + + const exists = this.existsSync(file.path); + if (!exists) continue; + + const currentDisk = hsm.state.disk; + + // Check disk state only after the cheap stat comparison. Reading and + // hashing every document on every poll is too expensive for large vaults. + try { + if (this.shouldReadDiskForPoll(currentDisk, file)) { + const diskState = await file.readDiskContent(); + + if (this.shouldSendDiskChanged(currentDisk, diskState)) { + hsm.send({ + type: "DISK_CHANGED", + contents: diskState.content, + mtime: diskState.mtime, + hash: diskState.hash, + }); + } + } + } catch (e) { + // File might have been deleted - ignore + } + + this.connectForkedIdleDocument(file); + } + } + + private connectForkedIdleDocuments(): void { + for (const file of this.files.values()) { + if (!isDocument(file)) continue; + this.connectForkedIdleDocument(file); + } + } + + private connectForkedIdleDocument(file: Document): void { + const hsm = file.hsm; + if (!hsm) return; + if (hsm.state.fork === null) return; + if (!hsm.matches("idle.localAhead")) return; + if (file.hasProviderIntegration() && file.intent === "connected") return; + if (!this.shouldConnect) return; + + file.connectForForkReconcile().catch(() => {}); + } + + private shouldReadDiskForPoll( + currentDisk: { hash: string; mtime: number } | null, + file: Document, + ): boolean { + if (!currentDisk) return true; + + const cachedDisk = this.getCachedDiskState(file); + if (cachedDisk) { + return cachedDisk.hash !== currentDisk.hash; + } + + const tfile = this.getTFile(file); + if (!tfile) return false; + + return tfile.stat.mtime !== currentDisk.mtime; + } + + private getCachedDiskState( + file: Document, + ): { hash: string; mtime: number } | null { + const tfile = this.getTFile(file); + if (!tfile) return null; + return this.getCachedDiskStateForTFile(tfile); + } + + private getCachedDiskStateForTFile(tfile: TFile): { hash: string; mtime: number } | null { + const fileCache = (this.metadataCache as any)?.fileCache; + const cached = + typeof fileCache?.get === "function" + ? fileCache.get(tfile.path) + : fileCache?.[tfile.path]; + if ( + !cached || + typeof cached.hash !== "string" || + typeof cached.mtime !== "number" + ) { + return null; + } + + if (cached.mtime !== tfile.stat.mtime) return null; + + return { hash: cached.hash, mtime: cached.mtime }; + } + + private getStartupDiskMetadata(tfile: TFile): { mtime: number; hash?: string } { + return this.getCachedDiskStateForTFile(tfile) ?? { mtime: tfile.stat.mtime }; + } + + getCurrentDiskMetadata(file: IFile): { mtime: number; hash?: string } | null { + const tfile = this.getTFile(file); + if (!tfile) return null; + return this.getStartupDiskMetadata(tfile); + } + + /** + * Determine if DISK_CHANGED event should be sent based on current vs new disk state. + * Returns true if disk state has changed, false if unchanged. + */ + private shouldSendDiskChanged( + currentDisk: { hash: string; mtime: number } | null, + newDiskState: { mtime: number; hash: string }, + ): boolean { + // No current disk state - always send + if (!currentDisk) return true; + + // Compare mtime first (fast check) + if (currentDisk.mtime !== newDiskState.mtime) return true; + + // Compare hash as fallback (handles clock skew edge cases) + if (currentDisk.hash !== newDiskState.hash) return true; + + return false; } private addLocalDocs = (types?: SyncType[]) => { @@ -319,6 +1006,13 @@ export class SharedFolder extends HasProvider { // Reserve GUIDs for new files before processing this.placeHold(syncTFiles); syncTFiles.forEach((tfile) => { + const vpath = this.getVirtualPath(tfile.path); + const guid = this.syncStore.get(vpath); + const existing = guid ? this.files.get(guid) : undefined; + if (existing) { + files.push(existing); + return; + } const file = this.getFile(tfile, false); if (file) { files.push(file); @@ -337,7 +1031,9 @@ export class SharedFolder extends HasProvider { if (value === this._server) { return; } - this.warn("server changed -- reinitializing all connections"); + if (this._server !== undefined) { + this.warn("server changed -- reinitializing all connections"); + } const shouldConnect = this.shouldConnect; this.reset(); const reconnect: HasProvider[] = []; @@ -383,7 +1079,22 @@ export class SharedFolder extends HasProvider { const isExtensionEnabled = this.syncSettingsManager.isExtensionEnabled(vpath); - return inFolder && isSupportedFileType && isExtensionEnabled; + return ( + inFolder && + isSupportedFileType && + isExtensionEnabled && + !this.isStorageBlockedTFile(tfile) + ); + } + + public isStorageBlockedTFile(tfile: TAbstractFile): boolean { + if (!(tfile instanceof TFile)) return false; + if (!this.checkPath(tfile.path)) return false; + const quota = this.remote?.relay.storageQuota?.quota ?? this.storageQuota; + if (quota !== 0) return false; + return this.syncSettingsManager.requiresStorage( + this.getVirtualPath(tfile.path), + ); } private getSyncFiles(): TAbstractFile[] { @@ -416,34 +1127,161 @@ export class SharedFolder extends HasProvider { this._shouldConnect = connect; } + public get localOnly(): boolean { + return this._localOnly; + } + + public set localOnly(value: boolean) { + if (this._localOnly === value) return; + this._localOnly = value; + this._settings.update((current) => ({ + ...current, + localOnly: value, + })); + const guids = Array.from(this.files.keys()); + this.mergeManager?.setLocalOnly(guids, value); + } + async netSync() { await this.whenReady(); + await this.mergeManager.initialize(); + if (this.destroyed) return; this.addLocalDocs(); - await this.syncFileTree(this.syncStore); + await this.syncFileTree(); this.backgroundSync.enqueueSharedFolderSync(this); } + async resync(): Promise { + if (!this.connected || this.localOnly) return; + const finishResync = this.backgroundSync.beginFolderResync(this); + try { + await this.netSync(); + } finally { + finishResync(); + } + } + public get settings(): SharedFolderSettings { return this._settings.get(); } + public getRecentRemoteActivity(limit = 30): RemoteActivityEntry[] { + return this.remoteActivityIndex.entries(limit); + } + + public getRemoteActivity(guid: string): RemoteActivityEntry | undefined { + return this.remoteActivityIndex.get(guid); + } + + public subscribeToRemoteActivity(callback: () => void): () => void { + if (this.destroyed) { + return () => {}; + } + this.remoteActivitySubscribers.add(callback); + return () => { + this.remoteActivitySubscribers.delete(callback); + }; + } + + private recordRemoteActivities(entries: readonly RemoteActivityEntry[]): void { + if (this.destroyed || entries.length === 0) return; + + let changed = false; + for (const entry of entries) { + changed = this.remoteActivityIndex.upsert(entry) || changed; + } + changed = this.pruneRemoteActivity() || changed; + if (!changed) return; + + this.persistRemoteActivity(); + this.notifyRemoteActivitySubscribers(); + } + + private pruneRemoteActivity(): boolean { + return this.remoteActivityIndex.pruneOlderThan( + this.currentTime() - REMOTE_ACTIVITY_RETENTION_MS, + ); + } + + private persistRemoteActivity(): void { + const persist = this._settings + .update((current) => ({ + ...current, + remoteActivity: this.remoteActivityIndex.serialize(), + })) + .catch((error) => { + this.warn("unable to persist remote activity", error); + }); + trackAsyncCleanup(persist); + trackPromise(`folder:remoteActivityPersist:${this.guid}`, persist); + } + + private notifyRemoteActivitySubscribers(): void { + for (const subscriber of [...this.remoteActivitySubscribers]) { + subscriber(); + } + } + + private currentTime(): number { + return this.timeProvider?.now() ?? Date.now(); + } + async sync() { - await this.syncFileTree(this.syncStore); + await this.syncFileTree(); } - connect(): Promise { + async connect(): Promise { if (this.s3rn instanceof S3RemoteFolder) { - if (this.connected || this.shouldConnect) { - return super.connect(); + if (this.connected) { + if (this.shouldConnect) this.enqueueLCABackfill("already-connected"); + return true; + } + if (this.shouldConnect) { + const result = await super.connect(); + if (result && this.mergeManager) { + // Clear server-advertised reconnect metadata so the next + // subdoc-index response reflects the current connection's + // server view. The applied remote baseline stays intact + // because it reflects state already incorporated locally. + // The provider preserves eventCallbacks across reconnects + // and re-sends the server subscribe frame itself, so the + // callbacks registered by the constructor's + // setupEventSubscriptions() call stay live. + this.mergeManager.clearServerAdvertisedSVs(); + this.enqueueLCABackfill("connect"); + this.connectForkedIdleDocuments(); + } + return result; } } - return Promise.resolve(false); + return false; + } + + private enqueueLCABackfill(reason: string): void { + if (this.destroyed || this.localOnly || !this.connected) return; + const queued = this.backgroundSync.enqueueLCABackfill(this); + if (queued > 0) { + this.debug(`[lca-backfill] queued ${queued} documents (${reason})`); + } } public get name(): string { return this.path.split("/").pop() || ""; } + public getUserDisplayName(userId: string): string | undefined { + const name = this.relayManager?.users.get(userId)?.name?.trim(); + return name || undefined; + } + + public isLocalUserId(userId: string): boolean { + return [ + this.loginManager?.user?.id, + this.relayManager?.user?.id, + this._provider?.awareness.getLocalState()?.user?.id, + ].some((id) => id === userId); + } + public get location(): string { return this.path.split("/").slice(0, -1).join("/"); } @@ -474,7 +1312,6 @@ export class SharedFolder extends HasProvider { })); if (value) { - this._server = value.relay.providerId; this.unsubscribes.push( value.relay.subscribe((relay) => { if (relay.guid === this.relayId) { @@ -525,8 +1362,8 @@ export class SharedFolder extends HasProvider { if (awaitingUpdates) { // If this is a brand new shared folder, we want to wait for a connection before we start reserving new guids for local files. this.connect(); - await this.onceConnected(); - await this.onceProviderSynced(); + await trackPromise(`folderConnected:${this.guid}`, this.onceConnected()); + await trackPromise(`folderReady:${this.guid}`, this.onceProviderSynced()); return this; } // If this is a shared folder with edits, then we can behave as though we're just offline. @@ -536,41 +1373,30 @@ export class SharedFolder extends HasProvider { this.readyPromise || new Dependency(promiseFn, (): [boolean, SharedFolder] => { return [this.ready, this]; - }); - return this.readyPromise.getPromise(); + }, this.timeProvider); + return trackPromise(`folder:whenReady:${this.guid}`, this.readyPromise.getPromise()); } whenSynced(): Promise { const promiseFn = async (): Promise => { - // Check if already synced first - if (this._persistence.synced) { - this.persistenceSynced = true; - return; - } - - return new Promise((resolve) => { - this._persistence.once("synced", () => { - this.persistenceSynced = true; - resolve(); - }); - }); + await this._persistence.whenSynced; + this.persistenceSynced = true; }; this.whenSyncedPromise = this.whenSyncedPromise || new Dependency(promiseFn, (): [boolean, void] => { - if (this._persistence.synced) { - this.persistenceSynced = true; - } return [this.persistenceSynced, undefined]; - }); - return this.whenSyncedPromise.getPromise(); + }, this.timeProvider); + return trackPromise(`folder:whenSynced:${this.guid}`, this.whenSyncedPromise.getPromise()); } public get intent(): ConnectionIntent { return this.shouldConnect ? "connected" : "disconnected"; } + + async _handleServerRename( doc: IFile, path: string, @@ -601,7 +1427,7 @@ export class SharedFolder extends HasProvider { vpath: string, meta: Meta, diffLog?: string[], - ): Promise { + ): Promise { // Create directories as needed const dir = dirname(vpath); if (!this.existsSync(dir)) { @@ -609,8 +1435,13 @@ export class SharedFolder extends HasProvider { diffLog?.push(`creating directory ${dir}`); } if (meta.type === "markdown") { - diffLog?.push(`created local .md file for remotely added doc ${vpath}`); + diffLog?.push(`creating local .md file for remotely added doc ${vpath}`); const doc = await this.downloadDoc(vpath, false); + if (!doc) { + diffLog?.push( + `deferred local .md file for remotely added doc ${vpath} (server has guid but no content yet)`, + ); + } return doc; } if (meta.type === "canvas") { @@ -644,6 +1475,139 @@ export class SharedFolder extends HasProvider { } } + /** + * Swap or rebuild a document's local CRDT identity. Called when the folder's + * meta CRDT resolves a path to a GUID that differs from the one we enrolled + * locally, and when the same GUID has unusable local CRDT state. Tears down + * the local Y.Doc + IDB + HSM state, downloads the winning CRDT from the + * server, and creates a fresh Document under the canonical GUID. + * + * Folder-level: does not require a living Document instance at fromGuid. + * On failure, leaves pendingUpload intact so the next observer event or + * startup scan re-detects and retries. + */ + private async executeRemap({ path, fromGuid, toGuid }: { + path: string; + fromGuid: string; + toGuid: string; + }): Promise { + const sameGuid = fromGuid === toGuid; + const operation = sameGuid ? "rebuild" : "remap"; + metrics.incDocumentRebuild(operation, "started"); + let operationTerminalRecorded = false; + const recordOperationTerminal = ( + result: "completed" | "deferred" | "failed", + ) => { + if (operationTerminalRecorded) return; + metrics.incDocumentRebuild(operation, result); + operationTerminalRecorded = true; + }; + if (!this.connected) { + recordOperationTerminal("deferred"); + this.log(`[${path}] ${operation} deferred: folder offline`); + return; + } + + let updateBytes: Uint8Array | undefined; + try { + updateBytes = await this.backgroundSync.downloadByGuid(this, toGuid, path); + } catch (e) { + recordOperationTerminal("deferred"); + this.warn(`[${path}] ${operation} download failed, deferring`, e); + return; + } + + if (!updateBytes) { + recordOperationTerminal("deferred"); + this.log(`[${path}] ${operation} deferred: server has guid but no content yet`); + return; + } + + if (this.destroyed) { + recordOperationTerminal("deferred"); + this.log(`[${path}] ${operation} aborted: folder destroyed during download`); + return; + } + + try { + const existingFile = this.files.get(fromGuid); + const existingHsm = existingFile && isDocument(existingFile) + ? existingFile.hsm + : null; + if (sameGuid) { + try { + await existingHsm?.resetLocalPersistenceForRebuild(); + } catch (e) { + this.warn(`[${path}] rebuild local cleanup failed`, e); + throw e; + } + await this._hsmStore.deleteState(fromGuid); + } else { + try { + indexedDB.deleteDatabase(`${this.appId}-relay-doc-${fromGuid}`); + } catch { /* best effort stale database cleanup */ } + const p = this._hsmStore.deleteState(fromGuid).catch(() => {}); + trackAsyncCleanup(p); + } + + this.backgroundSync.cancelDocumentWork(fromGuid); + + if (existingFile) { + this.files.delete(fromGuid); + this.fset.delete(existingFile); + existingFile.cleanup(); + existingFile.destroy(); + } + + this.syncStore.pendingUpload.delete(path); + + const newDoc = this.getOrCreateDoc(toGuid, path); + this.files.set(toGuid, newDoc); + this.fset.add(newDoc, true); + const isCurrentDoc = () => + !this.destroyed && !newDoc.destroyed && this.files.get(toGuid) === newDoc; + + if (!isCurrentDoc()) { + recordOperationTerminal("deferred"); + this.log(`[${path}] ${operation} aborted: new document is stale`); + return; + } + + if (updateBytes) { + await newDoc.hsm?.initializeFromRemote(updateBytes); + const remoteDoc = newDoc.ensureRemoteDoc(); + Y.applyUpdate(remoteDoc, updateBytes, remoteDoc); + newDoc.hsm?.setRemoteDoc(remoteDoc); + } + if (!isCurrentDoc()) { + recordOperationTerminal("deferred"); + this.log(`[${path}] ${operation} aborted after enroll: new document is stale`); + return; + } + if (newDoc.hsm && !newDoc.hsm.state.lca) { + await newDoc.hsm.awaitIdle(); + const diskState = await newDoc.readDiskContent(); + await newDoc.hsm.bootstrapLCAFromDisk(diskState); + } + await this.poll([toGuid]); + + recordOperationTerminal("completed"); + + this.log( + sameGuid + ? `Rebuilt Document ${path}: ${toGuid}` + : `Remapped Document ${path}: ${fromGuid} → ${toGuid}`, + ); + } catch (e) { + recordOperationTerminal("failed"); + throw e; + } + } + + async rebuildDocumentFromRemote(guid: string, path: string): Promise { + await this.executeRemap({ path, fromGuid: guid, toGuid: guid }); + } + private applyRemoteState( guid: string, path: string, @@ -670,13 +1634,14 @@ export class SharedFolder extends HasProvider { return { op: "update", path, promise: file.pull() }; } - // Check for GUID mismatch - file exists but not mapped to remote GUID - if (!file && isSyncFileMeta(meta)) { + // GUID mismatch — file at this path is mapped under a different + // guid locally than meta.id. Reconcile by swapping identity to + // the canonical meta.id. + if (!file) { const localGuid = this.syncStore.get(path); const localFile = localGuid ? this.files.get(localGuid) : null; - if (localGuid && localFile && isSyncFile(localFile)) { - // We have a local file with different GUID - check if content matches + if (localGuid && localFile && isSyncFile(localFile) && isSyncFileMeta(meta)) { const promise = this.remapIfHashMatches( localFile, localGuid, @@ -686,6 +1651,18 @@ export class SharedFolder extends HasProvider { ); return { op: "update", path, promise }; } + + if (localGuid && localGuid !== guid && isDocumentMeta(meta)) { + return { + op: "update", + path, + promise: this.executeRemap({ + path, + fromGuid: localGuid, + toGuid: guid, + }), + }; + } } return { op: "noop", path, promise: Promise.resolve() }; @@ -803,10 +1780,7 @@ export class SharedFolder extends HasProvider { private getDesiredRemotePaths(): Set { const paths = new Set(); - this.syncStore.forEach((_meta, path) => { - paths.add(path); - }); - this.pendingUpload.forEach((_guid, path) => { + this.syncStore.forEachWithPending((_meta, path) => { paths.add(path); }); return expandDesiredRemotePaths(paths); @@ -818,18 +1792,76 @@ export class SharedFolder extends HasProvider { ops: Operation[], types: SyncType[], ) { - syncStore.forEach((meta, path) => { + syncStore.forEachWithPending((meta, path) => { this._assertNamespacing(path); - if (types.contains(meta.type)) { - this._assertNamespacing(path); + if (meta && types.contains(meta.type)) { ops.push( this.applyRemoteState(meta.id, path, syncStore.remoteIds, diffLog), ); + } else if (!meta && types.contains(SyncType.Document)) { + // Pending upload only — no meta yet. Retry the upload so + // syncFileTree's sweep covers outbound reconciliation alongside + // the inbound remap/update work above. + ops.push(this.applyPendingUpload(path)); } }); } - syncFileTree(syncStore: SyncStore): Promise { + /** + * Retry a pending upload for a path whose local meta was never written + * (the initial enqueueSync failed or was deferred). Resolves the file via + * pendingUpload's guid, re-enqueues sync, and calls markUploaded on success + * so the local meta gets written and pendingUpload is cleared. + */ + private applyPendingUpload(path: string): OperationType { + const pendingGuid = this.syncStore.pendingUpload.get(path); + if (!pendingGuid) { + return { op: "noop", path, promise: Promise.resolve() }; + } + + // Server-authoritative rule: if committed filemeta already points at a + // different GUID for this path, do not publish/overwrite local pending + // metadata. Adopt the committed GUID instead. + const committedMeta = this.syncStore.getCommittedMeta(path); + if (committedMeta && committedMeta.id !== pendingGuid) { + this.warn( + "[applyPendingUpload] committed GUID differs from pending upload", + { + path, + pendingGuid, + committedGuid: committedMeta.id, + }, + ); + const pendingFile = this.files.get(pendingGuid); + if (isDocumentMeta(committedMeta) && pendingFile && isDocument(pendingFile)) { + return { + op: "update", + path, + promise: this.executeRemap({ + path, + fromGuid: pendingGuid, + toGuid: committedMeta.id, + }), + }; + } + return { op: "noop", path, promise: Promise.resolve() }; + } + + const file = this.files.get(pendingGuid); + if (!file || !(isDocument(file) || isCanvas(file) || isSyncFile(file))) { + return { op: "noop", path, promise: Promise.resolve() }; + } + return { + op: "update", + path, + promise: (async () => { + await this.backgroundSync.enqueueUpload(file); + await this.markUploaded(file); + })(), + }; + } + + syncFileTree(): Promise { // If a sync is already running, mark that we want another sync after if (this.syncFileTreePromise) { this.syncRequestedDuringSync = true; @@ -837,7 +1869,7 @@ export class SharedFolder extends HasProvider { promise.then(() => { if (this.syncRequestedDuringSync) { this.syncRequestedDuringSync = false; - return this.syncFileTree(syncStore); + return this.syncFileTree(); } }); return promise; @@ -845,6 +1877,10 @@ export class SharedFolder extends HasProvider { const promiseFn = async (): Promise => { try { + if (!this.mergeManager || this.destroyed) return; + await this.mergeManager.initialize(); + if (this.destroyed) return; + // When file types are newly enabled, enqueue their local // files for syncing before the rest of the tree sync runs. const currentTypes = this.syncStore.typeRegistry.getEnabledFileSyncTypes(); @@ -862,12 +1898,12 @@ export class SharedFolder extends HasProvider { this.ydoc.transact(async () => { // Sync folder operations first because renames/moves also affect files this.syncStore.migrateUp(); - this.syncByType(syncStore, diffLog, ops, [SyncType.Folder]); + this.syncByType(this.syncStore, diffLog, ops, [SyncType.Folder]); }, this); await Promise.all(ops.map((op) => op.promise)); this.ydoc.transact(async () => { this.syncByType( - syncStore, + this.syncStore, diffLog, ops, this.syncStore.typeRegistry.getEnabledFileSyncTypes(), @@ -881,7 +1917,11 @@ export class SharedFolder extends HasProvider { // Ensure these complete before checking for deletions await Promise.all( [...creates, ...renames].map((op) => - withTimeoutWarning(op.promise, op), + withTimeoutWarning( + op.promise, + this.timeProvider, + op, + ), ), ); @@ -905,9 +1945,12 @@ export class SharedFolder extends HasProvider { } }; - this.syncFileTreePromise = new SharedPromise(promiseFn); + this.syncFileTreePromise = new SharedPromise( + promiseFn, + this.timeProvider, + ); - return this.syncFileTreePromise.getPromise(); + return trackPromise(`folder:syncFileTree:${this.guid}`, this.syncFileTreePromise.getPromise()); } move(path: string) { @@ -1050,6 +2093,42 @@ export class SharedFolder extends HasProvider { if (!this.syncStore) { return; } + + // Server-authoritative rule: never overwrite an existing committed + // GUID for this path with a local pending GUID. + const committedMeta = this.syncStore.getCommittedMeta(file.path); + if (committedMeta && committedMeta.id !== meta.id) { + this.warn( + "[markUploaded] committed GUID differs from local upload metadata", + { + path: file.path, + localGuid: meta.id, + committedGuid: committedMeta.id, + }, + ); + // Server metadata already chose a different GUID for this path. + // The local upload succeeded, but the path must adopt the + // committed identity instead of leaving pendingUpload to shadow + // every later path lookup. + if ( + isDocument(file) && + isDocumentMeta(committedMeta) && + !this._pendingRemaps.has(file.path) + ) { + this._pendingRemaps.add(file.path); + this.executeRemap({ + path: file.path, + fromGuid: file.guid, + toGuid: committedMeta.id, + }).catch((e) => { + this.warn(`[${file.path}] remap retry from markUploaded failed`, e); + }).finally(() => { + this._pendingRemaps.delete(file.path); + }); + } + return; + } + if (this.syncStore.willSet(file.path, meta)) { this.log("new meta", file.path, meta); this.ydoc.transact(() => { @@ -1130,7 +2209,10 @@ export class SharedFolder extends HasProvider { if (Document.checkExtension(vpath)) { return this.getDoc(vpath); } - if (Canvas.checkExtension(vpath) && flags().enableCanvasSync) { + if ( + Canvas.checkExtension(vpath) && + this.syncSettingsManager.isExtensionEnabled(vpath) + ) { return this.getCanvas(vpath); } if (this.syncStore.canSync(vpath)) { @@ -1183,7 +2265,7 @@ export class SharedFolder extends HasProvider { const canvas = this.getOrCreateCanvas(guid, vpath); canvas.markOrigin("remote"); - this.backgroundSync.enqueueCanvasDownload(canvas); + this.backgroundSync.enqueueCanvasDownload(canvas, update); this.files.set(guid, canvas); this.fset.add(canvas, update); @@ -1228,8 +2310,8 @@ export class SharedFolder extends HasProvider { }, this._persistence); canvas.markOrigin("local"); this.log(`[${canvas.path}] Uploading file`); - await this.backgroundSync.enqueueSync(canvas); - this.markUploaded(canvas); + await this.backgroundSync.enqueueUpload(canvas); + await this.markUploaded(canvas); } })(); @@ -1252,12 +2334,13 @@ export class SharedFolder extends HasProvider { const canvas = this.getOrCreateCanvas(guid, vpath); (async () => { - this.whenReady().then(async () => { + trackPromise(`folder:canvasReady:${canvas.guid}`, this.whenReady()).then(async () => { const synced = await canvas.getServerSynced(); if (canvas.stat.size === 0 && !synced) { this.backgroundSync.enqueueCanvasDownload(canvas); } else if (this.pendingUpload.get(canvas.path)) { - this.backgroundSync.enqueueSync(canvas); + await this.backgroundSync.enqueueUpload(canvas); + await this.markUploaded(canvas); } }); })(); @@ -1309,10 +2392,36 @@ export class SharedFolder extends HasProvider { throw new Error("unexpected ifile type"); } doc.move(vpath, this); + + if (this._localOnly && doc.hsm) { + doc.hsm.setLocalOnly(true); + } + return doc; } - async downloadDoc(vpath: string, update = true): Promise { + async downloadDoc( + vpath: string, + update = true, + ): Promise { + const pending = this._pendingDownloadPromises.get(vpath); + if (pending) return pending; + + const promise = this.downloadDocOnce(vpath, update); + this._pendingDownloadPromises.set(vpath, promise); + this._pendingDownloads.add(vpath); + try { + return await promise; + } finally { + this._pendingDownloadPromises.delete(vpath); + this._pendingDownloads.delete(vpath); + } + } + + private async downloadDocOnce( + vpath: string, + update: boolean, + ): Promise { if (!Document.checkExtension(vpath)) { throw new Error("unexpected extension"); } @@ -1323,12 +2432,29 @@ export class SharedFolder extends HasProvider { if (!guid) { throw new Error(`called download on item that is not in ids ${vpath}`); } + const updateBytes = await this.backgroundSync.downloadByGuid(this, guid, vpath); + + if (!updateBytes) { + this.log(`[${vpath}] download deferred: server has guid but no content yet`); + return undefined; + } + + const tempDoc = new Y.Doc(); + Y.applyUpdate(tempDoc, updateBytes); + const contents = tempDoc.getText("contents").toString(); const doc = this.getOrCreateDoc(guid, vpath); - doc.markOrigin("remote"); + await doc.hsm?.initializeFromRemote(updateBytes); + const remoteDoc = doc.ensureRemoteDoc(); + doc.hsm?.setRemoteDoc(remoteDoc); + await doc.hsm?.awaitIdle(); + await doc.hsm?.completeInitialEnrollmentFromRemote(contents); - this.backgroundSync.enqueueDownload(doc); + if (!this.syncStore.has(doc.path)) { + throw new Error("file no longer wanted"); + } this.files.set(guid, doc); + await this.flush(doc, contents); this.fset.add(doc, update); return doc; @@ -1347,29 +2473,18 @@ export class SharedFolder extends HasProvider { } const doc = this.getOrCreateDoc(guid, vpath); - const originPromise = doc.getOrigin(); - const awaitingUpdatesPromise = this.awaitingUpdates(); - (async () => { - const exists = await this.exists(doc); + const [exists, awaitingUpdates] = await Promise.all([ + this.exists(doc), + this.awaitingUpdates(), + ]); if (!exists) { throw new Error(`Upload failed, doc does not exist at ${vpath}`); } - const [contents, origin, awaitingUpdates] = await Promise.all([ - this.read(doc), - originPromise, - awaitingUpdatesPromise, - ]); - const text = doc.ydoc.getText("contents"); - if (!awaitingUpdates && origin === undefined) { - this.log(`[${doc.path}] No Known Peers: Syncing file into ytext.`); - this.ydoc.transact(() => { - text.insert(0, contents); - }, this._persistence); - doc.markOrigin("local"); - this.log(`[${doc.path}] Uploading file`); - await this.backgroundSync.enqueueSync(doc); - this.markUploaded(doc); + if (!awaitingUpdates) { + await doc.hsm?.initializeWithContent(); + await this.backgroundSync.enqueueUpload(doc); + await this.markUploaded(doc); } })(); @@ -1392,12 +2507,13 @@ export class SharedFolder extends HasProvider { const doc = this.getOrCreateDoc(guid, vpath); (async () => { - this.whenReady().then(async () => { + trackPromise(`folder:docReady:${doc.guid}`, this.whenReady()).then(async () => { const synced = await doc.getServerSynced(); if (doc.tfile?.stat.size === 0 && !synced) { - this.backgroundSync.enqueueDownload(doc); + this.backgroundSync.enqueueDownload(doc, false); } else if (this.pendingUpload.get(doc.path)) { - this.backgroundSync.enqueueSync(doc); + await this.backgroundSync.enqueueUpload(doc); + await this.markUploaded(doc); } }); })(); @@ -1520,7 +2636,11 @@ export class SharedFolder extends HasProvider { } const file = this.getOrCreateSyncFile(guid, vpath, tfile); - this.backgroundSync.enqueueSync(file); + void (async () => { + if (!this.pendingUpload.get(file.path)) return; + await this.backgroundSync.enqueueUpload(file); + await this.markUploaded(file); + })(); this.fset.add(file, update); return file; @@ -1549,7 +2669,11 @@ export class SharedFolder extends HasProvider { const meta = this.syncStore.getMeta(vpath); if (!meta) { this.log("get syncfile missing meta"); - file.push(); + void (async () => { + if (!this.pendingUpload.get(file.path)) return; + await this.backgroundSync.enqueueUpload(file); + await this.markUploaded(file); + })(); } else { file.pull(); } @@ -1567,7 +2691,10 @@ export class SharedFolder extends HasProvider { if (Document.checkExtension(vpath)) { return this.uploadDoc(vpath, update); } - if (Canvas.checkExtension(vpath) && flags().enableCanvasSync) { + if ( + Canvas.checkExtension(vpath) && + this.syncSettingsManager.isExtensionEnabled(vpath) + ) { return this.uploadCanvas(vpath, update); } if (this.syncStore.canSync(vpath)) { @@ -1591,18 +2718,40 @@ export class SharedFolder extends HasProvider { return this.pendingDeletes.has(vpath); } + isPendingUpload(vpath: string): boolean { + return this.pendingUpload.has(vpath); + } + deleteFile(vpath: string) { + this.pendingUpload.delete(vpath); const guid = this.syncStore?.get(vpath); if (guid) { this.ydoc.transact(() => { this.syncStore.delete(vpath); const doc = this.files.get(guid); if (doc) { - doc.cleanup(); this.fset.delete(doc); + this.files.delete(guid); + doc.cleanup(); + doc.destroy(); } - this.files.delete(guid); }, this); + indexedDB.deleteDatabase(`${this.appId}-relay-doc-${guid}`); + const p = this._hsmStore.deleteState(guid).catch(() => {}); + trackAsyncCleanup(p); + } else { + // syncStore entry already gone (remote delete) - find by path + const doc = this.fset.find((f) => f.path === vpath); + if (doc) { + const docGuid = doc.guid; + this.fset.delete(doc); + this.files.delete(docGuid); + doc.cleanup(); + doc.destroy(); + indexedDB.deleteDatabase(`${this.appId}-relay-doc-${docGuid}`); + const p = this._hsmStore.deleteState(docGuid).catch(() => {}); + trackAsyncCleanup(p); + } } } @@ -1685,36 +2834,68 @@ export class SharedFolder extends HasProvider { } } + onDestroy(cb: () => void): void { + if (this.destroyed) { + try { cb(); } catch { /* caller's problem */ } + return; + } + this.unsubscribes.push(cb); + } + destroy() { this.destroyed = true; this.unsubscribes.forEach((unsub) => { unsub(); }); + this.unsubscribes = []; + + // Mark the merge manager as shutting down before destroying docs so + // per-doc unloads don't schedule hibernate timers we'd just orphan. + this.mergeManager?.beginShutdown(); + this.files.forEach((doc: IFile) => { doc.destroy(); this.files.delete(doc.guid); }); + + this.recordingBridge?.dispose(); + this.cas.destroy(); this.syncStore.destroy(); this.syncSettingsManager.destroy(); + this.mergeManager?.destroy(); + // IndexeddbPersistence self-destructs on the ydoc's 'destroy' event, + // but its async teardown promise (awaiting pending writes and + // compaction before closing the DB) is dropped inside that event + // handler. Capture it here so failures are logged. Calling destroy() + // removes the 'destroy' + // listener synchronously, so super.destroy() below won't double-fire. + if (this._persistence) { + const p = this._persistence.destroy().catch(() => {}); + trackAsyncCleanup(p); + } super.destroy(); - this.ydoc.destroy(); this.fset.clear(); this._settings.destroy(); this._settings = null as any; - this.diskBufferStore = null as any; + this.revokeProxy?.(); + this.revokeProxy = null; + this.proxy = null as any; this.relayManager = null as any; this.backgroundSync = null as any; this.loginManager = null as any; this.tokenStore = null as any; this.fileManager = null as any; + this.cas = null as any; this.syncStore = null as any; this.syncSettingsManager = null as any; + this.mergeManager = null as any; this.whenSyncedPromise?.destroy(); this.whenSyncedPromise = null as any; this.readyPromise?.destroy(); this.readyPromise = null as any; this.syncFileTreePromise?.destroy(); this.syncFileTreePromise = null as any; + } } @@ -1723,7 +2904,7 @@ export class SharedFolders extends ObservableSet { path: string, guid: string, relayId?: string, - awaitingUpdates?: boolean, + authoritative?: boolean, ) => SharedFolder; private _offRemoteUpdates?: () => void; @@ -1734,9 +2915,10 @@ export class SharedFolders extends ObservableSet { path: string, guid: string, relayId?: string, - awaitingUpdates?: boolean, + authoritative?: boolean, ) => SharedFolder, private settings: NamespacedSettings, + private _hsmStore: HSMStore, ) { super(); this.folderBuilder = folderBuilder; @@ -1761,11 +2943,23 @@ export class SharedFolders extends ObservableSet { } public delete(item: SharedFolder): boolean { + // Collect IDB database names before destroy nulls references + const dbNames: string[] = []; + if (item) { + item.files.forEach((doc: IFile) => { + dbNames.push(`${item.appId}-relay-doc-${doc.guid}`); + }); + dbNames.push(item.guid); + } item?.destroy(); const deleted = super.delete(item); this.settings.update((current) => { return current.filter((settings) => settings.guid !== item.guid); }); + // Delete IDB databases after in-memory objects are destroyed + for (const name of dbNames) { + indexedDB.deleteDatabase(name); + } return deleted; } @@ -1794,9 +2988,7 @@ export class SharedFolders extends ObservableSet { if (this._offRemoteUpdates) { this._offRemoteUpdates(); } - this.unsubscribes.forEach((unsub) => { - unsub(); - }); + super.destroy(); this.relayManager = null as any; this.folderBuilder = null as any; } @@ -1808,13 +3000,30 @@ export class SharedFolders extends ObservableSet { private _load(folders: SharedFolderSettings[]) { let updated = false; folders.forEach((folder: SharedFolderSettings) => { + // Validate required fields + if (!folder.path) { + this.warn(`Invalid settings: folder missing path, skipping`); + return; + } + if (!folder.guid || !S3RN.validateUUID(folder.guid)) { + this.warn( + `Invalid settings: folder "${folder.path}" has invalid guid "${folder.guid}", skipping`, + ); + return; + } const tFolder = this.vault.getFolderByPath(folder.path); if (!tFolder) { this.warn(`Invalid settings, ${folder.path} does not exist`); return; } - this._new(folder.path, folder.guid, folder?.relay); - updated = true; + try { + this._new(folder.path, folder.guid, folder?.relay); + updated = true; + } catch (e) { + this.warn( + `Failed to load folder "${folder.path}": ${e instanceof Error ? e.message : String(e)}`, + ); + } }); if (updated) { @@ -1826,8 +3035,21 @@ export class SharedFolders extends ObservableSet { path: string, guid: string, relayId?: string, - awaitingUpdates?: boolean, + authoritative?: boolean, ): SharedFolder { + // Validate inputs + if (!path) { + throw new Error("Cannot create shared folder: path is required"); + } + if (!guid || !S3RN.validateUUID(guid)) { + throw new Error(`Cannot create shared folder: invalid guid "${guid}"`); + } + if (relayId && !S3RN.validateUUID(relayId)) { + throw new Error( + `Cannot create shared folder: invalid relayId "${relayId}"`, + ); + } + const existing = this.find( (folder) => folder.path == path && folder.guid == guid, ); @@ -1842,13 +3064,22 @@ export class SharedFolders extends ObservableSet { if (samePath) { throw new Error("Conflict: Tracked folder exists at this location."); } - const folder = this.folderBuilder(path, guid, relayId, awaitingUpdates); + const folder = this.folderBuilder(path, guid, relayId, authoritative); this._set.add(folder); return folder; } - new(path: string, guid: string, relayId?: string, awaitingUpdates?: boolean) { - const folder = this._new(path, guid, relayId, awaitingUpdates); + /** Share a local folder — user is authoritative (source of truth). */ + init(path: string, relayId?: string): SharedFolder { + const guid = uuidv4(); + const folder = this._new(path, guid, relayId, true); + this.notifyListeners(); + return folder; + } + + /** Download a remote folder — server is authoritative. */ + clone(path: string, guid: string, relayId?: string): SharedFolder { + const folder = this._new(path, guid, relayId, false); this.notifyListeners(); return folder; } diff --git a/src/SyncFile.ts b/src/SyncFile.ts index fee98bd6..d18185b6 100644 --- a/src/SyncFile.ts +++ b/src/SyncFile.ts @@ -6,7 +6,7 @@ import { S3Folder, type S3RNType, } from "./S3RN"; -import { SharedFolder } from "./SharedFolder"; +import type { SharedFolder } from "./SharedFolder"; import { HasLogging } from "./debug"; import { type FileMeta, type FileMetas, type SyncFileType } from "./SyncTypes"; import { TFile, type Vault, type TFolder, type FileStats } from "obsidian"; @@ -271,6 +271,7 @@ export class SyncFile caf: ContentAddressedFile; ready: boolean = false; connected: boolean = true; + destroyed: boolean = false; offFileInfo: Unsubscriber = () => {}; uploadError?: string = undefined; @@ -335,10 +336,10 @@ export class SyncFile } public get tag() { - return this.inMeta - ? "" - : this.uploadError - ? this.uploadError + return this.uploadError + ? this.uploadError + : this.inMeta + ? "" : this.pending ? "pending" : "unknown"; @@ -383,7 +384,6 @@ export class SyncFile if (!this.meta || (hash && this.meta.hash !== hash) || force) { try { await this.sharedFolder.cas.writeFile(this); - await this.sharedFolder.markUploaded(this); this.uploadError = undefined; this.notifyListeners(); } catch (error) { @@ -395,6 +395,7 @@ export class SyncFile } this.uploadError = errorMessage.replace(/^Error:/, "").trim(); this.notifyListeners(); + throw new Error(this.uploadError); } } return; @@ -445,6 +446,9 @@ export class SyncFile return; } } catch (err) { + if (this.uploadError) { + throw err; + } this.warn("unable to compute hash", err); } } @@ -556,6 +560,7 @@ export class SyncFile cleanup() {} destroy() { + this.destroyed = true; this.offFileInfo?.(); this.offFileInfo = null as any; diff --git a/src/SyncFolder.ts b/src/SyncFolder.ts index 104f8c3c..08922e58 100644 --- a/src/SyncFolder.ts +++ b/src/SyncFolder.ts @@ -3,7 +3,6 @@ import type { SharedFolder } from "./SharedFolder"; import { HasLogging } from "./debug"; import { type Vault, TFolder } from "obsidian"; import type { Unsubscriber } from "./observable/Observable"; -import type { RelayManager } from "./RelayManager"; import { uuidv4 } from "lib0/random"; import type { IFile } from "./IFile"; diff --git a/src/SyncSettings.ts b/src/SyncSettings.ts index d6eb1ddc..fe96ade6 100644 --- a/src/SyncSettings.ts +++ b/src/SyncSettings.ts @@ -1,11 +1,12 @@ import { NamespacedSettings, Settings } from "./SettingsStorage"; -import { flags } from "./flagManager"; export interface SyncCategory { enabled: boolean; extensions: string[]; description: string; name: string; + requiresStorage: boolean; + canToggle: boolean; } export interface SyncFlags { @@ -13,26 +14,56 @@ export interface SyncFlags { audio?: boolean; videos?: boolean; pdfs?: boolean; + canvas?: boolean; + bases?: boolean; otherTypes?: boolean; } +export type SyncCategoryKey = keyof SyncFlags | "markdown"; + interface TypeSettings { name: string; description: string; extensions: string[]; defaultEnabled: boolean; + requiresStorage: boolean; } export class SyncSettingsManager extends NamespacedSettings< Record > { + private static readonly alwaysEnabledSchema: Record<"markdown", TypeSettings> = { + markdown: { + name: "Markdown", + description: "Sync Markdown files (.md)", + extensions: ["md"], + defaultEnabled: true, + requiresStorage: false, + }, + }; + private static readonly schema: Record = { + canvas: { + name: "Canvas", + description: "Sync Canvas files (.canvas)", + extensions: ["canvas"], + defaultEnabled: true, + requiresStorage: false, + }, + bases: { + name: "Bases", + description: "Sync Bases files (.base)", + extensions: ["base"], + defaultEnabled: true, + requiresStorage: true, + }, images: { name: "Images", description: "Sync image files (.bmp, .png, .jpg, .jpeg, .gif, .svg, .webp, .avif)", extensions: ["bmp", "png", "jpg", "jpeg", "gif", "svg", "webp", "avif"], defaultEnabled: true, + requiresStorage: true, }, audio: { name: "Audio", @@ -40,24 +71,28 @@ export class SyncSettingsManager extends NamespacedSettings< "Sync audio files (.mp3, .wav, .m4a, .3gp, .flac, .ogg, .oga, .opus)", extensions: ["mp3", "wav", "m4a", "3gp", "flac", "ogg", "oga", "opus"], defaultEnabled: true, + requiresStorage: true, }, videos: { name: "Videos", description: "Sync video files (.mp4, .webm, .ogv, .mov, .mkv)", extensions: ["mp4", "webm", "ogv", "mov", "mkv"], defaultEnabled: true, + requiresStorage: true, }, pdfs: { name: "PDFs", description: "Sync PDF files (.pdf)", extensions: ["pdf"], defaultEnabled: true, + requiresStorage: true, }, otherTypes: { name: "Other files", description: "Sync unsupported file types", extensions: [], defaultEnabled: false, + requiresStorage: true, }, }; @@ -69,7 +104,7 @@ export class SyncSettingsManager extends NamespacedSettings< ) as Record; constructor( - settings: Settings, + settings: Settings>, path: string, public enabled = true, ) { @@ -82,8 +117,6 @@ export class SyncSettingsManager extends NamespacedSettings< if (normalizedExt === "md") return true; - if (flags().enableCanvasSync && normalizedExt === "canvas") return true; - if (!this.enabled) { return false; } @@ -102,7 +135,34 @@ export class SyncSettingsManager extends NamespacedSettings< ); } - getCategory(key: keyof SyncFlags): SyncCategory { + public requiresStorage(path: string): boolean { + const extension = path.split(".").pop() || ""; + const normalizedExt = extension.toLowerCase(); + + if (normalizedExt === "md") return false; + + for (const schema of Object.values(SyncSettingsManager.schema)) { + if (schema.extensions.includes(normalizedExt)) { + return schema.requiresStorage; + } + } + + return SyncSettingsManager.schema.otherTypes.requiresStorage; + } + + getCategory(key: SyncCategoryKey): SyncCategory { + if (key === "markdown") { + const schema = SyncSettingsManager.alwaysEnabledSchema[key]; + return { + enabled: true, + name: schema.name, + description: schema.description, + extensions: schema.extensions, + requiresStorage: schema.requiresStorage, + canToggle: false, + }; + } + const schema = SyncSettingsManager.schema[key]; const enabled = this.get()[key] ?? schema.defaultEnabled; return { @@ -110,17 +170,23 @@ export class SyncSettingsManager extends NamespacedSettings< name: schema.name, description: schema.description, extensions: schema.extensions, + requiresStorage: schema.requiresStorage, + canToggle: true, }; } - getCategories(): Record { - return Object.keys(SyncSettingsManager.schema).reduce( - (acc, key) => { - acc[key as keyof SyncFlags] = this.getCategory(key as keyof SyncFlags); - return acc; - }, - {} as Record, - ); + getCategories(): Record { + const categories = { + markdown: this.getCategory("markdown"), + } as Record; + + for (const key of Object.keys(SyncSettingsManager.schema)) { + categories[key as keyof SyncFlags] = this.getCategory( + key as keyof SyncFlags, + ); + } + + return categories; } public async toggleCategory( diff --git a/src/SyncStore.ts b/src/SyncStore.ts index 491c4349..2d185c20 100644 --- a/src/SyncStore.ts +++ b/src/SyncStore.ts @@ -8,6 +8,7 @@ import { flag } from "./flags"; import { SyncType, TypeRegistry, + isCanvasMeta, isDocumentMeta, isSyncFolderMeta, makeDocumentMeta, @@ -125,6 +126,47 @@ export class SyncStore extends Observable { }); } + getCommittedSubdocGuids(): string[] { + const guids = new Set(); + this.meta.forEach((meta, path) => { + if (this.deleteSet.has(path)) return; + if (isDocumentMeta(meta) || isCanvasMeta(meta)) { + guids.add(meta.id); + } + }); + this.legacyIds.forEach((guid, path) => { + if (this.deleteSet.has(path)) return; + guids.add(guid); + }); + return Array.from(guids).sort(); + } + + /** + * Like forEach, but also yields pending-upload-only paths (where no meta + * has been written yet). The callback receives `null` for those entries. + * Used by reconciliation sweeps that need to retry uploads which never + * completed (and thus never wrote meta locally). + */ + forEachWithPending( + callbackFn: (meta: Meta | null, path: string) => void, + ) { + const seen = new Set(); + this.meta.forEach((meta, path) => { + if (this.deleteSet.has(path)) return; + seen.add(path); + callbackFn(meta, path); + }); + this.overlay.forEach((meta, path) => { + if (seen.has(path) || this.deleteSet.has(path)) return; + seen.add(path); + callbackFn(meta, path); + }); + this.pendingUpload.forEach((_guid, path) => { + if (seen.has(path) || this.deleteSet.has(path)) return; + callbackFn(null, path); + }); + } + has(path: string) { if (this.renames.has(path)) { path = this.renames.get(path)!; @@ -181,7 +223,8 @@ export class SyncStore extends Observable { } this.log("metadata write (path, existing, meta)", vpath, existing, meta); this.meta.set(vpath, meta); - if (this.pendingUpload.has(vpath)) { + const pendingGuid = this.pendingUpload.get(vpath); + if (pendingGuid && pendingGuid === meta.id) { this.pendingUpload.delete(vpath); } } @@ -252,10 +295,10 @@ export class SyncStore extends Observable { this.legacyIds.observe(logObserver); this.meta.observe(logObserver); this.unsubscribes.push(() => { - this.legacyIds.unobserve(logObserver); + this.legacyIds?.unobserve(logObserver); }); this.unsubscribes.push(() => { - this.meta.unobserve(logObserver); + this.meta?.unobserve(logObserver); }); }); @@ -278,10 +321,10 @@ export class SyncStore extends Observable { this.legacyIds.observe(legacyListener); this.meta.observe(syncFileObserver); this.unsubscribes.push(() => { - this.legacyIds.unobserve(legacyListener); + this.legacyIds?.unobserve(legacyListener); }); this.unsubscribes.push(() => { - this.meta.unobserve(syncFileObserver); + this.meta?.unobserve(syncFileObserver); }); this.unsubscribes.push( this.typeRegistry.subscribe(() => { @@ -340,6 +383,21 @@ export class SyncStore extends Observable { return meta; } + /** + * Get committed file metadata from the shared Y.Map only. + * Does not include pending uploads, overlay migration entries, or legacy ids. + */ + getCommittedMeta(vpath: string): Meta | undefined { + this.assertVPath(vpath); + if (this.renames.has(vpath)) { + vpath = this.renames.get(vpath)!; + } + if (this.deleteSet.has(vpath)) { + return undefined; + } + return this.meta.get(vpath); + } + delete(vpath: string) { this.assertVPath(vpath); this.legacyIds.delete(vpath); diff --git a/src/SyncTypes.ts b/src/SyncTypes.ts index e680d2a5..51f4b321 100644 --- a/src/SyncTypes.ts +++ b/src/SyncTypes.ts @@ -1,5 +1,4 @@ import { type SyncFlags, type SyncSettingsManager } from "./SyncSettings"; -import { flags } from "./flagManager"; import { getMimeType } from "./mimetypes"; import { Observable } from "./observable/Observable"; @@ -11,6 +10,7 @@ export enum SyncType { PDF = "pdf", Audio = "audio", Video = "video", + Base = "base", File = "file", } @@ -19,6 +19,7 @@ export type SyncFileType = | SyncType.PDF | SyncType.Audio | SyncType.Video + | SyncType.Base | SyncType.File; interface MetaBase { @@ -69,11 +70,21 @@ export interface VideoMeta extends BaseFileMeta { type: SyncType.Video; } +export interface BaseMeta extends BaseFileMeta { + type: SyncType.Base; +} + export interface FileMeta extends BaseFileMeta { type: SyncType.File; } -export type FileMetas = ImageMeta | PDFMeta | AudioMeta | VideoMeta | FileMeta; +export type FileMetas = + | ImageMeta + | PDFMeta + | AudioMeta + | VideoMeta + | BaseMeta + | FileMeta; export type Meta = FolderMeta | DocumentMeta | FileMetas | CanvasMeta; @@ -85,6 +96,7 @@ type SyncTypeToMeta = { [SyncType.Image]: ImageMeta; [SyncType.Audio]: AudioMeta; [SyncType.Video]: VideoMeta; + [SyncType.Base]: BaseMeta; [SyncType.File]: FileMeta; }; @@ -93,17 +105,20 @@ export const SyncFlagToTypeMap: Record = { audio: SyncType.Audio, videos: SyncType.Video, pdfs: SyncType.PDF, + canvas: SyncType.Canvas, + bases: SyncType.Base, otherTypes: SyncType.File, }; export const SyncTypeToFlagMap: Record = { [SyncType.Document]: null, // Always enabled - [SyncType.Canvas]: null, // Always enabled + [SyncType.Canvas]: "canvas", [SyncType.Folder]: null, // Always enabled [SyncType.Image]: "images", [SyncType.Audio]: "audio", [SyncType.Video]: "videos", [SyncType.PDF]: "pdfs", + [SyncType.Base]: "bases", [SyncType.File]: "otherTypes", }; @@ -139,6 +154,10 @@ export function isVideoMeta(meta?: Meta): meta is VideoMeta { return meta?.type === SyncType.Video; } +export function isBaseMeta(meta?: Meta): meta is BaseMeta { + return meta?.type === SyncType.Base; +} + export function makeDocumentMeta(guid: string): DocumentMeta { return { version: 0, @@ -199,9 +218,10 @@ export class TypeRegistry extends Observable { super(); configs = configs || TypeRegistry.defaults; configs.forEach(([type, config]) => this.protocols.set(type, config)); + this.updateFromSettings(); this.unsubscribes.push( - syncSettings.subscribe((settings) => { - this.updateFromSettings(settings); + syncSettings.subscribe(() => { + this.updateFromSettings(); }), ); } @@ -285,6 +305,14 @@ export class TypeRegistry extends Observable { enabled: true, }, ], + [ + SyncType.Base, + { + maxVersion: 0, + mimetypes: ["application/vnd.obsidian.base+yaml"], + enabled: true, + }, + ], [ SyncType.File, { @@ -304,9 +332,6 @@ export class TypeRegistry extends Observable { canSync(vpath: string, meta?: Meta): boolean { if (vpath.endsWith(".md")) return true; - if (flags().enableCanvasSync) { - if (vpath.endsWith(".canvas")) return true; - } // For new folders const hasExtension = vpath.split("/").pop()?.includes("."); @@ -326,9 +351,11 @@ export class TypeRegistry extends Observable { return !!this.protocols.get(type)?.enabled; } - private updateFromSettings(settings: Record): void { + private updateFromSettings(): void { + const categories = this.syncSettings.getCategories(); Object.entries(SyncFlagToTypeMap).forEach(([flagKey, syncType]) => { - this.setEnabled(syncType, settings[flagKey as keyof SyncFlags]); + const enabled = categories[flagKey as keyof SyncFlags].enabled; + this.setEnabled(syncType, enabled); }); } @@ -350,9 +377,6 @@ export class TypeRegistry extends Observable { for (const [type, config] of this.protocols) { if (config.mimetypes.includes(mimetype)) { - if (!flags().enableCanvasSync && type === SyncType.Canvas) { - return SyncType.File; - } return type; } } diff --git a/src/TextViewPlugin.ts b/src/TextViewPlugin.ts index 219c9522..092bd3f6 100644 --- a/src/TextViewPlugin.ts +++ b/src/TextViewPlugin.ts @@ -6,7 +6,7 @@ import { ViewHookPlugin } from "./plugins/ViewHookPlugin"; import { isLive, type LiveView } from "./LiveViews"; import { YText, YTextEvent, Transaction } from "yjs/dist/src/internals"; -import { diffMatchPatch } from "./y-diffMatchPatch"; +import { trackPromise } from "./trackPromise"; export class TextFileViewPlugin extends HasLogging { view: LiveView; @@ -33,14 +33,21 @@ export class TextFileViewPlugin extends HasLogging { file.path, ); if (folder) { - const newDoc = folder.proxy.getDoc(file.path); - this.warn("[TextViewPlugin] getDocument() found:", { - newDocPath: newDoc.path, - newDocGuid: newDoc.guid, - newDocTFile: newDoc._tfile?.path, - }); - this.doc = newDoc; - return this.doc; + try { + const newDoc = folder.proxy.getDoc(file.path); + this.warn("[TextViewPlugin] getDocument() found:", { + newDocPath: newDoc.path, + newDocGuid: newDoc.guid, + newDocTFile: newDoc._tfile?.path, + }); + this.doc = newDoc; + return this.doc; + } catch (error) { + this.warn("[TextViewPlugin] getDocument() failed:", { + filePath: file.path, + error: error instanceof Error ? error.message : String(error), + }); + } } } // Fallback to the LiveView's document @@ -90,6 +97,15 @@ export class TextFileViewPlugin extends HasLogging { this.install(); } + private requestNativeViewSave(): void { + this.saving = true; + try { + this.view.view.requestSave(); + } finally { + this.saving = false; + } + } + async resync() { if ( isLive(this.view) && @@ -104,8 +120,14 @@ export class TextFileViewPlugin extends HasLogging { return; } - await this.doc.whenSynced(); - if (this.doc.text === this.view.view.getViewData()) { + const hsm = this.doc.hsm; + if (hsm?.awaitState) { + await trackPromise(`textView:awaitActive:${this.doc?.guid}`, hsm.awaitState((s) => s.startsWith("active."))); + } else { + return; + } + const docText = this.doc.localText; + if (docText === this.view.view.getViewData()) { // Document and view content already match - set tracking immediately this.view.tracking = true; this.warn("resync() - content matches, setting tracking=true"); @@ -115,27 +137,23 @@ export class TextFileViewPlugin extends HasLogging { documentPath: this.doc.path, documentTFilePath: this.doc._tfile?.path, viewFilePath: this.view.view.file?.path, - documentText: this.doc.text, + documentText: docText, viewData: this.view.view.getViewData(), documentGuid: this.doc.guid, tFilesMatching: this.doc._tfile === this.view.view.file, - documentTextLength: this.doc.text?.length || 0, + documentTextLength: docText?.length || 0, viewDataLength: this.view.view.getViewData()?.length || 0, }); } - if (!this.doc.hasLocalDB() && this.doc.text === "") { + if (!this.doc.hasLocalDB() && docText === "") { this.warn("local db missing, not setting buffer"); return; } - // Check if document is stale before overwriting view content - const stale = await this.doc.checkStale(); - if (stale && this.view) { - this.warn("Document is stale - showing merge banner"); - this.view.checkStale().then(async (stale) => { - if (!stale) { - await this.syncViewToCRDT(); - } - }); // This will show the merge banner + // Check if document has HSM conflict before overwriting view content + const hasConflict = this.doc.hasHSMConflict(); + if (hasConflict && this.view) { + this.warn("Document has HSM conflict - showing merge banner"); + this.view.checkStale(); // This will show the merge banner via HSM } else { // Document is authoritative, force view to match CRDT state (like getKeyFrame in LiveEditPlugin) this.warn("Document is authoritative - syncing view to CRDT state"); @@ -154,9 +172,12 @@ export class TextFileViewPlugin extends HasLogging { ) { this.warn("Syncing view to CRDT - setViewData"); this.saving = true; - this.view.view.setViewData(this.doc.text, false); - this.doc.save(); - this.saving = false; + try { + this.view.view.setViewData(this.doc.localText, false); + } finally { + this.saving = false; + } + this.requestNativeViewSave(); this.view.tracking = true; } } @@ -168,7 +189,7 @@ export class TextFileViewPlugin extends HasLogging { if (!this.view.view.file) { this.warn("view file not ready, deferring install"); // Retry installation after a short delay - setTimeout(() => { + window.setTimeout(() => { if (!this.destroyed && this.view?.view?.file) { this.install(); } @@ -205,7 +226,7 @@ export class TextFileViewPlugin extends HasLogging { that.doc && that.view.view.file === that.doc.tfile ) { - if (that.view.document.text === data) { + if (that.doc.localText === data) { that.view.tracking = true; } } @@ -227,13 +248,18 @@ export class TextFileViewPlugin extends HasLogging { if (isLive(that.view) && !that.saving && that.doc) { if (that.view.tracking && !that.saving) { that.warn("tracking - applying diff"); - diffMatchPatch( - that.doc.ydoc, - that.view.view.getViewData(), - that.doc, - ); - that.doc.save(); - return; + const hsm = that.doc.hsm; + const docText = that.view.view.getViewData(); + if (!hsm) { + throw new Error("TextFileViewPlugin requestSave: no HSM"); + } + hsm.send({ + type: "CM6_CHANGE", + changes: hsm.computeDiffChanges(that.doc.localText, docText), + docText, + userEvent: "set", + }); + return old.call(this); } else { that.warn("not tracking - resync"); that.resync(); @@ -264,26 +290,35 @@ export class TextFileViewPlugin extends HasLogging { // Called when a yjs event is received. Results in view update if (tr.origin !== this.doc) { + if (tr.origin === this.doc.hsm) { + return; + } if (!this.view.tracking) { this.warn("resync from update, not tracking"); this.resync(); } this.warn("setting view data"); this.saving = true; - this.view.view.setViewData(this.doc.text, false); - this.view.view.requestSave(); - this.saving = false; + try { + this.view.view.setViewData(this.doc.localText, false); + } finally { + this.saving = false; + } + this.requestNativeViewSave(); this.view.tracking = true; } - }; + }; this.resync(); // Use the dynamically retrieved document for ytext this.doc = this.getDocument(); if (this.doc) { - this._ytext = this.doc.ytext; - this._ytext.observe(this.observer); + const localDoc = this.doc.localDoc; + if (localDoc) { + this._ytext = localDoc.getText("contents"); + this._ytext.observe(this.observer); + } } // Initialize ViewHookPlugin after sync state is established diff --git a/src/TimeProvider.ts b/src/TimeProvider.ts index 7ddca69d..b0eb492c 100644 --- a/src/TimeProvider.ts +++ b/src/TimeProvider.ts @@ -8,7 +8,7 @@ export class DefaultTimeProvider implements TimeProvider { this.intervals = []; } - getTime(): number { + now(): number { return Date.now(); } @@ -46,16 +46,17 @@ export class DefaultTimeProvider implements TimeProvider { this.intervals = []; } - debounce void>( - func: T, + debounce( + func: (...args: Args) => void, delay: number = 500, - ): (...args: Parameters) => void { - let timer: ReturnType; - return (...args: Parameters) => { - if (timer) { - clearTimeout(timer); + ): (...args: Args) => void { + let timer: number | undefined; + return (...args: Args) => { + if (timer !== undefined) { + this.clearTimeout(timer); } - timer = setTimeout(() => { + timer = this.setTimeout(() => { + timer = undefined; func(...args); }, delay); }; @@ -63,14 +64,14 @@ export class DefaultTimeProvider implements TimeProvider { } export interface TimeProvider { - getTime: () => number; + now: () => number; setInterval: (callback: () => void, ms: number) => number; clearInterval: (timerId: number) => void; setTimeout: (callback: () => void, ms: number) => number; clearTimeout: (timerId: number) => void; destroy: () => void; - debounce: void>( - func: T, + debounce: ( + func: (...args: Args) => void, delay: number, - ) => (...args: Parameters) => void; + ) => (...args: Args) => void; } diff --git a/src/TokenStore.ts b/src/TokenStore.ts index 6b925fcf..4e5e6692 100644 --- a/src/TokenStore.ts +++ b/src/TokenStore.ts @@ -3,6 +3,7 @@ import { decodeJwt } from "jose"; import type { TimeProvider } from "./TimeProvider"; import { RelayInstances } from "./debug"; +import { trackAsyncCleanup } from "./reloadUtils"; interface TokenStoreConfig { log: (message: string) => void; @@ -14,6 +15,8 @@ interface TokenStoreConfig { getTimeProvider: () => TimeProvider; getJwtExpiry?: (token: NetToken) => number; getStorage?: () => Map; + refreshJitterSeed?: string; + refreshJitterOffsetsMs?: readonly number[]; } function formatTime(milliseconds: number): string { @@ -28,6 +31,41 @@ function formatTime(milliseconds: number): string { } } +export const TOKEN_REFRESH_JITTER_OFFSETS_MS = Object.freeze([ + 0, + 5 * 1000, + 20 * 1000, + 45 * 1000, + 55 * 1000, +]); + +function hashStringToUint32(text: string): number { + let hash = 2166136261; + for (let i = 0; i < text.length; i++) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} + +const REFRESH_JITTER_DAY_MS = 24 * 60 * 60 * 1000; + +export function getTokenRefreshJitterMs( + seed: string | undefined, + documentId: string, + expiryTime: number, + offsets: readonly number[] = TOKEN_REFRESH_JITTER_OFFSETS_MS, +): number { + if (!seed || offsets.length === 0) { + return 0; + } + const expiryDay = Math.floor(expiryTime / REFRESH_JITTER_DAY_MS); + const index = + hashStringToUint32(`${seed}:${documentId}:${expiryDay}`) % + offsets.length; + return offsets[index]; +} + interface HasToken { token: string; } @@ -61,6 +99,8 @@ export class TokenStore { private timeProvider: TimeProvider; private refreshInterval: number | null; private readonly expiryMargin: number = 5 * 60 * 1000; // 5 minutes in milliseconds + private readonly refreshJitterSeed?: string; + private readonly refreshJitterOffsetsMs: readonly number[]; private activeConnections = 0; private maxConnections: number; protected getJwtExpiry: (token: TokenType) => number; @@ -87,6 +127,9 @@ export class TokenStore { this._log = config.log; this.refresh = config.refresh; this.timeProvider = config.getTimeProvider(); + this.refreshJitterSeed = config.refreshJitterSeed; + this.refreshJitterOffsetsMs = + config.refreshJitterOffsetsMs ?? TOKEN_REFRESH_JITTER_OFFSETS_MS; if (config.getJwtExpiry) { this.getJwtExpiry = config.getJwtExpiry; } else { @@ -147,7 +190,10 @@ export class TokenStore { this.log("check and refresh tokens"); this._cleanupInvalidTokens(); for (const [documentId, tokenInfo] of this.tokenMap.entries()) { - if (this.callbacks.has(documentId) && this.shouldRefresh(tokenInfo)) { + if ( + this.callbacks.has(documentId) && + this.shouldRefresh(tokenInfo, documentId) + ) { this.log("adding to refresh queue"); this.addToRefreshQueue(documentId); } @@ -237,14 +283,39 @@ export class TokenStore { } } + private getRefreshJitterMs( + token: TokenInfo, + documentId?: string, + ): number { + if (!documentId) { + return 0; + } + return getTokenRefreshJitterMs( + this.refreshJitterSeed, + documentId, + token.expiryTime, + this.refreshJitterOffsetsMs, + ); + } + + private getRefreshLeadTime( + token: TokenInfo, + documentId?: string, + ): number { + return this.expiryMargin + this.getRefreshJitterMs(token, documentId); + } + isTokenValid(token: TokenInfo): boolean { - const currentTime = this.timeProvider.getTime(); + const currentTime = this.timeProvider.now(); return currentTime < token.expiryTime; } - shouldRefresh(token: TokenInfo): boolean { - const currentTime = this.timeProvider.getTime(); - return currentTime + this.expiryMargin > token.expiryTime; + shouldRefresh(token: TokenInfo, documentId?: string): boolean { + const currentTime = this.timeProvider.now(); + return ( + currentTime + this.getRefreshLeadTime(token, documentId) > + token.expiryTime + ); } getTokenSync(documentId: string) { @@ -313,20 +384,23 @@ export class TokenStore { _reportWithFilter(filter: (documentId: string) => boolean) { const reportLines: string[] = []; - const currentTime = this.timeProvider.getTime(); + const currentTime = this.timeProvider.now(); const tokens = Array.from(this.tokenMap.entries()).sort((a, b) => { return a[1].expiryTime - b[1].expiryTime; }); - for (const [documentId, { friendlyName, expiryTime, attempts }] of tokens) { + for (const [documentId, tokenInfo] of tokens) { + const { friendlyName, expiryTime, attempts } = tokenInfo; if (!filter(documentId)) { continue; } const timeUntilExpiry = expiryTime - currentTime; + const refreshLeadTime = this.getRefreshLeadTime(tokenInfo, documentId); + const timeUntilRefresh = timeUntilExpiry - refreshLeadTime; let timeReport = ""; - if (timeUntilExpiry > 0) { - timeReport = `expires in ${formatTime( - timeUntilExpiry - this.expiryMargin, - )}`; + if (timeUntilRefresh > 0) { + timeReport = `refreshes in ${formatTime(timeUntilRefresh)}`; + } else if (timeUntilExpiry > 0) { + timeReport = "refresh due"; } else { timeReport = "expired"; } @@ -341,6 +415,13 @@ export class TokenStore { const reportLines: string[] = []; reportLines.push("Token Store Report:"); reportLines.push(`Expiry Margin: ${formatTime(this.expiryMargin)}`); + if (this.refreshJitterSeed && this.refreshJitterOffsetsMs.length > 0) { + reportLines.push( + `Refresh Jitter: per document/day, up to ${formatTime( + Math.max(...this.refreshJitterOffsetsMs), + )}`, + ); + } reportLines.push("Active Tokens:"); reportLines.push( ...this._reportWithFilter((documentId) => { @@ -359,8 +440,9 @@ export class TokenStore { async waitForQueue(): Promise { return new Promise((resolve) => { - setInterval(() => { + const interval = this.timeProvider.setInterval(() => { if (this.refreshQueue.size == 0) { + this.timeProvider.clearInterval(interval); return resolve(); } }, 100); @@ -393,6 +475,13 @@ export class TokenStore { } destroy() { + // Track active token refresh promises before clearing + if (this._activePromises.size > 0) { + trackAsyncCleanup( + Promise.all(this._activePromises.values()).then(() => {}), + ); + } + this.clear(); this.timeProvider.destroy(); this.timeProvider = null as any; diff --git a/src/UpdateManager.ts b/src/UpdateManager.ts index 6effaa35..2bb74195 100644 --- a/src/UpdateManager.ts +++ b/src/UpdateManager.ts @@ -1,4 +1,4 @@ -import type { Plugin } from "obsidian"; +import { Plugin } from "obsidian"; import type { TimeProvider } from "./TimeProvider"; import { Observable } from "./observable/Observable"; import { customFetch } from "./customFetch"; @@ -72,6 +72,7 @@ export class UpdateManager extends Observable { private lastReleaseCheck: number = 0; private lastChannelCheck: number = 0; private readonly CHECK_INTERVAL = 1000 * 60 * 60 * 24; + private updateGeneration: number = 0; observableName = "UpdateManager"; constructor( @@ -85,6 +86,9 @@ export class UpdateManager extends Observable { } public get releases(): Release[] { + if (this.destroyed) { + return []; + } return [...this.githubReleases.values()].sort((a, b) => b.tag_name.localeCompare(a.tag_name, undefined, { numeric: true, @@ -94,13 +98,23 @@ export class UpdateManager extends Observable { } public get beta(): Release | undefined { + if (this.destroyed) { + return undefined; + } return this.releaseChannels.get("beta"); } public get stable(): Release | undefined { + if (this.destroyed) { + return undefined; + } return this.releaseChannels.get("stable"); } + private isActiveGeneration(generation: number): boolean { + return !this.destroyed && generation === this.updateGeneration; + } + private async fetchReleases(): Promise { const apiUrl = `https://api.github.com/repos/${REPOSITORY}/releases`; @@ -150,6 +164,9 @@ export class UpdateManager extends Observable { public async fetchLatestTagFromChannel( channel: "beta" | "stable", ): Promise { + if (this.destroyed) { + return null; + } try { const manifestPath = { beta: "manifest-beta.json", @@ -171,6 +188,10 @@ export class UpdateManager extends Observable { } public async getReleases(): Promise { + const generation = this.updateGeneration; + if (!this.isActiveGeneration(generation)) { + return []; + } try { // Only fetch if we haven't fetched in a while to avoid rate limiting const now = Date.now(); @@ -186,6 +207,9 @@ export class UpdateManager extends Observable { this.lastReleaseCheck = now; const releases = await this.fetchReleases(); + if (!this.isActiveGeneration(generation)) { + return []; + } const localReleases = new Set([...this.githubReleases.keys()]); const remoteReleases = new Set(); releases.forEach((release: Release) => { @@ -201,6 +225,9 @@ export class UpdateManager extends Observable { }); const latest = await this.fetchLatestRelease(); + if (!this.isActiveGeneration(generation)) { + return []; + } if (latest) { latest.latest = true; this.githubReleases.set(latest.tag_name, latest); @@ -209,12 +236,21 @@ export class UpdateManager extends Observable { // Notify subscribers that we have new releases data this.notifyListeners(); } catch (error) { - this.error("Failed to fetch GitHub releases:", error); + if (this.isActiveGeneration(generation)) { + this.error("Failed to fetch GitHub releases:", error); + } + } + if (!this.isActiveGeneration(generation)) { + return []; } return [...this.githubReleases.values()]; } private async getChannelRelease(): Promise { + const generation = this.updateGeneration; + if (!this.isActiveGeneration(generation)) { + return null; + } const now = Date.now(); const channelRelease = this.releaseChannels.get( this.releaseSettings.get().channel, @@ -225,6 +261,9 @@ export class UpdateManager extends Observable { this.lastReleaseCheck = now; const releases = await this.getReleases(); + if (!this.isActiveGeneration(generation)) { + return null; + } if (releases.length === 0) { this.debug("No releases found"); return null; @@ -234,6 +273,9 @@ export class UpdateManager extends Observable { if (!channel) return null; const version = await this.fetchLatestTagFromChannel(channel); + if (!this.isActiveGeneration(generation)) { + return null; + } if (!version) return null; const release = releases.find((release) => { @@ -248,14 +290,22 @@ export class UpdateManager extends Observable { } private async update() { + const generation = this.updateGeneration; await this.getReleases(); + if (!this.isActiveGeneration(generation)) { + return; + } await this.getChannelRelease(); } public start(): void { - this.update(); + if (this.destroyed) { + return; + } + this.updateGeneration += 1; + void this.update(); this.updateCheckInterval = this.timeProvider.setInterval( - () => this.update(), + () => void this.update(), this.CHECK_INTERVAL, ); } @@ -364,8 +414,12 @@ export class UpdateManager extends Observable { } override destroy(): void { + if (this.destroyed) { + return; + } // Stop periodic update checking this.stop(); + this.updateGeneration += 1; // Set state to null before destroying this.githubReleases = null as any; diff --git a/src/UserFacingError.ts b/src/UserFacingError.ts new file mode 100644 index 00000000..53a94d01 --- /dev/null +++ b/src/UserFacingError.ts @@ -0,0 +1,155 @@ +const OBJECT_STRING = "[object Object]"; +const MAX_ERROR_MESSAGE_LENGTH = 300; + +export function formatUserFacingError( + error: unknown, + fallback = "Sync failed", +): string { + const message = extractErrorMessage(error, new Set()); + return normalizeMessage(message) ?? fallback; +} + +export function errorFromUnknown( + error: unknown, + fallback = "Sync failed", +): Error { + const message = formatUserFacingError(error, fallback); + if (error instanceof Error && message === error.message) return error; + return new Error(message); +} + +function extractErrorMessage( + value: unknown, + seen: Set, +): string | null { + if (value === null || value === undefined) return null; + + if (typeof value === "string") { + const parsed = parseJsonObject(value); + if (parsed !== null) { + return extractErrorMessage(parsed, seen) ?? value; + } + return value; + } + + if ( + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ) { + return String(value); + } + + if (value instanceof Error) { + return normalizeMessage(value.message) ?? normalizeMessage(value.name); + } + + if (typeof value !== "object") return null; + if (seen.has(value)) return null; + seen.add(value); + + const record = value as Record; + const directMessage = extractDirectMessage(record, seen); + if (directMessage) return directMessage; + + const nestedMessage = extractNestedMessage(record, seen); + if (nestedMessage) return nestedMessage; + + const statusMessage = extractStatusMessage(record); + if (statusMessage) return statusMessage; + + const code = primitiveToString(record.code); + if (code) return `Sync failed (${code})`; + + return null; +} + +function extractDirectMessage( + record: Record, + seen: Set, +): string | null { + for (const key of [ + "message", + "error", + "reason", + "detail", + "description", + ]) { + if (!(key in record)) continue; + const message = normalizeMessage(extractErrorMessage(record[key], seen)); + if (message) return message; + } + return null; +} + +function extractNestedMessage( + record: Record, + seen: Set, +): string | null { + for (const key of ["response", "data", "body", "cause"]) { + if (!(key in record)) continue; + const message = normalizeMessage(extractErrorMessage(record[key], seen)); + if (message) return message; + } + return null; +} + +function extractStatusMessage(record: Record): string | null { + const status = primitiveToString(record.status) ?? primitiveToString(record.statusCode); + const statusText = primitiveToString(record.statusText); + if (status && statusText) return `Request failed with status ${status}: ${statusText}`; + if (status) return `Request failed with status ${status}`; + return null; +} + +function parseJsonObject(value: string): object | null { + const trimmed = value.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null; + try { + const parsed: unknown = JSON.parse(trimmed); + return typeof parsed === "object" && parsed !== null ? parsed : null; + } catch { + return null; + } +} + +function normalizeMessage(message: unknown): string | null { + if (typeof message !== "string") return null; + const normalized = message.replace(/\s+/g, " ").trim(); + if (!normalized || normalized === OBJECT_STRING || normalized === "Object") { + return null; + } + const humanReadable = humanizeInternalSyncMessage(normalized); + return humanReadable.length > MAX_ERROR_MESSAGE_LENGTH + ? `${humanReadable.slice(0, MAX_ERROR_MESSAGE_LENGTH - 3)}...` + : humanReadable; +} + +function primitiveToString(value: unknown): string | null { + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ) { + return normalizeMessage(String(value)); + } + return null; +} + +function humanizeInternalSyncMessage(message: string): string { + const withoutPrefix = message.replace(/^(?:\[[^\]]+\]\s*)+/, ""); + const documentSyncFailure = withoutPrefix.match( + /^Document sync failed:\s+(.+?)(?:\s+\([^)]+\))?$/, + ); + if (documentSyncFailure) { + return `Unable to sync ${filenameFromPath(documentSyncFailure[1])}`; + } + return withoutPrefix; +} + +function filenameFromPath(path: string): string { + const normalized = path.replace(/\\/g, "/").trim(); + const parts = normalized.split("/").filter(Boolean); + return parts[parts.length - 1] || "file"; +} diff --git a/src/client/provider.ts b/src/client/provider.ts index 29ce2df7..5c4efb9e 100644 --- a/src/client/provider.ts +++ b/src/client/provider.ts @@ -15,11 +15,24 @@ import * as awarenessProtocol from "y-protocols/awareness"; import { Observable } from "lib0/observable"; import * as math from "lib0/math"; import * as url from "lib0/url"; +import { decode as decodeCBOR } from "cbor-x"; +import { metrics, curryLog } from "../debug"; +import type { TimeProvider } from "../TimeProvider"; + +const providerError = curryLog("[YSweetProvider]", "error"); +const providerLog = curryLog("[YSweetProvider]", "log"); export const messageSync = 0; export const messageQueryAwareness = 3; export const messageAwareness = 1; export const messageAuth = 2; +export const messageEvent = 4; +export const messageEventSubscribe = 5; +export const messageEventUnsubscribe = 6; +export const messageQuerySubdocs = 7; +export const messageSubdocs = 8; + +const SUBDOC_QUERY_PAGE_SIZE = 100; export type HandlerFunction = ( encoder: encoding.Encoder, @@ -38,13 +51,29 @@ messageHandlers[messageSync] = ( emitSynced, _messageType, ) => { + const syncMessageType = decoding.readVarUint(decoder); + if ( + provider.readOnly && + syncMessageType === syncProtocol.messageYjsSyncStep1 + ) { + decoding.readVarUint8Array(decoder); + return; + } + encoding.writeVarUint(encoder, messageSync); - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - provider.doc, - provider, - ); + switch (syncMessageType) { + case syncProtocol.messageYjsSyncStep1: + syncProtocol.readSyncStep1(decoder, encoder, provider.doc); + break; + case syncProtocol.messageYjsSyncStep2: + syncProtocol.readSyncStep2(decoder, provider.doc, provider); + break; + case syncProtocol.messageYjsUpdate: + syncProtocol.readUpdate(decoder, provider.doc, provider); + break; + default: + throw new Error("Unknown sync message type"); + } if ( emitSynced && syncMessageType === syncProtocol.messageYjsSyncStep2 && @@ -61,6 +90,9 @@ messageHandlers[messageQueryAwareness] = ( _emitSynced, _messageType, ) => { + if (provider.readOnly) { + return; + } encoding.writeVarUint(encoder, messageAwareness); encoding.writeVarUint8Array( encoder, @@ -97,6 +129,46 @@ messageHandlers[messageAuth] = ( ); }; +messageHandlers[messageEvent] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType, +) => { + const cborLength = decoding.readVarUint(decoder); + const cborData = decoding.readUint8Array(decoder, cborLength); + + try { + const eventMessage = decodeCBOR(cborData); + + // Only process if we're subscribed to this event type + if (provider.eventSubscriptions.has(eventMessage.event_type)) { + provider.processEvent(eventMessage); + } + } catch (error) { + providerError(`Failed to decode event message: ${error}`); + } +}; + +messageHandlers[messageSubdocs] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType, +) => { + const cborLength = decoding.readVarUint(decoder); + const cborData = decoding.readUint8Array(decoder, cborLength); + + try { + const subdocIndex = normalizeSubdocIndex(decodeCBOR(cborData)); + provider.handleSubdocIndex(subdocIndex); + } catch (error) { + providerError(`Failed to decode subdoc index: ${error}`); + } +}; + // @todo - this should depend on awareness.outdatedTime const messageReconnectTimeout = 30000; @@ -113,6 +185,13 @@ const readMessage = ( const messageType = decoding.readVarUint(decoder); const messageHandler = provider.messageHandlers[messageType]; if (/** @type {any} */ messageHandler) { + if (messageType === messageSync) { + metrics.recordProtocolMessage("sync", "in", buf.length); + } else if (messageType === messageEvent) { + metrics.recordProtocolMessage("event", "in", buf.length); + } else if (messageType === messageSubdocs) { + metrics.recordProtocolMessage("subdoc_index", "in", buf.length); + } messageHandler(encoder, decoder, provider, emitSynced, messageType); } else { console.error("Unable to compute message"); @@ -127,9 +206,13 @@ const setupWS = (provider: YSweetProvider) => { provider.ws = websocket; provider.wsconnecting = true; provider.wsconnected = false; + provider.wsConnectStartTime = time.getUnixTime(); provider.synced = false; websocket.onmessage = (event) => { + if (provider.ws !== websocket) { + return; + } provider.wsLastMessageReceived = time.getUnixTime(); const encoder = readMessage(provider, new Uint8Array(event.data), true); if (encoding.length(encoder) > 1) { @@ -137,12 +220,20 @@ const setupWS = (provider: YSweetProvider) => { } }; websocket.onerror = (event) => { + if (provider.ws !== websocket) { + return; + } provider.emit("connection-error", [event, provider]); }; websocket.onclose = (event) => { + if (provider.ws !== websocket) { + return; + } provider.emit("connection-close", [event, provider]); provider.ws = null; provider.wsconnecting = false; + provider.wsConnectStartTime = 0; + const wasConnected = provider.wsconnected; if (provider.wsconnected) { provider.wsconnected = false; provider.synced = false; @@ -154,32 +245,38 @@ const setupWS = (provider: YSweetProvider) => { ), provider, ); - provider.emit("status", [ - { - status: "disconnected", - intent: provider.intent, - }, - ]); } else { provider.wsUnsuccessfulReconnects++; } + provider.emit("status", [ + { + status: "disconnected", + intent: provider.intent, + }, + ]); // Start with no reconnect timeout and increase timeout by // using exponential backoff starting with 100ms if (provider.canReconnect()) { - setTimeout( - setupWS, - math.min( - math.pow(2, provider.wsUnsuccessfulReconnects) * 100, - provider.maxBackoffTime, - ), - provider, + const delay = math.min( + math.pow(2, provider.wsUnsuccessfulReconnects) * 100, + provider.maxBackoffTime, ); + provider._reconnectTimeout = provider._setTimeout(() => { + provider._reconnectTimeout = null; + setupWS(provider); + }, delay); + } else if (!wasConnected) { + provider.wsUnsuccessfulReconnects = provider.maxConnectionErrors; } }; websocket.onopen = () => { + if (provider.ws !== websocket) { + return; + } provider.wsLastMessageReceived = time.getUnixTime(); provider.wsconnecting = false; provider.wsconnected = true; + provider.wsConnectStartTime = 0; provider.wsUnsuccessfulReconnects = 0; provider.emit("status", [ { @@ -192,17 +289,45 @@ const setupWS = (provider: YSweetProvider) => { encoding.writeVarUint(encoder, messageSync); syncProtocol.writeSyncStep1(encoder, provider.doc); websocket.send(encoding.toUint8Array(encoder)); + // Flush messages that were buffered while disconnected. + // These are sync update frames that broadcastMessage couldn't + // send because the WebSocket wasn't ready. The opening sync + // exchange handles catch-up, while this flush delivers + // real-time updates that arrived during the disconnect window. + if (provider._pendingMessages.length > 0) { + if (provider.readOnly) { + provider._pendingMessages = []; + } else { + for (const pending of provider._pendingMessages) { + websocket.send(pending); + } + provider._pendingMessages = []; + } + } + // Re-subscribe to events after reconnection + if (provider.eventSubscriptions.size > 0) { + const eventTypes = Array.from(provider.eventSubscriptions); + provider.sendEventSubscribe(eventTypes); + } + // Query subdoc snapshots after the parent sync handshake completes. + provider.once("synced", (synced) => { + if (synced && provider.ws === websocket && provider.onSubdocIndex) { + provider.sendQuerySubdocs(); + } + }); // broadcast local awareness state if (provider.awareness.getLocalState() !== null) { - const encoderAwarenessState = encoding.createEncoder(); - encoding.writeVarUint(encoderAwarenessState, messageAwareness); - encoding.writeVarUint8Array( - encoderAwarenessState, - awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ - provider.doc.clientID, - ]), - ); - websocket.send(encoding.toUint8Array(encoderAwarenessState)); + if (!provider.readOnly) { + const encoderAwarenessState = encoding.createEncoder(); + encoding.writeVarUint(encoderAwarenessState, messageAwareness); + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ + provider.doc.clientID, + ]), + ); + websocket.send(encoding.toUint8Array(encoderAwarenessState)); + } } }; provider.emit("status", [ @@ -218,6 +343,9 @@ const broadcastMessage = (provider: YSweetProvider, buf: ArrayBuffer) => { const ws = provider.ws; if (provider.wsconnected && ws && ws.readyState === ws.OPEN) { ws.send(buf); + } else { + // Buffer the message — flushed in onopen when WebSocket connects + provider._pendingMessages.push(buf); } if (provider.bcconnected) { bc.publish(provider.bcChannel, buf, provider); @@ -244,6 +372,8 @@ export type YSweetProviderParams = { maxBackoffTime?: number; disableBc?: boolean; maxConnectionErrors?: number; + readOnly?: boolean; + timeProvider?: TimeProvider; }; export type ConnectionStatus = @@ -258,6 +388,118 @@ export interface ConnectionState { intent: ConnectionIntent; } +export interface EventMessage { + event_id: string; + event_type: string; + doc_id: string; + timestamp: number; + user?: string; + metadata?: Record; + update?: Uint8Array; +} + +export type EventCallback = (event: EventMessage) => void; + +export interface SubdocIndexEntry { + snapshot: Uint8Array; + lastSeen?: number; +} + +export type SubdocIndex = Record; +export type SubdocIndexCallback = (serverIndex: SubdocIndex) => void; +export type SubdocQueryDocIdsProvider = () => string[]; + +export function normalizeSubdocIndex(rawIndex: unknown): SubdocIndex { + const index: SubdocIndex = {}; + const rawEntries = readSubdocIndexEntries(readSubdocIndexData(rawIndex)); + + for (const [rawDocId, rawEntry] of rawEntries) { + if (typeof rawDocId !== "string" || rawDocId.length === 0) continue; + const entry = normalizeSubdocIndexEntry(rawEntry); + if (entry) { + index[rawDocId] = entry; + } + } + + return index; +} + +function readSubdocIndexData(rawIndex: unknown): unknown { + const data = readSubdocIndexField(rawIndex, "data"); + return data ?? rawIndex; +} + +function readSubdocIndexEntries(rawIndex: unknown): Array<[unknown, unknown]> { + if (rawIndex instanceof Map) { + return Array.from(rawIndex.entries()); + } + if (!rawIndex || typeof rawIndex !== "object") { + return []; + } + return Object.entries(rawIndex as Record); +} + +function normalizeSubdocIndexEntry(rawEntry: unknown): SubdocIndexEntry | null { + const rawBytes = asUint8Array(rawEntry); + if (rawBytes) { + return { snapshot: rawBytes }; + } + + const snapshot = + asUint8Array(readSubdocIndexField(rawEntry, "snapshot")) ?? + asUint8Array(readSubdocIndexField(rawEntry, "state_snapshot")) ?? + asUint8Array(readSubdocIndexField(rawEntry, "stateSnapshot")); + if (!snapshot) return null; + + const lastSeen = normalizeSubdocLastSeen( + readSubdocIndexField(rawEntry, "last_seen") ?? + readSubdocIndexField(rawEntry, "lastSeen"), + ); + const entry: SubdocIndexEntry = { snapshot }; + if (lastSeen !== undefined) entry.lastSeen = lastSeen; + return entry; +} + +function readSubdocIndexField(rawEntry: unknown, field: string): unknown { + if (rawEntry instanceof Map) { + return rawEntry.get(field); + } + if (!rawEntry || typeof rawEntry !== "object") { + return undefined; + } + return (rawEntry as Record)[field]; +} + +function asUint8Array(value: unknown): Uint8Array | null { + return value instanceof Uint8Array ? value : null; +} + +function normalizeSubdocLastSeen(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "bigint") { + if ( + value >= BigInt(Number.MIN_SAFE_INTEGER) && + value <= BigInt(Number.MAX_SAFE_INTEGER) + ) { + return Number(value); + } + return undefined; + } + if (value instanceof Date) { + const timestamp = value.getTime(); + return Number.isFinite(timestamp) ? timestamp : undefined; + } + if (typeof value === "string" && value.length > 0) { + const numeric = Number(value); + if (Number.isFinite(numeric)) return numeric; + const parsed = Date.parse(value); + if (Number.isFinite(parsed)) return parsed; + } + return undefined; +} + /** * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document. * The document name is attached to the provided url. I.e. the following example @@ -284,22 +526,82 @@ export class YSweetProvider extends Observable { disableBc: boolean; wsUnsuccessfulReconnects: number; messageHandlers: Array; + /** Messages buffered while WebSocket was not ready. Flushed on next send. */ + _pendingMessages: ArrayBuffer[]; _synced: boolean; ws: WebSocket | null; wsLastMessageReceived: number; + wsConnectStartTime: number; shouldConnect: boolean; _resyncInterval: ReturnType | number; // TODO: is setting this to 0 used as null? - _bcSubscriber: Function; + _bcSubscriber: (...args: any[]) => any; _updateHandler: ( arg0: Uint8Array, arg1: any, arg2: Y.Doc, arg3: Y.Transaction, ) => void; - _awarenessUpdateHandler: Function; - _unloadHandler: Function; + _awarenessUpdateHandler: (...args: any[]) => any; + _unloadHandler: (...args: any[]) => any; _checkInterval: ReturnType | number; + _reconnectTimeout: ReturnType | null; maxConnectionErrors: number; + readOnly: boolean; + eventSubscriptions: Set; + eventCallbacks: Map; + onSubdocIndex: SubdocIndexCallback | null; + subdocIndexCallbacks: Set; + lastSubdocIndex: SubdocIndex | null; + getSubdocQueryDocIds: SubdocQueryDocIdsProvider | null; + private _pendingSubdocIndexResponses: number; + private _pendingSubdocIndex: SubdocIndex | null; + private _timeProvider: TimeProvider | null; + + _setInterval( + callback: () => void, + ms: number, + ): ReturnType { + return this._timeProvider + ? (this._timeProvider.setInterval( + callback, + ms, + ) as unknown as ReturnType) + : (window.setInterval( + callback, + ms, + ) as unknown as ReturnType); + } + + _clearInterval(timerId: ReturnType | number): void { + if (this._timeProvider) { + this._timeProvider.clearInterval(timerId as number); + } else { + window.clearInterval(timerId as number); + } + } + + _setTimeout( + callback: () => void, + ms: number, + ): ReturnType { + return this._timeProvider + ? (this._timeProvider.setTimeout( + callback, + ms, + ) as unknown as ReturnType) + : (window.setTimeout( + callback, + ms, + ) as unknown as ReturnType); + } + + _clearTimeout(timerId: ReturnType | number): void { + if (this._timeProvider) { + this._timeProvider.clearTimeout(timerId as number); + } else { + window.clearTimeout(timerId as number); + } + } /** * @param serverUrl - server url @@ -327,9 +629,12 @@ export class YSweetProvider extends Observable { maxBackoffTime = 2500, disableBc = false, maxConnectionErrors = 3, + readOnly = false, + timeProvider, }: YSweetProviderParams = {}, ) { super(); + this._timeProvider = timeProvider ?? null; // ensure that url is always ends with / while (serverUrl[serverUrl.length - 1] === "/") { serverUrl = serverUrl.slice(0, serverUrl.length - 1); @@ -347,20 +652,31 @@ export class YSweetProvider extends Observable { this._WS = WebSocketPolyfill; this.awareness = awareness; this.wsconnected = false; + this._pendingMessages = []; this.wsconnecting = false; this.bcconnected = false; this.disableBc = disableBc; this.wsUnsuccessfulReconnects = 0; + this.readOnly = readOnly; this.messageHandlers = messageHandlers.slice(); this._synced = false; this.ws = null; this.wsLastMessageReceived = 0; + this.wsConnectStartTime = 0; this.shouldConnect = connect; this.maxConnectionErrors = maxConnectionErrors; + this.eventSubscriptions = new Set(); + this.eventCallbacks = new Map(); + this.onSubdocIndex = null; + this.subdocIndexCallbacks = new Set(); + this.lastSubdocIndex = null; + this.getSubdocQueryDocIds = null; + this._pendingSubdocIndexResponses = 0; + this._pendingSubdocIndex = null; this._resyncInterval = 0; if (resyncInterval > 0) { - this._resyncInterval = setInterval(() => { + this._resyncInterval = this._setInterval(() => { if (this.ws && this.ws.readyState === WebSocket.OPEN) { // resend sync step 1 const encoder = encoding.createEncoder(); @@ -385,10 +701,16 @@ export class YSweetProvider extends Observable { */ this._updateHandler = (update: Uint8Array, origin: any) => { if (origin !== this) { + if (this.readOnly) { + return; + } + metrics.recordProtocolMessage("sync", "out", update.length); const encoder = encoding.createEncoder(); encoding.writeVarUint(encoder, messageSync); syncProtocol.writeUpdate(encoder, update); broadcastMessage(this, encoding.toUint8Array(encoder)); + } else { + // Skipped because origin === this (our own sync response) } }; @@ -405,6 +727,9 @@ export class YSweetProvider extends Observable { }: { added: Array; updated: Array; removed: Array }, _origin: any, ) => { + if (this.readOnly) { + return; + } const changedClients = added.concat(updated).concat(removed); const encoder = encoding.createEncoder(); encoding.writeVarUint(encoder, messageAwareness); @@ -430,7 +755,7 @@ export class YSweetProvider extends Observable { } awareness.on("update", this._awarenessUpdateHandler); - this._checkInterval = setInterval(() => { + this._checkInterval = this._setInterval(() => { if ( this.wsconnected && messageReconnectTimeout < @@ -440,7 +765,19 @@ export class YSweetProvider extends Observable { // updates (which are updated every 15 seconds) this.ws?.close(); } + if ( + this.wsconnecting && + this.ws?.readyState === WebSocket.CONNECTING && + this.wsConnectStartTime > 0 && + messageReconnectTimeout < + time.getUnixTime() - this.wsConnectStartTime + ) { + // Connection attempt is stuck in CONNECTING with no transition. + // Force-close so onclose can run backoff/retry logic. + this.ws?.close(); + } }, messageReconnectTimeout / 10); + this._reconnectTimeout = null; if (connect) { this.connect(); } @@ -466,7 +803,7 @@ export class YSweetProvider extends Observable { */ once(name: string, f: (...args: any[]) => void) { if (name === "synced" && this._synced) { - setTimeout(() => f(this._synced), 0); + this._setTimeout(() => f(this._synced), 0); return this; } return super.once(name, f); @@ -506,9 +843,13 @@ export class YSweetProvider extends Observable { destroy() { if (this._resyncInterval !== 0) { - clearInterval(this._resyncInterval); + this._clearInterval(this._resyncInterval); + } + this._clearInterval(this._checkInterval); + if (this._reconnectTimeout !== null) { + this._clearTimeout(this._reconnectTimeout); + this._reconnectTimeout = null; } - clearInterval(this._checkInterval); if (this.ws) { this.ws.onopen = null; @@ -527,6 +868,10 @@ export class YSweetProvider extends Observable { this.disconnect(); this.awareness.destroy(); this._observers.clear(); + this.subdocIndexCallbacks.clear(); + this.onSubdocIndex = null; + this.getSubdocQueryDocIds = null; + this.lastSubdocIndex = null; if (typeof window !== "undefined") { window.removeEventListener("unload", this._unloadHandler as any); @@ -553,11 +898,13 @@ export class YSweetProvider extends Observable { encoding.writeVarUint(encoderSync, messageSync); syncProtocol.writeSyncStep1(encoderSync, this.doc); bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this); - // broadcast local state - const encoderState = encoding.createEncoder(); - encoding.writeVarUint(encoderState, messageSync); - syncProtocol.writeSyncStep2(encoderState, this.doc); - bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this); + if (!this.readOnly) { + // broadcast local state + const encoderState = encoding.createEncoder(); + encoding.writeVarUint(encoderState, messageSync); + syncProtocol.writeSyncStep2(encoderState, this.doc); + bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this); + } // write queryAwareness const encoderAwarenessQuery = encoding.createEncoder(); encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness); @@ -583,6 +930,13 @@ export class YSweetProvider extends Observable { } disconnectBc() { + if (this.readOnly) { + if (this.bcconnected) { + bc.unsubscribe(this.bcChannel, this._bcSubscriber as any); + this.bcconnected = false; + } + return; + } // broadcast message with local awareness state set to null (indicating disconnect) const encoder = encoding.createEncoder(); encoding.writeVarUint(encoder, messageAwareness); @@ -603,6 +957,14 @@ export class YSweetProvider extends Observable { disconnect() { this.shouldConnect = false; + this.wsconnected = false; + this.wsconnecting = false; + this.wsConnectStartTime = 0; + this.synced = false; + if (this._reconnectTimeout !== null) { + this._clearTimeout(this._reconnectTimeout); + this._reconnectTimeout = null; + } this.disconnectBc(); if (this.ws !== null) { this.ws.close(); @@ -610,10 +972,29 @@ export class YSweetProvider extends Observable { } } + setReadOnly(readOnly: boolean) { + this.readOnly = readOnly; + if (readOnly) { + this._pendingMessages = []; + } + } connect() { + const wasDisconnected = !this.shouldConnect; this.shouldConnect = true; + if (this._reconnectTimeout !== null) { + return; + } if (!this.wsconnected && this.ws === null) { + // User-initiated reconnects should start a fresh retry budget. + // Without this, a previous exhausted reconnect cycle can leave the + // provider permanently offline until the plugin is recreated. + if ( + wasDisconnected || + this.wsUnsuccessfulReconnects >= this.maxConnectionErrors + ) { + this.wsUnsuccessfulReconnects = 0; + } setupWS(this); this.connectBc(); } @@ -630,7 +1011,11 @@ export class YSweetProvider extends Observable { serverUrl: string, roomname: string, token: string, + readOnly?: boolean, ): { urlChanged: boolean; newUrl: string } { + if (readOnly !== undefined) { + this.setReadOnly(readOnly); + } // ensure that url is always ends with / while (serverUrl[serverUrl.length - 1] === "/") { serverUrl = serverUrl.slice(0, serverUrl.length - 1); @@ -662,4 +1047,157 @@ export class YSweetProvider extends Observable { return this.url === expectedUrl; } + subscribeToEvents(eventTypes: string[], callback: EventCallback) { + eventTypes.forEach(type => { + this.eventSubscriptions.add(type); + + if (!this.eventCallbacks.has(type)) { + this.eventCallbacks.set(type, []); + } + this.eventCallbacks.get(type)!.push(callback); + }); + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.sendEventSubscribe(eventTypes); + } + } + + unsubscribeFromEvents(eventTypes: string[], callback?: EventCallback) { + eventTypes.forEach(type => { + if (callback && this.eventCallbacks.has(type)) { + const callbacks = this.eventCallbacks.get(type)!; + const index = callbacks.indexOf(callback); + if (index > -1) { + callbacks.splice(index, 1); + } + if (callbacks.length === 0) { + this.eventSubscriptions.delete(type); + this.eventCallbacks.delete(type); + } + } else { + this.eventSubscriptions.delete(type); + this.eventCallbacks.delete(type); + } + }); + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.sendEventUnsubscribe(eventTypes); + } + } + + sendEventSubscribe(eventTypes: string[]) { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageEventSubscribe); + encoding.writeVarUint(encoder, eventTypes.length); + + eventTypes.forEach(type => { + encoding.writeVarString(encoder, type); + }); + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(encoding.toUint8Array(encoder)); + } + } + + sendEventUnsubscribe(eventTypes: string[]) { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageEventUnsubscribe); + encoding.writeVarUint(encoder, eventTypes.length); + + eventTypes.forEach(type => { + encoding.writeVarString(encoder, type); + }); + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(encoding.toUint8Array(encoder)); + } + } + + processEvent(eventMessage: EventMessage) { + this.emit('event', [eventMessage]); + + const callbacks = this.eventCallbacks.get(eventMessage.event_type) || []; + callbacks.forEach(callback => { + try { + callback(eventMessage); + } catch (error) { + providerError(`Event callback error: ${error}`); + } + }); + } + + subscribeToSubdocIndex(callback: SubdocIndexCallback): () => void { + this.subdocIndexCallbacks.add(callback); + return () => { + this.subdocIndexCallbacks.delete(callback); + }; + } + + sendQuerySubdocs(docIds?: string[]) { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + const queryDocIds = Array.from( + new Set((docIds ?? this.getSubdocQueryDocIds?.() ?? []).filter(Boolean)), + ); + if (queryDocIds.length === 0) { + providerLog("skipped MSG_QUERY_SUBDOCS (no doc IDs)"); + return; + } + const pageCount = Math.ceil(queryDocIds.length / SUBDOC_QUERY_PAGE_SIZE); + this._pendingSubdocIndexResponses = pageCount > 1 ? pageCount : 0; + this._pendingSubdocIndex = pageCount > 1 ? {} : null; + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const pageDocIds = queryDocIds.slice( + pageIndex * SUBDOC_QUERY_PAGE_SIZE, + (pageIndex + 1) * SUBDOC_QUERY_PAGE_SIZE, + ); + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageQuerySubdocs); + encoding.writeVarUint(encoder, pageDocIds.length); + pageDocIds.forEach((docId) => { + encoding.writeVarString(encoder, docId); + }); + this.ws.send(encoding.toUint8Array(encoder)); + providerLog(`sent MSG_QUERY_SUBDOCS (${pageDocIds.length})`); + } + } + } + + handleSubdocIndex(serverIndex: SubdocIndex) { + if (this._pendingSubdocIndexResponses > 0) { + this._pendingSubdocIndex = { + ...(this._pendingSubdocIndex ?? {}), + ...serverIndex, + }; + this._pendingSubdocIndexResponses -= 1; + if (this._pendingSubdocIndexResponses > 0) { + providerLog( + `received MSG_SUBDOCS page (${Object.keys(serverIndex).length} docs; waiting for ${this._pendingSubdocIndexResponses} pages)`, + ); + return; + } + const mergedIndex = this._pendingSubdocIndex; + this._pendingSubdocIndex = null; + this.notifySubdocIndex(mergedIndex ?? {}); + return; + } + this.notifySubdocIndex(serverIndex); + } + + private notifySubdocIndex(serverIndex: SubdocIndex) { + this.lastSubdocIndex = serverIndex; + const entries = Object.values(serverIndex); + const snapshotCount = entries.filter((entry) => entry.snapshot).length; + providerLog( + `received MSG_SUBDOCS (${entries.length} docs; ${snapshotCount} snapshots)`, + ); + this.onSubdocIndex?.(serverIndex); + for (const callback of Array.from(this.subdocIndexCallbacks)) { + try { + callback(serverIndex); + } catch (error) { + providerError(`Subdoc index callback error: ${error}`); + } + } + } + } diff --git a/src/components/AddToVaultModalContent.svelte b/src/components/AddToVaultModalContent.svelte index 00bd869c..8a223c3e 100644 --- a/src/components/AddToVaultModalContent.svelte +++ b/src/components/AddToVaultModalContent.svelte @@ -99,7 +99,7 @@ {app} {sharedFolders} selectedFolder={folderLocation} - disabled={availableFolders.length === 0} + disabled={!selectedRemoteFolder} /> diff --git a/src/components/FeatureFlagModalContent.svelte b/src/components/FeatureFlagModalContent.svelte index fedabebf..6ccdf4eb 100644 --- a/src/components/FeatureFlagModalContent.svelte +++ b/src/components/FeatureFlagModalContent.svelte @@ -1,18 +1,59 @@ -
-

Feature Flags

- {#each Object.entries(settings) - .filter(([k, v]) => isKeyOfFeatureFlags(k)) - .sort() as [flagName, value]} -
-
-
{flagName}
-
Toggle {flagName} on or off
-
-
-
{ - if (!isKeyOfFeatureFlags(flagName)) - throw new Error("Unexpected feature flag!"); - toggleFlag(flagName); - }} - class="checkbox-container" - class:is-enabled={value} - on:click={() => { - if (!isKeyOfFeatureFlags(flagName)) - throw new Error("Unexpected feature flag!"); - toggleFlag(flagName); - }} - > - -
-
+
+
+ {#each tabs as tab} + + {/each} +
+ +
+ {#if selectedTab} +
+ + {#each selectedTab.flags as flagName} + {@const entry = FeatureFlagSchema[flagName]} + {@const value = settings[flagName] ?? entry.default} +
+
+
+ + {entry.title} + {flagName} + +
+
+
{entry.description}
+
+
+
+
toggleFlag(flagName)} + on:keydown={(event) => handleToggleKey(event, flagName)} + > + +
+
+
+
+ {/each} +
-
- {/each} + {/if} +
{#if hasChanges} @@ -70,20 +161,98 @@
diff --git a/src/components/Relays.svelte b/src/components/Relays.svelte index e3704f92..b993a4b5 100644 --- a/src/components/Relays.svelte +++ b/src/components/Relays.svelte @@ -117,7 +117,7 @@ } // Add the folder to SharedFolders - const folder = plugin.sharedFolders.new( + const folder = plugin.sharedFolders.clone( vaultRelativePath, remoteFolder.guid, remoteFolder.relay.guid, @@ -301,14 +301,14 @@ -{#if subscriptions.values().length > 0} +{#if $subscriptions.values().filter((s) => $relays.has(s.relayId)).length > 0}
- {#each $subscriptions.values() as subscription} + {#each $subscriptions.values().filter((s) => $relays.has(s.relayId)) as subscription}
- {#if $manifestVersions[$selectedManifestTag] === "stable-alias" || $manifestVersions[$selectedManifestTag] === "beta-alias" || $activeChannel === "development"} - {#if $manifestVersions[$selectedManifestTag] === "stable-alias"} + {#if $releaseChannels[$selectedManifestTag] === "stable-alias" || $releaseChannels[$selectedManifestTag] === "beta-alias" || $activeChannel === "development"} + {#if $releaseChannels[$selectedManifestTag] === "stable-alias"}

Current stable release from main branch manifest.json

- {:else if $manifestVersions[$selectedManifestTag] === "beta-alias"} + {:else if $releaseChannels[$selectedManifestTag] === "beta-alias"}

Current beta release from main branch manifest-beta.json

{:else if $selectedManifestTag === plugin.version}

Current development version

diff --git a/src/components/ResourceMeterContent.svelte b/src/components/ResourceMeterContent.svelte new file mode 100644 index 00000000..2b795550 --- /dev/null +++ b/src/components/ResourceMeterContent.svelte @@ -0,0 +1,268 @@ + + +