chore: document db

.github/workflows/ci-test-e2e.yml

@@ -57,6 +57,14 @@ on:
         required: false
       CODECOV_TOKEN:
         required: false
+      CI_MONGO_URL:
+        required: false
+      NETBIRD_SETUP_KEY:
+        required: false
+      NETBIRD_PSK:
+        required: false
+      NETBIRD_TEST_IP:
+        required: false
       REPORTER_JIRA_ROCKETCHAT_API_KEY:
         required: false
 
@@ -71,10 +79,12 @@ jobs:
     runs-on: ubuntu-24.04
 
     env:
+      # secrets are not available in step `if:`; use for steps that must run only with CI_MONGO_URL
+      HAS_CI_MONGO_URL: ${{ secrets.CI_MONGO_URL != '' && 'true' || 'false' }}
       # if building for production on develop branch or release, add suffix for coverage images
       DOCKER_TAG_SUFFIX_ROCKETCHAT: ${{ inputs.coverage == matrix.mongodb-version && (github.event_name == 'release' || github.ref == 'refs/heads/develop') && '-cov' || '' }}
       MONGODB_VERSION: ${{ matrix.mongodb-version }}
-      COVERAGE_DIR: '/tmp/coverage/${{ startsWith(inputs.type, ''api'') && ''api'' || inputs.type }}'
+      COVERAGE_DIR: "/tmp/coverage/${{ startsWith(inputs.type, 'api') && 'api' || inputs.type }}"
       COVERAGE_FILE_NAME: '${{ inputs.type }}-${{ matrix.shard }}.json'
       COVERAGE_REPORTER: ${{ inputs.coverage == matrix.mongodb-version && 'json' || '' }}
 
@@ -155,6 +165,29 @@ jobs:
           # List loaded images
           docker images
 
+      - name: Connect to NetBird
+        if: env.HAS_CI_MONGO_URL == 'true'
+        uses: RocketChat/netbird-connect@main
+        with:
+          setup-key: ${{ secrets.NETBIRD_SETUP_KEY }}
+          preshared-key: ${{ secrets.NETBIRD_PSK }}
+          test-ip: ${{ secrets.NETBIRD_TEST_IP }}
+
+      - name: Configure external MongoDB URL
+        env:
+          CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+        if: env.CI_MONGO_URL != ''
+        run: |
+          DB_NAME="rocketchat_${{ inputs.type }}_${{ matrix.shard }}_${{ github.run_id }}_${{ github.run_attempt }}"
+          BASE="${CI_MONGO_URL%%\?*}"
+          BASE="${BASE%/}"
+          QUERY="${CI_MONGO_URL#"$BASE"}"
+          FULL_URL="${BASE}/${DB_NAME}${QUERY}"
+          echo "MONGO_URL=${FULL_URL}" >> $GITHUB_ENV
+          echo "DOCKER_MONGO_URL=${FULL_URL}" >> $GITHUB_ENV
+          echo "EXTERNAL_MONGO=true" >> $GITHUB_ENV
+          echo "DOCUMENTDB=true" >> $GITHUB_ENV
+
       - name: Set DEBUG_LOG_LEVEL (debug enabled)
         if: runner.debug == '1'
         run: echo "DEBUG_LOG_LEVEL=2" >> $GITHUB_ENV
@@ -175,15 +208,23 @@ jobs:
         if: inputs.release == 'ce'
         run: |
           # when we are testing CE, we only need to start the rocketchat container
-          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat --wait
+          EXTRA_ARGS=""
+          if [ "$EXTERNAL_MONGO" = "true" ]; then
+            EXTRA_ARGS="--scale mongo=0"
+          fi
+          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat $EXTRA_ARGS --wait
 
       - name: Start containers for EE
         if: inputs.release == 'ee'
         env:
           ENTERPRISE_LICENSE: ${{ inputs.enterprise-license }}
           TRANSPORTER: ${{ inputs.transporter }}
         run: |
-          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d --wait
+          EXTRA_ARGS=""
+          if [ "$EXTERNAL_MONGO" = "true" ]; then
+            EXTRA_ARGS="--scale mongo=0"
+          fi
+          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d $EXTRA_ARGS --wait
 
       - uses: ./.github/actions/setup-playwright
         if: inputs.type == 'ui'

.github/workflows/ci.yml

@@ -500,6 +500,10 @@ jobs:
     secrets:
       CR_USER: ${{ secrets.CR_USER }}
       CR_PAT: ${{ secrets.CR_PAT }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-api-livechat:
     name: 🔨 Test API Livechat (CE)
@@ -516,6 +520,10 @@ jobs:
     secrets:
       CR_USER: ${{ secrets.CR_USER }}
       CR_PAT: ${{ secrets.CR_PAT }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-ui:
     name: 🔨 Test UI (CE)
@@ -541,6 +549,10 @@ jobs:
       REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }}
       REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }}
       REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-api-ee:
     name: 🔨 Test API (EE)
@@ -561,6 +573,10 @@ jobs:
     secrets:
       CR_USER: ${{ secrets.CR_USER }}
       CR_PAT: ${{ secrets.CR_PAT }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-api-livechat-ee:
     name: 🔨 Test API Livechat (EE)
@@ -581,6 +597,10 @@ jobs:
     secrets:
       CR_USER: ${{ secrets.CR_USER }}
       CR_PAT: ${{ secrets.CR_PAT }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-ui-ee:
     name: 🔨 Test UI (EE)
@@ -609,6 +629,10 @@ jobs:
       REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }}
+      CI_MONGO_URL: ${{ secrets.CI_MONGO_URL }}
+      NETBIRD_SETUP_KEY: ${{ secrets.NETBIRD_SETUP_KEY }}
+      NETBIRD_PSK: ${{ secrets.NETBIRD_PSK }}
+      NETBIRD_TEST_IP: ${{ secrets.NETBIRD_TEST_IP }}
 
   test-federation-matrix:
     name: 🔨 Test Federation Matrix

apps/meteor/app/api/server/lib/users.ts

@@ -1,5 +1,5 @@
 import type { IUser } from '@rocket.chat/core-typings';
-import { Users, Subscriptions } from '@rocket.chat/models';
+import { Users, Subscriptions, getAllowDiskUse } from '@rocket.chat/models';
 import { escapeRegExp } from '@rocket.chat/string-helpers';
 import type { Mongo } from 'meteor/mongo';
 import type { Filter, FindOptions, RootFilterOperators } from 'mongodb';
@@ -229,7 +229,7 @@ export async function findPaginatedUsersByStatus({
 			skip: offset,
 			limit: count,
 			projection,
-			allowDiskUse: true,
+			...getAllowDiskUse(),
 		},
 	);
 	const [users, total] = await Promise.all([cursor.toArray(), totalCount]);

apps/meteor/app/api/server/v1/users.ts

@@ -543,42 +543,28 @@ API.v1.addRoute(
 						]
 					: [];
 
-			const result = await Users.col
-				.aggregate<{ sortedResults: IUser[]; totalCount: { total: number }[] }>([
-					{
-						$match: nonEmptyQuery,
-					},
-					{
-						$project: inclusiveFields,
-					},
-					{
-						$addFields: {
-							nameInsensitive: {
-								$toLower: '$name',
-							},
-						},
-					},
-					{
-						$facet: {
-							sortedResults: [
-								{
-									$sort: actualSort,
-								},
-								{
-									$skip: offset,
-								},
-								...limit,
-							],
-							totalCount: [{ $group: { _id: null, total: { $sum: 1 } } }],
+			const baseQuery = [
+				{
+					$match: nonEmptyQuery,
+				},
+				{
+					$project: inclusiveFields,
+				},
+				{
+					$addFields: {
+						nameInsensitive: {
+							$toLower: '$name',
 						},
 					},
-				])
-				.toArray();
+				},
+			];
+
+			const [users, countResult] = await Promise.all([
+				Users.col.aggregate<IUser>([...baseQuery, { $sort: actualSort }, { $skip: offset }, ...limit]).toArray(),
+				Users.col.aggregate<{ total: number }>([...baseQuery, { $count: 'total' }]).toArray(),
+			]);
 
-			const {
-				sortedResults: users,
-				totalCount: [{ total } = { total: 0 }],
-			} = result[0];
+			const total = countResult[0]?.total || 0;
 
 			return API.v1.success({
 				users,

apps/meteor/app/livechat/server/lib/QueueManager.ts

@@ -27,7 +27,7 @@ import { afterInquiryQueued, afterRoomQueued, beforeDelegateAgent, beforeRouteCh
 import { checkOnlineAgents, getOnlineAgents } from './service-status';
 import { getInquirySortMechanismSetting } from './settings';
 import { dispatchInquiryPosition } from '../../../../ee/app/livechat-enterprise/server/lib/Helper';
-import { client, shouldRetryTransaction } from '../../../../server/database/utils';
+import { client, shouldRetryTransaction, transactionOptions } from '../../../../server/database/utils';
 import { sendNotification } from '../../../lib/server';
 import { notifyOnLivechatInquiryChangedById, notifyOnLivechatInquiryChanged } from '../../../lib/server/lib/notifyListener';
 import { settings } from '../../../settings/server';
@@ -257,7 +257,7 @@ export class QueueManager {
 	): Promise<{ room: IOmnichannelRoom; inquiry: ILivechatInquiryRecord }> {
 		const session = client.startSession();
 		try {
-			session.startTransaction();
+			session.startTransaction(transactionOptions);
 			const room = await createLivechatRoom(insertionRoom, session);
 			logger.debug({ msg: 'Room created for visitor', visitorId: guest._id, roomId: room._id });
 			const inquiry = await createLivechatInquiry({

apps/meteor/app/livechat/server/lib/closeRoom.ts

@@ -9,7 +9,7 @@ import type { ClientSession } from 'mongodb';
 import type { CloseRoomParams, CloseRoomParamsByUser, CloseRoomParamsByVisitor } from './localTypes';
 import { livechatLogger as logger } from './logger';
 import { parseTranscriptRequest } from './parseTranscriptRequest';
-import { client, shouldRetryTransaction } from '../../../../server/database/utils';
+import { client, shouldRetryTransaction, transactionOptions } from '../../../../server/database/utils';
 import { callbacks } from '../../../../server/lib/callbacks';
 import {
 	notifyOnLivechatInquiryChanged,
@@ -33,7 +33,7 @@ export async function closeRoom(params: CloseRoomParams, attempts = 2): Promise<
 
 	const session = client.startSession();
 	try {
-		session.startTransaction();
+		session.startTransaction(transactionOptions);
 		const { room, closedBy, removedInquiry } = await doCloseRoom(params, session);
 		await session.commitTransaction();
 

apps/meteor/app/settings/server/SettingsRegistry.ts

@@ -198,7 +198,17 @@ export class SettingsRegistry {
 
 		const setting = isOverwritten ? settingFromCodeOverwritten : settingOverwrittenDefault;
 
-		await this.model.insertOne(setting); // no need to emit unless we remove the oplog
+		try {
+			await this.model.insertOne(setting); // no need to emit unless we remove the oplog
+		} catch (e: any) {
+			// Another process inserted the same setting first (e.g. main app + EE
+			// microservices booting in parallel). The check above is in-memory and
+			// not atomic, so concurrent boots race on the unique _id index.
+			// E11000 here is safe to ignore — the setting now exists.
+			if (e?.code !== 11000) {
+				throw e;
+			}
+		}
 
 		this.store.set(setting);
 	}
@@ -224,7 +234,16 @@ export class SettingsRegistry {
 		if (!this.store.has(_id)) {
 			options.ts = new Date();
 			this.store.set(options as ISetting);
-			await this.model.insertOne(options as ISetting);
+			try {
+				await this.model.insertOne(options as ISetting);
+			} catch (e: any) {
+				// Race with another process (main app + EE microservices boot in
+				// parallel). Same rationale as in `add` above — E11000 here means
+				// another caller wrote the group first; the desired state holds.
+				if (e?.code !== 11000) {
+					throw e;
+				}
+			}
 		}
 
 		if (!callback) {

apps/meteor/ee/server/api/abac/index.ts

@@ -1,7 +1,7 @@
 import { Abac } from '@rocket.chat/core-services';
 import type { AbacActor } from '@rocket.chat/core-services';
 import type { IServerEvents, IUser } from '@rocket.chat/core-typings';
-import { ServerEvents, Users } from '@rocket.chat/models';
+import { ServerEvents, Users, getAllowDiskUse } from '@rocket.chat/models';
 import { validateUnauthorizedErrorResponse } from '@rocket.chat/rest-typings/src/v1/Ajv';
 import { convertSubObjectsIntoPaths } from '@rocket.chat/tools';
 
@@ -393,7 +393,7 @@ const abacEndpoints = API.v1
 					sort: _sort,
 					skip: offset,
 					limit: count,
-					allowDiskUse: true,
+					...getAllowDiskUse(),
 				},
 			);
 

apps/meteor/ee/server/api/audit.ts

@@ -1,5 +1,5 @@
 import type { IUser, IRoom } from '@rocket.chat/core-typings';
-import { Rooms, AuditLog, ServerEvents } from '@rocket.chat/models';
+import { Rooms, AuditLog, ServerEvents, getAllowDiskUse } from '@rocket.chat/models';
 import { isServerEventsAuditSettingsProps, ajv, ajvQuery } from '@rocket.chat/rest-typings';
 import type { PaginatedRequest, PaginatedResult } from '@rocket.chat/rest-typings';
 import { convertSubObjectsIntoPaths } from '@rocket.chat/tools';
@@ -180,7 +180,7 @@ API.v1.get(
 				sort: _sort,
 				skip: offset,
 				limit: count,
-				allowDiskUse: true,
+				...getAllowDiskUse(),
 			},
 		);
 

apps/meteor/ee/server/lib/engagementDashboard/channels.ts

@@ -44,7 +44,7 @@ export const findChannelsWithNumberOfMessages = async ({
 		startOfLastWeek: convertDateToInt(startOfLastWeek),
 		endOfLastWeek: convertDateToInt(endOfLastWeek),
 		options,
-	}).toArray();
+	});
 
 	// The aggregation result may be undefined if there are no matching analytics or corresponding rooms in the period
 	if (!aggregationResult.length) {