diff --git a/REFERENCE.md b/REFERENCE.md index b949a3b4..33c0ab79 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -466,6 +466,37 @@ Defines the structure of event data that is sent from the external extractor to Optional. A **string** representing the stats file artifact ID. +- _pre_extraction_item_counts_ + + Optional. An array of **ItemTypeCount** objects reporting per-record-type counts collected during the metadata phase. Used for sync duration estimation. + +### `ItemInputType` enum + +Defines which sync duration estimation input a counted record type feeds into. + +#### Values + +- `MAIN` = `'main'` +- `USERS` = `'users'` + +### `ItemTypeCount` interface + +Represents a per-record-type count reported during the metadata phase, used for sync duration estimation. + +#### Properties + +- _record_type_ + + Required. A **string** identifying the external record type being counted. + +- _count_ + + Required. A **number** with the count of records of this type. + +- _model_input_type_ + + Required. An **ItemInputType** value indicating which estimation input this record type feeds into. + ### `EventType` enum Defines the different types of events that can be sent to the external extractor from ADaaS. The external extractor uses these events to know what to do next in the extraction process. diff --git a/package-lock.json b/package-lock.json index 300ee9a9..6f0798a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@devrev/ts-adaas", - "version": "1.20.1", + "version": "1.20.2-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@devrev/ts-adaas", - "version": "1.20.1", + "version": "1.20.2-beta.0", "license": "ISC", "dependencies": { "@devrev/typescript-sdk": "^1.1.78", diff --git a/package.json b/package.json index 8d03e228..7a5767f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/ts-adaas", - "version": "1.20.1", + "version": "1.20.2-beta.0", "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.", "type": "commonjs", "main": "./dist/index.js", diff --git a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts b/src/multithreading/worker-adapter/worker-adapter.emit.test.ts index 2da61f73..6812c369 100644 --- a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts +++ b/src/multithreading/worker-adapter/worker-adapter.emit.test.ts @@ -8,6 +8,7 @@ import { Artifact, EventType, ExtractorEventType, + ItemInputType, LoaderEventType, } from '../../types'; import { ActionType, LoaderReport } from '../../types/loading'; @@ -240,6 +241,36 @@ describe(`${WorkerAdapter.name}.emit`, () => { expect(callData).not.toHaveProperty('processed_files'); }); + it('should include pre_extraction_item_counts passed on the metadata-done event', async () => { + // Arrange + const { emit: mockEmit } = require('../../common/control-protocol'); + adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); + adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); + + // Act + await adapter.emit(ExtractorEventType.MetadataExtractionDone, { + pre_extraction_item_counts: [ + { + record_type: 'tickets', + count: 0, + model_input_type: ItemInputType.MAIN, + }, + { + record_type: 'customers', + count: 1200, + model_input_type: ItemInputType.USERS, + }, + ], + }); + + // Assert + const callData = mockEmit.mock.calls[0][0].data; + expect(callData.pre_extraction_item_counts).toEqual([ + { record_type: 'tickets', count: 0, model_input_type: 'main' }, + { record_type: 'customers', count: 1200, model_input_type: 'users' }, + ]); + }); + it('should include artifacts for all ExtractorEventType values', async () => { // Arrange const { emit: mockEmit } = require('../../common/control-protocol'); diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 64136e90..1de0bee6 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -387,6 +387,23 @@ export interface ConnectionData { key_type: string; } +/** + * ItemInputType is the sync-duration-estimation model input a counted record type feeds into. + */ +export enum ItemInputType { + MAIN = 'main', + USERS = 'users', +} + +/** + * ItemTypeCount is a per-record-type count reported during the metadata phase, used for sync-duration estimation. + */ +export interface ItemTypeCount { + record_type: string; + count: number; + model_input_type: ItemInputType; +} + /** * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS. */ @@ -412,6 +429,8 @@ export interface EventData { reports?: LoaderReport[]; processed_files?: string[]; stats_file?: string; + // Optional per-record-type counts reported on the metadata-done event, used for sync-duration estimation. + pre_extraction_item_counts?: ItemTypeCount[]; } /** diff --git a/src/types/index.ts b/src/types/index.ts index f49ef309..26d48405 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -30,6 +30,8 @@ export { ExtractorEvent, ExtractorEventType, InitialSyncScope, + ItemInputType, + ItemTypeCount, ProcessAttachmentReturnType, TimeUnit, TimeValue,