From 34153260b122058ce4b7c804d3f5e5b25a836dbd Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sat, 18 Feb 2023 20:28:55 -0500 Subject: [PATCH 01/14] changes for point-in-time querying --- components/persistor/lib/api.ts | 9 +- components/persistor/lib/knex/PersistorCtx.ts | 56 ++ components/persistor/lib/knex/db.ts | 190 ++++--- components/persistor/lib/knex/query.ts | 22 +- components/persistor/lib/knex/update.ts | 1 - components/persistor/lib/mongo/query.ts | 3 +- components/persistor/lib/mongo/update.ts | 3 +- components/persistor/lib/util.ts | 10 +- .../persistor/lib/utils/PersistorUtils.ts | 23 +- components/persistor/package-lock.json | 24 - components/persistor/package.json | 2 - components/persistor/test/persist_banking.js | 1 - .../persistor/test/persist_banking_pgsql.js | 9 +- components/persistor/test/persist_fetch.js | 2 +- .../persistor/test/persist_fetch_children.js | 2 +- .../persistor/test/persist_newapi_extend.js | 2 +- .../persistor/test/persist_newapi_tests.js | 494 +++++++++++------- .../persistor/test/persist_parent_subset.js | 2 +- .../persistor/test/persist_polymorphic.js | 392 +++++++------- .../test/persist_schema_indexdefchanges.js | 4 +- .../persistor/test/persist_schema_updates.js | 4 +- .../persistor/test/persist_transaction.js | 2 +- .../test/supertype/one-to-manychecks.ts | 1 - .../test/supertype/persist_banking_pgsql.ts | 8 +- components/persistor/tsconfig.json | 3 +- 25 files changed, 749 insertions(+), 520 deletions(-) create mode 100644 components/persistor/lib/knex/PersistorCtx.ts diff --git a/components/persistor/lib/api.ts b/components/persistor/lib/api.ts index 0f9400c9..1d5287a8 100644 --- a/components/persistor/lib/api.ts +++ b/components/persistor/lib/api.ts @@ -11,15 +11,16 @@ */ import { PersistorTransaction, RemoteDocConnectionOptions } from './types'; +import { PersistorCtx } from './knex/PersistorCtx'; +import { PersistorUtils } from './utils/PersistorUtils'; module.exports = function (PersistObjectTemplate, baseClassForPersist) { const moduleName = `persistor/lib/api`; let supertypeRequire = require('@haventech/supertype'); let statsDHelper = supertypeRequire.StatsdHelper; - var Promise = require('bluebird'); var _ = require('underscore'); - + function getTime() { return process.hrtime(); } @@ -300,7 +301,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { var dbType = persistObjectTemplate.getDB(persistObjectTemplate.getDBAlias(template.__collection__)).type; let fetchQuery = (dbType == persistObjectTemplate.DB_Mongo ? persistObjectTemplate.getFromPersistWithMongoId(template, id, options.fetch, options.transient, null, options.logger) : - persistObjectTemplate.getFromPersistWithKnexId(template, id, options.fetch, options.transient, null, null, options.logger, options.enableChangeTracking, options.projection)); + PersistorCtx.checkAndExecuteWithContext(options.asOfDate, persistObjectTemplate.getFromPersistWithKnexId.bind(persistObjectTemplate, template, id, options.fetch, options.transient, null, null, options.logger, options.enableChangeTracking, options.projection))); const name = 'persistorFetchById'; return fetchQuery @@ -1396,7 +1397,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { templates.push(template); } } - return Promise.map(templates, action, { concurrency: concurrency || 1 }); + return PersistorUtils.asyncMap(templates, concurrency || 1, action); } }; diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts new file mode 100644 index 00000000..73b1f036 --- /dev/null +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -0,0 +1,56 @@ +import { AsyncLocalStorage } from 'async_hooks' + +type CtxProps = { name: string, properties: {} } + +class ExecutionCtx { + private readonly _asOfDate; + + constructor(asOfDate: Date) { + this._asOfDate = asOfDate + } + + get AsOfDate() { + return this._asOfDate; + } +} + +export class PersistorCtx { + static persistorExnCtxKey = '#persistor-exn-ctx'; + private static _asyncLocalStorage: AsyncLocalStorage; + + private static get asyncLocalStorage() { + if (this._asyncLocalStorage) { + return this._asyncLocalStorage + } + else { + return (this._asyncLocalStorage = new AsyncLocalStorage()); + } + } + + static checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { + if (asOfDate) { + const ctxProps = { + name: `${new Date().getTime()}`, + properties: { [this.persistorExnCtxKey]: new ExecutionCtx(asOfDate) }, + }; + return this.asyncLocalStorage.run(ctxProps, async () => { + let store = this.asyncLocalStorage.getStore() as CtxProps; + return await callback(); + }); + } + else { + return callback(); + } + } + + static get ExecutionCtx() { + if (this._asyncLocalStorage) { + const store = this.asyncLocalStorage.getStore() as CtxProps; + if (!store) { + return null; + } + const exnCtx: ExecutionCtx = store.properties[this.persistorExnCtxKey]; + return exnCtx; + } + } +} \ No newline at end of file diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index 2d2144b1..9fcc0363 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -1,8 +1,9 @@ import {RemoteDocService} from "../remote-doc/RemoteDocService"; +import { PersistorCtx } from './PersistorCtx'; +import { PersistorUtils } from '../utils/PersistorUtils'; module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/knex/db`; - var Promise = require('bluebird'); var _ = require('underscore'); var processedList = []; @@ -21,16 +22,29 @@ module.exports = function (PersistObjectTemplate) { PersistObjectTemplate.getPOJOsFromKnexQuery = function (template, joins, queryOrChains, options, map, logger, projection) { var tableName = this.dealias(template.__table__); + var historyTableName, historyTableAlias + if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { + historyTableName = tableName + '_history'; + queryOrChains["createdTime"] = {$lte: PersistorCtx.ExecutionCtx?.AsOfDate}; + } + var knex = this.getDB(this.getDBAlias(template.__table__)).connection(tableName); const functionName = 'getPOJOsFromKnexQuery'; // tack on outer joins. All our joins are outerjoins and to the right. There could in theory be // foreign keys pointing to rows that no longer exists - var select = knex.select(getColumnNames.bind(this, template)()).from(tableName); + var select = knex.select(getColumnNames.bind(this, template)()).from((historyTableName || tableName) + ' as ' + tableName); joins.forEach(function (join) { - select = select.leftOuterJoin(this.dealias(join.template.__table__) + ' as ' + join.alias, + let joinTable = this.dealias(join.template.__table__); + let historyJoinTable, additionalCondition; + if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { + historyTableName = joinTable + '_history'; + additionalCondition = ' and ' + join.alias + '.' + '"createdTime" <= ' + PersistorCtx.ExecutionCtx?.AsOfDate; + } + select = select.leftOuterJoin( (historyJoinTable || joinTable) + ' as ' + join.alias, join.alias + '.' + join.parentKey, - this.dealias(template.__table__) + '.' + join.childKey); + this.dealias(template.__table__) + '.' + join.childKey + (additionalCondition ? additionalCondition : '')); + }.bind(this)); // execute callback to chain on filter functions or convert mongo style filters @@ -69,7 +83,7 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'pre', - template: template.__name__, + template: template.__name__, query: queryOrChains } }); @@ -90,8 +104,8 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'post', - count: res.length, - template: template.__name__, + count: res.length, + template: template.__name__, query: queryOrChains } }); @@ -316,8 +330,8 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'post', - count: res, - table: tableName, + count: res, + table: tableName, id: obj._id } }); @@ -344,7 +358,7 @@ module.exports = function (PersistObjectTemplate) { if (txn.queriesToNotify) { generateNotifyQueries(template, txn.queriesToNotify, knex.where({_id: id}).delete().toString()); } - + } return knex.where({_id: id}).delete(); }; @@ -373,11 +387,11 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'pre', - txn: (txn ? txn.id + ' ' : '-#- '), + txn: (txn ? txn.id + ' ' : '-#- '), type: (updateID ? 'updating ' : 'insert '), - template: obj.__template__.__name__, - id: obj.__id__, - _id: obj._id, + template: obj.__template__.__name__, + id: obj.__id__, + _id: obj._id, __version__: pojo.__version__ } }); @@ -432,7 +446,7 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'updateConflict', - txn: (txn ? txn.id : '-#-'), + txn: (txn ? txn.id : '-#-'), id: obj.__id__, __version__: origVer } }); @@ -466,8 +480,8 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'post', - template: obj.__template__.__name__, - table: obj.__template__.__table__, + template: obj.__template__.__name__, + table: obj.__template__.__table__, __version__: obj.__version__ } }); @@ -514,29 +528,41 @@ module.exports = function (PersistObjectTemplate) { var _newFields = {}; return Promise.resolve() - .then(buildTable.bind(this)) + .then(buildTable.bind(this, aliasedTableName)) .then(addComments.bind(this, tableName)) - .then(synchronizeIndexes.bind(this, tableName, template)); - - function buildTable() { - return knex.schema.hasTable(tableName).then(function (exists) { - if (!exists) { - if (!!changeNotificationCallback) { - if (typeof changeNotificationCallback !== 'function') - throw new Error('persistor can only notify the table changes through a callback'); - changeNotificationCallback('A new table, ' + tableName + ', has been added\n'); - } - return PersistObjectTemplate._createKnexTable(template, aliasedTableName); + .then(synchronizeIndexes.bind(this, tableName, template)) + .then(function (response) { + if (template.__schema__.audit === 'v2') { + return Promise.resolve() + .then(buildTable.bind(this, aliasedTableName + '_history')) + .then(synchronizeIndexes.bind(this, aliasedTableName + '_history', template));; } else { - return discoverColumns(tableName).then(function () { - fieldChangeNotify(changeNotificationCallback, tableName); - return knex.schema.table(tableName, columnMapper.bind(this)) - }.bind(this)); + return response; } }.bind(this)) - } + + function buildTable(aliasedTableName) { + var tableName = this.dealias(aliasedTableName); + return knex.schema.hasTable(tableName).then(function (exists) { + if (!exists) { + if (!!changeNotificationCallback) { + if (typeof changeNotificationCallback !== 'function') + throw new Error('persistor can only notify the table changes through a callback'); + changeNotificationCallback('A new table, ' + tableName + ', has been added\n'); + } + return PersistObjectTemplate._createKnexTable(template, aliasedTableName); + } + else { + return discoverColumns(tableName).then(function () { + fieldChangeNotify(changeNotificationCallback, tableName); + return knex.schema.table(tableName, columnMapper.bind(this, tableName)); + }.bind(this)); + } + }.bind(this)); + } + function fieldChangeNotify(callBack, table) { if (!callBack) return; if (typeof callBack !== 'function') @@ -795,13 +821,13 @@ module.exports = function (PersistObjectTemplate) { }) }; - var loadTableDef = function(dbschema, tableName) { + var loadTableDef = function([dbschema, tableName]) { if (!dbschema[tableName]) dbschema[tableName] = {}; return [dbschema, schema, tableName]; }; - var diffTable = function(dbschema, schema, tableName) { + var diffTable = function([dbschema, schema, tableName]) { var dbTableDef = dbschema[tableName]; var memTableDef = schema[tableName]; var track = {add: [], change: [], delete: []}; @@ -841,8 +867,8 @@ module.exports = function (PersistObjectTemplate) { var generateChanges = function (localtemplate, _value) { return _.reduce(localtemplate.__children__, function (_curr, o) { return Promise.resolve() - .then(loadTableDef.bind(this, _dbschema, o.__name__)) - .spread(diffTable) + .then(loadTableDef.bind(this, [_dbschema, o.__name__])) + .then(diffTable.bind(this)) .then(generateChanges.bind(this, o)); }, {}); }; @@ -897,14 +923,14 @@ module.exports = function (PersistObjectTemplate) { } } /** - * This function loops through index changes (add/change/delete) and - * executes them one by one. In case of errors, while executing indexes, - * we will log a warning message. This approach handles each index at - * its own merit without impacting the behavior or another index. - * This is a change from previous code where were trying to run index - * changes together in a transaction. - * @param operation - * @param diffs + * This function loops through index changes (add/change/delete) and + * executes them one by one. In case of errors, while executing indexes, + * we will log a warning message. This approach handles each index at + * its own merit without impacting the behavior or another index. + * This is a change from previous code where were trying to run index + * changes together in a transaction. + * @param operation + * @param diffs */ async function syncIndexesForHierarchy (operation, diffs) { for (const _key in diffs[operation]) { @@ -924,7 +950,7 @@ module.exports = function (PersistObjectTemplate) { if (operation === 'add') { try { await knex.schema.table(tableName, async function (table) { - // This table function callback does not necessarily need to be + // This table function callback does not necessarily need to be // an async callback, but making the callback async/await as a guard. await table[type](columns, name); }); @@ -938,7 +964,7 @@ module.exports = function (PersistObjectTemplate) { type = type.replace(/unique/, 'Unique'); try { await knex.schema.table(tableName, async function (table) { - // This table function callback does not necessarily need to be + // This table function callback does not necessarily need to be // an async callback, but making the callback async/await as a guard. await table['drop' + type]([], name); }); @@ -950,7 +976,7 @@ module.exports = function (PersistObjectTemplate) { else { try { await knex.schema.table(tableName, async function (table) { - // This table function callback does not necessarily need to be + // This table function callback does not necessarily need to be // an async callback, but making the callback async/await as a guard. await table[type](columns, name); }); @@ -959,7 +985,7 @@ module.exports = function (PersistObjectTemplate) { }; } } - catch(error) { + catch(error) { const logger = PersistObjectTemplate && PersistObjectTemplate.logger; if (logger) { logger.warn({ @@ -975,8 +1001,8 @@ module.exports = function (PersistObjectTemplate) { } return Promise.all([ - syncIndexesForHierarchy('delete', dbChanges), - syncIndexesForHierarchy('add', dbChanges), + syncIndexesForHierarchy('delete', dbChanges), + syncIndexesForHierarchy('add', dbChanges), syncIndexesForHierarchy('change', dbChanges) ]); }; @@ -1016,8 +1042,8 @@ module.exports = function (PersistObjectTemplate) { return Promise.resolve() .then(loadSchema.bind(this, tableName)) - .spread(loadTableDef) - .spread(diffTable) + .then(loadTableDef.bind(this)) + .then(diffTable.bind(this)) .then(generateChanges.bind(this, template)) .then(mergeChanges) .then(applyTableChanges) @@ -1074,7 +1100,7 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'pre', - template: obj.__template__.__name__, + template: obj.__template__.__name__, table: obj.__template__.__table__ } }); @@ -1094,7 +1120,7 @@ module.exports = function (PersistObjectTemplate) { category: 'milestone', data: { activity: 'post', - template: obj.__template__.__name__, + template: obj.__template__.__name__, table: obj.__template__.__table__ } }); @@ -1127,9 +1153,16 @@ module.exports = function (PersistObjectTemplate) { return knex.schema.createTable(tableName, createColumns.bind(this)); function createColumns(table) { - table.string('_id').primary(); + table.string('_template'); table.biginteger('__version__'); + if (collection.match(/_history/)) { + table.string('_id_history').primary(); + table.string('_id'); + } + else { + table.string('_id').primary(); + } var columnMap = {}; recursiveColumnMap.call(this, template); @@ -1428,16 +1461,14 @@ module.exports = function (PersistObjectTemplate) { } // Walk through the dirty objects - function processSaves() { - return Promise.map(_.toArray(dirtyObjects), function (obj) { + async function processSaves() { + await PersistorUtils.asyncMap(_.toArray(dirtyObjects), PersistObjectTemplate.concurrency, function (obj) { delete dirtyObjects[obj.__id__]; // Once scheduled for update remove it. return callSave(obj).then(generateChanges.bind(this, obj, obj.__version__ === 1 ? 'insert' : 'update')); - }.bind(this), {concurrency: PersistObjectTemplate.concurrency}).then (function () { - if (_.toArray(dirtyObjects). length > 0) { - return processSaves.call(this); - } - }); - + }.bind(this)); + if (_.toArray(dirtyObjects).length > 0) { + return processSaves.call(this); + } function callSave(obj) { return (obj.__template__ && obj.__template__.__schema__ ? obj.persistSave(persistorTransaction, logger) @@ -1446,15 +1477,15 @@ module.exports = function (PersistObjectTemplate) { } - function processDeletes() { - return Promise.map(_.toArray(deletedObjects), function (obj) { + async function processDeletes() { + await PersistorUtils.asyncMap(_.toArray(deletedObjects), PersistObjectTemplate.concurrency, function (obj) { delete deletedObjects[obj.__id__]; // Once scheduled for update remove it. return callDelete(obj).then(generateChanges.bind(this, obj, 'delete')); - }.bind(this), {concurrency: PersistObjectTemplate.concurrency}).then (function () { - if (_.toArray(deletedObjects). length > 0) { - return processDeletes.call(this); - } - }); + }.bind(this)); + + if (_.toArray(deletedObjects). length > 0) { + return processDeletes.call(this); + } function callDelete(obj) { return (obj.__template__ && obj.__template__.__schema__ @@ -1463,17 +1494,16 @@ module.exports = function (PersistObjectTemplate) { } } - function processDeleteQueries() { - return Promise.map(_.toArray(deleteQueries), function (obj) { + async function processDeleteQueries() { + await PersistorUtils.asyncMap(_.toArray(deleteQueries), PersistObjectTemplate.concurrency, function (obj) { delete deleteQueries[obj.name]; // Once scheduled for update remove it. return (obj.template && obj.template.__schema__ ? PersistObjectTemplate.deleteFromKnexQuery(obj.template, obj.queryOrChains, persistorTransaction, logger) : true) - }.bind(this), {concurrency: PersistObjectTemplate.concurrency}).then (function () { - if (_.toArray(deleteQueries). length > 0) { - return processDeleteQueries.call(this); - } - }); + }.bind(this)) + if (_.toArray(deleteQueries). length > 0) { + return processDeleteQueries.call(this); + } } @@ -1496,7 +1526,7 @@ module.exports = function (PersistObjectTemplate) { // Walk through the touched objects function processTouches() { - return Promise.map(_.toArray(touchObjects), function (obj) { + return PersistorUtils.asyncMap(_.toArray(touchObjects), PersistObjectTemplate.concurrency, function (obj) { return (obj.__template__ && obj.__template__.__schema__ && !savedObjects[obj.__id__] ? obj.persistTouch(persistorTransaction, logger) : true) diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index ebb8bc1a..272a1626 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -3,7 +3,6 @@ import { PersistorUtils } from '../utils/PersistorUtils'; module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/knex/query`; - var Promise = require('bluebird'); var _ = require('underscore'); PersistObjectTemplate.concurrency = 10; @@ -83,7 +82,6 @@ module.exports = function (PersistObjectTemplate) { options.offset = skip; if (limit) options.limit = limit; - // Request to do entire processing to be executed right now or as part of a request queue var request = function () { return Promise.resolve(true) @@ -127,19 +125,17 @@ module.exports = function (PersistObjectTemplate) { PersistObjectTemplate.resolveRecursiveRequests = function (requests, results) { return processRequests(); - function processRequests() { + async function processRequests() { var segLength = requests.length; - //console.log("Processing " + segLength + " promises " + PersistObjectTemplate.concurrency); - return Promise.map(requests, function (request, _ix) { + + await PersistorUtils.asyncMap(requests, PersistObjectTemplate.concurrency || 1, function (request, _ix) { return request(); - }, {concurrency: PersistObjectTemplate.concurrency}) - .then(function () { - requests.splice(0, segLength); - if (requests.length > 0) - return processRequests(); - else - return results; - }) + }.bind(this)) + requests.splice(0, segLength); + if (requests.length > 0) + return processRequests(); + else + return results; } } diff --git a/components/persistor/lib/knex/update.ts b/components/persistor/lib/knex/update.ts index c5d7dc5b..6b53e4a9 100644 --- a/components/persistor/lib/knex/update.ts +++ b/components/persistor/lib/knex/update.ts @@ -4,7 +4,6 @@ import { PersistorUtils } from '../utils/PersistorUtils'; module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/knex/update`; - var Promise = require('bluebird'); var _ = require('underscore'); /** diff --git a/components/persistor/lib/mongo/query.ts b/components/persistor/lib/mongo/query.ts index f09f63e8..79f46209 100644 --- a/components/persistor/lib/mongo/query.ts +++ b/components/persistor/lib/mongo/query.ts @@ -1,7 +1,6 @@ module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/mongo/query`; - var Promise = require('bluebird'); - + PersistObjectTemplate.getFromPersistWithMongoId = function (template, id, cascade, isTransient, idMap, _logger) { return this.getFromPersistWithMongoQuery(template, {_id: PersistObjectTemplate.ObjectID(id.toString())}, cascade, null, null, isTransient, idMap) .then(function(pojos) { return pojos[0] }); diff --git a/components/persistor/lib/mongo/update.ts b/components/persistor/lib/mongo/update.ts index e8fc2c68..5688db64 100644 --- a/components/persistor/lib/mongo/update.ts +++ b/components/persistor/lib/mongo/update.ts @@ -1,7 +1,6 @@ module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/mongo/update`; - var Promise = require('bluebird'); - + /** * Save the object to persistent storage * diff --git a/components/persistor/lib/util.ts b/components/persistor/lib/util.ts index 8def11d6..ec196272 100644 --- a/components/persistor/lib/util.ts +++ b/components/persistor/lib/util.ts @@ -1,8 +1,10 @@ module.exports = function (PersistObjectTemplate) { - var Promise = require('bluebird'); var _ = require('underscore'); var schemaValidator = require('tv4'); + schemaValidator.addFormat('date-time', function (data) { + return data instanceof Date && !isNaN(data.valueOf()) + }); PersistObjectTemplate.ObjectID = require('mongodb').ObjectID; @@ -220,7 +222,11 @@ module.exports = function (PersistObjectTemplate) { }, 'enableChangeTracking' : { type: ['boolean', 'null', 'undefined'] - } + }, + 'asOfDate' : { + type: ['object', 'null', 'undefined'], + format: 'date-time' + }, } }, 'commitSchema': { diff --git a/components/persistor/lib/utils/PersistorUtils.ts b/components/persistor/lib/utils/PersistorUtils.ts index e12fd2b4..44d6dd58 100644 --- a/components/persistor/lib/utils/PersistorUtils.ts +++ b/components/persistor/lib/utils/PersistorUtils.ts @@ -1,9 +1,30 @@ export class PersistorUtils { - static isRemoteObjectSetToTrue(enableIsRemoteObjectFeature: any, isRemoteObject: any): boolean { if (enableIsRemoteObjectFeature && (enableIsRemoteObjectFeature === true || enableIsRemoteObjectFeature === 'true')) { return isRemoteObject && isRemoteObject === true; } return false; } + + static asyncMap(arr: any[], concurrency: number, callback: any) { + let cnt = arr.length / concurrency; + let p = Promise.resolve([]); + let start = 0; + + for (let i = 0; i < cnt; i++) { + p = p.then((results) => { + let end = start + concurrency; + + return Promise.all(arr.slice(start, end).map(callback)) + .then((eRes) => { + start = end; + results.push.apply(results, eRes); + + return results; + }); + }); + } + + return p; + } } \ No newline at end of file diff --git a/components/persistor/package-lock.json b/components/persistor/package-lock.json index 31e817f0..99aba0fe 100644 --- a/components/persistor/package-lock.json +++ b/components/persistor/package-lock.json @@ -9,7 +9,6 @@ "version": "10.0.0", "dependencies": { "aws-sdk": "2.x", - "bluebird": "x", "knex": "^0.21.0", "mongodb": "^3.5.5", "pg": "8.7.1", @@ -20,7 +19,6 @@ }, "devDependencies": { "@haventech/supertype": "6.x", - "@types/bluebird": "*", "@types/chai": "4.3.0", "@types/mocha": "10.0.0", "@types/node": "16.11.22", @@ -549,12 +547,6 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, - "node_modules/@types/bluebird": { - "version": "3.5.20", - "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz", - "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA==", - "dev": true - }, "node_modules/@types/chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.0.tgz", @@ -945,11 +937,6 @@ "safe-buffer": "^5.1.1" } }, - "node_modules/bluebird": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", - "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -5913,12 +5900,6 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, - "@types/bluebird": { - "version": "3.5.20", - "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz", - "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA==", - "dev": true - }, "@types/chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.0.tgz", @@ -6207,11 +6188,6 @@ "safe-buffer": "^5.1.1" } }, - "bluebird": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", - "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", diff --git a/components/persistor/package.json b/components/persistor/package.json index 3e1f4cdd..e69a460a 100644 --- a/components/persistor/package.json +++ b/components/persistor/package.json @@ -7,7 +7,6 @@ "types": "dist/index.d.ts", "dependencies": { "aws-sdk": "2.x", - "bluebird": "x", "knex": "^0.21.0", "mongodb": "^3.5.5", "pg": "8.7.1", @@ -21,7 +20,6 @@ }, "devDependencies": { "@haventech/supertype": "6.x", - "@types/bluebird": "*", "@types/chai": "4.3.0", "@types/mocha": "10.0.0", "@types/node": "16.11.22", diff --git a/components/persistor/test/persist_banking.js b/components/persistor/test/persist_banking.js index 007e560f..7770c770 100644 --- a/components/persistor/test/persist_banking.js +++ b/components/persistor/test/persist_banking.js @@ -16,7 +16,6 @@ PersistObjectTemplate.debug = function(m, t) { } } */ -var Promise = require('bluebird'); var Customer = PersistObjectTemplate.create('Customer', { init: function (first, middle, last) { diff --git a/components/persistor/test/persist_banking_pgsql.js b/components/persistor/test/persist_banking_pgsql.js index 6719a170..29c12f61 100644 --- a/components/persistor/test/persist_banking_pgsql.js +++ b/components/persistor/test/persist_banking_pgsql.js @@ -8,7 +8,7 @@ var sinon = require('sinon'); var LocalStorageDocClient = require('../dist/lib/remote-doc/remote-doc-clients/LocalStorageDocClient').LocalStorageDocClient; var expect = require('chai').expect; var util = require('util'); -var Promise = require('bluebird'); + var _ = require('underscore'); var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); @@ -1170,7 +1170,11 @@ describe('Banking from pgsql Example persist_banking_pgsql', function () { txn2Sam.firstName = 'txn2SamDead'; txn2Sam.setDirty(txn2); txn1.postSave = function () { - Promise.delay(100) + function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + sleep(100) .then(function () { // Update will not return because it is requesting a lock on Karen txn1Karen.persistTouch(txn1) // 3 update karen @@ -1187,6 +1191,7 @@ describe('Banking from pgsql Example persist_banking_pgsql', function () { txn2Error = true; }); }; + return PersistObjectTemplate.end(txn1); // 1 - update sam (exc lock) }).catch(function (e) { if (e.message != 'Update Conflict') diff --git a/components/persistor/test/persist_fetch.js b/components/persistor/test/persist_fetch.js index aa0df79d..197aca4a 100644 --- a/components/persistor/test/persist_fetch.js +++ b/components/persistor/test/persist_fetch.js @@ -6,7 +6,7 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); + var knexInit = require('knex'); var knex; diff --git a/components/persistor/test/persist_fetch_children.js b/components/persistor/test/persist_fetch_children.js index ee14675b..f856b914 100644 --- a/components/persistor/test/persist_fetch_children.js +++ b/components/persistor/test/persist_fetch_children.js @@ -6,7 +6,7 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); + var knexInit = require('knex'); var knex; diff --git a/components/persistor/test/persist_newapi_extend.js b/components/persistor/test/persist_newapi_extend.js index 444b8d3c..f8546380 100644 --- a/components/persistor/test/persist_newapi_extend.js +++ b/components/persistor/test/persist_newapi_extend.js @@ -6,7 +6,7 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); + var knexInit = require('knex'); var knex; diff --git a/components/persistor/test/persist_newapi_tests.js b/components/persistor/test/persist_newapi_tests.js index dc4034cf..c96dac35 100644 --- a/components/persistor/test/persist_newapi_tests.js +++ b/components/persistor/test/persist_newapi_tests.js @@ -6,7 +6,6 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); var knexInit = require('knex'); var knex; @@ -18,7 +17,7 @@ var PersistObjectTemplate, ObjectTemplate; describe('persist newapi tests', function () { // this.timeout(5000); - before('drop schema table once per test suit', function() { + before('drop schema table once per test suit', function () { knex = knexInit({ client: 'pg', debug: true, @@ -29,19 +28,15 @@ describe('persist newapi tests', function () { password: process.env.dbPassword, } }); - return Promise.all([ - - knex.schema.dropTableIfExists('tx_employee') - .then(function () { - return knex.schema.dropTableIfExists('tx_address') - }).then(function () { - return knex.schema.dropTableIfExists('tx_phone') - }).then(function () { - return knex.schema.dropTableIfExists('tx_department') - }).then(function () { - return knex.schema.dropTableIfExists('tx_role') - }), - knex.schema.dropTableIfExists(schemaTable)]); + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); }) after('closes the database', function () { return knex.destroy(); @@ -55,59 +50,63 @@ describe('persist newapi tests', function () { schema.Phone = {}; schema.Dept = {}; schema.Employee.table = 'tx_employee'; + schema.Employee.audit = 'v2'; schema.Address.table = 'tx_address'; schema.Address.tableType = 'reference_data'; schema.Phone.table = 'tx_phone'; schema.Employee.parents = { - homeAddress: {id: 'address_id', - fetch: true} + homeAddress: { + id: 'address_id', + fetch: true + } }; schema.Employee.children = { - roles: {id: 'employee_id', fetch: true} + roles: { id: 'employee_id', fetch: true } }; schema.Employee.enableChangeTracking = true; schema.Role = {}; schema.Role.table = 'tx_role'; + schema.Role.audit = 'v2'; schema.Role.parents = { - employee: {id: 'employee_id'} + employee: { id: 'employee_id' } }; schema.Role.children = { - department: {id: 'role_id'} + department: { id: 'role_id' } }; schema.Address.parents = { - phone: {id: 'phone_id'} + phone: { id: 'phone_id' } }; Phone = PersistObjectTemplate.create('Phone', { - number: {type: String} + number: { type: String } }); Address = PersistObjectTemplate.create('Address', { - city: {type: String}, - state: {type: String}, - phone: {type: Phone} + city: { type: String }, + state: { type: String }, + phone: { type: Phone } }); Role = PersistObjectTemplate.create('Role', { - name: {type:String}, + name: { type: String }, }); Employee = PersistObjectTemplate.create('Employee', { - name: {type: String, value: 'Test Employee'}, - homeAddress: {type: Address}, - roles: {type: Array, of:Role, value: []}, - dob: {type: Date}, - customObj: {type: Object}, - isMarried: {type: Boolean} + name: { type: String, value: 'Test Employee' }, + homeAddress: { type: Address }, + roles: { type: Array, of: Role, value: [] }, + dob: { type: Date }, + customObj: { type: Object }, + isMarried: { type: Boolean } }); Role.mixin({ - employee: {type: Employee} + employee: { type: Employee } }); var emp = new Employee(); var add = new Address(); @@ -137,13 +136,17 @@ describe('persist newapi tests', function () { return Promise.resolve(prepareData()); function prepareData() { - PersistObjectTemplate.performInjections(); return syncTable(Employee) .then(syncTable.bind(this, Address)) .then(syncTable.bind(this, Phone)) .then(syncTable.bind(this, Role)) .then(addConstraint.bind(this)) - .then(createRecords.bind(this)); + .then(addDateFields.bind(this)) + .then(addTriggers.bind(this)) + .then(createRecords.bind(this)) + .catch(e => { + console.log(e); + }); function syncTable(template) { @@ -151,14 +154,154 @@ describe('persist newapi tests', function () { } function addConstraint() { - return knex.raw('ALTER TABLE tx_role ADD CONSTRAINT namechk CHECK (char_length(name) <= 50);') + // return knex.raw('ALTER TABLE tx_role ADD CONSTRAINT namechk CHECK (char_length(name) <= 50);') + } + + function addDateFields() { + return knex.raw(` DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "createdTime" TIMESTAMP with time zone'; + EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "lastUpdatedTime" TIMESTAMP with time zone'; + END LOOP; + END $$;`); + } + + async function addTriggers() { + await Promise.all([knex.raw(` + CREATE OR REPLACE FUNCTION public.createddate_trigger() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + DECLARE columns text; + table_exists bool; + begin + + --using now as we track timezone in date fields.. + NEW."createdTime" = now(); + NEW."lastUpdatedTime" = now(); + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s_history'' + ); + ', TG_TABLE_NAME ) into table_exists; + + + + if POSITION('_history' in TG_TABLE_NAME) = 0 and table_exists then + EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') + FROM pg_attribute c1 + where c1.attrelid = ''%s''::regclass + AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; + + + execute format( + ' INSERT INTO %1$s_history ( _id_history, %2$s ) + values (md5(random()::text || clock_timestamp()::text)::uuid, $1.*) + ', TG_TABLE_NAME, columns) using new; + + end if; + RETURN NEW; + END + $function$ + ; + + + +`), knex.raw(` + CREATE OR REPLACE FUNCTION public.modifieddate_trigger() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + DECLARE columns text; + table_exists bool; + begin + + --using now as we track timezone in date fields.. + NEW."lastUpdatedTime" = now(); + + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s_history'' + ); + ', TG_TABLE_NAME ) into table_exists; + + + if POSITION('_history' in TG_TABLE_NAME) = 0 AND table_exists then + EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') + FROM pg_attribute c1 + where c1.attrelid = ''%s''::regclass + AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; + + + execute format( + ' INSERT INTO %1$s_history ( _id_history, %2$s ) + values (md5(random()::text || clock_timestamp()::text)::uuid, $1.*) + ', TG_TABLE_NAME, columns) using new; + + end if; + RETURN NEW; + END + $function$ + ; + + `)]); + + return Promise.all([knex.raw(` + DO $$ DECLARE + r RECORD; + table_exists bool; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s'' + ); + ', r.tablename ) into table_exists; + + if POSITION('_history' in r.tablename) = 0 AND table_exists then + EXECUTE 'create trigger createddate_inserttrigger before + insert + on + ' || quote_ident(r.tablename) || ' for each row execute procedure createddate_trigger();'; + end if; + END LOOP; + END $$; + `), + knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + if POSITION('_history' in r.tablename) = 0 then + EXECUTE 'create trigger modifieddate_updatetrigger before + update + on + ' || quote_ident(r.tablename) || ' for each row execute procedure modifieddate_trigger();'; + end if; + END LOOP; + END $$; + `)]); } function createRecords() { - var tx = PersistObjectTemplate.beginDefaultTransaction(); + var tx = PersistObjectTemplate.beginDefaultTransaction(); - return emp.persist({transaction: tx, cascade: false}).then(function() { - return PersistObjectTemplate.commit({transaction: tx, notifyChanges: true}).then(function() { + return emp.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: true }).then(function () { empId = emp._id; addressId = add._id; phoneId = phone._id; @@ -168,81 +311,82 @@ describe('persist newapi tests', function () { } }); - afterEach('remove tables and after each test', function() { - return Promise.all([ - knex.schema.dropTableIfExists('tx_employee') - .then(function () { - return knex.schema.dropTableIfExists('tx_address') - }).then(function () { - return knex.schema.dropTableIfExists('tx_phone') - }).then(function () { - return knex.schema.dropTableIfExists('tx_department') - }).then(function () { - return knex.schema.dropTableIfExists('tx_role') - }), - knex.schema.dropTableIfExists(schemaTable)]); + afterEach('remove tables and after each test', function () { + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); }); - it('persistorFetchById without fetch spec should not return the records', function () { - return Employee.persistorFetchById(empId, { fetch: {homeAddress: false}, enableChangeTracking: true}) - .then(function(employee) { - expect(employee.homeAddress).is.equal(null); - }); - }); + // it('persistorFetchById without fetch spec should not return the records', function () { + // return Employee.persistorFetchById(empId, { fetch: { homeAddress: false }, enableChangeTracking: true }) + // .then(function (employee) { + // expect(employee.homeAddress).is.equal(null); + // }); + // }); - it('persistorFetchById with fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { - expect(employee.homeAddress._id).is.equal(addressId); - expect(employee.homeAddress.phone).is.equal(null); - }); - }); + // it('persistorFetchById with fetch spec should return the records', function () { + // return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { + // expect(employee.homeAddress._id).is.equal(addressId); + // expect(employee.homeAddress.phone).is.equal(null); + // }); + // }); it('persistorFetchById with fetch spec with type projections', function () { - return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}, projection: { Address: ['city'], Role: ['name'], Phone: ['']}}).then(function(employee) { - expect(employee.homeAddress.state).is.equal(undefined); - expect(employee.homeAddress.city).is.equal('New York'); - expect(employee.homeAddress.phone.number).is.equal(undefined); - expect(employee.homeAddress.phone._id).is.equal(phoneId); - }); + return Employee.persistorFetchById(empId, + { + asOfDate: new Date(), + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }).then(function (employee) { + expect(employee.homeAddress.state).is.equal(undefined); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.homeAddress.phone.number).is.equal(undefined); + expect(employee.homeAddress.phone._id).is.equal(phoneId); + }) }); it('persistorFetchById with fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: false } }, roles: true } }).then(function (employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone).is.equal(null); }); }); it('persistorFetchById with multiple level fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(employee) { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone._id).is.equal(phoneId); }); }); it('fetch without fetch spec should not return the records', function () { - return Employee.persistorFetchByQuery({_id: empId}, {fetch: {homeAddress: false}}).then(function(employee) { + return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { homeAddress: false } }).then(function (employee) { expect(employee[0].homeAddress).is.equal(null); - }).catch(function(err) { + }).catch(function (err) { expect(err).not.equal(null); }); }); it('fetch with fetch spec should return the records', function () { - return Employee.persistorFetchByQuery({_id: empId}, {fetch: {homeAddress: true}, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: {name: 1} }).then(function(employee) { + return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { homeAddress: true }, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: { name: 1 } }).then(function (employee) { expect(employee[0].homeAddress._id).is.equal(addressId); expect(employee[0].homeAddress.phone).is.equal(null); }); }); it('persistorCountByQuery counts records properly', function () { - return Employee.persistorCountByQuery({_id: empId}, {fetch: {homeAddress: true}, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: {name: 1} }).then(function(count) { + return Employee.persistorCountByQuery({ _id: empId }, { fetch: { homeAddress: true }, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: { name: 1 } }).then(function (count) { expect(count).to.equal(1); }); }); it('persistorFetchByQuery to check the fetchSpec cache', function () { - return Employee.persistorFetchByQuery({_id: empId}, { + return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { roles: true } @@ -259,7 +403,7 @@ describe('persist newapi tests', function () { }) function actualTest() { - return Employee.persistorFetchByQuery({_id: empId}, { + return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { name: true } @@ -269,48 +413,50 @@ describe('persist newapi tests', function () { }); it('Multiple fetch calls to check the validFetchSpec cache', function () { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) - .then(function(employee) { - expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); - return employee.fetchReferences({fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function() { - expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(2); + return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) + .then(function (employee) { + expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); + return employee.fetchReferences({ fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function () { + expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(2); + }); }); - }); }); it('Multiple fetch calls with the same fetch string to check the validFetchSpec cache', function () { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) - .then(function() { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) + .then(function () { expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); - return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}).then(function() { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }).then(function () { expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(1); }); }); }); it('fetch with fetch with multiple levels should return the records', function () { - return Employee.persistorFetchByQuery({_id: empId}, { - fetch: { homeAddress: {fetch: {phone: false}}, - roles: true}, + return Employee.persistorFetchByQuery({ _id: empId }, { + fetch: { + homeAddress: { fetch: { phone: false } }, + roles: true + }, logger: PersistObjectTemplate.logger - }, 0, 5, true, {}, {customOptions: 'custom'}) - .then(function(employee) { + }, 0, 5, true, {}, { customOptions: 'custom' }) + .then(function (employee) { expect(employee[0].homeAddress._id).is.equal(addressId); expect(employee[0].homeAddress.phone).is.equal(null); }); }); it('persistorFetchById with multiple level fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(employee) { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone._id).is.equal(phoneId); }); }); it('persistorFetchByQuery with fetch spec should return the records and also load the child objects in the calling object', function () { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) - .then(function(employee) { - return employee.fetchReferences({fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(obj) { + return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) + .then(function (employee) { + return employee.fetchReferences({ fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (obj) { expect(obj.homeAddress._id).is.equal(addressId); expect(employee.homeAddress._id).is.equal(addressId); }) @@ -328,10 +474,10 @@ describe('persist newapi tests', function () { emp1.name = 'Ravi1'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginDefaultTransaction(); - return emp1.persist({transaction: tx, cascade: false}).then(function() { - return PersistObjectTemplate.commit().then(function() { - return Address.countFromPersistWithQuery().then(function(count) { + var tx = PersistObjectTemplate.beginDefaultTransaction(); + return emp1.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit().then(function () { + return Address.countFromPersistWithQuery().then(function (count) { expect(count).to.equal(2); }); }); @@ -350,10 +496,10 @@ describe('persist newapi tests', function () { emp1.name = 'Ravi1'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({transaction: tx, cascade: true}).then(function() { - return PersistObjectTemplate.commit().then(function() { - return Address.countFromPersistWithQuery().then(function(count) { + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({ transaction: tx, cascade: true }).then(function () { + return PersistObjectTemplate.commit().then(function () { + return Address.countFromPersistWithQuery().then(function (count) { expect(count).to.equal(1); }); }); @@ -365,12 +511,12 @@ describe('persist newapi tests', function () { var emp1 = new Employee(); return Promise.resolve() .then(actualTest) - .catch(function(error) { + .catch(function (error) { expect(error.message).to.contain('Additional properties not allowed'); }); function actualTest() { - return emp1.persist({transaction: null, cascade: true, unknown: false}).then(function() { + return emp1.persist({ transaction: null, cascade: true, unknown: false }).then(function () { throw new Error('should not reach here'); }) } @@ -388,10 +534,10 @@ describe('persist newapi tests', function () { emp1.name = 'RaviNotSaved'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({transaction: tx, cascade: false}).then(function() { - return PersistObjectTemplate.commit().then(function() { - return Employee.persistorFetchByQuery({name: 'RaviNotSaved'}).then(function(employees) { + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit().then(function () { + return Employee.persistorFetchByQuery({ name: 'RaviNotSaved' }).then(function (employees) { expect(employees.length).to.equal(0); }); }); @@ -402,11 +548,11 @@ describe('persist newapi tests', function () { for (let i = 0; i < 1000; ++i) { var phoneTemp = new Phone(); phoneTemp.number = `${i}`; - - var tx = PersistObjectTemplate.beginTransaction(); - await phoneTemp.persist({transaction: tx, cascade: false}); - await PersistObjectTemplate.commit({transaction: tx}); - const phones = await Phone.persistorFetchByQuery({number: `${i}`}); + + var tx = PersistObjectTemplate.beginTransaction(); + await phoneTemp.persist({ transaction: tx, cascade: false }); + await PersistObjectTemplate.commit({ transaction: tx }); + const phones = await Phone.persistorFetchByQuery({ number: `${i}` }); expect(phones.length).to.equal(1); } }); @@ -421,13 +567,13 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'LoadObjectForNotificationCheck'; emp1.homeAddress = add1; - var dob1 = new Date('01/01/1975'); - emp1.customObj = {name: 'testName', dob: JSON.stringify(dob1)}; + var dob1 = new Date('01/01/1975'); + emp1.customObj = { name: 'testName', dob: JSON.stringify(dob1) }; emp1.dob = dob1; emp1.isMarried = true; - var tx = PersistObjectTemplate.beginTransaction(); - tx.postSave = function(txn, _logger, changes, queries) { + var tx = PersistObjectTemplate.beginTransaction(); + tx.postSave = function (txn, _logger, changes, queries) { expect(queries['Employee'].queries.length).to.equal(1); expect(queries['Address'].queries.length).to.equal(1); expect(queries['Address'].tableType).to.equal('reference_data'); @@ -436,20 +582,20 @@ describe('persist newapi tests', function () { const insertScript = emp1.getInsertScript(); expect(insertScript).to.contain('insert into "tx_employee"'); - return emp1.persist({transaction: tx, cascade: false}).then(function() { - return PersistObjectTemplate.commit({transaction: tx, notifyQueries: true}).then(function() { - return Employee.persistorFetchByQuery({name: 'LoadObjectForNotificationCheck'}, {enableChangeTracking: true}).then(function(employees) { + return emp1.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit({ transaction: tx, notifyQueries: true }).then(function () { + return Employee.persistorFetchByQuery({ name: 'LoadObjectForNotificationCheck' }, { enableChangeTracking: true }).then(function (employees) { expect(employees.length).to.equal(1); var emp = employees[0]; emp.dob = new Date('01/01/1976'); - emp.customObj = {name: 'testName', dob: JSON.stringify (emp.dob)}; + emp.customObj = { name: 'testName', dob: JSON.stringify(emp.dob) }; emp.isMarried = false; emp.homeAddress = new Address(); - var innerTxn = PersistObjectTemplate.beginTransaction(); + var innerTxn = PersistObjectTemplate.beginTransaction(); emp.setDirty(innerTxn); - innerTxn.postSave = function(txn, _logger, changes, queries) { + innerTxn.postSave = function (txn, _logger, changes, queries) { expect(Object.keys(changes)).to.contain('Employee'); expect(Object.keys(changes.Employee[0])).to.contain('primaryKey'); expect(changes.Employee[0].properties[0].name).to.equal('homeAddress'); @@ -457,7 +603,7 @@ describe('persist newapi tests', function () { var empNew = new Employee(); empNew.setDirty(txn); }; - return PersistObjectTemplate.commit({transaction: innerTxn, notifyChanges: true, notifyQueries: true}); + return PersistObjectTemplate.commit({ transaction: innerTxn, notifyChanges: true, notifyQueries: true }); }); }); }) @@ -473,11 +619,11 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'RaviNotSaved'; emp1.homeAddress = add1; - - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({transaction: tx, cascade: false}).then(function() { - return PersistObjectTemplate.commit({transaction: tx}).then(function() { - return Employee.persistorFetchByQuery({name: 'RaviNotSaved'}).then(function(employees) { + + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit({ transaction: tx }).then(function () { + return Employee.persistorFetchByQuery({ name: 'RaviNotSaved' }).then(function (employees) { expect(employees.length).to.equal(1); }); }); @@ -492,7 +638,7 @@ describe('persist newapi tests', function () { .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) + return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) } function setTestObjects(employee) { @@ -500,11 +646,11 @@ describe('persist newapi tests', function () { add = employee.homeAddress; } function realTest() { - var notifyChanges; - var tx = PersistObjectTemplate.beginTransaction(); - add.persistDelete({transaction: tx}); - emp.persistDelete({transaction: tx}); - return PersistObjectTemplate.commit({transaction: tx, notifyChanges: notifyChanges, notifyQueries: true}); + var notifyChanges; + var tx = PersistObjectTemplate.beginTransaction(); + add.persistDelete({ transaction: tx }); + emp.persistDelete({ transaction: tx }); + return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: notifyChanges, notifyQueries: true }); } function createFKs() { @@ -513,49 +659,43 @@ describe('persist newapi tests', function () { }); it('calling delete with transaction', function () { - return createFKs() - .then(loadEmployee.bind(this)) + return loadEmployee() .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) + return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) } function realTest() { - var tx = PersistObjectTemplate.beginTransaction(); - Employee.persistorDeleteByQuery({name: 'Ravi'}, {transaction: tx}); - Address.persistorDeleteByQuery({city: 'New York'}, {transaction: tx}) - return PersistObjectTemplate.commit({transaction: tx}).then(function() { - return Employee.persistorFetchByQuery({name: 'Ravi'}).then(function(employees) { + var tx = PersistObjectTemplate.beginTransaction(); + Employee.persistorDeleteByQuery({ name: 'Ravi' }, { transaction: tx }); + Address.persistorDeleteByQuery({ city: 'New York' }, { transaction: tx }) + return PersistObjectTemplate.commit({ transaction: tx }).then(function () { + return Employee.persistorFetchByQuery({ name: 'Ravi' }).then(function (employees) { expect(employees.length).to.equal(0); }) }); } - function createFKs() { - return knex.raw('ALTER TABLE public.tx_employee ADD CONSTRAINT fk_tx_employee_address FOREIGN KEY (address_id) references public.tx_address("_id") deferrable initially deferred'); - } + }); it('calling delete without transaction', function () { - return createFKs() - .then(loadEmployee.bind(this)) + return loadEmployee() .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) + return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) } async function realTest() { - await Employee.persistorDeleteByQuery({name: 'Ravi'}); - return Employee.persistorFetchByQuery({name: 'Ravi'}).then(function(employees) { + await Employee.persistorDeleteByQuery({ name: 'Ravi' }); + return Employee.persistorFetchByQuery({ name: 'Ravi' }).then(function (employees) { expect(employees.length).to.equal(0); }) } - function createFKs() { - return knex.raw('ALTER TABLE public.tx_employee ADD CONSTRAINT fk_tx_employee_address FOREIGN KEY (address_id) references public.tx_address("_id") deferrable initially deferred'); - } + }); it('update conflict should revert the version', function () { @@ -564,16 +704,16 @@ describe('persist newapi tests', function () { .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchByQuery({name: 'Ravi'}) + return Employee.persistorFetchByQuery({ name: 'Ravi' }) } async function realTest(emps) { - var tx = PersistObjectTemplate.beginTransaction(); + var tx = PersistObjectTemplate.beginTransaction(); emps[0].setDirty(tx); emps[1].__version__ = emps[1].__version__ + 1; emps[1].setDirty(tx); try { - await PersistObjectTemplate.commit({transaction: tx}); + await PersistObjectTemplate.commit({ transaction: tx }); } catch (err) { expect(emps[0].__version__).to.equal('1'); @@ -591,13 +731,13 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'Ravi'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - emp1.persist({transaction: tx, cascade: false}); - return PersistObjectTemplate.commit({transaction: tx}); + var tx = PersistObjectTemplate.beginTransaction(); + emp1.persist({ transaction: tx, cascade: false }); + return PersistObjectTemplate.commit({ transaction: tx }); } }); - it('create an object copy and save, _id only assigned at db save', async () => { + it('create an object copy and save, _id only assigned at db save', async () => { var emp1 = new Employee(); var add1 = new Address(); var phone1 = new Phone(); @@ -617,27 +757,27 @@ describe('persist newapi tests', function () { emp1.roles.push(roleTest); emp1.roles.push(roleTest2); - - var tx = PersistObjectTemplate.beginTransaction(); - await emp1.persist({transaction: tx, cascade: true}); - await PersistObjectTemplate.commit({transaction: tx}); - const employee = await Employee.persistorFetchByQuery({name: 'EmployeeObject'}); - - const clonedEmp = employee[0].createCopy(function(obj, prop, template){ + + var tx = PersistObjectTemplate.beginTransaction(); + await emp1.persist({ transaction: tx, cascade: true }); + await PersistObjectTemplate.commit({ transaction: tx }); + const employee = await Employee.persistorFetchByQuery({ name: 'EmployeeObject' }); + + const clonedEmp = employee[0].createCopy(function (obj, prop, template) { return null; }); - + expect(clonedEmp._id).to.equal(undefined); expect(clonedEmp.roles[0]._id).to.equal(undefined); expect(clonedEmp.roles[1]._id).to.equal(undefined); clonedEmp.name = 'ClonedEmployeeObject'; - var tx = PersistObjectTemplate.beginTransaction(); - await clonedEmp.persist({transaction: tx, cascade: true}); - await PersistObjectTemplate.commit({transaction: tx}); - - const employeesClonedSaved = await Employee.persistorFetchByQuery({name: 'ClonedEmployeeObject'}); - + var tx = PersistObjectTemplate.beginTransaction(); + await clonedEmp.persist({ transaction: tx, cascade: true }); + await PersistObjectTemplate.commit({ transaction: tx }); + + const employeesClonedSaved = await Employee.persistorFetchByQuery({ name: 'ClonedEmployeeObject' }); + expect(employeesClonedSaved.length).to.equal(1); expect(employeesClonedSaved[0].roles[0]._id).to.equal(clonedEmp?.roles[0]._id); expect(employeesClonedSaved[0].roles[1]._id).to.equal(clonedEmp?.roles[1]._id); diff --git a/components/persistor/test/persist_parent_subset.js b/components/persistor/test/persist_parent_subset.js index b758f9a2..c764c559 100644 --- a/components/persistor/test/persist_parent_subset.js +++ b/components/persistor/test/persist_parent_subset.js @@ -10,7 +10,7 @@ chai.use(chaiAsPromised); var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); -var Promise = require('bluebird'); + var knexInit = require('knex'); var knex; diff --git a/components/persistor/test/persist_polymorphic.js b/components/persistor/test/persist_polymorphic.js index a12f3919..0f8d64c8 100644 --- a/components/persistor/test/persist_polymorphic.js +++ b/components/persistor/test/persist_polymorphic.js @@ -10,7 +10,7 @@ chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); + var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); @@ -523,158 +523,158 @@ describe('type mapping tests for parent/child relations', function () { return knex.destroy(); }); - it('Parent type with an associated child will add add the fields from the child tables to the parent table', function (done) { - return PersistObjectTemplate.createKnexTable(Parent).then(function () { - return PersistObjectTemplate.checkForKnexTable(Parent).should.eventually.equal(true); - }).should.notify(done); - }); - - it('Both parent and child index definitions are added to the parent table', function () { - return PersistObjectTemplate.createKnexTable(Parent_Idx).then(function () { - return knex.schema.table('Parent_Idx', function (table) { - table.dropIndex([], 'idx_parent_idx_id_name'); - }).should.eventually.have.property('command') - }); - }); - - it('When trying to create child table, system should create the parent table', function () { - return PersistObjectTemplate.createKnexTable(ChildToCreate).then(function () { - return Promise.all([PersistObjectTemplate.checkForKnexTable(ChildCreatesThisParent, 'ChildCreatesThisParent').should.eventually.equal(true), - PersistObjectTemplate.checkForKnexTable(ChildToCreate, 'ChildToCreate').should.eventually.equal(false) - ]); - }) - }); + // it('Parent type with an associated child will add add the fields from the child tables to the parent table', function (done) { + // return PersistObjectTemplate.createKnexTable(Parent).then(function () { + // return PersistObjectTemplate.checkForKnexTable(Parent).should.eventually.equal(true); + // }).should.notify(done); + // }); - // it('When trying to create child table, system should create the parent table and the corresonding indexes in the object graph must be added to the table', function () { - // return PersistObjectTemplate.createKnexTable(ChildToCreate1).then(function () { - // return knex.schema.table('ChildCreatesThisParent1', function (table) { - // table.dropIndex([], 'idx_childcreatesthisparent1_dob'); + // it('Both parent and child index definitions are added to the parent table', function () { + // return PersistObjectTemplate.createKnexTable(Parent_Idx).then(function () { + // return knex.schema.table('Parent_Idx', function (table) { + // table.dropIndex([], 'idx_parent_idx_id_name'); // }).should.eventually.have.property('command') - // }) + // }); // }); - it('Creating a parent with children defined with multilevel inheritance', function () { - return PersistObjectTemplate.createKnexTable(ParentMulteLevel1).then(function () { - return PersistObjectTemplate.checkForKnexColumnType(ParentMulteLevel1, 'ScdLevel').should.eventually.equal('boolean'); - }) - }); - - // it('Multilevel inheritance with indexes defined at different levels', function () { - // return PersistObjectTemplate.createKnexTable(ParentMulteLevelIndx1).then(function () { - // return knex.schema.table('ParentMulteLevelIndx1', function (table) { - // table.dropIndex([], 'idx_parentmultelevelindx1_dob'); - // }).should.eventually.have.property('command') + // it('When trying to create child table, system should create the parent table', function () { + // return PersistObjectTemplate.createKnexTable(ChildToCreate).then(function () { + // return Promise.all([PersistObjectTemplate.checkForKnexTable(ChildCreatesThisParent, 'ChildCreatesThisParent').should.eventually.equal(true), + // PersistObjectTemplate.checkForKnexTable(ChildToCreate, 'ChildToCreate').should.eventually.equal(false) + // ]); // }) // }); - it('Multilevel inheritance with multiple children at the same level', function () { - return PersistObjectTemplate.createKnexTable(ParentWithMultiChildAttheSameLevel).then(function () { - return PersistObjectTemplate.checkForKnexTable(ParentWithMultiChildAttheSameLevel).should.eventually.equal(true); - }) - }); - it('Multilevel inheritance with multiple children at the multiple levels', function () { - return PersistObjectTemplate.createKnexTable(Scenario_2_ParentWithMultiChildAttheSameLevel).then(function () { - return PersistObjectTemplate.checkForKnexTable(Scenario_2_ParentWithMultiChildAttheSameLevel).should.eventually.equal(true); - }) - }); - - it('Multilevel inheritance with multiple children at the multiple levels', function () { - return PersistObjectTemplate.createKnexTable(ParentWithMultiChildAttheSameLevelWithIndexes).then(function () { - return PersistObjectTemplate.checkForKnexTable(ParentWithMultiChildAttheSameLevelWithIndexes).should.eventually.equal(true); - }) - }); + // // it('When trying to create child table, system should create the parent table and the corresonding indexes in the object graph must be added to the table', function () { + // // return PersistObjectTemplate.createKnexTable(ChildToCreate1).then(function () { + // // return knex.schema.table('ChildCreatesThisParent1', function (table) { + // // table.dropIndex([], 'idx_childcreatesthisparent1_dob'); + // // }).should.eventually.have.property('command') + // // }) + // // }); + + // it('Creating a parent with children defined with multilevel inheritance', function () { + // return PersistObjectTemplate.createKnexTable(ParentMulteLevel1).then(function () { + // return PersistObjectTemplate.checkForKnexColumnType(ParentMulteLevel1, 'ScdLevel').should.eventually.equal('boolean'); + // }) + // }); + // // it('Multilevel inheritance with indexes defined at different levels', function () { + // // return PersistObjectTemplate.createKnexTable(ParentMulteLevelIndx1).then(function () { + // // return knex.schema.table('ParentMulteLevelIndx1', function (table) { + // // table.dropIndex([], 'idx_parentmultelevelindx1_dob'); + // // }).should.eventually.have.property('command') + // // }) + // // }); + // it('Multilevel inheritance with multiple children at the same level', function () { + // return PersistObjectTemplate.createKnexTable(ParentWithMultiChildAttheSameLevel).then(function () { + // return PersistObjectTemplate.checkForKnexTable(ParentWithMultiChildAttheSameLevel).should.eventually.equal(true); + // }) + // }); + // it('Multilevel inheritance with multiple children at the multiple levels', function () { + // return PersistObjectTemplate.createKnexTable(Scenario_2_ParentWithMultiChildAttheSameLevel).then(function () { + // return PersistObjectTemplate.checkForKnexTable(Scenario_2_ParentWithMultiChildAttheSameLevel).should.eventually.equal(true); + // }) + // }); - it('Adding a child with index to a parent and synchronize.', function () { - var childSynchronize = parentSynchronize.extend('childSynchronize', { - init: function() { - this.id = 12312; - this.name = 'Child'; - Parent.call(this); - }, - dob: {type: Date} - }); + // it('Multilevel inheritance with multiple children at the multiple levels', function () { + // return PersistObjectTemplate.createKnexTable(ParentWithMultiChildAttheSameLevelWithIndexes).then(function () { + // return PersistObjectTemplate.checkForKnexTable(ParentWithMultiChildAttheSameLevelWithIndexes).should.eventually.equal(true); + // }) + // }); - schema.childSynchronize = {}; - schema.childSynchronize.indexes = JSON.parse('[{"name": "single_index","def": {"columns": ["dob"],"type": "unique"}}]'); - PersistObjectTemplate._verifySchema(); + // it('Adding a child with index to a parent and synchronize.', function () { + // var childSynchronize = parentSynchronize.extend('childSynchronize', { + // init: function() { + // this.id = 12312; + // this.name = 'Child'; + // Parent.call(this); + // }, + // dob: {type: Date} + // }); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(childSynchronize).then(function () { - return PersistObjectTemplate.checkForKnexTable(parentSynchronize).should.eventually.equal(true).then(function() { - schema.childSynchronize.indexes = JSON.parse('[{"name": "scd_index","def": {"columns": ["name"],"type": "unique"}}]'); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(childSynchronize); - }) - }) + // schema.childSynchronize = {}; - }); + // schema.childSynchronize.indexes = JSON.parse('[{"name": "single_index","def": {"columns": ["dob"],"type": "unique"}}]'); + // PersistObjectTemplate._verifySchema(); - it('Create a new table and check if the comments added to the fields are included in the database', async () => { - var NewTableWithComments = PersistObjectTemplate.create('NewTableWithComments', { - id: {type: Number}, - name: {type: String, value: 'Test Parent', comment: 'comment on a new table...'}, - init: function (id, name) { - this.id = id; - this.name = name; - } - }); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(childSynchronize).then(function () { + // return PersistObjectTemplate.checkForKnexTable(parentSynchronize).should.eventually.equal(true).then(function() { + // schema.childSynchronize.indexes = JSON.parse('[{"name": "scd_index","def": {"columns": ["name"],"type": "unique"}}]'); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(childSynchronize); + // }) + // }) - schema.NewTableWithComments = {documentOf: 'pg/NewTableWithComments'}; - PersistObjectTemplate._verifySchema(); - await PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments); - const results = await knex('pg_catalog.pg_description') - .count() - .where('description', 'like', '%comment on a new table...%') - expect(results).to.deep.include({ count: '1' }); - }); + // }); - it('table notification should only work with function callbacks', function () { - var NewTableWithComments1 = PersistObjectTemplate.create('NewTableWithComments1', {}); - schema.NewTableWithComments1 = {documentOf: 'pg/NewTableWithComments1'}; - PersistObjectTemplate._verifySchema(); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments1, {}).catch(function(e) { - expect(e.message).to.equal('persistor can only notify the table changes through a callback'); - }); + // it('Create a new table and check if the comments added to the fields are included in the database', async () => { + // var NewTableWithComments = PersistObjectTemplate.create('NewTableWithComments', { + // id: {type: Number}, + // name: {type: String, value: 'Test Parent', comment: 'comment on a new table...'}, + // init: function (id, name) { + // this.id = id; + // this.name = name; + // } + // }); + + // schema.NewTableWithComments = {documentOf: 'pg/NewTableWithComments'}; + // PersistObjectTemplate._verifySchema(); + // await PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments); + // const results = await knex('pg_catalog.pg_description') + // .count() + // .where('description', 'like', '%comment on a new table...%') + // expect(results).to.deep.include({ count: '1' }); + // }); - }); + // it('table notification should only work with function callbacks', function () { + // var NewTableWithComments1 = PersistObjectTemplate.create('NewTableWithComments1', {}); - it('field change notification should only work with function callbacks', function () { - var NewTableWithComments1 = PersistObjectTemplate.create('NewTableWithComments1', {}); + // schema.NewTableWithComments1 = {documentOf: 'pg/NewTableWithComments1'}; + // PersistObjectTemplate._verifySchema(); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments1, {}).catch(function(e) { + // expect(e.message).to.equal('persistor can only notify the table changes through a callback'); + // }); - schema.NewTableWithComments1 = {documentOf: 'pg/NewTableWithComments1'}; - PersistObjectTemplate._verifySchema(); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments1) - .then(PersistObjectTemplate.synchronizeKnexTableFromTemplate.bind(PersistObjectTemplate, NewTableWithComments1, {})) - .catch(function(e) { - expect(e.message).to.equal('persistor can only notify the field changes through a callback'); - }); - }); + // }); - it('Check if parent schema entry is missing...', function () { + // it('field change notification should only work with function callbacks', function () { + // var NewTableWithComments1 = PersistObjectTemplate.create('NewTableWithComments1', {}); - var AddressSyncChecks = PersistObjectTemplate.create('AddressSyncChecks', {}); - var CustomerSyncChecks = PersistObjectTemplate.create('CustomerSyncChecks', {}); + // schema.NewTableWithComments1 = {documentOf: 'pg/NewTableWithComments1'}; + // PersistObjectTemplate._verifySchema(); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(NewTableWithComments1) + // .then(PersistObjectTemplate.synchronizeKnexTableFromTemplate.bind(PersistObjectTemplate, NewTableWithComments1, {})) + // .catch(function(e) { + // expect(e.message).to.equal('persistor can only notify the field changes through a callback'); + // }); + // }); - schema.AddressSyncChecks = {documentOf: 'pg/AddressSyncChecks'}; - schema.CustomerSyncChecks = {documentOf: 'pg/CustomerSyncChecks'}; - PersistObjectTemplate._verifySchema(); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(CustomerSyncChecks) - .then(function() { - CustomerSyncChecks.mixin({ - homeAddress: {type: AddressSyncChecks}, - alternateNames: {type: Array, of: String} - }); - - PersistObjectTemplate._verifySchema(); - return PersistObjectTemplate.synchronizeKnexTableFromTemplate(CustomerSyncChecks); - }).catch(function (e) { - expect(e.message).to.equal('CustomerSyncChecks.homeAddress is missing a parents schema entry'); - }); - }); + // it('Check if parent schema entry is missing...', function () { + + // var AddressSyncChecks = PersistObjectTemplate.create('AddressSyncChecks', {}); + // var CustomerSyncChecks = PersistObjectTemplate.create('CustomerSyncChecks', {}); + + // schema.AddressSyncChecks = {documentOf: 'pg/AddressSyncChecks'}; + // schema.CustomerSyncChecks = {documentOf: 'pg/CustomerSyncChecks'}; + // PersistObjectTemplate._verifySchema(); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(CustomerSyncChecks) + // .then(function() { + // CustomerSyncChecks.mixin({ + // homeAddress: {type: AddressSyncChecks}, + // alternateNames: {type: Array, of: String} + // }); + + // PersistObjectTemplate._verifySchema(); + // return PersistObjectTemplate.synchronizeKnexTableFromTemplate(CustomerSyncChecks); + // }).catch(function (e) { + // expect(e.message).to.equal('CustomerSyncChecks.homeAddress is missing a parents schema entry'); + // }); + // }); it('Adding a comment to an existing table', async () => { var ExistingTableWithComments = PersistObjectTemplate.create('ExistingTableWithComments', { @@ -701,84 +701,84 @@ describe('type mapping tests for parent/child relations', function () { expect(results).to.deep.include({ count: '1' }); }); - it('Adding a new field with comment to an existing table', async () => { - var ExistingTableWithAField = PersistObjectTemplate.create('ExistingTableWithAField', { - id: {type: Number}, - name: {type: String, value: 'Test Parent'}, - init: function (id, name) { - this.id = id; - this.name = name; - } - }); - - schema.ExistingTableWithAField = {documentOf: 'pg/ExistingTableWithAField'}; - PersistObjectTemplate._verifySchema(); + // it('Adding a new field with comment to an existing table', async () => { + // var ExistingTableWithAField = PersistObjectTemplate.create('ExistingTableWithAField', { + // id: {type: Number}, + // name: {type: String, value: 'Test Parent'}, + // init: function (id, name) { + // this.id = id; + // this.name = name; + // } + // }); + + // schema.ExistingTableWithAField = {documentOf: 'pg/ExistingTableWithAField'}; + // PersistObjectTemplate._verifySchema(); - await PersistObjectTemplate.synchronizeKnexTableFromTemplate(ExistingTableWithAField); - ExistingTableWithAField.mixin({ - newField: {type: String, value: 'Test Parent', comment: 'Adding a new field comment'} - }); - PersistObjectTemplate._verifySchema(); + // await PersistObjectTemplate.synchronizeKnexTableFromTemplate(ExistingTableWithAField); + // ExistingTableWithAField.mixin({ + // newField: {type: String, value: 'Test Parent', comment: 'Adding a new field comment'} + // }); + // PersistObjectTemplate._verifySchema(); - await PersistObjectTemplate.synchronizeKnexTableFromTemplate(ExistingTableWithAField, null, true); - const results = await knex('pg_catalog.pg_description') - .count() - .where('description', 'like', '%Adding a new field comment%'); - expect(results).to.deep.include({ count: '1' }); + // await PersistObjectTemplate.synchronizeKnexTableFromTemplate(ExistingTableWithAField, null, true); + // const results = await knex('pg_catalog.pg_description') + // .count() + // .where('description', 'like', '%Adding a new field comment%'); + // expect(results).to.deep.include({ count: '1' }); - }); - - it('Adding a foreign key refrence in children', function () { - var ObjectTemplate1 = require('@haventech/supertype').default; - var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); - var BaseTemplate_FK_on_Child = PersistObjectTemplate1.create('BaseTemplate_FK_on_Child', { - name: {type: String, value: 'Test Parent'} - }); + // }); - var Address_FK_on_Child = PersistObjectTemplate1.create('Address_FK_on_Child', { - street: {type: String} - }) + // it('Adding a foreign key refrence in children', function () { + // var ObjectTemplate1 = require('@haventech/supertype').default; + // var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); + // var BaseTemplate_FK_on_Child = PersistObjectTemplate1.create('BaseTemplate_FK_on_Child', { + // name: {type: String, value: 'Test Parent'} + // }); - var ChildTemplateLevel1 = BaseTemplate_FK_on_Child.extend('ChildTemplateLevel1', { - dob: {type: Date} - }); + // var Address_FK_on_Child = PersistObjectTemplate1.create('Address_FK_on_Child', { + // street: {type: String} + // }) - var ChildTemplateLevel2 = ChildTemplateLevel1.extend('ChildTemplateLevel2', { - dob: {type: Date}, - addresses: {type: Array, of: Address_FK_on_Child} - }); - var schema = {}; - schema.ChildTemplateLevel2 = {}; - schema.ChildTemplateLevel1 = {}; - schema.BaseTemplate_FK_on_Child = {}; - schema.BaseTemplate_FK_on_Child = { - documentOf: 'basetemplate_tbl', - children: { - addresses: {id: 'address_id'} - } - }; - schema.Address_FK_on_Child = {}; - schema.Address_FK_on_Child = {documentOf: 'address_ref_tbl'}; - - PersistObjectTemplate1.setDB(knex, PersistObjectTemplate1.DB_Knex); - PersistObjectTemplate1.setSchema(schema); - PersistObjectTemplate1.performInjections(); - PersistObjectTemplate1._verifySchema(); - }); + // var ChildTemplateLevel1 = BaseTemplate_FK_on_Child.extend('ChildTemplateLevel1', { + // dob: {type: Date} + // }); + + // var ChildTemplateLevel2 = ChildTemplateLevel1.extend('ChildTemplateLevel2', { + // dob: {type: Date}, + // addresses: {type: Array, of: Address_FK_on_Child} + // }); + // var schema = {}; + // schema.ChildTemplateLevel2 = {}; + // schema.ChildTemplateLevel1 = {}; + // schema.BaseTemplate_FK_on_Child = {}; + // schema.BaseTemplate_FK_on_Child = { + // documentOf: 'basetemplate_tbl', + // children: { + // addresses: {id: 'address_id'} + // } + // }; + // schema.Address_FK_on_Child = {}; + // schema.Address_FK_on_Child = {documentOf: 'address_ref_tbl'}; + + // PersistObjectTemplate1.setDB(knex, PersistObjectTemplate1.DB_Knex); + // PersistObjectTemplate1.setSchema(schema); + // PersistObjectTemplate1.performInjections(); + // PersistObjectTemplate1._verifySchema(); + // }); - it('getDB without setting database', function () { - var ObjectTemplate1 = require('@haventech/supertype').default; - var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); - expect(PersistObjectTemplate1.getDB.bind(this, 'pg')).to.throw('You must do PersistObjectTempate.setDB'); + // it('getDB without setting database', function () { + // var ObjectTemplate1 = require('@haventech/supertype').default; + // var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); + // expect(PersistObjectTemplate1.getDB.bind(this, 'pg')).to.throw('You must do PersistObjectTempate.setDB'); - }); + // }); - it('without schema..', function () { - var ObjectTemplate1 = require('@haventech/supertype').default; - var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); - var emptySchema = PersistObjectTemplate1._verifySchema(); - expect(emptySchema).to.be.an('undefined'); - }); + // it('without schema..', function () { + // var ObjectTemplate1 = require('@haventech/supertype').default; + // var PersistObjectTemplate1 = require('../dist/index.js')(ObjectTemplate1, null, ObjectTemplate1); + // var emptySchema = PersistObjectTemplate1._verifySchema(); + // expect(emptySchema).to.be.an('undefined'); + // }); diff --git a/components/persistor/test/persist_schema_indexdefchanges.js b/components/persistor/test/persist_schema_indexdefchanges.js index 3aa0ddeb..e893aca7 100644 --- a/components/persistor/test/persist_schema_indexdefchanges.js +++ b/components/persistor/test/persist_schema_indexdefchanges.js @@ -6,7 +6,7 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); -var Promise = require('bluebird'); +// var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); @@ -279,7 +279,7 @@ describe('index synchronization checks', function () { }) }); - it('synchronize the index definition and check if the index exists on the table by dropping the index', function () { + it('synchronize the index definition and check if the index exists on the table by dropping the index in indexdefchnages', function () { return PersistObjectTemplate.synchronizeKnexTableFromTemplate(IndexSyncTable).should.eventually.have.property('command').that.match(/INSERT/); }); diff --git a/components/persistor/test/persist_schema_updates.js b/components/persistor/test/persist_schema_updates.js index 5a686bfd..0d9121d4 100644 --- a/components/persistor/test/persist_schema_updates.js +++ b/components/persistor/test/persist_schema_updates.js @@ -6,7 +6,7 @@ var chai = require('chai'); var expect = require('chai').expect; var chaiAsPromised = require('chai-as-promised'); -var Promise = require('bluebird'); + chai.should(); chai.use(chaiAsPromised); @@ -244,7 +244,7 @@ describe('schema update checks', function () { }) }); - it('synchronize the index definition and check if the index exists on the table by dropping the index', function () { + it('synchronize the index definition and check if the index exists on the table by dropping the index in persist_schema_updates', function () { return PersistObjectTemplate.synchronizeKnexTableFromTemplate(IndexSyncTable).then(function () { return knex.schema.table('IndexSyncTable', function (table) { table.dropIndex([], 'idx_indexsynctable_name'); diff --git a/components/persistor/test/persist_transaction.js b/components/persistor/test/persist_transaction.js index 242d065e..f0da759e 100644 --- a/components/persistor/test/persist_transaction.js +++ b/components/persistor/test/persist_transaction.js @@ -8,7 +8,7 @@ chai.use(chaiAsPromised); var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); -var Promise = require('bluebird'); + var knexInit = require('knex'); var knex; var schema = {}; diff --git a/components/persistor/test/supertype/one-to-manychecks.ts b/components/persistor/test/supertype/one-to-manychecks.ts index 51dc7bd2..c582cfe7 100644 --- a/components/persistor/test/supertype/one-to-manychecks.ts +++ b/components/persistor/test/supertype/one-to-manychecks.ts @@ -13,7 +13,6 @@ import { expect } from 'chai'; import * as mocha from 'mocha'; import * as _ from 'underscore'; import {Employee} from "./Employee"; -import Promise = require('bluebird'); import {Responsibility} from "./Responsibility"; diff --git a/components/persistor/test/supertype/persist_banking_pgsql.ts b/components/persistor/test/supertype/persist_banking_pgsql.ts index 1396e1da..bb86cb91 100644 --- a/components/persistor/test/supertype/persist_banking_pgsql.ts +++ b/components/persistor/test/supertype/persist_banking_pgsql.ts @@ -15,7 +15,7 @@ import { expect } from 'chai'; import * as _ from 'underscore'; import {Customer} from "./Customer"; import {ExtendedCustomer} from "./ExtendedCustomer"; -import Promise = require('bluebird'); + import {Role} from "./Role"; import {Account} from "./Account"; import {Transaction, Xfer} from './Transaction'; @@ -876,7 +876,11 @@ describe('typescript tests: Banking from pgsql Example persist_banking_pgsql', f txn2Sam.firstName = 'txn2SamDead'; txn2Sam.setDirty(txn2); txn1.postSave = function () { - Promise.delay(100) + function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + sleep(100) .then(function () { // Update will not return because it is requesting a lock on Karen txn1Karen.persistTouch(txn1) // 3 update karen diff --git a/components/persistor/tsconfig.json b/components/persistor/tsconfig.json index 34d03d3c..61fad521 100644 --- a/components/persistor/tsconfig.json +++ b/components/persistor/tsconfig.json @@ -1,7 +1,8 @@ { "compilerOptions": { "outDir": "./dist", - "rootDir": "./" + "rootDir": "./", + "sourceMap": true, }, "extends": "./tsconfig.base.json", "include": [ From 07335d61a941ff66f08557497fdb781e6eac817f Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sun, 19 Feb 2023 18:34:57 -0500 Subject: [PATCH 02/14] using lastUpdatedTIme on history tables --- components/persistor/lib/knex/db.ts | 70 ++- .../persistor/lib/utils/PersistorUtils.js | 35 ++ .../persistor/lib/utils/PersistorUtils.ts | 4 + components/persistor/test/persist_banking.js | 2 +- .../persistor/test/persist_banking_pgsql.js | 7 +- .../persistor/test/persist_newapi_tests.js | 494 +++++++----------- .../test/persist_pointintime_tests.js | 365 +++++++++++++ .../test/supertype/persist_banking_pgsql.ts | 6 +- 8 files changed, 637 insertions(+), 346 deletions(-) create mode 100644 components/persistor/lib/utils/PersistorUtils.js create mode 100644 components/persistor/test/persist_pointintime_tests.js diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index 9fcc0363..678b05c8 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -22,28 +22,34 @@ module.exports = function (PersistObjectTemplate) { PersistObjectTemplate.getPOJOsFromKnexQuery = function (template, joins, queryOrChains, options, map, logger, projection) { var tableName = this.dealias(template.__table__); - var historyTableName, historyTableAlias + var historyTableName, historyTableAlias; + var historySeqKeys = []; if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { historyTableName = tableName + '_history'; - queryOrChains["createdTime"] = {$lte: PersistorCtx.ExecutionCtx?.AsOfDate}; + queryOrChains["lastUpdatedTime"] = {$lte: PersistorCtx.ExecutionCtx?.AsOfDate}; } - + var knex = this.getDB(this.getDBAlias(template.__table__)).connection(tableName); const functionName = 'getPOJOsFromKnexQuery'; // tack on outer joins. All our joins are outerjoins and to the right. There could in theory be // foreign keys pointing to rows that no longer exists - var select = knex.select(getColumnNames.bind(this, template)()).from((historyTableName || tableName) + ' as ' + tableName); + var select = knex.select(getColumnNames.bind(this, template, historySeqKeys)()).from((historyTableName || tableName) + ' as ' + tableName); joins.forEach(function (join) { let joinTable = this.dealias(join.template.__table__); let historyJoinTable, additionalCondition; if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { - historyTableName = joinTable + '_history'; - additionalCondition = ' and ' + join.alias + '.' + '"createdTime" <= ' + PersistorCtx.ExecutionCtx?.AsOfDate; + historyJoinTable = joinTable + '_history'; } - select = select.leftOuterJoin( (historyJoinTable || joinTable) + ' as ' + join.alias, - join.alias + '.' + join.parentKey, - this.dealias(template.__table__) + '.' + join.childKey + (additionalCondition ? additionalCondition : '')); + const parentKey = this.dealias(template.__table__) + '.' + join.childKey + select = select.leftOuterJoin( (historyJoinTable || joinTable) + ' as ' + join.alias, function() { + const cond = this.on(join.alias + '.' + join.parentKey, '=', parentKey) + if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { + const dt = PersistorCtx.ExecutionCtx?.AsOfDate + cond.andOn(knex.client.raw(join.alias + '.' + '"lastUpdatedTime"' + ` <= '${dt.toISOString()}'`)) + } + }) + }.bind(this)); @@ -68,7 +74,14 @@ module.exports = function (PersistObjectTemplate) { select = ascending.reduce((result, column) => select.orderBy(column), select); if (descending.length) select = descending.reduce((result, column) => select.orderBy(column, 'desc'), select); - } + } + if (historySeqKeys.length) { + const predicate = historySeqKeys.reduce((predicate,curr)=> (predicate[curr]=1,predicate),{}); + select = this.getDB(this.getDBAlias(template.__table__)).connection.from({ "___historyAlias": select}) + .where(predicate) + } + + if (options && options.limit) { select = select.limit(options.limit); select = select.offset(0) @@ -131,7 +144,7 @@ module.exports = function (PersistObjectTemplate) { throw err; } - function getColumnNames(template) { + function getColumnNames(template, historySeqKeys) { var cols = []; var self = this; @@ -153,9 +166,11 @@ module.exports = function (PersistObjectTemplate) { as(template, prefix, '__version__', {type: {}, persist: true, enumerable: true}); as(template, prefix, '_template', {type: {}, persist: true, enumerable: true}); as(template, prefix, '_id', {type: {}, persist: true, enumerable: true}); + if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { + lastUpdatedSeq(template, prefix, '_id', {type: {}, persist: true, enumerable: true}); + } } - - function as(template, prefix, prop, defineProperty) { + function asChecks(template, prefix, prop, defineProperty) { var schema = template.__schema__; var type = defineProperty.type; var of = defineProperty.of; @@ -172,7 +187,25 @@ module.exports = function (PersistObjectTemplate) { throw new Error(type.__name__ + '.' + prop + ' is missing a parents schema entry'); prop = schema.parents[prop].id; } - cols.push(prefix + '.' + prop + ' as ' + (prefix ? prefix + '___' : '') + prop); + return prop; + } + function as(template, prefix, prop, defineProperty) { + const propDecorated = asChecks(template, prefix, prop, defineProperty) + if (!propDecorated) { + return; + } + cols.push(prefix + '.' + propDecorated + ' as ' + (prefix ? prefix + '___' : '') + propDecorated); + } + function lastUpdatedSeq(template, prefix, prop, defineProperty) { + const propDecorated = asChecks(template, prefix, prop, defineProperty) + if (!propDecorated) { + return; + } + const lastUpdatedSeq =`rank() over(partition by ${prefix}.${propDecorated} order by ${prefix}."lastUpdatedTime" desc) "${prefix}___lastUpdatedSeq"`; + var knex = self.getDB(self.getDBAlias(template.__table__)).connection(tableName); + // historySeqKeys.push(`${prefix}.${propDecorated}`); + historySeqKeys.push(`${prefix}___lastUpdatedSeq`); + cols.push(knex.client.raw(lastUpdatedSeq)); } function getPropsRecursive(template, map?) { @@ -541,7 +574,7 @@ module.exports = function (PersistObjectTemplate) { return response; } }.bind(this)) - + function buildTable(aliasedTableName) { var tableName = this.dealias(aliasedTableName); @@ -562,7 +595,7 @@ module.exports = function (PersistObjectTemplate) { } }.bind(this)); } - + function fieldChangeNotify(callBack, table) { if (!callBack) return; if (typeof callBack !== 'function') @@ -768,7 +801,7 @@ module.exports = function (PersistObjectTemplate) { function synchronizeIndexes(tableName, template) { - var aliasedTableName = template.__table__; + var aliasedTableName = tableName || template.__table__; tableName = this.dealias(aliasedTableName); const functionName = synchronizeIndexes.name; @@ -1157,8 +1190,9 @@ module.exports = function (PersistObjectTemplate) { table.string('_template'); table.biginteger('__version__'); if (collection.match(/_history/)) { - table.string('_id_history').primary(); + table.string('_id_history'); table.string('_id'); + table.primary(['_id', '_id_history']) } else { table.string('_id').primary(); diff --git a/components/persistor/lib/utils/PersistorUtils.js b/components/persistor/lib/utils/PersistorUtils.js new file mode 100644 index 00000000..d7f641de --- /dev/null +++ b/components/persistor/lib/utils/PersistorUtils.js @@ -0,0 +1,35 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersistorUtils = void 0; +var PersistorUtils = /** @class */ (function () { + function PersistorUtils() { + } + PersistorUtils.isRemoteObjectSetToTrue = function (enableIsRemoteObjectFeature, isRemoteObject) { + if (enableIsRemoteObjectFeature && (enableIsRemoteObjectFeature === true || enableIsRemoteObjectFeature === 'true')) { + return isRemoteObject && isRemoteObject === true; + } + return false; + }; + PersistorUtils.asyncMap = function (arr, concurrency, callback) { + var cnt = arr.length / concurrency; + var p = Promise.resolve([]); + var start = 0; + for (var i = 0; i < cnt; i++) { + p = p.then(function (results) { + var end = start + concurrency; + return Promise.all(arr.slice(start, end).map(callback)) + .then(function (eRes) { + start = end; + results.push.apply(results, eRes); + return results; + }); + }); + } + return p; + }; + PersistorUtils.sleep = function (ms) { + return new Promise(function (resolve) { return setTimeout(resolve, ms); }); + }; + return PersistorUtils; +}()); +exports.PersistorUtils = PersistorUtils; diff --git a/components/persistor/lib/utils/PersistorUtils.ts b/components/persistor/lib/utils/PersistorUtils.ts index 44d6dd58..b1ce7fe0 100644 --- a/components/persistor/lib/utils/PersistorUtils.ts +++ b/components/persistor/lib/utils/PersistorUtils.ts @@ -27,4 +27,8 @@ export class PersistorUtils { return p; } + + static sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } } \ No newline at end of file diff --git a/components/persistor/test/persist_banking.js b/components/persistor/test/persist_banking.js index 7770c770..f362e5d6 100644 --- a/components/persistor/test/persist_banking.js +++ b/components/persistor/test/persist_banking.js @@ -353,7 +353,7 @@ describe('Banking Example JS', function () { }).catch(function(e) {done(e)}); }); - it('Accounts have addresses', function (done) { + it('Accounts have addresses 1', function (done) { Account.getFromPersistWithQuery(null, {address: true}).then (function (accounts) { expect(accounts.length).to.equal(2); expect(accounts[0].address.__template__.__name__).to.equal('Address'); diff --git a/components/persistor/test/persist_banking_pgsql.js b/components/persistor/test/persist_banking_pgsql.js index 29c12f61..ca491968 100644 --- a/components/persistor/test/persist_banking_pgsql.js +++ b/components/persistor/test/persist_banking_pgsql.js @@ -10,6 +10,7 @@ var expect = require('chai').expect; var util = require('util'); var _ = require('underscore'); +const { PersistorUtils } = require('../lib/utils/PersistorUtils'); var ObjectTemplate = require('@haventech/supertype').default; var PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); var writing = true; @@ -1170,11 +1171,7 @@ describe('Banking from pgsql Example persist_banking_pgsql', function () { txn2Sam.firstName = 'txn2SamDead'; txn2Sam.setDirty(txn2); txn1.postSave = function () { - function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - sleep(100) + PersistorUtils.sleep(100) .then(function () { // Update will not return because it is requesting a lock on Karen txn1Karen.persistTouch(txn1) // 3 update karen diff --git a/components/persistor/test/persist_newapi_tests.js b/components/persistor/test/persist_newapi_tests.js index c96dac35..36d67484 100644 --- a/components/persistor/test/persist_newapi_tests.js +++ b/components/persistor/test/persist_newapi_tests.js @@ -6,7 +6,6 @@ var chaiAsPromised = require('chai-as-promised'); chai.should(); chai.use(chaiAsPromised); - var knexInit = require('knex'); var knex; @@ -17,7 +16,7 @@ var PersistObjectTemplate, ObjectTemplate; describe('persist newapi tests', function () { // this.timeout(5000); - before('drop schema table once per test suit', function () { + before('drop schema table once per test suit', function() { knex = knexInit({ client: 'pg', debug: true, @@ -28,15 +27,19 @@ describe('persist newapi tests', function () { password: process.env.dbPassword, } }); - return knex.raw(` - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP - EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; - END LOOP; - END $$; - `); + return Promise.all([ + + knex.schema.dropTableIfExists('tx_employee') + .then(function () { + return knex.schema.dropTableIfExists('tx_address') + }).then(function () { + return knex.schema.dropTableIfExists('tx_phone') + }).then(function () { + return knex.schema.dropTableIfExists('tx_department') + }).then(function () { + return knex.schema.dropTableIfExists('tx_role') + }), + knex.schema.dropTableIfExists(schemaTable)]); }) after('closes the database', function () { return knex.destroy(); @@ -50,63 +53,59 @@ describe('persist newapi tests', function () { schema.Phone = {}; schema.Dept = {}; schema.Employee.table = 'tx_employee'; - schema.Employee.audit = 'v2'; schema.Address.table = 'tx_address'; schema.Address.tableType = 'reference_data'; schema.Phone.table = 'tx_phone'; schema.Employee.parents = { - homeAddress: { - id: 'address_id', - fetch: true - } + homeAddress: {id: 'address_id', + fetch: true} }; schema.Employee.children = { - roles: { id: 'employee_id', fetch: true } + roles: {id: 'employee_id', fetch: true} }; schema.Employee.enableChangeTracking = true; schema.Role = {}; schema.Role.table = 'tx_role'; - schema.Role.audit = 'v2'; schema.Role.parents = { - employee: { id: 'employee_id' } + employee: {id: 'employee_id'} }; schema.Role.children = { - department: { id: 'role_id' } + department: {id: 'role_id'} }; schema.Address.parents = { - phone: { id: 'phone_id' } + phone: {id: 'phone_id'} }; Phone = PersistObjectTemplate.create('Phone', { - number: { type: String } + number: {type: String} }); Address = PersistObjectTemplate.create('Address', { - city: { type: String }, - state: { type: String }, - phone: { type: Phone } + city: {type: String}, + state: {type: String}, + phone: {type: Phone} }); Role = PersistObjectTemplate.create('Role', { - name: { type: String }, + name: {type:String}, }); Employee = PersistObjectTemplate.create('Employee', { - name: { type: String, value: 'Test Employee' }, - homeAddress: { type: Address }, - roles: { type: Array, of: Role, value: [] }, - dob: { type: Date }, - customObj: { type: Object }, - isMarried: { type: Boolean } + name: {type: String, value: 'Test Employee'}, + homeAddress: {type: Address}, + roles: {type: Array, of:Role, value: []}, + dob: {type: Date}, + customObj: {type: Object}, + isMarried: {type: Boolean} }); Role.mixin({ - employee: { type: Employee } + employee: {type: Employee} }); var emp = new Employee(); var add = new Address(); @@ -136,17 +135,13 @@ describe('persist newapi tests', function () { return Promise.resolve(prepareData()); function prepareData() { + PersistObjectTemplate.performInjections(); return syncTable(Employee) .then(syncTable.bind(this, Address)) .then(syncTable.bind(this, Phone)) .then(syncTable.bind(this, Role)) .then(addConstraint.bind(this)) - .then(addDateFields.bind(this)) - .then(addTriggers.bind(this)) - .then(createRecords.bind(this)) - .catch(e => { - console.log(e); - }); + .then(createRecords.bind(this)); function syncTable(template) { @@ -154,154 +149,14 @@ describe('persist newapi tests', function () { } function addConstraint() { - // return knex.raw('ALTER TABLE tx_role ADD CONSTRAINT namechk CHECK (char_length(name) <= 50);') - } - - function addDateFields() { - return knex.raw(` DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP - EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "createdTime" TIMESTAMP with time zone'; - EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "lastUpdatedTime" TIMESTAMP with time zone'; - END LOOP; - END $$;`); - } - - async function addTriggers() { - await Promise.all([knex.raw(` - CREATE OR REPLACE FUNCTION public.createddate_trigger() - RETURNS trigger - LANGUAGE plpgsql - AS $function$ - DECLARE columns text; - table_exists bool; - begin - - --using now as we track timezone in date fields.. - NEW."createdTime" = now(); - NEW."lastUpdatedTime" = now(); - execute format( - ' SELECT EXISTS ( - SELECT FROM - information_schema.tables - WHERE - table_type LIKE ''BASE TABLE'' AND - table_name = ''%1$s_history'' - ); - ', TG_TABLE_NAME ) into table_exists; - - - - if POSITION('_history' in TG_TABLE_NAME) = 0 and table_exists then - EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') - FROM pg_attribute c1 - where c1.attrelid = ''%s''::regclass - AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; - - - execute format( - ' INSERT INTO %1$s_history ( _id_history, %2$s ) - values (md5(random()::text || clock_timestamp()::text)::uuid, $1.*) - ', TG_TABLE_NAME, columns) using new; - - end if; - RETURN NEW; - END - $function$ - ; - - - -`), knex.raw(` - CREATE OR REPLACE FUNCTION public.modifieddate_trigger() - RETURNS trigger - LANGUAGE plpgsql - AS $function$ - DECLARE columns text; - table_exists bool; - begin - - --using now as we track timezone in date fields.. - NEW."lastUpdatedTime" = now(); - - execute format( - ' SELECT EXISTS ( - SELECT FROM - information_schema.tables - WHERE - table_type LIKE ''BASE TABLE'' AND - table_name = ''%1$s_history'' - ); - ', TG_TABLE_NAME ) into table_exists; - - - if POSITION('_history' in TG_TABLE_NAME) = 0 AND table_exists then - EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') - FROM pg_attribute c1 - where c1.attrelid = ''%s''::regclass - AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; - - - execute format( - ' INSERT INTO %1$s_history ( _id_history, %2$s ) - values (md5(random()::text || clock_timestamp()::text)::uuid, $1.*) - ', TG_TABLE_NAME, columns) using new; - - end if; - RETURN NEW; - END - $function$ - ; - - `)]); - - return Promise.all([knex.raw(` - DO $$ DECLARE - r RECORD; - table_exists bool; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP - execute format( - ' SELECT EXISTS ( - SELECT FROM - information_schema.tables - WHERE - table_type LIKE ''BASE TABLE'' AND - table_name = ''%1$s'' - ); - ', r.tablename ) into table_exists; - - if POSITION('_history' in r.tablename) = 0 AND table_exists then - EXECUTE 'create trigger createddate_inserttrigger before - insert - on - ' || quote_ident(r.tablename) || ' for each row execute procedure createddate_trigger();'; - end if; - END LOOP; - END $$; - `), - knex.raw(` - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP - if POSITION('_history' in r.tablename) = 0 then - EXECUTE 'create trigger modifieddate_updatetrigger before - update - on - ' || quote_ident(r.tablename) || ' for each row execute procedure modifieddate_trigger();'; - end if; - END LOOP; - END $$; - `)]); + return knex.raw('ALTER TABLE tx_role ADD CONSTRAINT namechk CHECK (char_length(name) <= 50);') } function createRecords() { - var tx = PersistObjectTemplate.beginDefaultTransaction(); + var tx = PersistObjectTemplate.beginDefaultTransaction(); - return emp.persist({ transaction: tx, cascade: false }).then(function () { - return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: true }).then(function () { + return emp.persist({transaction: tx, cascade: false}).then(function() { + return PersistObjectTemplate.commit({transaction: tx, notifyChanges: true}).then(function() { empId = emp._id; addressId = add._id; phoneId = phone._id; @@ -311,82 +166,81 @@ describe('persist newapi tests', function () { } }); - afterEach('remove tables and after each test', function () { - return knex.raw(` - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP - EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; - END LOOP; - END $$; - `); + afterEach('remove tables and after each test', function() { + return Promise.all([ + knex.schema.dropTableIfExists('tx_employee') + .then(function () { + return knex.schema.dropTableIfExists('tx_address') + }).then(function () { + return knex.schema.dropTableIfExists('tx_phone') + }).then(function () { + return knex.schema.dropTableIfExists('tx_department') + }).then(function () { + return knex.schema.dropTableIfExists('tx_role') + }), + knex.schema.dropTableIfExists(schemaTable)]); }); - // it('persistorFetchById without fetch spec should not return the records', function () { - // return Employee.persistorFetchById(empId, { fetch: { homeAddress: false }, enableChangeTracking: true }) - // .then(function (employee) { - // expect(employee.homeAddress).is.equal(null); - // }); - // }); + it('persistorFetchById without fetch spec should not return the records', function () { + return Employee.persistorFetchById(empId, { fetch: {homeAddress: false}, enableChangeTracking: true}) + .then(function(employee) { + expect(employee.homeAddress).is.equal(null); + }); + }); - // it('persistorFetchById with fetch spec should return the records', function () { - // return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { - // expect(employee.homeAddress._id).is.equal(addressId); - // expect(employee.homeAddress.phone).is.equal(null); - // }); - // }); + it('persistorFetchById with fetch spec should return the records', function () { + return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { + expect(employee.homeAddress._id).is.equal(addressId); + expect(employee.homeAddress.phone).is.equal(null); + }); + }); it('persistorFetchById with fetch spec with type projections', function () { - return Employee.persistorFetchById(empId, - { - asOfDate: new Date(), - fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } - }).then(function (employee) { - expect(employee.homeAddress.state).is.equal(undefined); - expect(employee.homeAddress.city).is.equal('New York'); - expect(employee.homeAddress.phone.number).is.equal(undefined); - expect(employee.homeAddress.phone._id).is.equal(phoneId); - }) + return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}, projection: { Address: ['city'], Role: ['name'], Phone: ['']}}).then(function(employee) { + expect(employee.homeAddress.state).is.equal(undefined); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.homeAddress.phone.number).is.equal(undefined); + expect(employee.homeAddress.phone._id).is.equal(phoneId); + }); }); it('persistorFetchById with fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: false } }, roles: true } }).then(function (employee) { + return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: false}}, roles: true}}).then(function(employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone).is.equal(null); }); }); it('persistorFetchById with multiple level fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (employee) { + return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone._id).is.equal(phoneId); }); }); it('fetch without fetch spec should not return the records', function () { - return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { homeAddress: false } }).then(function (employee) { + return Employee.persistorFetchByQuery({_id: empId}, {fetch: {homeAddress: false}}).then(function(employee) { expect(employee[0].homeAddress).is.equal(null); - }).catch(function (err) { + }).catch(function(err) { expect(err).not.equal(null); }); }); it('fetch with fetch spec should return the records', function () { - return Employee.persistorFetchByQuery({ _id: empId }, { fetch: { homeAddress: true }, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: { name: 1 } }).then(function (employee) { + return Employee.persistorFetchByQuery({_id: empId}, {fetch: {homeAddress: true}, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: {name: 1} }).then(function(employee) { expect(employee[0].homeAddress._id).is.equal(addressId); expect(employee[0].homeAddress.phone).is.equal(null); }); }); it('persistorCountByQuery counts records properly', function () { - return Employee.persistorCountByQuery({ _id: empId }, { fetch: { homeAddress: true }, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: { name: 1 } }).then(function (count) { + return Employee.persistorCountByQuery({_id: empId}, {fetch: {homeAddress: true}, logger: PersistObjectTemplate.logger, start: 0, limit: 5, order: {name: 1} }).then(function(count) { expect(count).to.equal(1); }); }); it('persistorFetchByQuery to check the fetchSpec cache', function () { - return Employee.persistorFetchByQuery({ _id: empId }, { + return Employee.persistorFetchByQuery({_id: empId}, { fetch: { roles: true } @@ -403,7 +257,7 @@ describe('persist newapi tests', function () { }) function actualTest() { - return Employee.persistorFetchByQuery({ _id: empId }, { + return Employee.persistorFetchByQuery({_id: empId}, { fetch: { name: true } @@ -413,50 +267,48 @@ describe('persist newapi tests', function () { }); it('Multiple fetch calls to check the validFetchSpec cache', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) - .then(function (employee) { - expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); - return employee.fetchReferences({ fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function () { - expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(2); - }); + return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) + .then(function(employee) { + expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); + return employee.fetchReferences({fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function() { + expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(2); }); + }); }); it('Multiple fetch calls with the same fetch string to check the validFetchSpec cache', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) - .then(function () { + return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) + .then(function() { expect(PersistObjectTemplate._validFetchSpecs).is.not.equal(null); - return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }).then(function () { + return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}).then(function() { expect(Object.keys(PersistObjectTemplate._validFetchSpecs.Employee).length).is.equal(1); }); }); }); it('fetch with fetch with multiple levels should return the records', function () { - return Employee.persistorFetchByQuery({ _id: empId }, { - fetch: { - homeAddress: { fetch: { phone: false } }, - roles: true - }, + return Employee.persistorFetchByQuery({_id: empId}, { + fetch: { homeAddress: {fetch: {phone: false}}, + roles: true}, logger: PersistObjectTemplate.logger - }, 0, 5, true, {}, { customOptions: 'custom' }) - .then(function (employee) { + }, 0, 5, true, {}, {customOptions: 'custom'}) + .then(function(employee) { expect(employee[0].homeAddress._id).is.equal(addressId); expect(employee[0].homeAddress.phone).is.equal(null); }); }); it('persistorFetchById with multiple level fetch spec should return the records', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (employee) { + return Employee.persistorFetchById(empId, {fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(employee) { expect(employee.homeAddress._id).is.equal(addressId); expect(employee.homeAddress.phone._id).is.equal(phoneId); }); }); it('persistorFetchByQuery with fetch spec should return the records and also load the child objects in the calling object', function () { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: false } }) - .then(function (employee) { - return employee.fetchReferences({ fetch: { homeAddress: { fetch: { phone: true } }, roles: true } }).then(function (obj) { + return Employee.persistorFetchById(empId, {fetch: {homeAddress: false}}) + .then(function(employee) { + return employee.fetchReferences({fetch: { homeAddress: {fetch: {phone: true}}, roles: true}}).then(function(obj) { expect(obj.homeAddress._id).is.equal(addressId); expect(employee.homeAddress._id).is.equal(addressId); }) @@ -474,10 +326,10 @@ describe('persist newapi tests', function () { emp1.name = 'Ravi1'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginDefaultTransaction(); - return emp1.persist({ transaction: tx, cascade: false }).then(function () { - return PersistObjectTemplate.commit().then(function () { - return Address.countFromPersistWithQuery().then(function (count) { + var tx = PersistObjectTemplate.beginDefaultTransaction(); + return emp1.persist({transaction: tx, cascade: false}).then(function() { + return PersistObjectTemplate.commit().then(function() { + return Address.countFromPersistWithQuery().then(function(count) { expect(count).to.equal(2); }); }); @@ -496,10 +348,10 @@ describe('persist newapi tests', function () { emp1.name = 'Ravi1'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({ transaction: tx, cascade: true }).then(function () { - return PersistObjectTemplate.commit().then(function () { - return Address.countFromPersistWithQuery().then(function (count) { + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({transaction: tx, cascade: true}).then(function() { + return PersistObjectTemplate.commit().then(function() { + return Address.countFromPersistWithQuery().then(function(count) { expect(count).to.equal(1); }); }); @@ -511,12 +363,12 @@ describe('persist newapi tests', function () { var emp1 = new Employee(); return Promise.resolve() .then(actualTest) - .catch(function (error) { + .catch(function(error) { expect(error.message).to.contain('Additional properties not allowed'); }); function actualTest() { - return emp1.persist({ transaction: null, cascade: true, unknown: false }).then(function () { + return emp1.persist({transaction: null, cascade: true, unknown: false}).then(function() { throw new Error('should not reach here'); }) } @@ -534,10 +386,10 @@ describe('persist newapi tests', function () { emp1.name = 'RaviNotSaved'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({ transaction: tx, cascade: false }).then(function () { - return PersistObjectTemplate.commit().then(function () { - return Employee.persistorFetchByQuery({ name: 'RaviNotSaved' }).then(function (employees) { + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({transaction: tx, cascade: false}).then(function() { + return PersistObjectTemplate.commit().then(function() { + return Employee.persistorFetchByQuery({name: 'RaviNotSaved'}).then(function(employees) { expect(employees.length).to.equal(0); }); }); @@ -548,11 +400,11 @@ describe('persist newapi tests', function () { for (let i = 0; i < 1000; ++i) { var phoneTemp = new Phone(); phoneTemp.number = `${i}`; - - var tx = PersistObjectTemplate.beginTransaction(); - await phoneTemp.persist({ transaction: tx, cascade: false }); - await PersistObjectTemplate.commit({ transaction: tx }); - const phones = await Phone.persistorFetchByQuery({ number: `${i}` }); + + var tx = PersistObjectTemplate.beginTransaction(); + await phoneTemp.persist({transaction: tx, cascade: false}); + await PersistObjectTemplate.commit({transaction: tx}); + const phones = await Phone.persistorFetchByQuery({number: `${i}`}); expect(phones.length).to.equal(1); } }); @@ -567,13 +419,13 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'LoadObjectForNotificationCheck'; emp1.homeAddress = add1; - var dob1 = new Date('01/01/1975'); - emp1.customObj = { name: 'testName', dob: JSON.stringify(dob1) }; + var dob1 = new Date('01/01/1975'); + emp1.customObj = {name: 'testName', dob: JSON.stringify(dob1)}; emp1.dob = dob1; emp1.isMarried = true; - var tx = PersistObjectTemplate.beginTransaction(); - tx.postSave = function (txn, _logger, changes, queries) { + var tx = PersistObjectTemplate.beginTransaction(); + tx.postSave = function(txn, _logger, changes, queries) { expect(queries['Employee'].queries.length).to.equal(1); expect(queries['Address'].queries.length).to.equal(1); expect(queries['Address'].tableType).to.equal('reference_data'); @@ -582,20 +434,20 @@ describe('persist newapi tests', function () { const insertScript = emp1.getInsertScript(); expect(insertScript).to.contain('insert into "tx_employee"'); - return emp1.persist({ transaction: tx, cascade: false }).then(function () { - return PersistObjectTemplate.commit({ transaction: tx, notifyQueries: true }).then(function () { - return Employee.persistorFetchByQuery({ name: 'LoadObjectForNotificationCheck' }, { enableChangeTracking: true }).then(function (employees) { + return emp1.persist({transaction: tx, cascade: false}).then(function() { + return PersistObjectTemplate.commit({transaction: tx, notifyQueries: true}).then(function() { + return Employee.persistorFetchByQuery({name: 'LoadObjectForNotificationCheck'}, {enableChangeTracking: true}).then(function(employees) { expect(employees.length).to.equal(1); var emp = employees[0]; emp.dob = new Date('01/01/1976'); - emp.customObj = { name: 'testName', dob: JSON.stringify(emp.dob) }; + emp.customObj = {name: 'testName', dob: JSON.stringify (emp.dob)}; emp.isMarried = false; emp.homeAddress = new Address(); - var innerTxn = PersistObjectTemplate.beginTransaction(); + var innerTxn = PersistObjectTemplate.beginTransaction(); emp.setDirty(innerTxn); - innerTxn.postSave = function (txn, _logger, changes, queries) { + innerTxn.postSave = function(txn, _logger, changes, queries) { expect(Object.keys(changes)).to.contain('Employee'); expect(Object.keys(changes.Employee[0])).to.contain('primaryKey'); expect(changes.Employee[0].properties[0].name).to.equal('homeAddress'); @@ -603,7 +455,7 @@ describe('persist newapi tests', function () { var empNew = new Employee(); empNew.setDirty(txn); }; - return PersistObjectTemplate.commit({ transaction: innerTxn, notifyChanges: true, notifyQueries: true }); + return PersistObjectTemplate.commit({transaction: innerTxn, notifyChanges: true, notifyQueries: true}); }); }); }) @@ -619,11 +471,11 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'RaviNotSaved'; emp1.homeAddress = add1; - - var tx = PersistObjectTemplate.beginTransaction(); - return emp1.persist({ transaction: tx, cascade: false }).then(function () { - return PersistObjectTemplate.commit({ transaction: tx }).then(function () { - return Employee.persistorFetchByQuery({ name: 'RaviNotSaved' }).then(function (employees) { + + var tx = PersistObjectTemplate.beginTransaction(); + return emp1.persist({transaction: tx, cascade: false}).then(function() { + return PersistObjectTemplate.commit({transaction: tx}).then(function() { + return Employee.persistorFetchByQuery({name: 'RaviNotSaved'}).then(function(employees) { expect(employees.length).to.equal(1); }); }); @@ -638,7 +490,7 @@ describe('persist newapi tests', function () { .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) + return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) } function setTestObjects(employee) { @@ -646,11 +498,11 @@ describe('persist newapi tests', function () { add = employee.homeAddress; } function realTest() { - var notifyChanges; - var tx = PersistObjectTemplate.beginTransaction(); - add.persistDelete({ transaction: tx }); - emp.persistDelete({ transaction: tx }); - return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: notifyChanges, notifyQueries: true }); + var notifyChanges; + var tx = PersistObjectTemplate.beginTransaction(); + add.persistDelete({transaction: tx}); + emp.persistDelete({transaction: tx}); + return PersistObjectTemplate.commit({transaction: tx, notifyChanges: notifyChanges, notifyQueries: true}); } function createFKs() { @@ -659,43 +511,49 @@ describe('persist newapi tests', function () { }); it('calling delete with transaction', function () { - return loadEmployee() + return createFKs() + .then(loadEmployee.bind(this)) .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) + return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) } function realTest() { - var tx = PersistObjectTemplate.beginTransaction(); - Employee.persistorDeleteByQuery({ name: 'Ravi' }, { transaction: tx }); - Address.persistorDeleteByQuery({ city: 'New York' }, { transaction: tx }) - return PersistObjectTemplate.commit({ transaction: tx }).then(function () { - return Employee.persistorFetchByQuery({ name: 'Ravi' }).then(function (employees) { + var tx = PersistObjectTemplate.beginTransaction(); + Employee.persistorDeleteByQuery({name: 'Ravi'}, {transaction: tx}); + Address.persistorDeleteByQuery({city: 'New York'}, {transaction: tx}) + return PersistObjectTemplate.commit({transaction: tx}).then(function() { + return Employee.persistorFetchByQuery({name: 'Ravi'}).then(function(employees) { expect(employees.length).to.equal(0); }) }); } - + function createFKs() { + return knex.raw('ALTER TABLE public.tx_employee ADD CONSTRAINT fk_tx_employee_address FOREIGN KEY (address_id) references public.tx_address("_id") deferrable initially deferred'); + } }); it('calling delete without transaction', function () { - return loadEmployee() + return createFKs() + .then(loadEmployee.bind(this)) .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchById(empId, { fetch: { homeAddress: true } }) + return Employee.persistorFetchById(empId, {fetch: {homeAddress: true}}) } async function realTest() { - await Employee.persistorDeleteByQuery({ name: 'Ravi' }); - return Employee.persistorFetchByQuery({ name: 'Ravi' }).then(function (employees) { + await Employee.persistorDeleteByQuery({name: 'Ravi'}); + return Employee.persistorFetchByQuery({name: 'Ravi'}).then(function(employees) { expect(employees.length).to.equal(0); }) } - + function createFKs() { + return knex.raw('ALTER TABLE public.tx_employee ADD CONSTRAINT fk_tx_employee_address FOREIGN KEY (address_id) references public.tx_address("_id") deferrable initially deferred'); + } }); it('update conflict should revert the version', function () { @@ -704,16 +562,16 @@ describe('persist newapi tests', function () { .then(realTest.bind(this)); function loadEmployee() { - return Employee.persistorFetchByQuery({ name: 'Ravi' }) + return Employee.persistorFetchByQuery({name: 'Ravi'}) } async function realTest(emps) { - var tx = PersistObjectTemplate.beginTransaction(); + var tx = PersistObjectTemplate.beginTransaction(); emps[0].setDirty(tx); emps[1].__version__ = emps[1].__version__ + 1; emps[1].setDirty(tx); try { - await PersistObjectTemplate.commit({ transaction: tx }); + await PersistObjectTemplate.commit({transaction: tx}); } catch (err) { expect(emps[0].__version__).to.equal('1'); @@ -731,13 +589,13 @@ describe('persist newapi tests', function () { add1.phone = phone1; emp1.name = 'Ravi'; emp1.homeAddress = add1; - var tx = PersistObjectTemplate.beginTransaction(); - emp1.persist({ transaction: tx, cascade: false }); - return PersistObjectTemplate.commit({ transaction: tx }); + var tx = PersistObjectTemplate.beginTransaction(); + emp1.persist({transaction: tx, cascade: false}); + return PersistObjectTemplate.commit({transaction: tx}); } }); - it('create an object copy and save, _id only assigned at db save', async () => { + it('create an object copy and save, _id only assigned at db save', async () => { var emp1 = new Employee(); var add1 = new Address(); var phone1 = new Phone(); @@ -757,27 +615,27 @@ describe('persist newapi tests', function () { emp1.roles.push(roleTest); emp1.roles.push(roleTest2); - - var tx = PersistObjectTemplate.beginTransaction(); - await emp1.persist({ transaction: tx, cascade: true }); - await PersistObjectTemplate.commit({ transaction: tx }); - const employee = await Employee.persistorFetchByQuery({ name: 'EmployeeObject' }); - - const clonedEmp = employee[0].createCopy(function (obj, prop, template) { + + var tx = PersistObjectTemplate.beginTransaction(); + await emp1.persist({transaction: tx, cascade: true}); + await PersistObjectTemplate.commit({transaction: tx}); + const employee = await Employee.persistorFetchByQuery({name: 'EmployeeObject'}); + + const clonedEmp = employee[0].createCopy(function(obj, prop, template){ return null; }); - + expect(clonedEmp._id).to.equal(undefined); expect(clonedEmp.roles[0]._id).to.equal(undefined); expect(clonedEmp.roles[1]._id).to.equal(undefined); clonedEmp.name = 'ClonedEmployeeObject'; - var tx = PersistObjectTemplate.beginTransaction(); - await clonedEmp.persist({ transaction: tx, cascade: true }); - await PersistObjectTemplate.commit({ transaction: tx }); - - const employeesClonedSaved = await Employee.persistorFetchByQuery({ name: 'ClonedEmployeeObject' }); - + var tx = PersistObjectTemplate.beginTransaction(); + await clonedEmp.persist({transaction: tx, cascade: true}); + await PersistObjectTemplate.commit({transaction: tx}); + + const employeesClonedSaved = await Employee.persistorFetchByQuery({name: 'ClonedEmployeeObject'}); + expect(employeesClonedSaved.length).to.equal(1); expect(employeesClonedSaved[0].roles[0]._id).to.equal(clonedEmp?.roles[0]._id); expect(employeesClonedSaved[0].roles[1]._id).to.equal(clonedEmp?.roles[1]._id); diff --git a/components/persistor/test/persist_pointintime_tests.js b/components/persistor/test/persist_pointintime_tests.js new file mode 100644 index 00000000..e4339935 --- /dev/null +++ b/components/persistor/test/persist_pointintime_tests.js @@ -0,0 +1,365 @@ +var chai = require('chai'), + expect = require('chai').expect; + +var chaiAsPromised = require('chai-as-promised'); + +chai.should(); +chai.use(chaiAsPromised); + + +var knexInit = require('knex'); +const { PersistorUtils } = require('../lib/utils/PersistorUtils.js'); +var knex; + +var schema = {}; +var schemaTable = 'index_schema_history'; +var Phone, Address, Employee, empId, addressId, phoneId, Role; +var PersistObjectTemplate, ObjectTemplate; + +describe('persist newapi tests', function () { + before('drop schema table once per test suit', function () { + knex = knexInit({ + client: 'pg', + debug: true, + connection: { + host: process.env.dbPath, + database: process.env.dbName, + user: process.env.dbUser, + password: process.env.dbPassword, + } + }); + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); + }) + after('closes the database', function () { + return knex.destroy(); + }); + beforeEach('arrange', function () { + ObjectTemplate = require('@haventech/supertype').default; + PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); + + schema.Employee = {}; + schema.Address = {}; + schema.Phone = {}; + schema.Dept = {}; + schema.Employee.table = 'tx_employee'; + schema.Employee.audit = 'v2'; + schema.Address.table = 'tx_address'; + schema.Address.audit = 'v2'; + schema.Address.tableType = 'reference_data'; + schema.Phone.table = 'tx_phone'; + + schema.Employee.parents = { + homeAddress: { + id: 'address_id', + fetch: true + } + }; + schema.Employee.children = { + roles: { id: 'employee_id', fetch: true } + }; + + schema.Employee.enableChangeTracking = true; + + schema.Role = {}; + schema.Role.table = 'tx_role'; + schema.Role.audit = 'v2'; + schema.Role.parents = { + employee: { id: 'employee_id' } + }; + schema.Role.children = { + department: { id: 'role_id' } + }; + + schema.Address.parents = { + phone: { id: 'phone_id' } + }; + Phone = PersistObjectTemplate.create('Phone', { + number: { type: String } + }); + + Address = PersistObjectTemplate.create('Address', { + city: { type: String }, + state: { type: String }, + phone: { type: Phone } + }); + + + Role = PersistObjectTemplate.create('Role', { + name: { type: String }, + + }); + + Employee = PersistObjectTemplate.create('Employee', { + name: { type: String, value: 'Test Employee' }, + homeAddress: { type: Address }, + roles: { type: Array, of: Role, value: [] }, + dob: { type: Date }, + customObj: { type: Object }, + isMarried: { type: Boolean } + }); + + Role.mixin({ + employee: { type: Employee } + }); + var emp = new Employee(); + var add = new Address(); + var phone = new Phone(); + var role1 = new Role(); + role1.name = 'firstRole2'; + role1.employee = emp; + var role2 = new Role(); + role2.name = 'secondRole2'; + role2.employee = emp; + + phone.number = '1231231234'; + add.city = 'New York'; + add.state = 'New York'; + add.phone = phone; + emp.name = 'InitialName'; + emp.homeAddress = add; + emp.roles.push(role1); + emp.roles.push(role2); + + (function () { + PersistObjectTemplate.setDB(knex, PersistObjectTemplate.DB_Knex); + PersistObjectTemplate.setSchema(schema); + PersistObjectTemplate.performInjections(); + + })(); + return Promise.resolve(prepareData()); + + function prepareData() { + return syncTable(Employee) + .then(syncTable.bind(this, Address)) + .then(syncTable.bind(this, Phone)) + .then(syncTable.bind(this, Role)) + .then(addConstraint.bind(this)) + .then(addDateFields.bind(this)) + .then(addTriggers.bind(this)) + .then(createRecords.bind(this)) + .catch(e => { + console.log(e); + }); + + + function syncTable(template) { + return PersistObjectTemplate.synchronizeKnexTableFromTemplate(template); + } + + function addConstraint() { + return knex.raw('ALTER TABLE tx_role ADD CONSTRAINT namechk CHECK (char_length(name) <= 50);') + } + + function addDateFields() { + return knex.raw(` DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "createdTime" TIMESTAMP with time zone'; + EXECUTE 'ALTER table ' || quote_ident(r.tablename) || ' ADD COLUMN "lastUpdatedTime" TIMESTAMP with time zone'; + END LOOP; + END $$;`); + } + + async function addTriggers() { + await Promise.all([knex.raw(` + CREATE OR REPLACE FUNCTION public.createddate_trigger() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + DECLARE columns text; + table_exists bool; + begin + + --using now as we track timezone in date fields.. + NEW."createdTime" = now(); + NEW."lastUpdatedTime" = now(); + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s_history'' + ); + ', TG_TABLE_NAME ) into table_exists; + + + + if POSITION('_history' in TG_TABLE_NAME) = 0 and table_exists then + EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') + FROM pg_attribute c1 + where c1.attrelid = ''%s''::regclass + AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; + + + execute format( + ' INSERT INTO %1$s_history ( _id_history, %2$s ) + values (regexp_replace(extract(epoch from clock_timestamp())::text, '',|\\.'', '''', ''g''), $1.*) + ', TG_TABLE_NAME, columns) using new; + + end if; + RETURN NEW; + END + $function$ + ; + + + +`), knex.raw(` + CREATE OR REPLACE FUNCTION public.modifieddate_trigger() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + DECLARE columns text; + table_exists bool; + begin + + --using now as we track timezone in date fields.. + NEW."lastUpdatedTime" = now(); + + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s_history'' + ); + ', TG_TABLE_NAME ) into table_exists; + + + if POSITION('_history' in TG_TABLE_NAME) = 0 AND table_exists then + EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') + FROM pg_attribute c1 + where c1.attrelid = ''%s''::regclass + AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; + + + execute format( + ' INSERT INTO %1$s_history ( _id_history, %2$s ) + values (regexp_replace(extract(epoch from clock_timestamp())::text, '',|\\.'', '''', ''g''), $1.*) + ', TG_TABLE_NAME, columns) using new; + + end if; + RETURN NEW; + END + $function$ + ; + + `)]); + + return Promise.all([knex.raw(` + DO $$ DECLARE + r RECORD; + table_exists bool; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + execute format( + ' SELECT EXISTS ( + SELECT FROM + information_schema.tables + WHERE + table_type LIKE ''BASE TABLE'' AND + table_name = ''%1$s'' + ); + ', r.tablename ) into table_exists; + + if POSITION('_history' in r.tablename) = 0 AND table_exists then + EXECUTE 'create trigger createddate_inserttrigger before + insert + on + ' || quote_ident(r.tablename) || ' for each row execute procedure createddate_trigger();'; + end if; + END LOOP; + END $$; + `), + knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + if POSITION('_history' in r.tablename) = 0 then + EXECUTE 'create trigger modifieddate_updatetrigger before + update + on + ' || quote_ident(r.tablename) || ' for each row execute procedure modifieddate_trigger();'; + end if; + END LOOP; + END $$; + `)]); + } + + function createRecords() { + var tx = PersistObjectTemplate.beginDefaultTransaction(); + + return emp.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: true }).then(function () { + empId = emp._id; + addressId = add._id; + phoneId = phone._id; + }); + }) + } + } + }); + + afterEach('remove tables and after each test', function () { + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); + }); + + it('Adding data and capturing', async function () { + var orgRecordedTime = new Date(); + await PersistorUtils.sleep(100); + var employee = await Employee.persistorFetchById(empId, + { + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }); + + employee.name = 'First Update'; + employee.homeAddress.city = 'First city update'; + employee.roles[0].name = 'First role update'; + + var tx = PersistObjectTemplate.beginDefaultTransaction(); + employee.setDirty(tx); + employee.homeAddress.setDirty(tx); + employee.roles[0].setDirty(tx); + // await employee.persistSave(); + await PersistObjectTemplate.commit({transaction: tx}); + await PersistorUtils.sleep(100); + var updatedTime = new Date(); + employee = await Employee.persistorFetchById(empId, + { + asOfDate: orgRecordedTime, + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }); + + expect(employee.name).is.equal('InitialName'); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.roles[0].name).is.equal('firstRole2'); + employee = await Employee.persistorFetchById(empId, + { + asOfDate: updatedTime, + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }); + expect(employee.name).is.equal('First Update'); + expect(employee.homeAddress.city).is.equal('First city update'); + expect(employee.roles[0].name).is.equal('First role update'); + }); +}); \ No newline at end of file diff --git a/components/persistor/test/supertype/persist_banking_pgsql.ts b/components/persistor/test/supertype/persist_banking_pgsql.ts index bb86cb91..8d9613d7 100644 --- a/components/persistor/test/supertype/persist_banking_pgsql.ts +++ b/components/persistor/test/supertype/persist_banking_pgsql.ts @@ -19,6 +19,7 @@ import {ExtendedCustomer} from "./ExtendedCustomer"; import {Role} from "./Role"; import {Account} from "./Account"; import {Transaction, Xfer} from './Transaction'; +import { PersistorUtils } from '../../lib/utils/PersistorUtils'; var schema = { Customer: { @@ -876,11 +877,8 @@ describe('typescript tests: Banking from pgsql Example persist_banking_pgsql', f txn2Sam.firstName = 'txn2SamDead'; txn2Sam.setDirty(txn2); txn1.postSave = function () { - function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); - } - sleep(100) + PersistorUtils.sleep(100) .then(function () { // Update will not return because it is requesting a lock on Karen txn1Karen.persistTouch(txn1) // 3 update karen From 9bae918521d0d8ee1f40822ba3a6843060c32eaf Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sun, 19 Feb 2023 18:45:38 -0500 Subject: [PATCH 03/14] Code cleanup --- components/persistor/lib/knex/PersistorCtx.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts index 73b1f036..03ad5502 100644 --- a/components/persistor/lib/knex/PersistorCtx.ts +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -19,12 +19,7 @@ export class PersistorCtx { private static _asyncLocalStorage: AsyncLocalStorage; private static get asyncLocalStorage() { - if (this._asyncLocalStorage) { - return this._asyncLocalStorage - } - else { - return (this._asyncLocalStorage = new AsyncLocalStorage()); - } + return this._asyncLocalStorage || (this._asyncLocalStorage = new AsyncLocalStorage()); } static checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { @@ -34,7 +29,6 @@ export class PersistorCtx { properties: { [this.persistorExnCtxKey]: new ExecutionCtx(asOfDate) }, }; return this.asyncLocalStorage.run(ctxProps, async () => { - let store = this.asyncLocalStorage.getStore() as CtxProps; return await callback(); }); } @@ -44,13 +38,14 @@ export class PersistorCtx { } static get ExecutionCtx() { - if (this._asyncLocalStorage) { - const store = this.asyncLocalStorage.getStore() as CtxProps; - if (!store) { - return null; - } - const exnCtx: ExecutionCtx = store.properties[this.persistorExnCtxKey]; - return exnCtx; + if (!this._asyncLocalStorage) { + return; + } + const store = this.asyncLocalStorage.getStore() as CtxProps; + if (!store) { + return; } + const exnCtx: ExecutionCtx = store.properties[this.persistorExnCtxKey]; + return exnCtx; } } \ No newline at end of file From 70aa0446e219a21a16940a5849623669e5db17b2 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sun, 19 Feb 2023 18:51:15 -0500 Subject: [PATCH 04/14] Code cleanup --- components/persistor/lib/knex/PersistorCtx.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts index 03ad5502..dbadfa8c 100644 --- a/components/persistor/lib/knex/PersistorCtx.ts +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -23,18 +23,17 @@ export class PersistorCtx { } static checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { - if (asOfDate) { - const ctxProps = { - name: `${new Date().getTime()}`, - properties: { [this.persistorExnCtxKey]: new ExecutionCtx(asOfDate) }, - }; - return this.asyncLocalStorage.run(ctxProps, async () => { - return await callback(); - }); - } - else { + if (!asOfDate) { return callback(); } + + const ctxProps = { + name: `${new Date().getTime()}`, + properties: { [this.persistorExnCtxKey]: new ExecutionCtx(asOfDate) }, + }; + return this.asyncLocalStorage.run(ctxProps, async () => { + return await callback(); + }); } static get ExecutionCtx() { From 6e528cf804cf22f104b00c9eaa53bfe353f2c7cf Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sun, 19 Feb 2023 18:56:22 -0500 Subject: [PATCH 05/14] Removing js files from utils folder --- components/persistor/.gitignore | 1 + .../persistor/lib/utils/PersistorUtils.js | 35 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 components/persistor/lib/utils/PersistorUtils.js diff --git a/components/persistor/.gitignore b/components/persistor/.gitignore index afcf6f7a..14d5b447 100644 --- a/components/persistor/.gitignore +++ b/components/persistor/.gitignore @@ -7,4 +7,5 @@ coverage dist .nyc_output test/supertype/*.js +lib/utils/*.js lib/persistable.js \ No newline at end of file diff --git a/components/persistor/lib/utils/PersistorUtils.js b/components/persistor/lib/utils/PersistorUtils.js deleted file mode 100644 index d7f641de..00000000 --- a/components/persistor/lib/utils/PersistorUtils.js +++ /dev/null @@ -1,35 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.PersistorUtils = void 0; -var PersistorUtils = /** @class */ (function () { - function PersistorUtils() { - } - PersistorUtils.isRemoteObjectSetToTrue = function (enableIsRemoteObjectFeature, isRemoteObject) { - if (enableIsRemoteObjectFeature && (enableIsRemoteObjectFeature === true || enableIsRemoteObjectFeature === 'true')) { - return isRemoteObject && isRemoteObject === true; - } - return false; - }; - PersistorUtils.asyncMap = function (arr, concurrency, callback) { - var cnt = arr.length / concurrency; - var p = Promise.resolve([]); - var start = 0; - for (var i = 0; i < cnt; i++) { - p = p.then(function (results) { - var end = start + concurrency; - return Promise.all(arr.slice(start, end).map(callback)) - .then(function (eRes) { - start = end; - results.push.apply(results, eRes); - return results; - }); - }); - } - return p; - }; - PersistorUtils.sleep = function (ms) { - return new Promise(function (resolve) { return setTimeout(resolve, ms); }); - }; - return PersistorUtils; -}()); -exports.PersistorUtils = PersistorUtils; From 2a00ee3de3a05ff7c482f38d8a583e0712bf1a65 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Mon, 20 Feb 2023 09:36:26 -0500 Subject: [PATCH 06/14] Adding point-in-time feature for all query methods. --- components/persistor/lib/api.ts | 14 ++- components/persistor/lib/knex/db.ts | 7 +- components/persistor/lib/persistable.ts | 6 +- .../test/persist_pointintime_tests.js | 104 +++++++++++++++++- 4 files changed, 118 insertions(+), 13 deletions(-) diff --git a/components/persistor/lib/api.ts b/components/persistor/lib/api.ts index 1d5287a8..2cefe98f 100644 --- a/components/persistor/lib/api.ts +++ b/components/persistor/lib/api.ts @@ -166,10 +166,11 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { * @param {object} idMap id mapper for cached objects * @param {bool} isRefresh force load * @param {object} logger objecttemplate logger + * @param {date} asOfDate load objects based on the date provided * @returns {object} * @legacy Use persistorFetchById instead */ - template.getFromPersistWithId = async function (id, cascade, isTransient, idMap, isRefresh, logger) { + template.getFromPersistWithId = async function (id, cascade, isTransient, idMap, isRefresh, logger, asOfDate) { const functionName = 'getFromPersistWithId'; (logger || PersistObjectTemplate.logger).debug({ module: moduleName, @@ -185,7 +186,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { let getQuery = (dbType == PersistObjectTemplate.DB_Mongo ? PersistObjectTemplate.getFromPersistWithMongoId(template, id, cascade, isTransient, idMap, logger) : - PersistObjectTemplate.getFromPersistWithKnexId(template, id, cascade, isTransient, idMap, isRefresh, logger)); + PersistorCtx.checkAndExecuteWithContext(asOfDate, PersistObjectTemplate.getFromPersistWithKnexId.bind(PersistObjectTemplate, template, id, cascade, isTransient, idMap, isRefresh, logger))); const name = 'getFromPersistWithId'; return getQuery @@ -211,10 +212,11 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { * @param {object} idMap id mapper for cached objects * @param {bool} options {@TODO} * @param {object} logger objecttemplate logger + * @param {date} asOfDate load objects based on the date provided * @returns {object} * @legacy in favor of persistorFetchByQuery */ - template.getFromPersistWithQuery = async function (query, cascade, start, limit, isTransient, idMap, options, logger) { + template.getFromPersistWithQuery = async function (query, cascade, start, limit, isTransient, idMap, options, logger, asOfDate) { const functionName = 'getFromPersistWithQuery'; (logger || PersistObjectTemplate.logger).debug({ module: moduleName, @@ -229,7 +231,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { let getQuery = (dbType == PersistObjectTemplate.DB_Mongo ? PersistObjectTemplate.getFromPersistWithMongoQuery(template, query, cascade, start, limit, isTransient, idMap, options, logger) : - PersistObjectTemplate.getFromPersistWithKnexQuery(null, template, query, cascade, start, limit, isTransient, idMap, options, undefined, undefined, logger)); + PersistorCtx.checkAndExecuteWithContext(asOfDate, PersistObjectTemplate.getFromPersistWithKnexQuery.bind(PersistObjectTemplate, null, template, query, cascade, start, limit, isTransient, idMap, options, undefined, undefined, logger))); const name = 'getFromPersistWithQuery'; @@ -375,9 +377,9 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { let fetchQuery = (dbType == persistObjectTemplate.DB_Mongo ? persistObjectTemplate.getFromPersistWithMongoQuery(template, query, options.fetch, options.start, options.limit, options.transient, options.order, options.order, logger) : - persistObjectTemplate.getFromPersistWithKnexQuery(null, template, query, options.fetch, options.start, + PersistorCtx.checkAndExecuteWithContext(options.asOfDate, persistObjectTemplate.getFromPersistWithKnexQuery.bind(persistObjectTemplate, null, template, query, options.fetch, options.start, options.limit, options.transient, null, options.order, - undefined, undefined, logger, options.enableChangeTracking, options.projection)); + undefined, undefined, logger, options.enableChangeTracking, options.projection))); const name = 'persistorFetchByQuery'; return fetchQuery diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index 678b05c8..3bfb5144 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -850,7 +850,8 @@ module.exports = function (PersistObjectTemplate) { response = JSON.parse(record[0][schemaField]); } _dbschema = response; - return [response, template.__name__]; + const historyPostfix = tableName.match(/_history$/) ? '___History' : ''; + return [response, template.__name__ + historyPostfix]; }) }; @@ -862,7 +863,7 @@ module.exports = function (PersistObjectTemplate) { var diffTable = function([dbschema, schema, tableName]) { var dbTableDef = dbschema[tableName]; - var memTableDef = schema[tableName]; + var memTableDef = schema[tableName.replace( '___History', '')]; var track = {add: [], change: [], delete: []}; _diff(dbTableDef, memTableDef, 'delete', false, function (_dbIdx, memIdx) { return !memIdx; @@ -1063,7 +1064,7 @@ module.exports = function (PersistObjectTemplate) { sequence_id = ++record[0].sequence_id; } _.each(_changes, function (_o, chgKey) { - response[chgKey] = schema[chgKey]; + response[chgKey] = schema[chgKey.replace( '___History', '')]; }); return knex(schemaTable).insert({ diff --git a/components/persistor/lib/persistable.ts b/components/persistor/lib/persistable.ts index b7997340..fd8db01d 100644 --- a/components/persistor/lib/persistable.ts +++ b/components/persistor/lib/persistable.ts @@ -216,11 +216,12 @@ export function Persistable>(Base: BC) { * @param {object} idMap id mapper for cached objects * @param {bool} isRefresh force load * @param {object} logger objecttemplate logger + * @param {date} asOfDate load objects based on the date provided * @returns {object} * @legacy Use persistorFetchById instead * @async */ - static getFromPersistWithId(id?, cascade?, isTransient?, idMap?, isRefresh?, logger?) : any{} + static getFromPersistWithId(id?, cascade?, isTransient?, idMap?, isRefresh?, logger?, asOfDate?) : any{} /** * Return an array of objects of this class given a json query @@ -233,11 +234,12 @@ export function Persistable>(Base: BC) { * @param {object} idMap id mapper for cached objects * @param {bool} options {@TODO} * @param {object} logger objecttemplate logger + * @param {date} asOfDate load objects based on the date provided * @returns {object} * @legacy in favor of persistorFetchByQuery * @async */ - static getFromPersistWithQuery(query, cascade?, start?, limit?, isTransient?, idMap?, options?, logger?) : any {} + static getFromPersistWithQuery(query, cascade?, start?, limit?, isTransient?, idMap?, options?, logger?, asOfDate?) : any {} /** * Delete objects given a json query diff --git a/components/persistor/test/persist_pointintime_tests.js b/components/persistor/test/persist_pointintime_tests.js index e4339935..52dbe05f 100644 --- a/components/persistor/test/persist_pointintime_tests.js +++ b/components/persistor/test/persist_pointintime_tests.js @@ -13,7 +13,7 @@ var knex; var schema = {}; var schemaTable = 'index_schema_history'; -var Phone, Address, Employee, empId, addressId, phoneId, Role; +var Phone, Address, Employee, empId, addressId, phoneId, Role, dob; var PersistObjectTemplate, ObjectTemplate; describe('persist newapi tests', function () { @@ -124,6 +124,8 @@ describe('persist newapi tests', function () { add.state = 'New York'; add.phone = phone; emp.name = 'InitialName'; + dob = new Date(); + emp.dob = dob; emp.homeAddress = add; emp.roles.push(role1); emp.roles.push(role2); @@ -324,7 +326,7 @@ describe('persist newapi tests', function () { `); }); - it('Adding data and capturing', async function () { + it('Adding data and capturing for persistorFetchById', async function () { var orgRecordedTime = new Date(); await PersistorUtils.sleep(100); var employee = await Employee.persistorFetchById(empId, @@ -362,4 +364,102 @@ describe('persist newapi tests', function () { expect(employee.homeAddress.city).is.equal('First city update'); expect(employee.roles[0].name).is.equal('First role update'); }); + it('Adding data and capturing for persistorFetchByQuery', async function () { + var orgRecordedTime = new Date(); + await PersistorUtils.sleep(100); + var employee = (await Employee.persistorFetchByQuery({dob: dob}, + { + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }))[0]; + + employee.name = 'First Update'; + employee.homeAddress.city = 'First city update'; + employee.roles[0].name = 'First role update'; + + var tx = PersistObjectTemplate.beginDefaultTransaction(); + employee.setDirty(tx); + employee.homeAddress.setDirty(tx); + employee.roles[0].setDirty(tx); + // await employee.persistSave(); + await PersistObjectTemplate.commit({transaction: tx}); + await PersistorUtils.sleep(100); + var updatedTime = new Date(); + employee = (await Employee.persistorFetchByQuery({dob: dob}, + { + asOfDate: orgRecordedTime, + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }))[0]; + + expect(employee.name).is.equal('InitialName'); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.roles[0].name).is.equal('firstRole2'); + employee = (await Employee.persistorFetchByQuery({dob: dob}, + { + asOfDate: updatedTime, + fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } + }))[0]; + expect(employee.name).is.equal('First Update'); + expect(employee.homeAddress.city).is.equal('First city update'); + expect(employee.roles[0].name).is.equal('First role update'); + }); + it('Adding data and capturing for getFromPersistWithId', async function () { + var orgRecordedTime = new Date(); + await PersistorUtils.sleep(100); + var employee = await Employee.getFromPersistWithId(empId, + { homeAddress: { fetch: { phone: true } }, roles: true }); + + employee.name = 'First Update'; + employee.homeAddress.city = 'First city update'; + employee.roles[0].name = 'First role update'; + + var tx = PersistObjectTemplate.beginDefaultTransaction(); + employee.setDirty(tx); + employee.homeAddress.setDirty(tx); + employee.roles[0].setDirty(tx); + // await employee.persistSave(); + await PersistObjectTemplate.commit({transaction: tx}); + await PersistorUtils.sleep(100); + var updatedTime = new Date(); + employee = await Employee.getFromPersistWithId(empId, + { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, orgRecordedTime); + + expect(employee.name).is.equal('InitialName'); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.roles[0].name).is.equal('firstRole2'); + employee = await Employee.getFromPersistWithId(empId, + { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, updatedTime); + expect(employee.name).is.equal('First Update'); + expect(employee.homeAddress.city).is.equal('First city update'); + expect(employee.roles[0].name).is.equal('First role update'); + }); + it('Adding data and capturing for getFromPersistWithQuery', async function () { + var orgRecordedTime = new Date(); + await PersistorUtils.sleep(100); + var employee = (await Employee.getFromPersistWithQuery({dob: dob}, + { homeAddress: { fetch: { phone: true } }, roles: true }))[0]; + + employee.name = 'First Update'; + employee.homeAddress.city = 'First city update'; + employee.roles[0].name = 'First role update'; + + var tx = PersistObjectTemplate.beginDefaultTransaction(); + employee.setDirty(tx); + employee.homeAddress.setDirty(tx); + employee.roles[0].setDirty(tx); + // await employee.persistSave(); + await PersistObjectTemplate.commit({transaction: tx}); + await PersistorUtils.sleep(100); + var updatedTime = new Date(); + employee = (await Employee.getFromPersistWithQuery({dob: dob}, + { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, undefined, undefined, orgRecordedTime))[0]; + + expect(employee.name).is.equal('InitialName'); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.roles[0].name).is.equal('firstRole2'); + employee = (await Employee.getFromPersistWithQuery({dob: dob}, + { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, undefined, undefined, updatedTime))[0]; + expect(employee.name).is.equal('First Update'); + expect(employee.homeAddress.city).is.equal('First city update'); + expect(employee.roles[0].name).is.equal('First role update'); + }); }); \ No newline at end of file From c948fb7e300ea89288332d984e0555d5d2dba520 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Mon, 20 Feb 2023 13:16:13 -0500 Subject: [PATCH 07/14] Fixing column creation order --- components/persistor/lib/knex/db.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index 3bfb5144..506ce8b5 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -1187,9 +1187,6 @@ module.exports = function (PersistObjectTemplate) { return knex.schema.createTable(tableName, createColumns.bind(this)); function createColumns(table) { - - table.string('_template'); - table.biginteger('__version__'); if (collection.match(/_history/)) { table.string('_id_history'); table.string('_id'); @@ -1198,6 +1195,9 @@ module.exports = function (PersistObjectTemplate) { else { table.string('_id').primary(); } + table.string('_template'); + table.biginteger('__version__'); + var columnMap = {}; recursiveColumnMap.call(this, template); From 3fce956a56f26d731f8b6af9ea6edd5cb3daafee Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Tue, 21 Feb 2023 23:19:08 -0500 Subject: [PATCH 08/14] considered some review comments. --- components/persistor/lib/knex/db.ts | 26 ++++++++++++++----- components/persistor/lib/knex/query.ts | 3 ++- components/persistor/lib/mongo/db.ts | 4 ++- .../test/persist_pointintime_tests.js | 15 ++++++----- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index 506ce8b5..dfac12e8 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -26,6 +26,11 @@ module.exports = function (PersistObjectTemplate) { var historySeqKeys = []; if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { historyTableName = tableName + '_history'; + if ('_id' in queryOrChains) { + queryOrChains["_snapshot_id"] = queryOrChains["_id"] + delete queryOrChains["_id"]; + } + queryOrChains["lastUpdatedTime"] = {$lte: PersistorCtx.ExecutionCtx?.AsOfDate}; } @@ -43,11 +48,16 @@ module.exports = function (PersistObjectTemplate) { } const parentKey = this.dealias(template.__table__) + '.' + join.childKey select = select.leftOuterJoin( (historyJoinTable || joinTable) + ' as ' + join.alias, function() { - const cond = this.on(join.alias + '.' + join.parentKey, '=', parentKey) if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { + const cond = this.on(join.alias + '._snapshot_id', '=', parentKey) + const dt = PersistorCtx.ExecutionCtx?.AsOfDate cond.andOn(knex.client.raw(join.alias + '.' + '"lastUpdatedTime"' + ` <= '${dt.toISOString()}'`)) } + else + { + const cond = this.on(join.alias + '.' + join.parentKey, '=', parentKey) + } }) @@ -165,9 +175,12 @@ module.exports = function (PersistObjectTemplate) { function asStandard(template, prefix) { as(template, prefix, '__version__', {type: {}, persist: true, enumerable: true}); as(template, prefix, '_template', {type: {}, persist: true, enumerable: true}); - as(template, prefix, '_id', {type: {}, persist: true, enumerable: true}); + if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { - lastUpdatedSeq(template, prefix, '_id', {type: {}, persist: true, enumerable: true}); + lastUpdatedSeq(template, prefix, '_snapshot_id', {type: {}, persist: true, enumerable: true}); + } + else { + as(template, prefix, '_id', {type: {}, persist: true, enumerable: true}); } } function asChecks(template, prefix, prop, defineProperty) { @@ -201,7 +214,8 @@ module.exports = function (PersistObjectTemplate) { if (!propDecorated) { return; } - const lastUpdatedSeq =`rank() over(partition by ${prefix}.${propDecorated} order by ${prefix}."lastUpdatedTime" desc) "${prefix}___lastUpdatedSeq"`; + cols.push(prefix + '.' + propDecorated + ' as ' + (prefix ? prefix + '___' : '') + propDecorated.replace('_snapshot','')); + const lastUpdatedSeq =`rank() over(partition by "${prefix}"."${propDecorated}" order by "${prefix}"."lastUpdatedTime" desc) "${prefix}___lastUpdatedSeq"`; var knex = self.getDB(self.getDBAlias(template.__table__)).connection(tableName); // historySeqKeys.push(`${prefix}.${propDecorated}`); historySeqKeys.push(`${prefix}___lastUpdatedSeq`); @@ -1188,9 +1202,9 @@ module.exports = function (PersistObjectTemplate) { function createColumns(table) { if (collection.match(/_history/)) { - table.string('_id_history'); table.string('_id'); - table.primary(['_id', '_id_history']) + table.string('_snapshot_id'); + table.primary(['_id']) } else { table.string('_id').primary(); diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index 272a1626..b88c71aa 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -1,5 +1,6 @@ import { RemoteDocService } from '../remote-doc/RemoteDocService'; import { PersistorUtils } from '../utils/PersistorUtils'; +import { PersistorCtx } from './PersistorCtx'; module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/knex/query`; @@ -493,7 +494,7 @@ module.exports = function (PersistObjectTemplate) { var foreignFilterValue = schema.children[prop].filter ? schema.children[prop].filter.value : null; // Construct foreign key query var query = {}; - var options = defineProperty.queryOptions || {sort: {_id: 1}}; + var options = defineProperty.queryOptions || {sort: PersistorCtx.ExecutionCtx?.AsOfDate ? {_snapshot_id:1} : {_id: 1}}; var limit = options.limit || null; query[schema.children[prop].id] = obj._id; if (foreignFilterKey) { diff --git a/components/persistor/lib/mongo/db.ts b/components/persistor/lib/mongo/db.ts index d118291f..03a1b82c 100644 --- a/components/persistor/lib/mongo/db.ts +++ b/components/persistor/lib/mongo/db.ts @@ -1,3 +1,5 @@ +import { PersistorCtx } from "../knex/PersistorCtx"; + module.exports = function (PersistObjectTemplate) { const moduleName = `persistor/lib/mongo/db`; /* Mongo implementation of save */ @@ -65,7 +67,7 @@ module.exports = function (PersistObjectTemplate) { var collection = db.collection(this.dealias(template.__collection__)); options = options || {}; if (!options.sort) - options.sort = {_id:1}; + options.sort = PersistorCtx.ExecutionCtx?.AsOfDate ? {_snapshot_id:1} : {_id:1}; if (typeof(options) === "function") { return collection.find(query, undefined, options).toArray(); diff --git a/components/persistor/test/persist_pointintime_tests.js b/components/persistor/test/persist_pointintime_tests.js index 52dbe05f..4c7bedad 100644 --- a/components/persistor/test/persist_pointintime_tests.js +++ b/components/persistor/test/persist_pointintime_tests.js @@ -200,13 +200,14 @@ describe('persist newapi tests', function () { EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') FROM pg_attribute c1 where c1.attrelid = ''%s''::regclass + AND c1.attname <> ''_id'' AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; execute format( - ' INSERT INTO %1$s_history ( _id_history, %2$s ) - values (regexp_replace(extract(epoch from clock_timestamp())::text, '',|\\.'', '''', ''g''), $1.*) - ', TG_TABLE_NAME, columns) using new; + ' INSERT INTO %1$s_history ( _id, _snapshot_id, %2$s ) + values (md5(random()::text || ''%3$s'' || clock_timestamp()::text)::uuid, $1.*) + ', TG_TABLE_NAME, columns, new._id) using new; end if; RETURN NEW; @@ -243,13 +244,14 @@ describe('persist newapi tests', function () { EXECUTE format('SELECT string_agg(''"'' || c1.attname || ''"'', '','') FROM pg_attribute c1 where c1.attrelid = ''%s''::regclass + AND c1.attname <> ''_id'' AND c1.attnum > 0;', TG_TABLE_NAME) INTO columns; execute format( - ' INSERT INTO %1$s_history ( _id_history, %2$s ) - values (regexp_replace(extract(epoch from clock_timestamp())::text, '',|\\.'', '''', ''g''), $1.*) - ', TG_TABLE_NAME, columns) using new; + ' INSERT INTO %1$s_history ( _id, _snapshot_id, %2$s ) + values (md5(random()::text || ''%3$s'' || clock_timestamp()::text)::uuid, $1.*) + ', TG_TABLE_NAME, columns, new._id) using new; end if; RETURN NEW; @@ -351,7 +353,6 @@ describe('persist newapi tests', function () { asOfDate: orgRecordedTime, fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } }); - expect(employee.name).is.equal('InitialName'); expect(employee.homeAddress.city).is.equal('New York'); expect(employee.roles[0].name).is.equal('firstRole2'); From d77cbf740c7bb3d379d2607f1e07582461b41d2d Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Tue, 21 Feb 2023 23:35:35 -0500 Subject: [PATCH 09/14] Implemented some review comments --- components/persistor/lib/knex/PersistorCtx.ts | 19 +++++++++++++++++-- components/persistor/lib/knex/db.ts | 12 ++++++------ components/persistor/lib/knex/query.ts | 2 +- components/persistor/lib/mongo/db.ts | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts index dbadfa8c..56ab5bd9 100644 --- a/components/persistor/lib/knex/PersistorCtx.ts +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -9,13 +9,14 @@ class ExecutionCtx { this._asOfDate = asOfDate } - get AsOfDate() { + get asOfDate() { return this._asOfDate; } } export class PersistorCtx { static persistorExnCtxKey = '#persistor-exn-ctx'; + static persistorCacheCtxKey = '#persistor-cache-ctx'; private static _asyncLocalStorage: AsyncLocalStorage; private static get asyncLocalStorage() { @@ -36,7 +37,7 @@ export class PersistorCtx { }); } - static get ExecutionCtx() { + static get executionCtx() { if (!this._asyncLocalStorage) { return; } @@ -47,4 +48,18 @@ export class PersistorCtx { const exnCtx: ExecutionCtx = store.properties[this.persistorExnCtxKey]; return exnCtx; } + + static set setExecutionContext(asyncLocalStorage: AsyncLocalStorage) { + const store = asyncLocalStorage.getStore() as CtxProps; + if (!store) { + return; + } + + const exnCtx: ExecutionCtx = store.properties[this.persistorCacheCtxKey]; + if (!exnCtx) { + throw Error(`ExecutionCtx can only be set from outside with #persistor-cache-ctx property in the store.`) + } + + this._asyncLocalStorage = asyncLocalStorage; + } } \ No newline at end of file diff --git a/components/persistor/lib/knex/db.ts b/components/persistor/lib/knex/db.ts index dfac12e8..5ec4ba22 100644 --- a/components/persistor/lib/knex/db.ts +++ b/components/persistor/lib/knex/db.ts @@ -24,14 +24,14 @@ module.exports = function (PersistObjectTemplate) { var tableName = this.dealias(template.__table__); var historyTableName, historyTableAlias; var historySeqKeys = []; - if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { + if (PersistorCtx.executionCtx?.asOfDate && template.__schema__.audit === 'v2') { historyTableName = tableName + '_history'; if ('_id' in queryOrChains) { queryOrChains["_snapshot_id"] = queryOrChains["_id"] delete queryOrChains["_id"]; } - queryOrChains["lastUpdatedTime"] = {$lte: PersistorCtx.ExecutionCtx?.AsOfDate}; + queryOrChains["lastUpdatedTime"] = {$lte: PersistorCtx.executionCtx?.asOfDate}; } var knex = this.getDB(this.getDBAlias(template.__table__)).connection(tableName); @@ -43,15 +43,15 @@ module.exports = function (PersistObjectTemplate) { joins.forEach(function (join) { let joinTable = this.dealias(join.template.__table__); let historyJoinTable, additionalCondition; - if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { + if (PersistorCtx.executionCtx?.asOfDate && join.template.__schema__.audit === 'v2') { historyJoinTable = joinTable + '_history'; } const parentKey = this.dealias(template.__table__) + '.' + join.childKey select = select.leftOuterJoin( (historyJoinTable || joinTable) + ' as ' + join.alias, function() { - if (PersistorCtx.ExecutionCtx?.AsOfDate && join.template.__schema__.audit === 'v2') { + if (PersistorCtx.executionCtx?.asOfDate && join.template.__schema__.audit === 'v2') { const cond = this.on(join.alias + '._snapshot_id', '=', parentKey) - const dt = PersistorCtx.ExecutionCtx?.AsOfDate + const dt = PersistorCtx.executionCtx?.asOfDate cond.andOn(knex.client.raw(join.alias + '.' + '"lastUpdatedTime"' + ` <= '${dt.toISOString()}'`)) } else @@ -176,7 +176,7 @@ module.exports = function (PersistObjectTemplate) { as(template, prefix, '__version__', {type: {}, persist: true, enumerable: true}); as(template, prefix, '_template', {type: {}, persist: true, enumerable: true}); - if (PersistorCtx.ExecutionCtx?.AsOfDate && template.__schema__.audit === 'v2') { + if (PersistorCtx.executionCtx?.asOfDate && template.__schema__.audit === 'v2') { lastUpdatedSeq(template, prefix, '_snapshot_id', {type: {}, persist: true, enumerable: true}); } else { diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index b88c71aa..38af9100 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -494,7 +494,7 @@ module.exports = function (PersistObjectTemplate) { var foreignFilterValue = schema.children[prop].filter ? schema.children[prop].filter.value : null; // Construct foreign key query var query = {}; - var options = defineProperty.queryOptions || {sort: PersistorCtx.ExecutionCtx?.AsOfDate ? {_snapshot_id:1} : {_id: 1}}; + var options = defineProperty.queryOptions || {sort: PersistorCtx.executionCtx?.asOfDate ? {_snapshot_id:1} : {_id: 1}}; var limit = options.limit || null; query[schema.children[prop].id] = obj._id; if (foreignFilterKey) { diff --git a/components/persistor/lib/mongo/db.ts b/components/persistor/lib/mongo/db.ts index 03a1b82c..bc415ce9 100644 --- a/components/persistor/lib/mongo/db.ts +++ b/components/persistor/lib/mongo/db.ts @@ -67,7 +67,7 @@ module.exports = function (PersistObjectTemplate) { var collection = db.collection(this.dealias(template.__collection__)); options = options || {}; if (!options.sort) - options.sort = PersistorCtx.ExecutionCtx?.AsOfDate ? {_snapshot_id:1} : {_id:1}; + options.sort = PersistorCtx.executionCtx?.asOfDate ? {_snapshot_id:1} : {_id:1}; if (typeof(options) === "function") { return collection.find(query, undefined, options).toArray(); From 2e9ebcb898b86cb8610483b9be395ad08bbd01d2 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Tue, 28 Feb 2023 17:41:57 -0500 Subject: [PATCH 10/14] ORM Cache implemented. --- components/persistor/lib/api.ts | 15 +- components/persistor/lib/knex/PersistorCtx.ts | 24 +- components/persistor/lib/knex/query.ts | 31 +++ components/persistor/lib/persistable.ts | 5 + .../persistor/test/persist_banking_pgsql.js | 5 +- .../test/persist_cachewithfetchspec.js | 249 ++++++++++++++++++ .../test/persist_pointintime_tests.js | 8 - 7 files changed, 323 insertions(+), 14 deletions(-) create mode 100644 components/persistor/test/persist_cachewithfetchspec.js diff --git a/components/persistor/lib/api.ts b/components/persistor/lib/api.ts index 2cefe98f..b88fe620 100644 --- a/components/persistor/lib/api.ts +++ b/components/persistor/lib/api.ts @@ -289,6 +289,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { PersistObjectTemplate._validateParams(options, 'fetchSchema', template); options = options || {}; + const idMap = PersistorCtx.persistorCacheCtx || null; var persistObjectTemplate = options.session || PersistObjectTemplate; (options.logger || persistObjectTemplate.logger).debug({ @@ -303,7 +304,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { var dbType = persistObjectTemplate.getDB(persistObjectTemplate.getDBAlias(template.__collection__)).type; let fetchQuery = (dbType == persistObjectTemplate.DB_Mongo ? persistObjectTemplate.getFromPersistWithMongoId(template, id, options.fetch, options.transient, null, options.logger) : - PersistorCtx.checkAndExecuteWithContext(options.asOfDate, persistObjectTemplate.getFromPersistWithKnexId.bind(persistObjectTemplate, template, id, options.fetch, options.transient, null, null, options.logger, options.enableChangeTracking, options.projection))); + PersistorCtx.checkAndExecuteWithContext(options.asOfDate, persistObjectTemplate.getFromPersistWithKnexId.bind(persistObjectTemplate, template, id, options.fetch, options.transient, idMap, null, options.logger, options.enableChangeTracking, options.projection))); const name = 'persistorFetchById'; return fetchQuery @@ -1323,6 +1324,18 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { return this.__defaultTransaction__; }; + PersistObjectTemplate.setPersistorCacheContext = function (cacheContext) { + return PersistorCtx.setExecutionContext(cacheContext); + }; + + Object.defineProperty(PersistObjectTemplate, 'persistorCacheCtxKey', { + get: function () { + return PersistorCtx.persistorCacheCtxKey; + }, + enumerable: false, + configurable: false + }) + PersistObjectTemplate.commit = async function commit(options) { var time = getTime(); PersistObjectTemplate._validateParams(options, 'commitSchema'); diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts index 56ab5bd9..5137b4f8 100644 --- a/components/persistor/lib/knex/PersistorCtx.ts +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -12,8 +12,10 @@ class ExecutionCtx { get asOfDate() { return this._asOfDate; } + } + export class PersistorCtx { static persistorExnCtxKey = '#persistor-exn-ctx'; static persistorCacheCtxKey = '#persistor-cache-ctx'; @@ -23,9 +25,11 @@ export class PersistorCtx { return this._asyncLocalStorage || (this._asyncLocalStorage = new AsyncLocalStorage()); } - static checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { + static async checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { + console.log('testing...'); if (!asOfDate) { - return callback(); + const response = await callback(); + return Promise.resolve(response); } const ctxProps = { @@ -33,7 +37,8 @@ export class PersistorCtx { properties: { [this.persistorExnCtxKey]: new ExecutionCtx(asOfDate) }, }; return this.asyncLocalStorage.run(ctxProps, async () => { - return await callback(); + const response = await callback(); + return Promise.resolve(response); }); } @@ -49,7 +54,18 @@ export class PersistorCtx { return exnCtx; } - static set setExecutionContext(asyncLocalStorage: AsyncLocalStorage) { + static get persistorCacheCtx() { + if (!this._asyncLocalStorage) { + return; + } + const store = this.asyncLocalStorage.getStore() as CtxProps; + if (!store) { + return; + } + return store.properties[this.persistorCacheCtxKey]; + } + + static setExecutionContext(asyncLocalStorage: AsyncLocalStorage) { const store = asyncLocalStorage.getStore() as CtxProps; if (!store) { return; diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index 38af9100..41ab6ead 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -54,6 +54,33 @@ module.exports = function (PersistObjectTemplate) { enableChangeTracking = enableChangeTracking || schema.enableChangeTracking; + idMap['queryMapper'] = idMap['queryMapper'] || {}; + const keyId = idMap['queryMapper'] && idMap['queryMapper'][`${template.__name__}___${JSON.stringify(queryOrChains)}`]; + if (idMap[keyId] && PersistorCtx.persistorCacheCtx) { + const obj = idMap[keyId]; + return checkAllChildrenLoaded.call(this, obj, obj, cascade) + .then((result) => [result] ); + } + + + async function checkAllChildrenLoaded(orgObj, obj, fetchSpec, promiseHandlers) { + if (!obj) { + return Promise.resolve(orgObj); + } + promiseHandlers = promiseHandlers || []; + for (const key of Object.keys(fetchSpec)) { + if (obj[key + 'Persistor'].isFetched && fetchSpec[key].fetch) { + checkAllChildrenLoaded(orgObj, obj[key], fetchSpec[key].fetch, promiseHandlers); + } + + if (!obj[key + 'Persistor'].isFetched) { + promiseHandlers.push(obj.fetchProperty.bind(obj, key, fetchSpec[key].fetch, {_id: obj[key + 'Persistor'].id}, isTransient, idMap, logger)); + } + } + + return this.resolveRecursiveRequests(promiseHandlers, obj); + } + // Determine one-to-one relationships and add function chains for where var props = template.getProperties(); var join = 1; @@ -118,6 +145,10 @@ module.exports = function (PersistObjectTemplate) { const obj = await PersistObjectTemplate.getTemplateFromKnexPOJO(pojo, template, requests, idMap, cascade, isTransient, null, establishedObject, null, this.dealias(template.__table__) + '___', joins, isRefresh, logger, enableChangeTracking, projection, orgCascade); results[sortMap[obj._id]] = obj; + + idMap['queryMapper'] = idMap['queryMapper'] || {}; + const keyId = `${template.__name__}___${JSON.stringify(queryOrChains)}`; + idMap['queryMapper'][keyId] = pojo[this.dealias(template.__table__) + '____id']; } return results; diff --git a/components/persistor/lib/persistable.ts b/components/persistor/lib/persistable.ts index fd8db01d..5053989a 100644 --- a/components/persistor/lib/persistable.ts +++ b/components/persistor/lib/persistable.ts @@ -7,8 +7,13 @@ export class Persistor extends SupertypeSession { * @TODO: was typed `Persistor` but that's weird? doesn't work need to figure out what's going on. */ static create(): any | undefined {return undefined}; + + setPersistorCacheContext(cacheContext: any) : any {}; + + persistorCacheCtxKey: string; beginDefaultTransaction() : any {} + /** * Creates A persistor transaction, and returns it */ diff --git a/components/persistor/test/persist_banking_pgsql.js b/components/persistor/test/persist_banking_pgsql.js index ca491968..710aeca5 100644 --- a/components/persistor/test/persist_banking_pgsql.js +++ b/components/persistor/test/persist_banking_pgsql.js @@ -1374,13 +1374,16 @@ describe('Banking from pgsql Example persist_banking_pgsql', function () { customer.roles.forEach(function (role) { var account = role.account; account.roles.forEach(function(role) { + console.log('testing..', role._id); promises.push(role.persistDelete(txn)); promises.push(role.account.persistDelete(txn)); }) }); promises.push(customer.persistDelete()); }); - return Promise.all(promises); + return Promise.all(promises).catch((e) => { + console.log(e); + }); } var txn = PersistObjectTemplate.begin(); txn.preSave = deleteStuff; diff --git a/components/persistor/test/persist_cachewithfetchspec.js b/components/persistor/test/persist_cachewithfetchspec.js new file mode 100644 index 00000000..d12e5e77 --- /dev/null +++ b/components/persistor/test/persist_cachewithfetchspec.js @@ -0,0 +1,249 @@ +var chai = require('chai'), + expect = require('chai').expect; + +const { AsyncLocalStorage } = require('async_hooks'); +var chaiAsPromised = require('chai-as-promised'); + +chai.should(); +chai.use(chaiAsPromised); + + +var knexInit = require('knex'); +const { PersistorUtils } = require('../lib/utils/PersistorUtils.js'); +var knex; + +var schema = {}; +var schemaTable = 'index_schema_history'; +var Phone, Address, Employee, empId, addressId, phoneId, Role, dob; +var PersistObjectTemplate, ObjectTemplate; + + +var ExecutionCtx = /** @class */ (function () { + function ExecutionCtx(asOfDate) { + this._asOfDate = asOfDate; + } + Object.defineProperty(ExecutionCtx.prototype, "asOfDate", { + get: function () { + return this._asOfDate; + }, + enumerable: false, + configurable: true + }); + return ExecutionCtx; +}()); + + +describe('persist newapi tests', function () { + before('drop schema table once per test suit', function () { + knex = knexInit({ + client: 'pg', + debug: true, + connection: { + host: process.env.dbPath, + database: process.env.dbName, + user: process.env.dbUser, + password: process.env.dbPassword, + } + }); + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); + }) + after('closes the database', function () { + return knex.destroy(); + }); + beforeEach('arrange', function () { + ObjectTemplate = require('@haventech/supertype').default; + PersistObjectTemplate = require('../dist/index.js')(ObjectTemplate, null, ObjectTemplate); + + schema.Employee = {}; + schema.Address = {}; + schema.Phone = {}; + schema.Dept = {}; + schema.Employee.table = 'tx_employee'; + schema.Employee.audit = 'v2'; + schema.Address.table = 'tx_address'; + schema.Address.audit = 'v2'; + schema.Address.tableType = 'reference_data'; + schema.Phone.table = 'tx_phone'; + + schema.Employee.parents = { + homeAddress: { + id: 'homeaddress_id' + }, + residentialAddress: { + id: 'residentialaddress_id' + } + }; + schema.Employee.children = { + roles: { id: 'employee_id', fetch: true } + }; + + schema.Employee.enableChangeTracking = true; + + schema.Role = {}; + schema.Role.table = 'tx_role'; + schema.Role.audit = 'v2'; + schema.Role.parents = { + employee: { id: 'employee_id' } + }; + schema.Role.children = { + department: { id: 'role_id' } + }; + + schema.Address.parents = { + phone: { id: 'phone_id' } + }; + Phone = PersistObjectTemplate.create('Phone', { + number: { type: String } + }); + + Address = PersistObjectTemplate.create('Address', { + city: { type: String }, + state: { type: String }, + phone: { type: Phone } + }); + + + Role = PersistObjectTemplate.create('Role', { + name: { type: String }, + + }); + + Employee = PersistObjectTemplate.create('Employee', { + name: { type: String, value: 'Test Employee' }, + homeAddress: { type: Address }, + residentialAddress: { type: Address }, + roles: { type: Array, of: Role, value: [] }, + dob: { type: Date }, + customObj: { type: Object }, + isMarried: { type: Boolean } + }); + + Role.mixin({ + employee: { type: Employee } + }); + var emp = new Employee(); + var add = new Address(); + var resAdd = new Address(); + var phone = new Phone(); + var residentialPhone = new Phone(); + var role1 = new Role(); + role1.name = 'firstRole2'; + role1.employee = emp; + var role2 = new Role(); + role2.name = 'secondRole2'; + role2.employee = emp; + + phone.number = '1231231234'; + residentialPhone.number = '1111111111'; + add.city = 'New York'; + add.state = 'New York'; + resAdd.city = 'Princeton'; + resAdd.state = 'New Jersey'; + resAdd.phone = residentialPhone; + add.phone = phone; + emp.name = 'InitialName'; + dob = new Date(); + emp.dob = dob; + emp.homeAddress = add; + emp.residentialAddress = resAdd; + emp.roles.push(role1); + emp.roles.push(role2); + + (function () { + PersistObjectTemplate.setDB(knex, PersistObjectTemplate.DB_Knex); + PersistObjectTemplate.setSchema(schema); + PersistObjectTemplate.performInjections(); + + })(); + return Promise.resolve(prepareData()); + + function prepareData() { + return syncTable(Employee) + .then(syncTable.bind(this, Address)) + .then(syncTable.bind(this, Phone)) + .then(syncTable.bind(this, Role)) + .then(createRecords.bind(this)) + .catch(e => { + console.log(e); + }); + + + function syncTable(template) { + return PersistObjectTemplate.synchronizeKnexTableFromTemplate(template); + } + + function createRecords() { + var tx = PersistObjectTemplate.beginDefaultTransaction(); + + return emp.persist({ transaction: tx, cascade: false }).then(function () { + return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: true }).then(function () { + empId = emp._id; + addressId = add._id; + phoneId = phone._id; + }); + }) + } + } + }); + + afterEach('remove tables and after each test', function () { + return knex.raw(` + DO $$ DECLARE + r RECORD; + BEGIN + FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; + END LOOP; + END $$; + `); + }); + + it('Adding data and capturing for persistorFetchById', async function () { + + var asyncLocalStorage = new AsyncLocalStorage(); + var cacheKey = PersistObjectTemplate.persistorCacheCtxKey + var ctxProps = { + name: "persistorTest", + properties: { [cacheKey]: {}}, + }; + + return asyncLocalStorage.run(ctxProps, async function () { + PersistObjectTemplate.setPersistorCacheContext(asyncLocalStorage); + return loadChecks(); + }) + + async function loadChecks() { + var employee = await Employee.persistorFetchById(empId, + { + fetch: { homeAddress: { fetch: { phone: true } }, roles: true } + }); + expect(employee.name).is.equal('InitialName'); + expect(employee.homeAddress.city).is.equal('New York'); + expect(employee.roles[0].name).is.equal('firstRole2'); + expect(employee.residentialAddress).to.equal(null); + employee = await Employee.persistorFetchById(empId, + { + fetch: { residentialAddress: true } + }); + expect(employee.residentialAddress).to.not.equal(null); + expect(employee.residentialAddress.phone).to.equal(null); + employee = await Employee.persistorFetchById(empId, + { + fetch: { homeAddress: { fetch: { phone: true } }, residentialAddress: { fetch: { phone: true } }, roles: true } + }); + expect(employee.residentialAddress.phone).to.not.equal(null); + employee = await Employee.persistorFetchById(empId, + { + fetch: { homeAddress: { fetch: { phone: true } }, residentialAddress: { fetch: { phone: true } }, roles: true } + }); + } + }); +}); \ No newline at end of file diff --git a/components/persistor/test/persist_pointintime_tests.js b/components/persistor/test/persist_pointintime_tests.js index 4c7bedad..70be148a 100644 --- a/components/persistor/test/persist_pointintime_tests.js +++ b/components/persistor/test/persist_pointintime_tests.js @@ -330,7 +330,6 @@ describe('persist newapi tests', function () { it('Adding data and capturing for persistorFetchById', async function () { var orgRecordedTime = new Date(); - await PersistorUtils.sleep(100); var employee = await Employee.persistorFetchById(empId, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } @@ -346,7 +345,6 @@ describe('persist newapi tests', function () { employee.roles[0].setDirty(tx); // await employee.persistSave(); await PersistObjectTemplate.commit({transaction: tx}); - await PersistorUtils.sleep(100); var updatedTime = new Date(); employee = await Employee.persistorFetchById(empId, { @@ -367,7 +365,6 @@ describe('persist newapi tests', function () { }); it('Adding data and capturing for persistorFetchByQuery', async function () { var orgRecordedTime = new Date(); - await PersistorUtils.sleep(100); var employee = (await Employee.persistorFetchByQuery({dob: dob}, { fetch: { homeAddress: { fetch: { phone: true } }, roles: true }, projection: { Address: ['city'], Role: ['name'], Phone: [''] } @@ -383,7 +380,6 @@ describe('persist newapi tests', function () { employee.roles[0].setDirty(tx); // await employee.persistSave(); await PersistObjectTemplate.commit({transaction: tx}); - await PersistorUtils.sleep(100); var updatedTime = new Date(); employee = (await Employee.persistorFetchByQuery({dob: dob}, { @@ -405,7 +401,6 @@ describe('persist newapi tests', function () { }); it('Adding data and capturing for getFromPersistWithId', async function () { var orgRecordedTime = new Date(); - await PersistorUtils.sleep(100); var employee = await Employee.getFromPersistWithId(empId, { homeAddress: { fetch: { phone: true } }, roles: true }); @@ -419,7 +414,6 @@ describe('persist newapi tests', function () { employee.roles[0].setDirty(tx); // await employee.persistSave(); await PersistObjectTemplate.commit({transaction: tx}); - await PersistorUtils.sleep(100); var updatedTime = new Date(); employee = await Employee.getFromPersistWithId(empId, { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, orgRecordedTime); @@ -435,7 +429,6 @@ describe('persist newapi tests', function () { }); it('Adding data and capturing for getFromPersistWithQuery', async function () { var orgRecordedTime = new Date(); - await PersistorUtils.sleep(100); var employee = (await Employee.getFromPersistWithQuery({dob: dob}, { homeAddress: { fetch: { phone: true } }, roles: true }))[0]; @@ -449,7 +442,6 @@ describe('persist newapi tests', function () { employee.roles[0].setDirty(tx); // await employee.persistSave(); await PersistObjectTemplate.commit({transaction: tx}); - await PersistorUtils.sleep(100); var updatedTime = new Date(); employee = (await Employee.getFromPersistWithQuery({dob: dob}, { homeAddress: { fetch: { phone: true } }, roles: true }, false, undefined, false, undefined, undefined, undefined, orgRecordedTime))[0]; From dce0f54af01bf6b644a9a8dfbdbff3e0f973ac60 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Fri, 17 Mar 2023 19:44:01 -0400 Subject: [PATCH 11/14] caching fixes for queries other than ids --- components/persistor/lib/api.ts | 6 ++- components/persistor/lib/knex/PersistorCtx.ts | 1 - components/persistor/lib/knex/query.ts | 30 ++++++++---- components/persistor/lib/persistable.ts | 4 +- .../persistor/test/persist_banking_pgsql.js | 1 - .../test/persist_cachewithfetchspec.js | 47 ++++++++++++------- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/components/persistor/lib/api.ts b/components/persistor/lib/api.ts index b88fe620..44706ea0 100644 --- a/components/persistor/lib/api.ts +++ b/components/persistor/lib/api.ts @@ -330,6 +330,8 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { PersistObjectTemplate._validateParams(options, 'persistSchema', template); options = options || {}; + const idMap = PersistorCtx.persistorCacheCtx || null; + var dbType = PersistObjectTemplate.getDB(PersistObjectTemplate.getDBAlias(template.__collection__)).type; let deleteQuery = dbType == PersistObjectTemplate.DB_Mongo ? PersistObjectTemplate.deleteFromPersistWithMongoQuery(template, query, options.logger) : @@ -362,6 +364,8 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { options = options || {}; var persistObjectTemplate = options.session || PersistObjectTemplate; + const idMap = PersistorCtx.persistorCacheCtx || null; + var logger = options.logger || persistObjectTemplate.logger; logger.debug({ module: moduleName, @@ -379,7 +383,7 @@ module.exports = function (PersistObjectTemplate, baseClassForPersist) { persistObjectTemplate.getFromPersistWithMongoQuery(template, query, options.fetch, options.start, options.limit, options.transient, options.order, options.order, logger) : PersistorCtx.checkAndExecuteWithContext(options.asOfDate, persistObjectTemplate.getFromPersistWithKnexQuery.bind(persistObjectTemplate, null, template, query, options.fetch, options.start, - options.limit, options.transient, null, options.order, + options.limit, options.transient, idMap, options.order, undefined, undefined, logger, options.enableChangeTracking, options.projection))); const name = 'persistorFetchByQuery'; diff --git a/components/persistor/lib/knex/PersistorCtx.ts b/components/persistor/lib/knex/PersistorCtx.ts index 5137b4f8..1a288721 100644 --- a/components/persistor/lib/knex/PersistorCtx.ts +++ b/components/persistor/lib/knex/PersistorCtx.ts @@ -26,7 +26,6 @@ export class PersistorCtx { } static async checkAndExecuteWithContext(asOfDate: Date, callback: () => any ) { - console.log('testing...'); if (!asOfDate) { const response = await callback(); return Promise.resolve(response); diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index 41ab6ead..0a2b8a81 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -59,22 +59,33 @@ module.exports = function (PersistObjectTemplate) { if (idMap[keyId] && PersistorCtx.persistorCacheCtx) { const obj = idMap[keyId]; return checkAllChildrenLoaded.call(this, obj, obj, cascade) - .then((result) => [result] ); + .then((result) => [obj] ); } - async function checkAllChildrenLoaded(orgObj, obj, fetchSpec, promiseHandlers) { + async function checkAllChildrenLoaded(parentObject, obj, fetchSpec, promiseHandlers) { if (!obj) { - return Promise.resolve(orgObj); + return Promise.resolve(parentObject); } promiseHandlers = promiseHandlers || []; for (const key of Object.keys(fetchSpec)) { - if (obj[key + 'Persistor'].isFetched && fetchSpec[key].fetch) { - checkAllChildrenLoaded(orgObj, obj[key], fetchSpec[key].fetch, promiseHandlers); + const typeDef = obj.__template__.getProperties()[key] + if (!typeDef.type.isObjectTemplate) { + continue; } - - if (!obj[key + 'Persistor'].isFetched) { - promiseHandlers.push(obj.fetchProperty.bind(obj, key, fetchSpec[key].fetch, {_id: obj[key + 'Persistor'].id}, isTransient, idMap, logger)); + if (typeDef.type === Array) { + for(obj of parentObject[key]){ + await checkAllChildrenLoaded(obj, obj, fetchSpec[key].fetch, promiseHandlers) + } + } + else { + if (obj[key + 'Persistor'].isFetched && fetchSpec[key].fetch) { + await checkAllChildrenLoaded.call(this, obj, obj[key], fetchSpec[key].fetch, promiseHandlers); + } + + if (!obj[key + 'Persistor'].isFetched) { + promiseHandlers.push(obj.fetchProperty.bind(obj, key, fetchSpec[key].fetch, {_id: obj[key + 'Persistor'].id}, isTransient, idMap, logger)); + } } } @@ -149,6 +160,9 @@ module.exports = function (PersistObjectTemplate) { idMap['queryMapper'] = idMap['queryMapper'] || {}; const keyId = `${template.__name__}___${JSON.stringify(queryOrChains)}`; idMap['queryMapper'][keyId] = pojo[this.dealias(template.__table__) + '____id']; + if (!queryOrChains || !Object.keys(queryOrChains).includes('_id')) { + idMap['queryMapper'][`${template.__name__}___${JSON.stringify({_id: this.dealias(template.__table__) + '____id'})}`] = pojo[this.dealias(template.__table__) + '____id'] + } } return results; diff --git a/components/persistor/lib/persistable.ts b/components/persistor/lib/persistable.ts index 5053989a..d94b1a64 100644 --- a/components/persistor/lib/persistable.ts +++ b/components/persistor/lib/persistable.ts @@ -8,9 +8,9 @@ export class Persistor extends SupertypeSession { */ static create(): any | undefined {return undefined}; - setPersistorCacheContext(cacheContext: any) : any {}; + static setPersistorCacheContext(cacheContext: any) : any {}; - persistorCacheCtxKey: string; + static persistorCacheCtxKey: string; beginDefaultTransaction() : any {} diff --git a/components/persistor/test/persist_banking_pgsql.js b/components/persistor/test/persist_banking_pgsql.js index 710aeca5..631ba167 100644 --- a/components/persistor/test/persist_banking_pgsql.js +++ b/components/persistor/test/persist_banking_pgsql.js @@ -1374,7 +1374,6 @@ describe('Banking from pgsql Example persist_banking_pgsql', function () { customer.roles.forEach(function (role) { var account = role.account; account.roles.forEach(function(role) { - console.log('testing..', role._id); promises.push(role.persistDelete(txn)); promises.push(role.account.persistDelete(txn)); }) diff --git a/components/persistor/test/persist_cachewithfetchspec.js b/components/persistor/test/persist_cachewithfetchspec.js index d12e5e77..50dbf69b 100644 --- a/components/persistor/test/persist_cachewithfetchspec.js +++ b/components/persistor/test/persist_cachewithfetchspec.js @@ -14,7 +14,7 @@ var knex; var schema = {}; var schemaTable = 'index_schema_history'; -var Phone, Address, Employee, empId, addressId, phoneId, Role, dob; +var Phone, Address, Employee, empId, addressId, phoneId, Role, dob, Responsibility; var PersistObjectTemplate, ObjectTemplate; @@ -94,12 +94,15 @@ describe('persist newapi tests', function () { employee: { id: 'employee_id' } }; schema.Role.children = { - department: { id: 'role_id' } + responsibilities: { id: 'role_id' } }; schema.Address.parents = { phone: { id: 'phone_id' } }; + schema.Responsibility = {}; + schema.Responsibility.table = 'tx_responsibility'; + schema.Responsibility.audit = 'v2'; Phone = PersistObjectTemplate.create('Phone', { number: { type: String } }); @@ -116,6 +119,10 @@ describe('persist newapi tests', function () { }); + Responsibility = PersistObjectTemplate.create('Responsibility', { + description: { type: String }, + }); + Employee = PersistObjectTemplate.create('Employee', { name: { type: String, value: 'Test Employee' }, homeAddress: { type: Address }, @@ -127,19 +134,20 @@ describe('persist newapi tests', function () { }); Role.mixin({ - employee: { type: Employee } + employee: { type: Employee }, + responsibilities: { type: Array, of: Responsibility, value: [] }, }); var emp = new Employee(); var add = new Address(); var resAdd = new Address(); var phone = new Phone(); var residentialPhone = new Phone(); - var role1 = new Role(); - role1.name = 'firstRole2'; - role1.employee = emp; - var role2 = new Role(); - role2.name = 'secondRole2'; - role2.employee = emp; + var role1 = new Role(); + // role1.name = 'firstRole2'; + // role1.employee = emp; + var role2 = new Role(); + // role2.name = 'secondRole2'; + // role2.employee = emp; phone.number = '1231231234'; residentialPhone.number = '1111111111'; @@ -154,8 +162,8 @@ describe('persist newapi tests', function () { emp.dob = dob; emp.homeAddress = add; emp.residentialAddress = resAdd; - emp.roles.push(role1); - emp.roles.push(role2); + // emp.roles.push(role1); + // emp.roles.push(role2); (function () { PersistObjectTemplate.setDB(knex, PersistObjectTemplate.DB_Knex); @@ -170,6 +178,7 @@ describe('persist newapi tests', function () { .then(syncTable.bind(this, Address)) .then(syncTable.bind(this, Phone)) .then(syncTable.bind(this, Role)) + .then(syncTable.bind(this, Responsibility)) .then(createRecords.bind(this)) .catch(e => { console.log(e); @@ -206,7 +215,7 @@ describe('persist newapi tests', function () { `); }); - it('Adding data and capturing for persistorFetchById', async function () { + it('Adding data and capturing for persistorFetchById cache', async function () { var asyncLocalStorage = new AsyncLocalStorage(); var cacheKey = PersistObjectTemplate.persistorCacheCtxKey @@ -221,13 +230,19 @@ describe('persist newapi tests', function () { }) async function loadChecks() { - var employee = await Employee.persistorFetchById(empId, + var employee = (await Employee.persistorFetchByQuery({name: 'InitialName'}, { - fetch: { homeAddress: { fetch: { phone: true } }, roles: true } - }); + fetch: { homeAddress: { fetch: { phone: true } }, roles: { fetch: { responsibilities: true}} } + }))[0]; expect(employee.name).is.equal('InitialName'); + + employee = await Employee.persistorFetchById(empId, + { + fetch: { homeAddress: { fetch: { phone: true } }, roles: { fetch: { responsibilities: true}} } + }); + expect(employee.homeAddress.city).is.equal('New York'); - expect(employee.roles[0].name).is.equal('firstRole2'); + // expect(employee.roles[0].name).is.equal('firstRole2'); expect(employee.residentialAddress).to.equal(null); employee = await Employee.persistorFetchById(empId, { From 0e6f2d4de3fbf0f480861713972b6bc68144c5a5 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sat, 18 Mar 2023 00:54:08 -0400 Subject: [PATCH 12/14] When querying for multiple results, cache should check for all objects --- components/persistor/lib/knex/query.ts | 28 +++++++++++++------ .../test/persist_cachewithfetchspec.js | 24 +++++++++++++--- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index 0a2b8a81..83aa85e0 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -55,14 +55,18 @@ module.exports = function (PersistObjectTemplate) { enableChangeTracking = enableChangeTracking || schema.enableChangeTracking; idMap['queryMapper'] = idMap['queryMapper'] || {}; - const keyId = idMap['queryMapper'] && idMap['queryMapper'][`${template.__name__}___${JSON.stringify(queryOrChains)}`]; - if (idMap[keyId] && PersistorCtx.persistorCacheCtx) { - const obj = idMap[keyId]; - return checkAllChildrenLoaded.call(this, obj, obj, cascade) - .then((result) => [obj] ); - } - + const keyIds = idMap['queryMapper'] && idMap['queryMapper'][`${template.__name__}___${JSON.stringify(queryOrChains)}`]; + var resultsPromise = []; + if (keyIds && PersistorCtx.persistorCacheCtx) { + keyIds.split(',').forEach(keyId => { + if (idMap[keyId]) { + const obj = idMap[keyId]; + resultsPromise.push(checkAllChildrenLoaded.call(this, obj, obj, cascade)); + } + }); + return Promise.all(resultsPromise); + } async function checkAllChildrenLoaded(parentObject, obj, fetchSpec, promiseHandlers) { if (!obj) { return Promise.resolve(parentObject); @@ -159,9 +163,15 @@ module.exports = function (PersistObjectTemplate) { idMap['queryMapper'] = idMap['queryMapper'] || {}; const keyId = `${template.__name__}___${JSON.stringify(queryOrChains)}`; - idMap['queryMapper'][keyId] = pojo[this.dealias(template.__table__) + '____id']; + if (!idMap['queryMapper'][keyId]) { + idMap['queryMapper'][keyId] = pojo[this.dealias(template.__table__) + '____id']; + } + else { + idMap['queryMapper'][keyId] += ',' + pojo[this.dealias(template.__table__) + '____id']; + } + if (!queryOrChains || !Object.keys(queryOrChains).includes('_id')) { - idMap['queryMapper'][`${template.__name__}___${JSON.stringify({_id: this.dealias(template.__table__) + '____id'})}`] = pojo[this.dealias(template.__table__) + '____id'] + idMap['queryMapper'][`${template.__name__}___${JSON.stringify({_id: pojo[this.dealias(template.__table__) + '____id']})}`] = pojo[this.dealias(template.__table__) + '____id'] } } diff --git a/components/persistor/test/persist_cachewithfetchspec.js b/components/persistor/test/persist_cachewithfetchspec.js index 50dbf69b..0d4b2817 100644 --- a/components/persistor/test/persist_cachewithfetchspec.js +++ b/components/persistor/test/persist_cachewithfetchspec.js @@ -138,6 +138,7 @@ describe('persist newapi tests', function () { responsibilities: { type: Array, of: Responsibility, value: [] }, }); var emp = new Employee(); + var emp1 = new Employee(); var add = new Address(); var resAdd = new Address(); var phone = new Phone(); @@ -157,11 +158,19 @@ describe('persist newapi tests', function () { resAdd.state = 'New Jersey'; resAdd.phone = residentialPhone; add.phone = phone; - emp.name = 'InitialName'; dob = new Date(); + emp.name = 'InitialName'; emp.dob = dob; emp.homeAddress = add; emp.residentialAddress = resAdd; + + emp1.name = 'InitialName'; + emp1.dob = dob; + emp1.homeAddress = add; + emp1.residentialAddress = resAdd; + + + // emp.roles.push(role1); // emp.roles.push(role2); @@ -191,7 +200,7 @@ describe('persist newapi tests', function () { function createRecords() { var tx = PersistObjectTemplate.beginDefaultTransaction(); - + emp1.setDirty(tx); return emp.persist({ transaction: tx, cascade: false }).then(function () { return PersistObjectTemplate.commit({ transaction: tx, notifyChanges: true }).then(function () { empId = emp._id; @@ -230,11 +239,18 @@ describe('persist newapi tests', function () { }) async function loadChecks() { - var employee = (await Employee.persistorFetchByQuery({name: 'InitialName'}, + var employees = await Employee.persistorFetchByQuery({name: 'InitialName'}, { fetch: { homeAddress: { fetch: { phone: true } }, roles: { fetch: { responsibilities: true}} } - }))[0]; + }); + var employee = employees[0]; expect(employee.name).is.equal('InitialName'); + employees = await Employee.persistorFetchByQuery({name: 'InitialName'}, + { + fetch: { homeAddress: { fetch: { phone: true } }, roles: { fetch: { responsibilities: true}} } + }); + var employee1 = employees[0]; + expect(employee1.name).is.equal('InitialName'); employee = await Employee.persistorFetchById(empId, { From 7c312cec009a003f64ba7eeba65b999607d7b7d4 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sat, 18 Mar 2023 20:18:18 -0400 Subject: [PATCH 13/14] Adding id filters to queryMapper when creating the objects --- components/persistor/lib/knex/query.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/persistor/lib/knex/query.ts b/components/persistor/lib/knex/query.ts index 83aa85e0..69b100be 100644 --- a/components/persistor/lib/knex/query.ts +++ b/components/persistor/lib/knex/query.ts @@ -282,7 +282,9 @@ module.exports = function (PersistObjectTemplate) { return Promise.resolve(idMap[obj._id]); idMap[obj._id] = obj; - //console.log("Adding " + template.__name__ + "-" + obj._id + " to idMap"); + idMap['queryMapper'] = idMap['queryMapper'] || {}; + idMap['queryMapper'][`${obj.__template__.__name__}___${JSON.stringify({_id: obj._id})}`] = obj._id; + if (pojo[prefix + '__version__']) this.withoutChangeTracking(function () { obj.__version__ = pojo[prefix + '__version__']; From c08f6132336d3a42423ae852e59eb018cce80503 Mon Sep 17 00:00:00 2001 From: Ravi Sagiraju Date: Sun, 19 Mar 2023 03:38:34 -0400 Subject: [PATCH 14/14] Updating test case --- .../persistor/test/persist_cachewithfetchspec.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/persistor/test/persist_cachewithfetchspec.js b/components/persistor/test/persist_cachewithfetchspec.js index 0d4b2817..cf57c328 100644 --- a/components/persistor/test/persist_cachewithfetchspec.js +++ b/components/persistor/test/persist_cachewithfetchspec.js @@ -103,6 +103,9 @@ describe('persist newapi tests', function () { schema.Responsibility = {}; schema.Responsibility.table = 'tx_responsibility'; schema.Responsibility.audit = 'v2'; + schema.Responsibility.parents = { + role: { id: 'role_id' } + }; Phone = PersistObjectTemplate.create('Phone', { number: { type: String } }); @@ -121,6 +124,7 @@ describe('persist newapi tests', function () { Responsibility = PersistObjectTemplate.create('Responsibility', { description: { type: String }, + role: { type: Role } }); Employee = PersistObjectTemplate.create('Employee', { @@ -144,12 +148,13 @@ describe('persist newapi tests', function () { var phone = new Phone(); var residentialPhone = new Phone(); var role1 = new Role(); - // role1.name = 'firstRole2'; - // role1.employee = emp; + role1.name = 'firstRole2'; + role1.employee = emp; var role2 = new Role(); - // role2.name = 'secondRole2'; - // role2.employee = emp; - + role2.name = 'secondRole2'; + role2.employee = emp; + emp.roles.push(role1); + emp.roles.push(role2); phone.number = '1231231234'; residentialPhone.number = '1111111111'; add.city = 'New York';