From 372bb4b3b495c1e09c796649d3941932b44e9f18 Mon Sep 17 00:00:00 2001 From: Gabriele Percoco <114912675+Percocco@users.noreply.github.com> Date: Thu, 25 Sep 2025 15:41:34 +0200 Subject: [PATCH 1/3] toarray operator added --- .gitignore | 1 + backend/src/mapHandler.js | 164 +++++++++++++++----------------------- 2 files changed, 66 insertions(+), 99 deletions(-) diff --git a/.gitignore b/.gitignore index 1c9b506..82ada0e 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ frontend/src/app/pages/data-model-mapper/testModule.ts backend/src/writers/orionWriter.old.js backend/output/ frontend/src/app/pages/data-model-mapper/stepper_test.html +*.sandbox diff --git a/backend/src/mapHandler.js b/backend/src/mapHandler.js index f555b09..a48fb40 100644 --- a/backend/src/mapHandler.js +++ b/backend/src/mapHandler.js @@ -24,6 +24,8 @@ const utils = require('./utils/utils.js'); const validator = require('./schemaHandler.js'); const unorm = require('unorm'); const staticPattern = /static:(.*)/; +const toArrayPattern = /toarray:(.*)/; +const forEachPattern = /foreach:(.*)/; const dotPattern = /(.*)\.(.*)/; const log = require('./utils/logger')//.app(module); @@ -108,10 +110,19 @@ const encodingHandler = (mapSourceSubField, source) => { */ const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) => { - + logger.debug({ parsedSourceKey, normSourceKey, schemaDestKey, source }) for (let key in normSourceKey) { + logger.debug({ key }) + + let schemaDestSubKey + if (schemaDestKey.properties && !schemaDestKey.oneOf) + schemaDestSubKey = schemaDestKey.properties[key]; + if (schemaDestKey.oneOf) + for (let oneOfElement of schemaDestKey.oneOf) + if (oneOfElement.properties && oneOfElement.properties[key]) + schemaDestSubKey = oneOfElement.properties[key]; + logger.debug({ schemaDestSubKey }) - let schemaDestSubKey = schemaDestKey.properties[key]; if (schemaDestSubKey) { let schemaFieldType = schemaDestSubKey.type; @@ -127,7 +138,7 @@ const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) => parsedSourceKey[key] = new Function("input", "return new Date(input['" + mapSourceSubField + "']).toISOString();"); else if (schemaFieldType === 'string' && Array.isArray(mapSourceSubField)) parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsArray(mapSourceSubField).result); - else if (schemaFieldType === 'array' && Array.isArray(mapSourceSubField)) + else if (schemaFieldType === 'array') parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsToDestArray(mapSourceSubField)); else if (schemaFieldType === 'string' && typeof mapSourceSubField === 'string' && (mapSourceSubField.startsWith("static:") || mapSourceSubField == "")) { if (mapSourceSubField == "") mapSourceSubField = "static:" @@ -171,111 +182,42 @@ const extractFromNestedField = (source, field) => { */ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service, group, entityIdField, NGSI_entity, minioObj, config, res) => { + logger.debug({ rowNumber, source, map }) var result = {}; // If the destKey is entityIdField and has only "static:" fields, the pair value indicates only an ID prefix // The resulting string will be concatenated with rowNumber var isIdPrefix = false; - for (var mapDestKey in map) { - - let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair + let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair let singleResult = undefined; let schemaDestKey = modelSchema.allOf[0].properties[mapDestKey]; - - //// If the map key has a . , it means that the source key is an object - //if (mapDestKey.test(dotPattern)){ - // var extrFields = mapDestKey.match(dotPattern); - // // Check if there are other subfields for this object field - // if (extrFields.length > 1) { - - - // Check if destKey is present in modelSchema ? - if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) { - + if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) {// Check if destKey is present in modelSchema ? if (config.ignoreValidation && source[map[mapDestKey]]) modelSchema.allOf[0].properties[mapDestKey] = { "type": typeof source[map[mapDestKey]] } - - // If the value of key-value maping pair is a function definition, eval it. - //if ( (typeof mapSourceField == "string") && mapSourceField.startsWith("function")) { - // map[destKey] = utils.parseFunction(mapSourceField); - - // Convert the single source field from map, to the final mapped single object or key-value pair, to be validated - // If valid, it is added to the final result object, otherwise is discarded - - // Normalize encoding, avoiding problems with fields name not recognized due to different source encoding - var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey))); - - // Initialize with normalized Source Key, can be replaced in the specific cases below - let parsedSourceKey = normSourceKey; - - // If the value type of mapped field is different from string, try first to extract it - // If destination Schema field is OneOf - if (schemaDestKey && !schemaDestKey.type && schemaDestKey.oneOf) { - - var oneOf = schemaDestKey.oneOf; - - /** If Destination Key is an object with coordinates it's a location type field or if it is "geometry" **/ - if (mapDestKey === 'location') { - if (normSourceKey.type && normSourceKey.coordinates && normSourceKey.type.startsWith('static:')) { - - var parsedStaticType = normSourceKey.type.match(staticPattern)[1]; - if (Array.isArray(oneOf) && oneOf.find(k => k.properties.type.enum.find(e => e == parsedStaticType))) { - - parsedSourceKey['type'] = new Function("input", "return '" + parsedStaticType + "'"); - parsedSourceKey['coordinates'] = new Function("input", "return " + "[Number(input['" + normSourceKey.coordinates[0] + "']),Number(input['" + normSourceKey.coordinates[1] + "'])]"); - } - - } else if (normSourceKey === 'geometry' || normSourceKey === 'location') { - parsedSourceKey = new Function("input", "return input['" + normSourceKey + "'];"); - } else - continue; - } - - /********************* Destination Key is an Object ****************************************/ - - } else if (schemaDestKey && schemaDestKey.type === 'object' && typeof normSourceKey === 'object') - + var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey)));// Normalize encoding, avoiding problems + let parsedSourceKey = normSourceKey;// Initialize with normalized Source Key, can be replaced in the specific cases below + logger.debug({ schemaDestKey, normSourceKey }) + if (schemaDestKey && schemaDestKey.type === 'object' || typeof normSourceKey === 'object') parsedSourceKey = objectHandler(parsedSourceKey, normSourceKey, schemaDestKey, source) - - /********************* Destination Field is an Array ********************************************/ - else if (schemaDestKey && schemaDestKey.type === 'array' && Array.isArray(normSourceKey)) - parsedSourceKey = new Function("input", "return " + handleSourceFieldsToDestArray(normSourceKey)); - - /********************* Destination Field is a Number ********************************************/ - else if (schemaDestKey && (schemaDestKey.type === 'number' || schemaDestKey.type === 'integer')) - if (Array.isArray(normSourceKey)) parsedSourceKey = new Function("input", "return " + handleSourceFieldsArray(normSourceKey, 'number').result); - else { - parsedSourceKey = handleDottedField(normSourceKey); - if (parsedSourceKey.startsWith('[')) { - let num = eval('source' + parsedSourceKey); - if (typeof num === 'string') parsedSourceKey = new Function("input", "return Number(input['" + normSourceKey + "'])"); - else if (typeof num === 'number') parsedSourceKey = new Function("input", "return input['" + normSourceKey + "']"); } } - - /********************* Destination Field is a String ********************************************/ - else if (schemaDestKey && (schemaDestKey.type === 'boolean')) - parsedSourceKey = new Function("input", "return (input['" + normSourceKey + "'].toLowerCase() == 'true' || input['" + normSourceKey + "'] == 1 || input['" + normSourceKey + "'] == '1' ) ? true: false"); - else if (schemaDestKey && schemaDestKey.type === 'string') { - if (schemaDestKey.format === 'date-time') { - var date = eval('source' + handleDottedField(normSourceKey)); if (date === undefined || date === '') continue; @@ -286,10 +228,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'"); else if (typeof normSourceKey === 'string' && normSourceKey.startsWith("encode:")) parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'"); - - /********************* Destination Key is a entityId field (according to definition in config.js) **/ } else if (mapDestKey == entityIdField) { - if (Array.isArray(normSourceKey) && normSourceKey.length !== 0) { let resIdFields = handleSourceFieldsArray(normSourceKey); parsedSourceKey = new Function("input", "return " + resIdFields.result); @@ -299,23 +238,11 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'"); else if (normSourceKey.startsWith("encode:")) parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'"); - - } - - // Add type to map field - //parsedNorm.type = new Function("input", "return '" + modelSchemaDestKey.type +"'"); - - /********************* Perform actual mapping with parsed and normalized source key (parsedNorm) **/ - if (typeof parsedSourceKey == "string") parsedSourceKey = parsedSourceKey.replaceAll('"', '') - - //logger.debug(parsedSourceKey) - //logger.debug(typeof parsedSourceKey) - + logger.debug({ mapDestKey, parsedSourceKey, coordinates: parsedSourceKey.coordinates?.toString() }) var converter = mapper.makeConverter({ [mapDestKey]: parsedSourceKey }); - try { singleResult = converter(source); } catch (error) { @@ -377,7 +304,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service result.id = result.id.replaceAll(" ", "") } catch (error) { logger.error(error) - + logger.error("UnknownEntity") } } @@ -482,6 +409,10 @@ const handleSourceFieldsArray = (sourceFieldArray, sourceFieldType) => { /* Map fields of the source array into a stringifed Array (source and dest are both arrays) */ const handleSourceFieldsToDestArray = (sourceFieldArray) => { + //let foreachIndex = [] + let foreachFound = false + + logger.debug({ sourceFieldArray }) if (sourceFieldArray.length > 0) { var finalArray = []; @@ -491,11 +422,40 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => { sourceFieldArray.forEach(function (value, index, array) { var staticMatch = value.match(staticPattern); + //let toArrayMatch = value.match(toArrayPattern); + let forEachMatch = value.match(forEachPattern); if (staticMatch && staticMatch.length > 0) { finalArray[index] = staticMatch[1]; - } else { + } + else if (forEachMatch && forEachMatch.length > 0) { + if (!foreachFound) foreachFound = true + //let arrayField = forEachMatch[1]; + //let arrayFieldCleaned = cleanValue(arrayField); + //foreachIndex.push(index) + let subOperator = forEachMatch[1]; + let forEachArgument = subOperator.split(',')[0]; + let forEachBody = subOperator.split(',').slice(1).join(','); + if (forEachArgument && forEachBody) { + let toArrayMatch = forEachBody.match(toArrayPattern); + if (toArrayMatch && toArrayMatch.length > 0) { + //let toArray = [] + let toArrayBodyField = toArrayMatch[1].split(","); + logger.debug({ toArrayMatch, toArrayBodyField }) + finalArray[index] = "input['" + forEachArgument + "'].map(o=> [o['" + toArrayBodyField[0] + "']" + //if(toArrayBodyField.length>2) + if (toArrayBodyField.length > 2) + for (let index = 1; index < toArrayBodyField.length - 1; index++) + finalArray[index] += ", o['" + toArrayBodyField[index] + "']]" + else + finalArray[index] += ", o['" + toArrayBodyField[1] + "']" + finalArray[index] += "])" + } + } + else finalArray[index] = ""; + } + else { var splittedDot = value.match(dotPattern); if (splittedDot) { @@ -511,6 +471,7 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => { }); // print Array String as output + logger.debug({ finalArray }) resultString = '['; finalArray.forEach(function (value, index) { if (value.startsWith("input")) { @@ -519,11 +480,16 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => { } else { //static resultString += '"' + value + '",'; } + //if (foreachIndex.includes(index)) resultString = resultString.substring(1, resultString.length - 1) }); resultString = resultString.slice(0, resultString.length - 1) + ']'; - return resultString; - } else return '[]'; + logger.debug({ resultString }) + if (foreachFound) + return resultString.substring(1, resultString.length - 1); + return resultString//.substring(1, resultString.length - 1); + } + else return '[]'; }; /* Returns array notation from dotten notation (without input) From f0a42fc3c33dc4896182cff12bbf4ce111c77f53 Mon Sep 17 00:00:00 2001 From: Gabriele Percoco <114912675+Percocco@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:57:17 +0200 Subject: [PATCH 2/3] keycloak configured in frontend --- frontend/src/assets/config.json | 4 ++-- keycloak.yml | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 keycloak.yml diff --git a/frontend/src/assets/config.json b/frontend/src/assets/config.json index 6e8c304..814576a 100644 --- a/frontend/src/assets/config.json +++ b/frontend/src/assets/config.json @@ -3,8 +3,8 @@ "checkConsentAtOperator": false, "dmmGuiUrl": "http://localhost:12345/data-model-mapper-gui", "auth": { - "idmHost": "https://platform.beopendep.it/auth", - "clientId": "beopen-dashboard", + "idmHost": "http://localhost:8080", + "clientId": "dmm", "disableAuth": "false", "authProfile": "oidc", "authRealm": "master" diff --git a/keycloak.yml b/keycloak.yml new file mode 100644 index 0000000..8441da0 --- /dev/null +++ b/keycloak.yml @@ -0,0 +1,37 @@ +version: "3.9" +# tips : OTP disablle : authentication -> browser -> browser - Conditional OTP -> Disabled +# tips : avoid CORS error : configure web origins in client +# tips : if keycloak does not allow login login valid redirect URIs need to be set in client settings +# tips : avoid expired sessions : realm settings -> tokens -> access token lifespan -> 1 Days, o clients -> advanced -> advanced settings -> first 4 lines to 1 Days +services: + keycloak: + image: quay.io/keycloak/keycloak:26.0.2 + container_name: keycloak + command: + - start-dev # only in development, for production use 'start', https protocol and cert + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_DB: postgres + KC_DB_URL_HOST: db + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: keycloak + ports: + - "8080:8080" + restart: unless-stopped + depends_on: + - db + + db: + image: postgres:16 + container_name: keycloak-db + environment: + POSTGRES_DB: keycloak + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: keycloak + volumes: + - keycloak-db-data:/var/lib/postgresql/data + restart: unless-stopped + +volumes: + keycloak-db-data: From 1b7b379aa68325d27db604f9820460dc35f9861b Mon Sep 17 00:00:00 2001 From: Percocco Date: Tue, 30 Sep 2025 15:07:00 +0200 Subject: [PATCH 3/3] fix --- backend/src/server/api/middlewares/auth.js | 50 ++++++++++++++-------- keycloak.yml | 7 ++- orion.yml | 2 +- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/backend/src/server/api/middlewares/auth.js b/backend/src/server/api/middlewares/auth.js index 6fc38cb..7943ad8 100644 --- a/backend/src/server/api/middlewares/auth.js +++ b/backend/src/server/api/middlewares/auth.js @@ -69,7 +69,7 @@ module.exports = { } catch (error) { - logger.error(error) + logger.error(error) if (error.message == "invalid token" || error.message == "jwt expired" || error.message == "jwt malformed") return send(res, 403); else @@ -110,22 +110,34 @@ module.exports = { if (common.isMinioWriterActive()) { - try { - var data = (await axios.get(config.authConfig.userInfoEndpoint, { headers: { "Authorization": authHeader } })).data - } - catch (error) { - logger.error(error?.toString()) - logger.error(error?.response?.data || error?.response) - //req.body.prefix = decodedToken.email - //config.group = decodedToken.email + + let data + + if (config.authConfig.userInfoEndpoint) try { - data = await minioWriter.getUserData(decodedToken.email) + data = (await axios.get(config.authConfig.userInfoEndpoint, { headers: { "Authorization": authHeader } })).data } catch (error) { - logger.error(error) - send(res, 500, error || error.toString()) + logger.error(error?.toString()) + logger.error(error?.response?.data || error?.response) + //req.body.prefix = decodedToken.email + //config.group = decodedToken.email + try { + data = await minioWriter.getUserData(decodedToken.email) + logger.debug(data) + } + catch (error) { + logger.error(error) + return send(res, 500, error || error.toString()) + } } - } + else + data = { + email: decodedToken.email, + username: decodedToken.preferred_username, + pilot: decodedToken.pilot?.toLowerCase() || "default" + } + logger.debug(data) let { pilot, username, email } = data if (!req.body.config) req.body.config = { @@ -139,14 +151,18 @@ module.exports = { req.body.email = email } else {//TODO test this - req.body.config.orionWriter.fiwareService = req.body.bucketName = decodedToken.pilot.toLowerCase() //+ "/" + email + "/" + config.minioWriter.defaultInputFolderName//{pilot, email} + if (!req.body.config) + req.body.config = { + orionWriter: {} + } + req.body.config.orionWriter.fiwareService = req.body.bucketName = decodedToken.pilot?.toLowerCase() || "default" //+ "/" + email + "/" + config.minioWriter.defaultInputFolderName//{pilot, email} req.body.prefix = (decodedToken.email || decodedToken.username) + "/" + config.minioWriter.defaultInputFolderName req.body.config.group = decodedToken.email || decodedToken.username - req.body.config.orionWriter.fiwareServicePath = "/" + decodedToken.pilot.toLowerCase() - req.body.pilot = decodedToken.pilot + req.body.config.orionWriter.fiwareServicePath = "/" + req.body.bucketName + req.body.pilot = req.body.bucketName req.body.email = decodedToken.email } - logger.debug(req.body.prefix) + //logger.debug({ body: req.body }) diff --git a/keycloak.yml b/keycloak.yml index 8441da0..8f6e265 100644 --- a/keycloak.yml +++ b/keycloak.yml @@ -2,7 +2,12 @@ version: "3.9" # tips : OTP disablle : authentication -> browser -> browser - Conditional OTP -> Disabled # tips : avoid CORS error : configure web origins in client # tips : if keycloak does not allow login login valid redirect URIs need to be set in client settings -# tips : avoid expired sessions : realm settings -> tokens -> access token lifespan -> 1 Days, o clients -> advanced -> advanced settings -> first 4 lines to 1 Days +# tips : avoid expired sessions : realm settings -> tokens -> access token lifespan -> 1 Days, or clients -> advanced -> advanced settings -> first 4 lines to 1 Days, +# or realm settings -> sessions -> SSO somethings -> 1 Days +# tips : public key : realm settings -> keys -> active keys -> RS256 +# tips : to allow minio login : client scopes -> create client scope -> consoleAdmin, +# then click consoleAdmin -> mappers -> add mapper -> by configuration -> Hardcoded claim -> tokenClaimName : policy, token claim value : consoleAdmin, +# then on main menu click clients -> your-client -> client scopes -> Add client scope -> consoleAdmin services: keycloak: image: quay.io/keycloak/keycloak:26.0.2 diff --git a/orion.yml b/orion.yml index a62328b..c01a87f 100644 --- a/orion.yml +++ b/orion.yml @@ -1,5 +1,5 @@ version: "3.8" - +# tips : inserted entities, without defined pilots, are inserted with a fiWareService default and a fiWareServicePath /default services: mongo: image: mongo:4.4