Skip to content

Frontend dev #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Percocco merged 3 commits into from
Sep 30, 2025
Merged

Frontend dev #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
372bb4b
toarray operator added
Percocco Sep 25, 2025
f0a42fc
keycloak configured in frontend
Percocco Sep 26, 2025
1b7b379
fix
Percocco Sep 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,4 @@ frontend/src/app/pages/data-model-mapper/testModule.ts
backend/src/writers/orionWriter.old.js
backend/output/
frontend/src/app/pages/data-model-mapper/stepper_test.html
*.sandbox
164 changes: 65 additions & 99 deletions backend/src/mapHandler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const utils = require('./utils/utils.js');
const validator = require('./schemaHandler.js');
const unorm = require('unorm');
const staticPattern = /static:(.*)/;
const toArrayPattern = /toarray:(.*)/;
const forEachPattern = /foreach:(.*)/;
const dotPattern = /(.*)\.(.*)/;

const log = require('./utils/logger')//.app(module);
Expand Down Expand Up @@ -108,10 +110,19 @@ const encodingHandler = (mapSourceSubField, source) => {
*/

const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) => {

logger.debug({ parsedSourceKey, normSourceKey, schemaDestKey, source })
for (let key in normSourceKey) {
logger.debug({ key })

let schemaDestSubKey
if (schemaDestKey.properties && !schemaDestKey.oneOf)
schemaDestSubKey = schemaDestKey.properties[key];
if (schemaDestKey.oneOf)
for (let oneOfElement of schemaDestKey.oneOf)
if (oneOfElement.properties && oneOfElement.properties[key])
schemaDestSubKey = oneOfElement.properties[key];
logger.debug({ schemaDestSubKey })

let schemaDestSubKey = schemaDestKey.properties[key];
if (schemaDestSubKey) {

let schemaFieldType = schemaDestSubKey.type;
Expand All @@ -127,7 +138,7 @@ const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) =>
parsedSourceKey[key] = new Function("input", "return new Date(input['" + mapSourceSubField + "']).toISOString();");
else if (schemaFieldType === 'string' && Array.isArray(mapSourceSubField))
parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsArray(mapSourceSubField).result);
else if (schemaFieldType === 'array' && Array.isArray(mapSourceSubField))
else if (schemaFieldType === 'array')
parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsToDestArray(mapSourceSubField));
else if (schemaFieldType === 'string' && typeof mapSourceSubField === 'string' && (mapSourceSubField.startsWith("static:") || mapSourceSubField == "")) {
if (mapSourceSubField == "") mapSourceSubField = "static:"
Expand Down Expand Up @@ -171,111 +182,42 @@ const extractFromNestedField = (source, field) => {
*/
const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service, group, entityIdField, NGSI_entity, minioObj, config, res) => {

logger.debug({ rowNumber, source, map })
var result = {};
// If the destKey is entityIdField and has only "static:" fields, the pair value indicates only an ID prefix
// The resulting string will be concatenated with rowNumber
var isIdPrefix = false;

for (var mapDestKey in map) {

let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair
let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair
let singleResult = undefined;
let schemaDestKey = modelSchema.allOf[0].properties[mapDestKey];

//// If the map key has a . , it means that the source key is an object
//if (mapDestKey.test(dotPattern)){
// var extrFields = mapDestKey.match(dotPattern);
// // Check if there are other subfields for this object field
// if (extrFields.length > 1) {


// Check if destKey is present in modelSchema ?
if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) {

if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) {// Check if destKey is present in modelSchema ?
if (config.ignoreValidation && source[map[mapDestKey]])
modelSchema.allOf[0].properties[mapDestKey] = { "type": typeof source[map[mapDestKey]] }

// If the value of key-value maping pair is a function definition, eval it.
//if ( (typeof mapSourceField == "string") && mapSourceField.startsWith("function")) {
// map[destKey] = utils.parseFunction(mapSourceField);

// Convert the single source field from map, to the final mapped single object or key-value pair, to be validated
// If valid, it is added to the final result object, otherwise is discarded

// Normalize encoding, avoiding problems with fields name not recognized due to different source encoding
var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey)));

// Initialize with normalized Source Key, can be replaced in the specific cases below
let parsedSourceKey = normSourceKey;

// If the value type of mapped field is different from string, try first to extract it
// If destination Schema field is OneOf
if (schemaDestKey && !schemaDestKey.type && schemaDestKey.oneOf) {

var oneOf = schemaDestKey.oneOf;

/** If Destination Key is an object with coordinates it's a location type field or if it is "geometry" **/
if (mapDestKey === 'location') {
if (normSourceKey.type && normSourceKey.coordinates && normSourceKey.type.startsWith('static:')) {

var parsedStaticType = normSourceKey.type.match(staticPattern)[1];
if (Array.isArray(oneOf) && oneOf.find(k => k.properties.type.enum.find(e => e == parsedStaticType))) {

parsedSourceKey['type'] = new Function("input", "return '" + parsedStaticType + "'");
parsedSourceKey['coordinates'] = new Function("input", "return " + "[Number(input['" + normSourceKey.coordinates[0] + "']),Number(input['" + normSourceKey.coordinates[1] + "'])]");
}

} else if (normSourceKey === 'geometry' || normSourceKey === 'location') {
parsedSourceKey = new Function("input", "return input['" + normSourceKey + "'];");
} else
continue;
}

/********************* Destination Key is an Object ****************************************/

} else if (schemaDestKey && schemaDestKey.type === 'object' && typeof normSourceKey === 'object')

var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey)));// Normalize encoding, avoiding problems
let parsedSourceKey = normSourceKey;// Initialize with normalized Source Key, can be replaced in the specific cases below
logger.debug({ schemaDestKey, normSourceKey })
if (schemaDestKey && schemaDestKey.type === 'object' || typeof normSourceKey === 'object')
parsedSourceKey = objectHandler(parsedSourceKey, normSourceKey, schemaDestKey, source)

/********************* Destination Field is an Array ********************************************/

else if (schemaDestKey && schemaDestKey.type === 'array' && Array.isArray(normSourceKey))

parsedSourceKey = new Function("input", "return " + handleSourceFieldsToDestArray(normSourceKey));

/********************* Destination Field is a Number ********************************************/

else if (schemaDestKey && (schemaDestKey.type === 'number' || schemaDestKey.type === 'integer'))

if (Array.isArray(normSourceKey))
parsedSourceKey = new Function("input", "return " + handleSourceFieldsArray(normSourceKey, 'number').result);

else {

parsedSourceKey = handleDottedField(normSourceKey);

if (parsedSourceKey.startsWith('[')) {

let num = eval('source' + parsedSourceKey);

if (typeof num === 'string')
parsedSourceKey = new Function("input", "return Number(input['" + normSourceKey + "'])");

else if (typeof num === 'number')
parsedSourceKey = new Function("input", "return input['" + normSourceKey + "']");
}
}

/********************* Destination Field is a String ********************************************/

else if (schemaDestKey && (schemaDestKey.type === 'boolean'))

parsedSourceKey = new Function("input", "return (input['" + normSourceKey + "'].toLowerCase() == 'true' || input['" + normSourceKey + "'] == 1 || input['" + normSourceKey + "'] == '1' ) ? true: false");

else if (schemaDestKey && schemaDestKey.type === 'string') {

if (schemaDestKey.format === 'date-time') {

var date = eval('source' + handleDottedField(normSourceKey));
if (date === undefined || date === '')
continue;
Expand All @@ -286,10 +228,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'");
else if (typeof normSourceKey === 'string' && normSourceKey.startsWith("encode:"))
parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'");

/********************* Destination Key is a entityId field (according to definition in config.js) **/
} else if (mapDestKey == entityIdField) {

if (Array.isArray(normSourceKey) && normSourceKey.length !== 0) {
let resIdFields = handleSourceFieldsArray(normSourceKey);
parsedSourceKey = new Function("input", "return " + resIdFields.result);
Expand All @@ -299,23 +238,11 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'");
else if (normSourceKey.startsWith("encode:"))
parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'");


}

// Add type to map field
//parsedNorm.type = new Function("input", "return '" + modelSchemaDestKey.type +"'");

/********************* Perform actual mapping with parsed and normalized source key (parsedNorm) **/

if (typeof parsedSourceKey == "string")
parsedSourceKey = parsedSourceKey.replaceAll('"', '')

//logger.debug(parsedSourceKey)
//logger.debug(typeof parsedSourceKey)

logger.debug({ mapDestKey, parsedSourceKey, coordinates: parsedSourceKey.coordinates?.toString() })
var converter = mapper.makeConverter({ [mapDestKey]: parsedSourceKey });

try {
singleResult = converter(source);
} catch (error) {
Expand Down Expand Up @@ -377,7 +304,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
result.id = result.id.replaceAll(" ", "")
} catch (error) {
logger.error(error)

logger.error("UnknownEntity")
}
}
Expand Down Expand Up @@ -482,6 +409,10 @@ const handleSourceFieldsArray = (sourceFieldArray, sourceFieldType) => {
/* Map fields of the source array into a stringifed Array (source and dest are both arrays)
*/
const handleSourceFieldsToDestArray = (sourceFieldArray) => {
//let foreachIndex = []
let foreachFound = false

logger.debug({ sourceFieldArray })

if (sourceFieldArray.length > 0) {
var finalArray = [];
Expand All @@ -491,11 +422,40 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
sourceFieldArray.forEach(function (value, index, array) {

var staticMatch = value.match(staticPattern);
//let toArrayMatch = value.match(toArrayPattern);
let forEachMatch = value.match(forEachPattern);
if (staticMatch && staticMatch.length > 0) {

finalArray[index] = staticMatch[1];

} else {
}
else if (forEachMatch && forEachMatch.length > 0) {
if (!foreachFound) foreachFound = true
//let arrayField = forEachMatch[1];
//let arrayFieldCleaned = cleanValue(arrayField);
//foreachIndex.push(index)
let subOperator = forEachMatch[1];
let forEachArgument = subOperator.split(',')[0];
let forEachBody = subOperator.split(',').slice(1).join(',');
if (forEachArgument && forEachBody) {
let toArrayMatch = forEachBody.match(toArrayPattern);
if (toArrayMatch && toArrayMatch.length > 0) {
//let toArray = []
let toArrayBodyField = toArrayMatch[1].split(",");
logger.debug({ toArrayMatch, toArrayBodyField })
finalArray[index] = "input['" + forEachArgument + "'].map(o=> [o['" + toArrayBodyField[0] + "']"
//if(toArrayBodyField.length>2)
if (toArrayBodyField.length > 2)
for (let index = 1; index < toArrayBodyField.length - 1; index++)
finalArray[index] += ", o['" + toArrayBodyField[index] + "']]"
else
finalArray[index] += ", o['" + toArrayBodyField[1] + "']"
finalArray[index] += "])"
}
}
else finalArray[index] = "";
}
else {

var splittedDot = value.match(dotPattern);
if (splittedDot) {
Expand All @@ -511,6 +471,7 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
});

// print Array String as output
logger.debug({ finalArray })
resultString = '[';
finalArray.forEach(function (value, index) {
if (value.startsWith("input")) {
Expand All @@ -519,11 +480,16 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
} else { //static
resultString += '"' + value + '",';
}
//if (foreachIndex.includes(index)) resultString = resultString.substring(1, resultString.length - 1)
});

resultString = resultString.slice(0, resultString.length - 1) + ']';
return resultString;
} else return '[]';
logger.debug({ resultString })
if (foreachFound)
return resultString.substring(1, resultString.length - 1);
return resultString//.substring(1, resultString.length - 1);
}
else return '[]';
};

/* Returns array notation from dotten notation (without input)
Expand Down
50 changes: 33 additions & 17 deletions backend/src/server/api/middlewares/auth.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ module.exports = {
}
catch (error) {

logger.error(error)
logger.error(error)
if (error.message == "invalid token" || error.message == "jwt expired" || error.message == "jwt malformed")
return send(res, 403);
else
Expand Down Expand Up @@ -110,22 +110,34 @@ module.exports = {


if (common.isMinioWriterActive()) {
try {
var data = (await axios.get(config.authConfig.userInfoEndpoint, { headers: { "Authorization": authHeader } })).data
}
catch (error) {
logger.error(error?.toString())
logger.error(error?.response?.data || error?.response)
//req.body.prefix = decodedToken.email
//config.group = decodedToken.email

let data

if (config.authConfig.userInfoEndpoint)
try {
data = await minioWriter.getUserData(decodedToken.email)
data = (await axios.get(config.authConfig.userInfoEndpoint, { headers: { "Authorization": authHeader } })).data
}
catch (error) {
logger.error(error)
send(res, 500, error || error.toString())
logger.error(error?.toString())
logger.error(error?.response?.data || error?.response)
//req.body.prefix = decodedToken.email
//config.group = decodedToken.email
try {
data = await minioWriter.getUserData(decodedToken.email)
logger.debug(data)
}
catch (error) {
logger.error(error)
return send(res, 500, error || error.toString())
}
}
}
else
data = {
email: decodedToken.email,
username: decodedToken.preferred_username,
pilot: decodedToken.pilot?.toLowerCase() || "default"
}
logger.debug(data)
let { pilot, username, email } = data
if (!req.body.config)
req.body.config = {
Expand All @@ -139,14 +151,18 @@ module.exports = {
req.body.email = email
}
else {//TODO test this
req.body.config.orionWriter.fiwareService = req.body.bucketName = decodedToken.pilot.toLowerCase() //+ "/" + email + "/" + config.minioWriter.defaultInputFolderName//{pilot, email}
if (!req.body.config)
req.body.config = {
orionWriter: {}
}
req.body.config.orionWriter.fiwareService = req.body.bucketName = decodedToken.pilot?.toLowerCase() || "default" //+ "/" + email + "/" + config.minioWriter.defaultInputFolderName//{pilot, email}
req.body.prefix = (decodedToken.email || decodedToken.username) + "/" + config.minioWriter.defaultInputFolderName
req.body.config.group = decodedToken.email || decodedToken.username
req.body.config.orionWriter.fiwareServicePath = "/" + decodedToken.pilot.toLowerCase()
req.body.pilot = decodedToken.pilot
req.body.config.orionWriter.fiwareServicePath = "/" + req.body.bucketName
req.body.pilot = req.body.bucketName
req.body.email = decodedToken.email
}
logger.debug(req.body.prefix)
//logger.debug({ body: req.body })



Expand Down
4 changes: 2 additions & 2 deletions frontend/src/assets/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
"checkConsentAtOperator": false,
"dmmGuiUrl": "http://localhost:12345/data-model-mapper-gui",
"auth": {
"idmHost": "https://platform.beopendep.it/auth",
"clientId": "beopen-dashboard",
"idmHost": "http://localhost:8080",
"clientId": "dmm",
"disableAuth": "false",
"authProfile": "oidc",
"authRealm": "master"
Expand Down
Loading
Loading