Skip to content

Custom operator paralellization #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kespera merged 6 commits into from
Jul 3, 2026
Merged

Custom operator paralellization #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8429057
Paralellize CustomOperators.
kespera Jun 29, 2026
96eb564
Add test measurement output.
kespera Jun 29, 2026
345048e
Fix patternmatching.
kespera Jun 29, 2026
c875029
Edit test measurement output.
kespera Jun 29, 2026
d976294
Add testcase.
kespera Jun 29, 2026
f48f327
Modify paralellization.
kespera Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions qualitypatternmodel/src/qualitypatternmodel/javaoperators/LinkOperatorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@

public class LinkOperatorUtil {

public static final int DEFAULT_TIMEOUT = 5000;
public static final String AGENT = "QPM/1.15; (https://github.com/Project-KONDA/pattern-based-quality-analysis)";
public static final int DEFAULT_TIMEOUT = 50000;
public static final String AGENT = "QPM/1.16; (https://github.com/Project-KONDA/pattern-based-quality-analysis)";

private static final HttpClient CLIENT = HttpClient.newBuilder()
.connectTimeout(Duration.ofMillis(DEFAULT_TIMEOUT))
Expand Down
61 changes: 46 additions & 15 deletions qualitypatternmodel/src/qualitypatternmodel/javaquery/impl/JavaFilterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@
package qualitypatternmodel.javaquery.impl;

import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.eclipse.emf.common.notify.Notification;
import org.eclipse.emf.common.notify.NotificationChain;
Expand Down Expand Up @@ -568,24 +572,51 @@ public void setPatternId(String newPatternId) {
@Override
public JSONArray filterQueryResults() throws InvalidityException {
EList<InterimResultContainer> interims = getInterimResults();
JSONArray results = new JSONArray();
for (InterimResultContainer ir: interims) {
try {
if (getFilter().apply(ir.getParameter())) {
InterimResult ret = ir.getReturn();
if (ret instanceof JsonResult) {
results.put(((JsonResult) ret).getValue());
} else {
results.put(resultjson(ret.toString()));
ExecutorService executor = Executors.newFixedThreadPool(100);
try {

List<Future<JSONObject>> futures = new ArrayList<Future<JSONObject>>();
for (InterimResultContainer ir: interims) {
futures.add(executor.submit(() -> {
try {
if (getFilter().apply(ir.getParameter())) {
InterimResult ret = ir.getReturn();
if (ret instanceof JsonResult) {
return ((JsonResult) ret).getValue();
} else {
return resultjson(ret.toString());
}
}
return null;
} catch (InvalidityException e) {
throw new InvalidityException(ir.toString() + "\n*\n" + ir.getParameter() + "\n" + e.getMessage(), e);
}
}
} catch (InvalidityException e) {
throw new InvalidityException(ir.toString() + "\n*\n" + ir.getParameter() + "\n" + e.getMessage(), e);
}));
}
}
return results;
}

JSONArray results = new JSONArray();

for (Future<JSONObject> future: futures) {
try {
JSONObject obj = future.get();
if (obj != null)
results.put(obj);
} catch (Exception e) {
Throwable cause = e.getCause();
if (cause instanceof InvalidityException) {
throw (InvalidityException) cause;
}
throw new RuntimeException(cause);
}

}
return results;

} finally {
executor.shutdown();
}
}

private JSONObject resultjson(String value) {
JSONObject res = new JSONObject();
res.put(ConstantsJSON.RESULT_SNIPPET, value);
Expand Down
13 changes: 10 additions & 3 deletions qualitypatternmodel/test/newservelettest/templatetests/TemplateTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public static void main(String[] args) throws IOException, InvalidityException,
static List<Arguments> argumentProvider() throws IOException {
JSONObject config = Util.loadJson(pathConfig);
List<Arguments> args = new ArrayList<Arguments>();
int no = 0;
for (String key : config.keySet()) {

JSONArray array = config.getJSONArray(key);
Expand All @@ -91,8 +92,11 @@ static List<Arguments> argumentProvider() throws IOException {
boolean isTest = json.has(TEST) && json.getBoolean(TEST);
boolean isActive = !json.has(DEACTIVATED) || !json.getBoolean(DEACTIVATED);

if ((ignoreDeactivated || isActive) && (!onlyTest || isTest))
args.add(Arguments.of(key, json.getJSONObject(PARAMS), json.getJSONObject(EXPECTED), json.has(TEST)));
if ((ignoreDeactivated || isActive) && (!onlyTest || isTest)) {
args.add(Arguments.of(key, no, json.getJSONObject(PARAMS), json.getJSONObject(EXPECTED), json.has(TEST)));
no++;
}

}
}
if (onlyTest && args.size() == 0) {
Expand All @@ -105,7 +109,7 @@ static List<Arguments> argumentProvider() throws IOException {

@ParameterizedTest
@MethodSource("argumentProvider")
public void testPattern(String id, JSONObject params, JSONObject expected, boolean debug) throws InvalidityException, OperatorCycleException, MissingPatternContainerException, JSONException, InvalidServletCallException, FailedServletCallException {
public void testPattern(String id, int no, JSONObject params, JSONObject expected, boolean debug) throws InvalidityException, OperatorCycleException, MissingPatternContainerException, JSONException, InvalidServletCallException, FailedServletCallException {
CompletePattern pattern;
try{
pattern = findPattern(id);
Expand All @@ -115,7 +119,10 @@ public void testPattern(String id, JSONObject params, JSONObject expected, boole
}
parameterizePattern(pattern, params);
JSONObject query = ServletUtilities.generateQueryJson(pattern);
Long time = System.currentTimeMillis();
JSONObject result = XQueryProcessorSaxon.queryConstraintsFilePaths(Arrays.asList(query), Arrays.asList(pathData));
time = System.currentTimeMillis() - time;
System.out.println(no + "\t" + id + "\t" + time); // + "\t" + params.toString());
if (debug) {
if (debugShowQuery) {
System.out.println("\nQUERY");
Expand Down
11 changes: 11 additions & 0 deletions qualitypatternmodel/test/newservelettest/templatetests/template-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1973,6 +1973,17 @@
"0": "true",
"1": "//validlink/link/text()"
}
},
{
"deactivated": true,
"expected": {
"totalIncidents": 11,
"totalFindings": 100
},
"params": {
"0": "true",
"1": "//hundredlinks/link/text()"
}
}
],
"ValidLinkIso": [
Expand Down
102 changes: 102 additions & 0 deletions qualitypatternmodel/test/newservelettest/templatetests/template-data.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,106 @@
<list id="4"><value>a</value><value>a</value></list>
<list id="5"></list>
</uniquelists>
<hundredlinks>
<link>_DSC3552_kleib.jpg</link>
<link>_DSC3554_klein.jpg</link>
<link>google.com</link>
<link>http://fotothek.slub-dresden.de/fotos/df_hauptkatalog_0355520.jpg</link>
<link>http://fotothek.slub-dresden.de/fotos/df_hauptkatalog_0355520_karte.jpg</link>
<link>http://fotothek.spsg.de/eas/partitions/2/0/249000/249178/cf50af12965de9406e38fb789f0fe988008dc74a/image/jpeg/249178.jpg</link>
<link>http://google.de</link>
<link>http://novalidlink.de</link>
<link>http://resources.digicult-museen.net/dam/provided_image/m5ea54539bef06</link>
<link>http://resources.digicult-museen.net/dam/provided_image/m5ea5453a0926f</link>
<link>http://resources.digicult-museen.net/dam/provided_image/m5ea5453a433bb</link>
<link>http://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-2.jpg</link>
<link>http://www.bildindex.de/bilder/d/fm931286</link>
<link>http://www.bildindex.de/bilder/d/fm931287</link>
<link>http://www.bildindex.de/bilder/d/lwl-c567588pad</link>
<link>http://www.bildindex.de/bilder/d/mi11813c13</link>
<link>http://www.bildindex.de/bilder/d/oe00189b05</link>
<link>http://www.bildindex.de/bilder/d/oe00189b09</link>
<link>http://www.bildindex.de/bilder/d/oe00189b10</link>
<link>http://www.bildindex.de/bilder/m/mi11813c13</link>
<link>http://www.github.com</link>
<link>http://www.google.de</link>
<link>http://www.invalid-link.de</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/202007/100h_fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/202007/200w_fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/202007/fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/100h_akte-i-11-78541-965304.jpg</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/200w_akte-i-11-78541-965304.jpg</link>
<link>https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/akte-i-11-78541-965304.jpg</link>
<link>https://digicult-web-west.digicult-verbund.de/entity-resources/images/digicult-web-west/1/DISPLAY600_8078.jpg</link>
<link>https://digitaldepot.dsm.museum/iiif/iiif/2//3ac32ac2-e451-496a-b16c-09c49adc69f0/full/full/0/default.jpg</link>
<link>https://digitaldepot.dsm.museum/iiif/iiif/2//6eaebd9e-60f7-478a-a19d-7249fd926978/full/full/0/default.jpg</link>
<link>https://digitaldepot.dsm.museum/iiif/iiif/2//c858d1af-4a02-40bc-9890-e1e8fe83df47/full/full/0/default.jpg</link>
<link>https://digitaldepot.dsm.museum/iiif/iiif/2//fe3c2739-bc3e-4d2e-a780-81097294de81/full/full/0/default.jpg</link>
<link>https://emuseum.duesseldorf.de/internal/media/dispatcher/176380/full</link>
<link>https://emuseum.duesseldorf.de/internal/media/dispatcher/176380/preview</link>
<link>https://fotothek.slub-dresden.de/fotos/df/hauptkatalog/0355000/df_hauptkatalog_0355520.jpg</link>
<link>https://ghwk-db.de/files/original/51b65bc08e48b3bc5c195b79be8bc1a7a8a41cc7.jpg</link>
<link>https://ghwk-db.de/files/original/8f1240ba411d0cf0c17a93aa81f20410e8146039.jpg</link>
<link>https://id.smb.museum/digital-asset/3371759</link>
<link>https://id.smb.museum/digital-asset/4948631</link>
<link>https://id.smb.museum/digital-asset/5816199</link>
<link>https://id.smb.museum/digital-asset/5821956</link>
<link>https://id.smb.museum/digital-asset/5822610</link>
<link>https://ikmk.smb.museum/image/18314033/rs_opt.jpg</link>
<link>https://ikmk.smb.museum/image/18314033/rs_org.jpg</link>
<link>https://ikmk.smb.museum/image/18314033/vs_opt.jpg</link>
<link>https://ikmk.smb.museum/image/18314033/vs_org.jpg</link>
<link>https://images.stockcake.com/public/a/8/e/a8ebd9b9-640c-4f6a-8877-4a91ea968549_medium/alert-black-white-cat-stockcake.jpg</link>
<link>https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_1355592/files/images/00000001.jpg/full/full/0/default.jpg</link>
<link>https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_1355592/files/images/00000002.jpg/full/full/0/default.jpg</link>
<link>https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_30154/files/images/00000001.jpg/full/full/0/default.jpg</link>
<link>https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_30922/files/images/00000001.jpg/full/full/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_%23_001/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_001_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_002_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_003_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_004_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_006_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_007_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20140922_008_D5_ND_CF/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20170308_001_SAMOA_ED_GFriedinger/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20170308_002_SAMOA_ED_GFriedinger/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20170308_003_SAMOA_ED_GFriedinger/full/max/0/default.jpg</link>
<link>https://media.deutsches-museum.de/id/1993-522_20220520_001_Fotoatelier_ND_CIlling/full/max/0/default.jpg</link>
<link>https://muenzportal.uni-koeln.de/JPG/VAR_0008_O.jpg</link>
<link>https://muenzportal.uni-koeln.de/JPG/VAR_0008_R.jpg</link>
<link>https://onlinekatalog.kunsthalle-bremen.de/DE-MUS-027614/object/3/166674</link>
<link>https://recherche.smb.museum/images/5822610_2500x2500.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-2.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-3.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-4.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/200w_de-ka-de-weltakkord-w-688-61394.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/200w_de-ka-de-weltakkord-w-688-61394-4.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-2.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-3.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-4.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/100h_knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/200w_knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/200w_knochenspatel-zur-herstellung-von-ratterdekor-93855-655513-3.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/knochenspatel-zur-herstellung-von-ratterdekor-93855-655513-3.jpg</link>
<link>https://rlp.museum-digital.de/data/rlp/resources/video/202006/03101359436.mp4</link>
<link>https://rlp.museum-digital.de/data/rlp/resources/video/202006/200w_03101730082.jpg</link>
<link>https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_rs.jpg/full/300,/0/default.jpg</link>
<link>https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_rs.jpg/full/full/0/default.jpg</link>
<link>https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_vs.jpg/full/300,/0/default.jpg</link>
<link>https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_vs.jpg/full/full/0/default.jpg</link>
<link>https://www.museum-digital.de/data/berlin/images/import_17/201601/100h_29173524067.jpg</link>
<link>https://www.museum-digital.de/data/berlin/images/import_17/201601/29173524067.jpg</link>
<link>https://www.runesdb.de/image-service/17125/full/max/0/default.jpg</link>
<link>https://www.tierschutzbund.de/fileadmin/_processed_/e/1/csm_Strassenkatze_getigert_liegend_Xena_c_Tierschutzverein_Schwalbach___Frankfurt-West_e.V._603e5195d1.jpg</link>
<link>I11.jpg</link>
<link>nix</link>
<link>nothing</link>
<link>no-valid-link.com</link>
<link>novalidlink.de</link>
<link>www.invalid-link.de</link>
</hundredlinks>
</root>
Loading