diff --git a/qualitypatternmodel/src/qualitypatternmodel/javaoperators/LinkOperatorUtil.java b/qualitypatternmodel/src/qualitypatternmodel/javaoperators/LinkOperatorUtil.java index 3d257a7e..ed9e418e 100644 --- a/qualitypatternmodel/src/qualitypatternmodel/javaoperators/LinkOperatorUtil.java +++ b/qualitypatternmodel/src/qualitypatternmodel/javaoperators/LinkOperatorUtil.java @@ -13,8 +13,8 @@ public class LinkOperatorUtil { - public static final int DEFAULT_TIMEOUT = 5000; - public static final String AGENT = "QPM/1.15; (https://github.com/Project-KONDA/pattern-based-quality-analysis)"; + public static final int DEFAULT_TIMEOUT = 50000; + public static final String AGENT = "QPM/1.16; (https://github.com/Project-KONDA/pattern-based-quality-analysis)"; private static final HttpClient CLIENT = HttpClient.newBuilder() .connectTimeout(Duration.ofMillis(DEFAULT_TIMEOUT)) diff --git a/qualitypatternmodel/src/qualitypatternmodel/javaquery/impl/JavaFilterImpl.java b/qualitypatternmodel/src/qualitypatternmodel/javaquery/impl/JavaFilterImpl.java index f3dfc4f1..b808f548 100644 --- a/qualitypatternmodel/src/qualitypatternmodel/javaquery/impl/JavaFilterImpl.java +++ b/qualitypatternmodel/src/qualitypatternmodel/javaquery/impl/JavaFilterImpl.java @@ -3,9 +3,13 @@ package qualitypatternmodel.javaquery.impl; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.eclipse.emf.common.notify.Notification; import org.eclipse.emf.common.notify.NotificationChain; @@ -568,24 +572,51 @@ public void setPatternId(String newPatternId) { @Override public JSONArray filterQueryResults() throws InvalidityException { EList interims = getInterimResults(); - JSONArray results = new JSONArray(); - for (InterimResultContainer ir: interims) { - try { - if (getFilter().apply(ir.getParameter())) { - InterimResult ret = ir.getReturn(); - if (ret instanceof JsonResult) { - results.put(((JsonResult) ret).getValue()); - } else { - results.put(resultjson(ret.toString())); + ExecutorService executor = Executors.newFixedThreadPool(100); + try { + + List> futures = new ArrayList>(); + for (InterimResultContainer ir: interims) { + futures.add(executor.submit(() -> { + try { + if (getFilter().apply(ir.getParameter())) { + InterimResult ret = ir.getReturn(); + if (ret instanceof JsonResult) { + return ((JsonResult) ret).getValue(); + } else { + return resultjson(ret.toString()); + } + } + return null; + } catch (InvalidityException e) { + throw new InvalidityException(ir.toString() + "\n*\n" + ir.getParameter() + "\n" + e.getMessage(), e); } - } - } catch (InvalidityException e) { - throw new InvalidityException(ir.toString() + "\n*\n" + ir.getParameter() + "\n" + e.getMessage(), e); + })); } - } - return results; - } + JSONArray results = new JSONArray(); + + for (Future future: futures) { + try { + JSONObject obj = future.get(); + if (obj != null) + results.put(obj); + } catch (Exception e) { + Throwable cause = e.getCause(); + if (cause instanceof InvalidityException) { + throw (InvalidityException) cause; + } + throw new RuntimeException(cause); + } + + } + return results; + + } finally { + executor.shutdown(); + } + } + private JSONObject resultjson(String value) { JSONObject res = new JSONObject(); res.put(ConstantsJSON.RESULT_SNIPPET, value); diff --git a/qualitypatternmodel/test/newservelettest/templatetests/TemplateTest.java b/qualitypatternmodel/test/newservelettest/templatetests/TemplateTest.java index 58479e36..d5726f80 100644 --- a/qualitypatternmodel/test/newservelettest/templatetests/TemplateTest.java +++ b/qualitypatternmodel/test/newservelettest/templatetests/TemplateTest.java @@ -83,6 +83,7 @@ public static void main(String[] args) throws IOException, InvalidityException, static List argumentProvider() throws IOException { JSONObject config = Util.loadJson(pathConfig); List args = new ArrayList(); + int no = 0; for (String key : config.keySet()) { JSONArray array = config.getJSONArray(key); @@ -91,8 +92,11 @@ static List argumentProvider() throws IOException { boolean isTest = json.has(TEST) && json.getBoolean(TEST); boolean isActive = !json.has(DEACTIVATED) || !json.getBoolean(DEACTIVATED); - if ((ignoreDeactivated || isActive) && (!onlyTest || isTest)) - args.add(Arguments.of(key, json.getJSONObject(PARAMS), json.getJSONObject(EXPECTED), json.has(TEST))); + if ((ignoreDeactivated || isActive) && (!onlyTest || isTest)) { + args.add(Arguments.of(key, no, json.getJSONObject(PARAMS), json.getJSONObject(EXPECTED), json.has(TEST))); + no++; + } + } } if (onlyTest && args.size() == 0) { @@ -105,7 +109,7 @@ static List argumentProvider() throws IOException { @ParameterizedTest @MethodSource("argumentProvider") - public void testPattern(String id, JSONObject params, JSONObject expected, boolean debug) throws InvalidityException, OperatorCycleException, MissingPatternContainerException, JSONException, InvalidServletCallException, FailedServletCallException { + public void testPattern(String id, int no, JSONObject params, JSONObject expected, boolean debug) throws InvalidityException, OperatorCycleException, MissingPatternContainerException, JSONException, InvalidServletCallException, FailedServletCallException { CompletePattern pattern; try{ pattern = findPattern(id); @@ -115,7 +119,10 @@ public void testPattern(String id, JSONObject params, JSONObject expected, boole } parameterizePattern(pattern, params); JSONObject query = ServletUtilities.generateQueryJson(pattern); + Long time = System.currentTimeMillis(); JSONObject result = XQueryProcessorSaxon.queryConstraintsFilePaths(Arrays.asList(query), Arrays.asList(pathData)); + time = System.currentTimeMillis() - time; + System.out.println(no + "\t" + id + "\t" + time); // + "\t" + params.toString()); if (debug) { if (debugShowQuery) { System.out.println("\nQUERY"); diff --git a/qualitypatternmodel/test/newservelettest/templatetests/template-config.json b/qualitypatternmodel/test/newservelettest/templatetests/template-config.json index f4a2e7bb..50696674 100644 --- a/qualitypatternmodel/test/newservelettest/templatetests/template-config.json +++ b/qualitypatternmodel/test/newservelettest/templatetests/template-config.json @@ -1973,6 +1973,17 @@ "0": "true", "1": "//validlink/link/text()" } + }, + { + "deactivated": true, + "expected": { + "totalIncidents": 11, + "totalFindings": 100 + }, + "params": { + "0": "true", + "1": "//hundredlinks/link/text()" + } } ], "ValidLinkIso": [ diff --git a/qualitypatternmodel/test/newservelettest/templatetests/template-data.xml b/qualitypatternmodel/test/newservelettest/templatetests/template-data.xml index b27518d3..cda244fc 100644 --- a/qualitypatternmodel/test/newservelettest/templatetests/template-data.xml +++ b/qualitypatternmodel/test/newservelettest/templatetests/template-data.xml @@ -144,4 +144,106 @@ aa + + _DSC3552_kleib.jpg + _DSC3554_klein.jpg + google.com + http://fotothek.slub-dresden.de/fotos/df_hauptkatalog_0355520.jpg + http://fotothek.slub-dresden.de/fotos/df_hauptkatalog_0355520_karte.jpg + http://fotothek.spsg.de/eas/partitions/2/0/249000/249178/cf50af12965de9406e38fb789f0fe988008dc74a/image/jpeg/249178.jpg + http://google.de + http://novalidlink.de + http://resources.digicult-museen.net/dam/provided_image/m5ea54539bef06 + http://resources.digicult-museen.net/dam/provided_image/m5ea5453a0926f + http://resources.digicult-museen.net/dam/provided_image/m5ea5453a433bb + http://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-2.jpg + http://www.bildindex.de/bilder/d/fm931286 + http://www.bildindex.de/bilder/d/fm931287 + http://www.bildindex.de/bilder/d/lwl-c567588pad + http://www.bildindex.de/bilder/d/mi11813c13 + http://www.bildindex.de/bilder/d/oe00189b05 + http://www.bildindex.de/bilder/d/oe00189b09 + http://www.bildindex.de/bilder/d/oe00189b10 + http://www.bildindex.de/bilder/m/mi11813c13 + http://www.github.com + http://www.google.de + http://www.invalid-link.de + https://brandenburg.museum-digital.de/data/brandenburg/images/202007/100h_fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg + https://brandenburg.museum-digital.de/data/brandenburg/images/202007/200w_fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg + https://brandenburg.museum-digital.de/data/brandenburg/images/202007/fotografie-der-fuhrunternehmer-krueger-und-sohn-43734-2.jpg + https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/100h_akte-i-11-78541-965304.jpg + https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/200w_akte-i-11-78541-965304.jpg + https://brandenburg.museum-digital.de/data/brandenburg/images/4/78541-v_1356_s/akte_i_11/akte-i-11-78541-965304.jpg + https://digicult-web-west.digicult-verbund.de/entity-resources/images/digicult-web-west/1/DISPLAY600_8078.jpg + https://digitaldepot.dsm.museum/iiif/iiif/2//3ac32ac2-e451-496a-b16c-09c49adc69f0/full/full/0/default.jpg + https://digitaldepot.dsm.museum/iiif/iiif/2//6eaebd9e-60f7-478a-a19d-7249fd926978/full/full/0/default.jpg + https://digitaldepot.dsm.museum/iiif/iiif/2//c858d1af-4a02-40bc-9890-e1e8fe83df47/full/full/0/default.jpg + https://digitaldepot.dsm.museum/iiif/iiif/2//fe3c2739-bc3e-4d2e-a780-81097294de81/full/full/0/default.jpg + https://emuseum.duesseldorf.de/internal/media/dispatcher/176380/full + https://emuseum.duesseldorf.de/internal/media/dispatcher/176380/preview + https://fotothek.slub-dresden.de/fotos/df/hauptkatalog/0355000/df_hauptkatalog_0355520.jpg + https://ghwk-db.de/files/original/51b65bc08e48b3bc5c195b79be8bc1a7a8a41cc7.jpg + https://ghwk-db.de/files/original/8f1240ba411d0cf0c17a93aa81f20410e8146039.jpg + https://id.smb.museum/digital-asset/3371759 + https://id.smb.museum/digital-asset/4948631 + https://id.smb.museum/digital-asset/5816199 + https://id.smb.museum/digital-asset/5821956 + https://id.smb.museum/digital-asset/5822610 + https://ikmk.smb.museum/image/18314033/rs_opt.jpg + https://ikmk.smb.museum/image/18314033/rs_org.jpg + https://ikmk.smb.museum/image/18314033/vs_opt.jpg + https://ikmk.smb.museum/image/18314033/vs_org.jpg + https://images.stockcake.com/public/a/8/e/a8ebd9b9-640c-4f6a-8877-4a91ea968549_medium/alert-black-white-cat-stockcake.jpg + https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_1355592/files/images/00000001.jpg/full/full/0/default.jpg + https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_1355592/files/images/00000002.jpg/full/full/0/default.jpg + https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_30154/files/images/00000001.jpg/full/full/0/default.jpg + https://kulturerbe.niedersachsen.de/api/v1/records/record_kuniweb_30922/files/images/00000001.jpg/full/full/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_%23_001/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_001_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_002_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_003_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_004_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_006_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_007_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20140922_008_D5_ND_CF/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20170308_001_SAMOA_ED_GFriedinger/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20170308_002_SAMOA_ED_GFriedinger/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20170308_003_SAMOA_ED_GFriedinger/full/max/0/default.jpg + https://media.deutsches-museum.de/id/1993-522_20220520_001_Fotoatelier_ND_CIlling/full/max/0/default.jpg + https://muenzportal.uni-koeln.de/JPG/VAR_0008_O.jpg + https://muenzportal.uni-koeln.de/JPG/VAR_0008_R.jpg + https://onlinekatalog.kunsthalle-bremen.de/DE-MUS-027614/object/3/166674 + https://recherche.smb.museum/images/5822610_2500x2500.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-2.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-3.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/100h_de-ka-de-weltakkord-w-688-61394-4.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/200w_de-ka-de-weltakkord-w-688-61394.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/200w_de-ka-de-weltakkord-w-688-61394-4.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-2.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-3.jpg + https://rlp.museum-digital.de/data/rlp/images/55/61394-rkf_255_2020/de_ka_de_weltakkord_w_688/de-ka-de-weltakkord-w-688-61394-4.jpg + https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/100h_knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg + https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/200w_knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg + https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/200w_knochenspatel-zur-herstellung-von-ratterdekor-93855-655513-3.jpg + https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/knochenspatel-zur-herstellung-von-ratterdekor-93855-458752.jpg + https://rlp.museum-digital.de/data/rlp/images/93/93855-tsm_2022-0157/knochenspatel_zur_herstel/knochenspatel-zur-herstellung-von-ratterdekor-93855-655513-3.jpg + https://rlp.museum-digital.de/data/rlp/resources/video/202006/03101359436.mp4 + https://rlp.museum-digital.de/data/rlp/resources/video/202006/200w_03101730082.jpg + https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_rs.jpg/full/300,/0/default.jpg + https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_rs.jpg/full/full/0/default.jpg + https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_vs.jpg/full/300,/0/default.jpg + https://www.kenom.de/iiif/image/record_DE-33_kenom_213833/record_DE-33_kenom_213833_vs.jpg/full/full/0/default.jpg + https://www.museum-digital.de/data/berlin/images/import_17/201601/100h_29173524067.jpg + https://www.museum-digital.de/data/berlin/images/import_17/201601/29173524067.jpg + https://www.runesdb.de/image-service/17125/full/max/0/default.jpg + https://www.tierschutzbund.de/fileadmin/_processed_/e/1/csm_Strassenkatze_getigert_liegend_Xena_c_Tierschutzverein_Schwalbach___Frankfurt-West_e.V._603e5195d1.jpg + I11.jpg + nix + nothing + no-valid-link.com + novalidlink.de + www.invalid-link.de + \ No newline at end of file