diff --git a/modules/invenio-oaiharvester/invenio_oaiharvester/harvester.py b/modules/invenio-oaiharvester/invenio_oaiharvester/harvester.py index 29f4f842ba..f9bfc9d881 100644 --- a/modules/invenio-oaiharvester/invenio_oaiharvester/harvester.py +++ b/modules/invenio-oaiharvester/invenio_oaiharvester/harvester.py @@ -1453,9 +1453,7 @@ def ddi_harvest_processing(self, harvest_data, res): """Process parsing DDI data.""" def get_mapping_ddi(): """Get DDI mapping.""" - item_type_id = self.itemtype.id - type_mapping = Mapping.get_record(item_type_id) - item_map = get_mapping(type_mapping, "ddi_mapping") + item_map = get_mapping(self.itemtype.id, "ddi_mapping") lst_keys_x = list(item_map.keys()) for i in lst_keys_x: lst_keys.append(i) @@ -1543,7 +1541,7 @@ def parse_to_obj_data_by_mapping_keys(vals, keys): def get_same_key_from_form(sub_key): """Get the same key with sub_key in form.""" for item_sub_key_form in item_sub_keys_form: - if item_sub_key_form.replace("[]", "") == sub_key: + if item_sub_key_form.replace("[]", "") in sub_key.split(','): sub_key = item_sub_key_form break return sub_key diff --git a/modules/invenio-oaiharvester/tests/test_harvester.py b/modules/invenio-oaiharvester/tests/test_harvester.py index ea2986af8d..d5b5934b8c 100644 --- a/modules/invenio-oaiharvester/tests/test_harvester.py +++ b/modules/invenio-oaiharvester/tests/test_harvester.py @@ -124,7 +124,7 @@ def test_list_sets(): sets = list_sets(url) specs = [s.xpath("./x:setSpec[1]",namespaces=namespaces)[0].text for s in sets] assert specs == ["test_repo1","test_repo2","test_repo3","test_repo4"] - + # def list_records( # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_list_records -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp @responses.activate @@ -167,7 +167,7 @@ def test_list_records(): result = [str(etree.tostring(record),"utf-8") for record in records] assert result == ['test_record1', 'test_record2'] assert rtoken == "test_token" - + # resumptiontoken is not none records, rtoken = list_records("https://test.org/","2023-01-10","2023-10-01","jpcoar_1.0","*",resumption_token="test_token") result = [str(etree.tostring(record),"utf-8") for record in records] @@ -186,11 +186,11 @@ def test_map_field(): } result = map_field(schema) assert result == {"Title":"item_1551264308487"} - + # def subitem_recs(schema, keys, value, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_subitem_recs -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_subitem_recs(db_itemtype): - + # "items.perperty" in schema. len(keys) > 2,_subitem is not false schema = {"items":{"properties":{"item_key1":{'type': 'array', 'items': {'type': 'object', 'format': 'object', 'properties': {'subitem_1551256006332': {'type': 'string', 'title': 'Creator Given Name', 'format': 'text', 'title_i18n': {'en': 'Creator Given Name', 'ja': '作成者名'}, 'title_i18n_temp': {'en': 'Creator Given Name', 'ja': '作成者名'}}, 'subitem_1551256007414': {'enum': [None,'ja','en'], 'type': ['null', 'string'], 'title': 'Language', 'format': 'select', 'currentEnum': ['ja','en']}}}, 'title': 'Creator Given Name', 'format': 'array'}}}} value = "jpcoar:givenName.#text" @@ -198,7 +198,7 @@ def test_subitem_recs(db_itemtype): metadata = OrderedDict([('jpcoar:givenName', OrderedDict([('@xml:lang', 'ja'), ('#text', '太郎')]))]) result = subitem_recs(schema, keys, value, metadata) assert result == [[{'subitem_1551256006332': '太郎'}]] - + # "items.perperty" in schema. len(keys) > 2,_subitem is false schema = {"items":{"properties":{"item_key1":{'type': 'array', 'items': {'type': 'object', 'format': 'object', 'properties': {'subitem_1551256006332': {'type': 'string', 'title': 'Creator Given Name', 'format': 'text', 'title_i18n': {'en': 'Creator Given Name', 'ja': '作成者名'}, 'title_i18n_temp': {'en': 'Creator Given Name', 'ja': '作成者名'}}, 'subitem_1551256007414': {'enum': [None,'ja','en'], 'type': ['null', 'string'], 'title': 'Language', 'format': 'select', 'currentEnum': ['ja','en']}}}, 'title': 'Creator Given Name', 'format': 'array'}}}} value = "jpcoar:givenName.#text" @@ -206,7 +206,7 @@ def test_subitem_recs(db_itemtype): metadata = OrderedDict([('jpcoar:givenName', "")]) result = subitem_recs(schema, keys, value, metadata) assert result == [] - + value = "jpcoar:givenName.#text" keys = ['subitem_1551256006332'] schema = {'type': 'array', 'items': {'type': 'object', 'format': 'object', 'properties': {'subitem_1551256006332': {'type': 'string', 'title': 'Creator Given Name', 'format': 'text', 'title_i18n': {'en': 'Creator Given Name', 'ja': '作成者名'}, 'title_i18n_temp': {'en': 'Creator Given Name', 'ja': '作成者名'}}, 'subitem_1551256007414': {'enum': [None,'ja','en'], 'type': ['null', 'string'], 'title': 'Language', 'format': 'select', 'currentEnum': ['ja','en']}}}, 'title': 'Creator Given Name', 'format': 'array'} @@ -214,7 +214,7 @@ def test_subitem_recs(db_itemtype): metadata = OrderedDict([('jpcoar:givenName', ["太郎",OrderedDict([('@xml:lang', 'ja'), ('#text', '次郎')])])]) result = subitem_recs(schema, keys, value, metadata) assert result == [{'subitem_1551256006332': '太郎'}, {'subitem_1551256006332': '次郎'}] - + # "." not in value, metadata is not str and OrderedDict schema = {'type': 'array', 'items': {'type': 'object', 'format': 'object', 'properties': {'subitem_1551257342360': {'type': 'string', 'title': 'Contributor Given Name', 'format': 'text', 'title_i18n': {'en': 'Contributor Given Name', 'ja': '寄与者名'}, 'title_i18n_temp': {'en': 'Contributor Given Name', 'ja': '寄与者名'}}, 'subitem_1551257343979': {'enum': [None, 'ja','en'], 'type': ['null', 'string'], 'title': 'Language', 'format': 'select', 'currentEnum': ['ja','en']}}}, 'title': 'Contributor Given Name', 'format': 'array'} keys = ['subitem_1551257342360'] @@ -222,7 +222,7 @@ def test_subitem_recs(db_itemtype): metadata = [] result = subitem_recs(schema, keys, value, metadata) assert result == [] - + # "properties" in schema ## len(keys) > 1 keys = ["test_key1","test_key2"] @@ -230,51 +230,51 @@ def test_subitem_recs(db_itemtype): metadata = OrderedDict([("#text","value")]) result = subitem_recs(schema, keys, value, metadata) assert result == {'test_key2': 'value'} - + ## len(keys) = 1 keys = ["test_key"] schema = {"properties":{"test_key":"value"}} result = subitem_recs(schema, keys, value, metadata) assert result == {'test_key': 'value'} - + ### "." in value, len(value.split(".")) > 2 value = "test.#text" metadata = OrderedDict([("#text","value")]) result = subitem_recs(schema, keys, value, metadata) assert result == None - + #### str metadata = OrderedDict([("test","value")]) result = subitem_recs(schema, keys, value, metadata) assert result == {"test_key":"value"} - + #### list metadata = OrderedDict([("test",[OrderedDict([("#text","value")]),OrderedDict([("#text","value2")])])]) result = subitem_recs(schema, keys, value, metadata) assert result == {"test_key":"value"} - + #### ordereddict metadata = OrderedDict([("test",OrderedDict([("#text","value")]))]) result = subitem_recs(schema, keys, value, metadata) assert result == {"test_key":"value"} - + #### other metadata = OrderedDict([("test",1)]) result = subitem_recs(schema, keys, value, metadata) assert result == {} - + ## "." not in value ### metadata is OrderedDict value = "#text" metadata = OrderedDict([("#text","value")]) result = subitem_recs(schema, keys, value, metadata) assert result == {'test_key': 'value'} - + ### metadata is not str, OrderedDict metadata = [] result = subitem_recs(schema, keys, value, metadata) assert result == {} - + # not item_key, "." in value ## len(value.split(".")) > 2, value[0] not in metadata schema = {'type': 'string', 'title': 'Version', 'format': 'text', 'title_i18n': {'en': 'Version', 'ja': 'バージョン情報'}, 'title_i18n_temp': {'en': 'Version', 'ja': 'バージョン情報'}} @@ -283,24 +283,24 @@ def test_subitem_recs(db_itemtype): metadata = OrderedDict([("datacite:version",[OrderedDict([("#text","1.2")]),OrderedDict([("#text","1.3")])])]) result = subitem_recs(schema, keys, value, metadata) assert result == None - + ## metadata.get(value[0]) is list value = "datacite:version.#text" metadata = OrderedDict([("datacite:version",[OrderedDict([("#text","1.2")]),OrderedDict([("#text","1.3")])])]) result = subitem_recs(schema, keys, value, metadata) assert result == "1.2" - + ## metadata.get(value[0]) is OrderedDict metadata = OrderedDict([("datacite:version",OrderedDict([("#text","1.2")]))]) result = subitem_recs(schema, keys, value, metadata) assert result == "1.2" - + ## metadata.get(value[0]) is not OrderedDict,list,str metadata = OrderedDict([("datacite:version",1)]) result = subitem_recs(schema, keys, value, metadata) assert result == None - - + + schema = {'type': 'string', 'title': 'Version', 'format': 'text', 'title_i18n': {'en': 'Version', 'ja': 'バージョン情報'}, 'title_i18n_temp': {'en': 'Version', 'ja': 'バージョン情報'}} keys = ["test_key"] result = subitem_recs(schema, keys, value, metadata) @@ -322,12 +322,12 @@ def test_parsing_metadata(db_itemtype): assert result1 == None assert result2 == None assert res == {} - + patterns = [ ('title.@value', '#text'), ('title.@attributes.xml:lang', '@xml:lang') ] - + mappin2 = {"title.@value":[".subitem_1551255647225"]} # item_key is false res = {} @@ -374,7 +374,7 @@ def test_parsing_metadata(db_itemtype): result1, result2 = parsing_metadata(mappin, props, patterns, metadata, res) assert result1 == "main_item" assert result2 == [{'test_item1': [[{'subitem_1551256006332': '太郎1'}], {'subitem_1551256006332': '太郎2'}, [{'subitem_1551256006332': '太郎3'}]]}] - + # submetadata is dict res = {} metadata = OrderedDict([("#text","value")]) @@ -399,12 +399,12 @@ def test_parsing_metadata(db_itemtype): assert result1 == "main_item" assert result2 == [{"test_item1":{"test_key":"value2"}}] - + @pytest.fixture() def mapper_jpcoar(db_itemtype): def factory(type): xml_str='
oai:weko3.example.org:000000012023-02-20T06:24:47Z1557819692844:15578197332761557820086539
test full itemother title1234テスト, 太郎テスト太郎テスト　別郎56785678test, smithtestsmithother smith1234metadata only accessPaidテスト権利情報テスト　太郎テスト主題this is test abstract.test publisher2022-10-202022-10-19jpnnewspaper1.1AO1111https://doi.org/1234/0000000001https://192.168.56.103/records/11234/00000000011111111関連情報テストhttps://192.168.56.103/records/31 to 212345678901234567891112テスト位置情報22222テスト助成機関1111テスト研究test source Identifiertest collectiblestest title book51223335551237894562349999テスト学位2022-10-19学位授与機関識別子テスト学位授与機関テスト会議12345テスト機関12テスト会場JPNhttps://weko3.example.org/record/1/files/test1.txttext/plain18 B2022-10-201.0https://weko3.example.org/record/1/files/test2application/octet-stream18 B1.2https://weko3.example.org/record/1/files/test3.pngimage/png18 B2.1' - + tree = etree.fromstring(xml_str) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() @@ -414,7 +414,7 @@ def factory(type): item_type_mapping = Mapping.get_record(item_type.id) item_map = get_full_mapping(item_type_mapping, "jpcoar_mapping") res = {"$schema":item_type.id,"pubdate":dateutil.parser.parse(str(self_json["record"]["header"].get("datestamp"))).date()} - + if not isinstance(tags[type],list): metadata = [tags[type]] else: @@ -449,19 +449,19 @@ def test_add_title(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("dc:title") res1 = copy.deepcopy(res) add_title(schema, mapping, res1, metadata) - + assert "item_1551264308487" in res1 assert res1["item_1551264308487"] == [{'subitem_1551255647225': 'test full item', 'subitem_1551255648112': 'ja'}] assert "title" in res1 assert res1["title"] == "test full item" - + res2 = copy.deepcopy(res) add_title(schema, mapping, res2, ["test full item"]) assert "item_1551264308487" in res2 assert res2["item_1551264308487"] == [{'subitem_1551255647225': 'test full item'}] assert "title" in res2 assert res2["title"] == "test full item" - + res3 = copy.deepcopy(res) add_title(schema, mapping, res3, [["test full item"]]) assert "item_1551264308487" in res3 @@ -489,17 +489,17 @@ def test_add_creator_jpcoar(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("jpcoar:creator") res1 = copy.deepcopy(res) add_creator_jpcoar(schema, mapping, res1, metadata) - + assert "item_1551264340087" in res1 assert res1["item_1551264340087"] == [{'subitem_1551255991424': [{'subitem_1551256006332': '太郎', 'subitem_1551256007414': 'ja'}], 'subitem_1551255929209': [{'subitem_1551255938498': 'テスト', 'subitem_1551255964991': 'ja'}], 'subitem_1551255898956': [{'subitem_1551255905565': 'テスト, 太郎', 'subitem_1551255907416': 'ja'}], 'subitem_1551256025394': [{'subitem_1551256035730': 'テスト\u3000別郎', 'subitem_1551256055588': 'ja'}]}] - + # def add_contributor_jpcoar(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_contributor_jpcoar -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_contributor_jpcoar(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("jpcoar:contributor") res1 = copy.deepcopy(res) add_contributor_jpcoar(schema, mapping, res1, metadata) - + assert "item_1551264418667" in res1 assert res1["item_1551264418667"] == [{'subitem_1551257036415': 'ContactPerson', 'subitem_1551257339190': [{'subitem_1551257342360': '', 'subitem_1551257343979': 'en'}], 'subitem_1551257272214': [{'subitem_1551257314588': 'test', 'subitem_1551257316910': 'en'}], 'subitem_1551257245638': [{'subitem_1551257276108': 'test, smith', 'subitem_1551257279831': 'en'}], 'subitem_1551257372442': [{'subitem_1551257374288': 'other smith', 'subitem_1551257375939': 'en'}]}] @@ -508,7 +508,7 @@ def test_add_contributor_jpcoar(mapper_jpcoar): def test_add_access_right(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("dcterms:accessRights") res1 = copy.deepcopy(res) - + add_access_right(schema, mapping, res1, metadata) assert "item_1551264447183" in res1 assert res1["item_1551264447183"] == [{'subitem_1551257553743': 'metadata only access', 'subitem_1551257578398': 'http://purl.org/coar/access_right/c_14cb'}] @@ -521,33 +521,33 @@ def test_add_apc(mapper_jpcoar): add_apc(schema, mapping, res1, metadata) assert "item_1551264605515" in res1 assert res1["item_1551264605515"] == [{'subitem_1551257776901': 'Paid'}] - + # def add_right(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_right -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_right(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("dc:rights") res1 = copy.deepcopy(res) add_right(schema,mapping,res1,metadata) - + assert "item_1551264629907" in res1 assert res1["item_1551264629907"] == [{'subitem_1551257025236': [{'subitem_1551257043769': 'テスト権利情報', 'subitem_1551257047388': 'ja'}], 'subitem_1551257030435': 'テスト権利情報Resource'}] - + # def add_subject(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_subject -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_subject(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("jpcoar:subject") res1 = copy.deepcopy(res) add_subject(schema, mapping, res1, metadata) - + assert "item_1551264822581" in res1 assert res1["item_1551264822581"] == [{'subitem_1551257315453': 'テスト主題', 'subitem_1551257323812': 'ja', 'subitem_1551257343002': 'http://bsh.com', 'subitem_1551257329877': 'BSH'}] - + # def add_description(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_description -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_description(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("datacite:description") res1 = copy.deepcopy(res) - + add_description(schema, mapping, res1, metadata) assert "item_1551264846237" in res1 assert res1["item_1551264846237"] == [{'subitem_1551255577890': 'this is test abstract.', 'subitem_1551255592625': 'en', 'subitem_1551255637472': 'Abstract'}] @@ -569,7 +569,7 @@ def test_add_date(mapper_jpcoar): add_date(schema, mapping, res1, metadata) assert "item_1551264974654" in res1 assert res1["item_1551264974654"] == [{'subitem_1551255753471': '2022-10-20', 'subitem_1551255775519': 'Accepted'}, {'subitem_1551255753471': '2022-10-19', 'subitem_1551255775519': 'Issued'}] - + # def add_language(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_language -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_language(mapper_jpcoar): @@ -578,7 +578,7 @@ def test_add_language(mapper_jpcoar): add_language(schema, mapping, res1, metadata) assert "item_1551265002099" in res1 assert res1["item_1551265002099"] == [{'subitem_1551255818386': 'jpn'}] - + # def add_version(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_version -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_version(mapper_jpcoar): @@ -587,7 +587,7 @@ def test_add_version(mapper_jpcoar): add_version(schema, mapping, res1, metadata) assert "item_1551265075370" in res1 assert res1["item_1551265075370"] == [{'subitem_1551255975405': '1.1'}] - + # def add_version_type(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_version_type -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_version_type(mapper_jpcoar): @@ -596,7 +596,7 @@ def test_add_version_type(mapper_jpcoar): add_version_type(schema, mapping, res1, metadata) assert "item_1551265118680" in res1 assert res1["item_1551265118680"] == [{'subitem_1551256025676': 'AO'}] - + # def add_identifier_registration(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_identifier_registration -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_identifier_registration(mapper_jpcoar): @@ -605,7 +605,7 @@ def test_add_identifier_registration(mapper_jpcoar): add_identifier_registration(schema, mapping, res1, metadata) assert "item_1581495499605" in res1 assert res1["item_1581495499605"]==[{'subitem_1551256250276': '1234/0000000001', 'subitem_1551256259586': 'JaLC'}] - + # def add_temporal(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_temporal -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_temporal(mapper_jpcoar): @@ -622,13 +622,13 @@ def test_add_source_identifier(mapper_jpcoar): add_source_identifier(schema, mapping, res, metadata) assert "item_1551265409089" in res assert res["item_1551265409089"] == [{'subitem_1551256405981': 'test source Identifier', 'subitem_1551256409644': 'PISSN'}] - + # def add_file(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_file -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_file(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("jpcoar:file") test = [ - {'subitem_1551255854908': '1.0', 'subitem_1551255750794': 'text/plain', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255820788': [{'subitem_1551255828320': '2022-10-20', 'subitem_1551255833133': 'Accepted'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test1.txt'}]}, + {'subitem_1551255854908': '1.0', 'subitem_1551255750794': 'text/plain', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255820788': [{'subitem_1551255828320': '2022-10-20', 'subitem_1551255833133': 'Accepted'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test1.txt'}]}, {'subitem_1551255854908': '1.2', 'subitem_1551255750794': 'application/octet-stream', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test2'}]}, {'subitem_1551255854908': '2.1', 'subitem_1551255750794': 'image/png', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test3.png'}]} ] @@ -651,15 +651,15 @@ def test_add_source_title(mapper_jpcoar): add_source_title(schema, mapping, res, metadata) assert "item_1551265438256" in res assert res["item_1551265438256"] == [{'subitem_1551256349044': 'test collectibles', 'subitem_1551256350188': 'ja'}, {'subitem_1551256349044': 'test title book', 'subitem_1551256350188': 'ja'}] - + # def add_volume(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_volume -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_volume(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("jpcoar:volume") add_volume(schema, mapping, res, metadata) - assert "item_1551265463411" in res + assert "item_1551265463411" in res assert res["item_1551265463411"] == [{'subitem_1551256328147': '5'}, {'subitem_1551256328147': '1'}] - + # def add_issue(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_issue -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_issue(mapper_jpcoar): @@ -691,7 +691,7 @@ def test_add_page_end(mapper_jpcoar): add_page_end(schema, mapping, res, metadata) assert "item_1551265569218" in res assert res["item_1551265569218"] == [{'subitem_1551256198917': '456'}, {'subitem_1551256198917': '234'}] - + # def add_dissertation_number(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_dissertation_number -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_dissertation_number(mapper_jpcoar): @@ -707,7 +707,7 @@ def test_add_date_granted(mapper_jpcoar): add_date_granted(schema, mapping, res, metadata) assert "item_1551265811989" in res assert res["item_1551265811989"] == [{'subitem_1551256096004': '2022-10-19'}] - + # def add_conference(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_conference -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_conference(mapper_jpcoar): @@ -715,7 +715,7 @@ def test_add_conference(mapper_jpcoar): add_conference(schema, mapping, res, metadata) assert "item_1551265973055" in res assert res["item_1551265973055"] == [{'subitem_1599711813532': 'JPN', 'subitem_1599711655652': '12345', 'subitem_1599711633003': [{'subitem_1599711636923': 'テスト会議', 'subitem_1599711645590': 'ja'}]}] - + # def add_degree_grantor(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_degree_grantor -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_degree_grantor(mapper_jpcoar): @@ -723,7 +723,7 @@ def test_add_degree_grantor(mapper_jpcoar): add_degree_grantor(schema, mapping, res, metadata) assert "item_1551265903092" in res assert res["item_1551265903092"] == [{'subitem_1551256015892': [{'subitem_1551256027296': '学位授与機関識別子テスト', 'subitem_1551256029891': 'kakenhi'}], 'subitem_1551256037922': [{'subitem_1551256042287': '学位授与機関', 'subitem_1551256047619': 'ja'}]}] - + # def add_degree_name(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_degree_name -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_degree_name(mapper_jpcoar): @@ -731,7 +731,7 @@ def test_add_degree_name(mapper_jpcoar): add_degree_name(schema, mapping, res, metadata) assert "item_1551265790591" in res assert res["item_1551265790591"] == [{'subitem_1551256126428': 'テスト学位', 'subitem_1551256129013': 'ja'}] - + # def add_funding_reference(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_funding_reference -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_funding_reference(mapper_jpcoar): @@ -739,14 +739,14 @@ def test_add_funding_reference(mapper_jpcoar): add_funding_reference(schema, mapping, res, metadata) assert "item_1551265385290" in res assert res["item_1551265385290"] == [{'subitem_1551256462220': [{'subitem_1551256653656': 'テスト助成機関', 'subitem_1551256657859': 'ja'}], 'subitem_1551256454316': [{'subitem_1551256614960': '22222', 'subitem_1551256619706': 'Crossref Funder'}], 'subitem_1551256688098': [{'subitem_1551256691232': 'テスト研究', 'subitem_1551256694883': 'ja'}], 'subitem_1551256665850': [{'subitem_1551256671920': '1111', 'subitem_1551256679403': 'https://test.research.com'}]}] - + # def add_geo_location(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_geo_location -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_geo_location(mapper_jpcoar): schema, mapping, res, metadata = mapper_jpcoar("datacite:geoLocation") add_geo_location(schema, mapping, res, metadata) assert "item_key:item_1570068313185" not in res - + # def add_relation(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_relation -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp @@ -755,7 +755,7 @@ def test_add_relation(mapper_jpcoar): add_relation(schema, mapping, res, metadata) assert "item_1551265227803" in res assert res["item_1551265227803"] == [{'subitem_1551256388439': 'isVersionOf', 'subitem_1551256480278': [{'subitem_1551256498531': '関連情報テスト', 'subitem_1551256513476': 'ja'}], 'subitem_1551256465077': [{'subitem_1551256478339': '1111111', 'subitem_1551256629524': 'ARK'}]}, {'subitem_1551256388439': 'isVersionOf', 'subitem_1551256465077': [{'subitem_1551256478339': 'https://192.168.56.103/records/3', 'subitem_1551256629524': 'URI'}]}] - + # def add_rights_holder(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_rights_holder -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_rights_holder(mapper_jpcoar): @@ -780,7 +780,7 @@ def test_add_creator_dc(mapper_dc): add_creator_dc(schema, mapping, res, metadata) assert "item_1593074267803" in res assert res["item_1593074267803"] == [{'creatorNames': [{'creatorName': 'テスト, 太郎'}]}, {'creatorNames': [{'creatorName': '1'}]}, {'creatorNames': [{'creatorName': '1234'}]}] - + # def add_data_by_key(schema, res, resource_list, key): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_data_by_key -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_data_by_key(): @@ -813,7 +813,7 @@ def test_add_data_by_key(): result = add_data_by_key(schema,res,[],"PubDate") assert result == None assert res == {} - + resource_list = [ OrderedDict([("#text","test_title"),("@xml:lang","ja")]), # it is OrderedDict {"test":"value"} # it is not OrderedDict, str @@ -821,12 +821,12 @@ def test_add_data_by_key(): test = {"item_1551264308487":[{"subitem_1551255647225":"test_title","subitem_1551255648112":"ja"},{}]} result = add_data_by_key(schema,res,resource_list,"Title") assert res == test - + resource_list = "other_title" test = {"item_1551264308487":[{"subitem_1551255647225":"test_title","subitem_1551255648112":"ja"},{},{"subitem_1551255647225":"other_title"}]} result = add_data_by_key(schema,res,resource_list,"Title") assert res == test - + # def add_source_dc(schema, res, source_list): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_source_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_source_dc(mapper_dc): @@ -834,7 +834,7 @@ def test_add_source_dc(mapper_dc): add_source_dc(schema, mapping, res, metadata) assert "item_1617186941041" in res assert res["item_1617186941041"] == [{'subitem_1522650091861': 'test collectibles'}] - + # def add_coverage_dc(schema, res, coverage_list): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_coverage_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_coverage_dc(mapper_dc): @@ -842,7 +842,7 @@ def test_add_coverage_dc(mapper_dc): add_coverage_dc(schema, mapping, res, metadata) assert "item_1617186859717" in res assert res["item_1617186859717"] == [{'subitem_1522658031721': '1 to 2'}] - + # def add_format_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_format_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_format_dc(mapper_dc): @@ -850,7 +850,7 @@ def test_add_format_dc(mapper_dc): add_format_dc(schema, mapping, res, metadata) assert "item_1617605131499" in res assert res["item_1617605131499"] == [{'format': 'text/plain'}] - + # def add_contributor_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_contributor_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_contributor_dc(mapper_dc): @@ -858,7 +858,7 @@ def test_add_contributor_dc(mapper_dc): add_contributor_dc(schema, mapping, res, metadata) assert "item_1617349709064" in res assert res["item_1617349709064"] == [{'contributorNames': [{'contributorName': 'test, smith'}]}, {'contributorNames': [{'contributorName': '2'}]}, {'contributorNames': [{'contributorName': '5678'}]}] - + # def add_relation_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_relation_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_relation_dc(mapper_dc): @@ -866,7 +866,7 @@ def test_add_relation_dc(mapper_dc): add_relation_dc(schema, mapping, res, metadata) assert "item_1617353299429" in res assert res["item_1617353299429"] == [{'subitem_1522306287251': {'subitem_1522306436033': '1111111'}}] - + # def add_rights_dc(schema, res, rights, lang='', rights_resource=''): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_rights_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_rights_dc(mapper_dc): @@ -883,7 +883,7 @@ def test_add_identifier_dc(mapper_dc): add_identifier_dc(schema, mapping, res, metadata) assert "item_1617186783814" in res assert res["item_1617186783814"] == [{'subitem_identifier_uri': '1111'}] - + # def add_description_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_description_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_description_dc(mapper_dc): @@ -891,7 +891,7 @@ def test_add_description_dc(mapper_dc): add_description_dc(schema, mapping, res, metadata) assert "item_1617186626617" in res assert res["item_1617186626617"] == [{'subitem_description': 'this is test abstract.'}] - + # def add_subject_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_subject_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_subject_dc(mapper_dc): @@ -899,7 +899,7 @@ def test_add_subject_dc(mapper_dc): add_subject_dc(schema, mapping, res, metadata) assert "item_1617186609386" in res assert res["item_1617186609386"] == [{'subitem_1523261968819': 'テスト主題'}] - + # def add_title_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_title_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_title_dc(mapper_dc): @@ -910,20 +910,20 @@ def test_add_title_dc(mapper_dc): assert res1["item_1617186331708"] == [{'subitem_1551255647225': 'test full item', 'subitem_1551255648112': 'ja'}] assert "title" in res1 assert res1["title"] == "test full item" - + res2 = copy.deepcopy(res) add_title_dc(schema, mapping, res2, ["test full item"]) assert "item_1617186331708" in res2 assert res2["item_1617186331708"] == [{'subitem_1551255647225': 'test full item'}] assert "title" in res2 assert res2["title"] == "test full item" - + res3 = copy.deepcopy(res) add_title_dc(schema, mapping, res3, [["test full item"]]) assert "item_1617186331708" in res3 assert res3["item_1617186331708"] == [{'subitem_1551255647225': 'test full item', 'subitem_1551255648112': 'test full item'}] assert "title" not in res3 - + with patch("invenio_oaiharvester.harvester.parsing_metadata",return_value=(None,None)): res4 = copy.deepcopy(res) add_title_dc(schema, mapping, res4, metadata) @@ -945,7 +945,7 @@ def test_add_date_dc(mapper_dc): add_date_dc(schema, mapping, res, metadata) assert "item_1617186660861" in res assert res["item_1617186660861"] == [{'subitem_1522300722591': '2022-10-20'}] - + # def add_publisher_dc(schema, mapping, res, metadata): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::test_add_publisher_dc -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_add_publisher_dc(mapper_dc): @@ -987,12 +987,12 @@ def test_map_sets(): tree = etree.fromstring(xml_str) sets = tree.findall("set") - + result = map_sets(sets) assert type(result) is OrderedDict assert result["user-test_community1"] == "test_community_title1" assert result["user-test_community2"] == " " - + # class BaseMapper: # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::TestBaseMapper -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp class TestBaseMapper: @@ -1016,7 +1016,7 @@ def test_init(self,app,db): db.session.commit() mapper = BaseMapper(xml) assert hasattr(mapper, "itemtype") == False - + # exist item_type with name "Multiple" or "Others" item_type_name2 = ItemTypeName( id=11, name="Multiple", has_site_license=True, is_active=True @@ -1048,16 +1048,16 @@ def test_map_itemtype(self,db): db.session.add(item_type_name1) db.session.add(item_type1) db.session.commit() - + # not exist type - xml_str='
oai:weko3.example.org:000000012023-02-20T06:24:47Z1557819692844:15578197332761557820086539
test full itemother title1234テスト, 太郎テスト太郎テスト　別郎56785678test, smithtestsmithother smith1234metadata only accessPaidテスト権利情報テスト　太郎テスト主題this is test abstract.test publisher2022-10-202022-10-19jpn1.1AO1111https://doi.org/1234/0000000001https://192.168.56.103/records/11234/00000000011111111関連情報テストhttps://192.168.56.103/records/31 to 212345678901234567891112テスト位置情報22222テスト助成機関1111テスト研究test source Identifiertest collectiblestest title book51223335551237894562349999テスト学位2022-10-19学位授与機関識別子テスト学位授与機関テスト会議12345テスト機関12テスト会場JPNhttps://weko3.example.org/record/1/files/test1.txttext/plain18 B2022-10-201.0https://weko3.example.org/record/1/files/test2application/octet-stream18 B1.2https://weko3.example.org/record/1/files/test3.pngimage/png18 B2.1' + xml_str='
oai:weko3.example.org:000000012023-02-20T06:24:47Z1557819692844:15578197332761557820086539
test full itemother title1234テスト, 太郎テスト太郎テスト　別郎56785678test, smithtestsmithother smith1234metadata only accessPaidテスト権利情報テスト　太郎テスト主題this is test abstract.test publisher2022-10-202022-10-19jpn1.1AO1111https://doi.org/1234/0000000001https://192.168.56.103/records/11234/00000000011111111関連情報テストhttps://192.168.56.103/records/31 to 212345678901234567891112テスト位置情報22222テスト助成機関1111テスト研究test source Identifiertest collectiblestest title book51223335551237894562349999テスト学位2022-10-19学位授与機関識別子テスト学位授与機関テスト会議12345テスト機関12テスト会場JPNhttps://weko3.example.org/record/1/files/test1.txttext/plain18 B2022-10-201.0https://weko3.example.org/record/1/files/test2application/octet-stream18 B1.2https://weko3.example.org/record/1/files/test3.pngimage/png18 B2.1' tree = etree.fromstring(xml_str) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() mapper = BaseMapper(xml) mapper.map_itemtype("jpcoar:jpcoar") assert hasattr(mapper, "itemtype") == False - + BaseMapper.update_itemtype_map() # "news paper" is in RESOURCE_TYPE_MAP and itemtype_map # "conference paper" is in RESOURCE_TYPE_MAP, not in itemtype_map @@ -1093,28 +1093,28 @@ def test_init(self,app,db): mapper = DCMapper(xml) assert hasattr(mapper,"itemtype") assert "Multiple" in BaseMapper.itemtype_map - + # def map(self): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::TestDCMapper::test_map -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_map(self,db_itemtype): - + deleted_xml = '2023-03-01T02:07:10Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000012023-02-20T06:24:47Z
' tree = etree.fromstring(deleted_xml) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() mapper = DCMapper(xml) mapper.itemtype = ItemType.query.filter_by(id=12).one() - + result = mapper.map() assert result == {} - + xml_str = '2023-03-01T02:07:10Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000012023-02-20T06:24:47Z1557819692844:15578197332761557820086539
test full itemテスト, 太郎11234テスト主題this is test abstract.test publishertest, smith256782022-10-20conference paper1111test collectiblesjpn11111111 to 2metadata only accesstext/plain' tree = etree.fromstring(xml_str) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() mapper = DCMapper(xml) mapper.itemtype = ItemType.query.filter_by(id=12).one() - + test = {'$schema': 12, 'pubdate': '2023-02-20', 'item_1617186331708': [{'subitem_1551255647225': 'test full item', 'subitem_1551255648112': 'ja'}], 'title': 'test full item', 'item_1593074267803': [{'creatorNames': [{'creatorName': 'テスト, 太郎'}]}, {'creatorNames': [{'creatorName': '1'}]}, {'creatorNames': [{'creatorName': '1234'}]}], 'item_1617186609386': [{'subitem_1523261968819': 'テスト主題'}], 'item_1617186626617': [{'subitem_description': 'this is test abstract.'}], 'item_1617186643794': [{'subitem_1522300316516': 'test publisher'}], 'item_1617349709064': [{'contributorNames': [{'contributorName': 'test, smith'}]}, {'contributorNames': [{'contributorName': '2'}]}, {'contributorNames': [{'contributorName': '5678'}]}], 'item_1617186660861': [{'subitem_1522300722591': '2022-10-20'}], 'item_1617258105262': [{'resourcetype': 'conference paper'}], 'item_1617186783814': [{'subitem_identifier_uri': '1111'}], 'item_1617186941041': [{'subitem_1522650091861': 'test collectibles'}], 'item_1617186702042': [{'subitem_1551255818386': 'jpn'}], 'item_1617353299429': [{'subitem_1522306287251': {'subitem_1522306436033': '1111111'}}], 'item_1617186859717': [{'subitem_1522658031721': '1 to 2'}], 'item_1617186499011': [{'subitem_1522650717957': 'metadata only access'}], 'item_1617605131499': [{'format': 'text/plain'}]} result = mapper.map() assert result == test @@ -1141,28 +1141,28 @@ def test_init(self,app,db): mapper = JPCOARMapper(xml) assert hasattr(mapper,"itemtype") assert "Multiple" in BaseMapper.itemtype_map - + # def map(self): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::TestJPCOARMapper::test_map -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_map(self,db_itemtype): - + deleted_xml = '2023-03-01T02:07:10Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000012023-02-20T06:24:47Z
' tree = etree.fromstring(deleted_xml) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() mapper = JPCOARMapper(xml) mapper.itemtype = ItemType.query.filter_by(id=12).one() - + result = mapper.map() assert result == {} - + xml_str = '
oai:weko3.example.org:000000012023-02-20T06:24:47Z1557819692844:15578197332761557820086539
test full itemother title1234テスト, 太郎テスト太郎テスト　別郎56785678test, smithtestsmithother smith1234metadata only accessPaidテスト権利情報テスト　太郎テスト主題this is test abstract.test publisher2022-10-202022-10-19jpnnewspaper1.1AO1111https://doi.org/1234/0000000001https://192.168.56.103/records/11234/00000000011111111関連情報テストhttps://192.168.56.103/records/31 to 212345678901234567891112テスト位置情報22222テスト助成機関1111テスト研究test source Identifiertest collectiblestest title book51223335551237894562349999テスト学位2022-10-19学位授与機関識別子テスト学位授与機関テスト会議12345テスト機関12テスト会場JPNhttps://weko3.example.org/record/1/files/test1.txttext/plain18 B2022-10-201.0https://weko3.example.org/record/1/files/test2application/octet-stream18 B1.2https://weko3.example.org/record/1/files/test3.pngimage/png18 B2.1' tree = etree.fromstring(xml_str) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] xml = etree.tostring(record,encoding="utf-8").decode() mapper = JPCOARMapper(xml) mapper.itemtype = ItemType.query.filter_by(id=10).one() - + test = {'$schema': 10, 'pubdate': '2023-02-20', 'item_1551264308487': [{'subitem_1551255647225': 'test full item', 'subitem_1551255648112': 'ja'}], 'title': 'test full item', 'item_1551264326373': [{'subitem_1551255720400': 'other title', 'subitem_1551255721061': 'en'}], 'item_1551264340087': [{'subitem_1551255991424': [{'subitem_1551256006332': '太郎', 'subitem_1551256007414': 'ja'}], 'subitem_1551255929209': [{'subitem_1551255938498': 'テスト', 'subitem_1551255964991': 'ja'}], 'subitem_1551255898956': [{'subitem_1551255905565': 'テスト, 太郎', 'subitem_1551255907416': 'ja'}], 'subitem_1551256025394': [{'subitem_1551256035730': 'テスト\u3000別郎', 'subitem_1551256055588': 'ja'}]}], 'item_1551264418667': [{'subitem_1551257036415': 'ContactPerson', 'subitem_1551257339190': [{'subitem_1551257342360': '', 'subitem_1551257343979': 'en'}], 'subitem_1551257272214': [{'subitem_1551257314588': 'test', 'subitem_1551257316910': 'en'}], 'subitem_1551257245638': [{'subitem_1551257276108': 'test, smith', 'subitem_1551257279831': 'en'}], 'subitem_1551257372442': [{'subitem_1551257374288': 'other smith', 'subitem_1551257375939': 'en'}]}], 'item_1551264447183': [{'subitem_1551257553743': 'metadata only access', 'subitem_1551257578398': 'http://purl.org/coar/access_right/c_14cb'}], 'item_1551264605515': [{'subitem_1551257776901': 'Paid'}], 'item_1551264629907': [{'subitem_1551257025236': [{'subitem_1551257043769': 'テスト権利情報', 'subitem_1551257047388': 'ja'}], 'subitem_1551257030435': 'テスト権利情報Resource'}], 'item_1551264767789': [{'subitem_1551257249371': [{'subitem_1551257255641': 'テスト\u3000太郎', 'subitem_1551257257683': 'ja'}]}], 'item_1551264822581': [{'subitem_1551257315453': 'テスト主題', 'subitem_1551257323812': 'ja', 'subitem_1551257343002': 'http://bsh.com', 'subitem_1551257329877': 'BSH'}], 'item_1551264846237': [{'subitem_1551255577890': 'this is test abstract.', 'subitem_1551255592625': 'en', 'subitem_1551255637472': 'Abstract'}], 'item_1551264917614': [{'subitem_1551255702686': 'test publisher', 'subitem_1551255710277': 'ja'}], 'item_1551264974654': [{'subitem_1551255753471': '2022-10-20', 'subitem_1551255775519': 'Accepted'}, {'subitem_1551255753471': '2022-10-19', 'subitem_1551255775519': 'Issued'}], 'item_1551265002099': [{'subitem_1551255818386': 'jpn'}], 'item_1551265032053': [{'resourcetype': 'newspaper', 'resourceuri': 'http://purl.org/coar/resource_type/c_2fe3'}], 'item_1551265075370': [{'subitem_1551255975405': '1.1'}], 'item_1551265118680': [{'subitem_1551256025676': 'AO'}], 'system_identifier_doi': [{'subitem_systemidt_identifier': '1111', 'subitem_systemidt_identifier_type': 'DOI'}, {'subitem_systemidt_identifier': 'https://doi.org/1234/0000000001', 'subitem_systemidt_identifier_type': 'DOI'}, {'subitem_systemidt_identifier': 'https://192.168.56.103/records/1', 'subitem_systemidt_identifier_type': 'URI'}], 'item_1581495499605': [{'subitem_1551256250276': '1234/0000000001', 'subitem_1551256259586': 'JaLC'}], 'item_1551265227803': [{'subitem_1551256388439': 'isVersionOf', 'subitem_1551256480278': [{'subitem_1551256498531': '関連情報テスト', 'subitem_1551256513476': 'ja'}], 'subitem_1551256465077': [{'subitem_1551256478339': '1111111', 'subitem_1551256629524': 'ARK'}]}, {'subitem_1551256388439': 'isVersionOf', 'subitem_1551256465077': [{'subitem_1551256478339': 'https://192.168.56.103/records/3', 'subitem_1551256629524': 'URI'}]}], 'item_1551265302120': [{'subitem_1551256918211': '1 to 2', 'subitem_1551256920086': 'ja'}], 'item_1551265385290': [{'subitem_1551256462220': [{'subitem_1551256653656': 'テスト助成機関', 'subitem_1551256657859': 'ja'}], 'subitem_1551256454316': [{'subitem_1551256614960': '22222', 'subitem_1551256619706': 'Crossref Funder'}], 'subitem_1551256688098': [{'subitem_1551256691232': 'テスト研究', 'subitem_1551256694883': 'ja'}], 'subitem_1551256665850': [{'subitem_1551256671920': '1111', 'subitem_1551256679403': 'https://test.research.com'}]}], 'item_1551265409089': [{'subitem_1551256405981': 'test source Identifier', 'subitem_1551256409644': 'PISSN'}], 'item_1551265438256': [{'subitem_1551256349044': 'test collectibles', 'subitem_1551256350188': 'ja'}, {'subitem_1551256349044': 'test title book', 'subitem_1551256350188': 'ja'}], 'item_1551265463411': [{'subitem_1551256328147': '5'}, {'subitem_1551256328147': '1'}], 'item_1551265520160': [{'subitem_1551256294723': '2'}, {'subitem_1551256294723': '2'}], 'item_1551265553273': [{'subitem_1551256248092': '333'}, {'subitem_1551256248092': '555'}], 'item_1551265569218': [{'subitem_1551256198917': '123'}, {'subitem_1551256198917': '789'}, {'subitem_1551256198917': '456'}, {'subitem_1551256198917': '234'}], 'item_1551265738931': [{'subitem_1551256171004': '9999'}], 'item_1551265790591': [{'subitem_1551256126428': 'テスト学位', 'subitem_1551256129013': 'ja'}], 'item_1551265811989': [{'subitem_1551256096004': '2022-10-19'}], 'item_1551265903092': [{'subitem_1551256015892': [{'subitem_1551256027296': '学位授与機関識別子テスト', 'subitem_1551256029891': 'kakenhi'}], 'subitem_1551256037922': [{'subitem_1551256042287': '学位授与機関', 'subitem_1551256047619': 'ja'}]}], 'item_1551265973055': [{'subitem_1599711813532': 'JPN', 'subitem_1599711655652': '12345', 'subitem_1599711633003': [{'subitem_1599711636923': 'テスト会議', 'subitem_1599711645590': 'ja'}]}], 'item_1570069138259': [{'subitem_1551255854908': '1.0', 'subitem_1551255750794': 'text/plain', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255820788': [{'subitem_1551255828320': '2022-10-20', 'subitem_1551255833133': 'Accepted'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test1.txt'}]}, {'subitem_1551255854908': '1.2', 'subitem_1551255750794': 'application/octet-stream', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test2'}]}, {'subitem_1551255854908': '2.1', 'subitem_1551255750794': 'image/png', 'subitem_1551255788530': [{'subitem_1570068579439': '18 B'}], 'subitem_1551255558587': [{'subitem_1551255570271': 'https://weko3.example.org/record/1/files/test3.png'}]}]} result = mapper.map() assert result == test @@ -1190,79 +1190,79 @@ def test_init(self,app,db): assert hasattr(mapper,"itemtype") assert "Multiple" in BaseMapper.itemtype_map assert mapper.record_title == "" - + # def ddi_harvest_processing(self, harvest_data, res): # .tox/c1/bin/pytest --cov=invenio_oaiharvester tests/test_harvester.py::TestDDIMapper::test_ddi_harvest_processing -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiharvester/.tox/c1/tmp def test_ddi_harvest_processing(self,db_itemtype): data = OrderedDict([ - ('@xmlns:dc', 'http://purl.org/dc/terms/'), - ('@xmlns:fn', 'http://www.w3.org/2005/xpath-functions'), - ('@xmlns:saxon', 'http://xml.apache.org/xslt'), - ('@xmlns:xhtml', 'http://www.w3.org/1999/xhtml'), - ('@xmlns:xs', 'http://www.w3.org/2001/XMLSchema'), - ('@xmlns', 'ddi:codebook:2_5'), - ('@xsi:schemaLocation', 'https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd'), + ('@xmlns:dc', 'http://purl.org/dc/terms/'), + ('@xmlns:fn', 'http://www.w3.org/2005/xpath-functions'), + ('@xmlns:saxon', 'http://xml.apache.org/xslt'), + ('@xmlns:xhtml', 'http://www.w3.org/1999/xhtml'), + ('@xmlns:xs', 'http://www.w3.org/2001/XMLSchema'), + ('@xmlns', 'ddi:codebook:2_5'), + ('@xsi:schemaLocation', 'https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd'), ('stdyDscr', OrderedDict([ ('citation', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('titlStmt', [ 'titlSmt_top1', 'titlSmt_top2', OrderedDict([ ('titl', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test ddi full item'), - ])), + ])), ('altTitl', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'other ddi title') - ])), + ])), ('IDNo', OrderedDict([ - ('@agency', 'test_id_agency'), - ('@xml:lang', 'ja'), + ('@agency', 'test_id_agency'), + ('@xml:lang', 'ja'), ('#text', 'test_study_id') ])) - ])]), + ])]), ('rspStmt', OrderedDict([ ('AuthEnty', OrderedDict([ - ('@ID', '4'), - ('@xml:lang', 'ja'), - ('@affiliation', 'author.affiliation'), + ('@ID', '4'), + ('@xml:lang', 'ja'), + ('@affiliation', 'author.affiliation'), ('#text', 'テスト, 太郎') ])) - ])), + ])), ('rspStmt', OrderedDict([ ('AuthEnty', OrderedDict([ - ('@ID', '4'), - ('@xml:lang', 'ja'), - ('@affiliation', 'author.affiliation'), + ('@ID', '4'), + ('@xml:lang', 'ja'), + ('@affiliation', 'author.affiliation'), ('#text', 'テスト, 太郎') ])) ])), ('prodStmt', OrderedDict([ ('producer', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test_publisher') - ])), + ])), ('copyright', OrderedDict([ ('@xml:lang', 'ja'), ('@description','this is rights description.'), ('@from','today'), ('#text', 'test_rights') - ])), + ])), ('fundAg', OrderedDict([ - ('@ID', 'test_found_ageny'), - ('@xml:lang', 'ja'), + ('@ID', 'test_found_ageny'), + ('@xml:lang', 'ja'), ('#text', 'test_founder_name') - ])), + ])), ('grantNo', 'test_grant_no') - ])), + ])), ('distStmt', OrderedDict([ ('distrbtr', OrderedDict([ - ('@URI', 'https://test.distributor.affiliation'), - ('@abbr', 'TDN'), - ('@xml:lang', 'ja'), - ('@affiliation', 'Test Distributor Affiliation'), + ('@URI', 'https://test.distributor.affiliation'), + ('@abbr', 'TDN'), + ('@xml:lang', 'ja'), + ('@affiliation', 'Test Distributor Affiliation'), ('#text', 'Test Distributor Name') ])) ])), @@ -1271,22 +1271,22 @@ def test_ddi_harvest_processing(self,db_itemtype): ('@xml:lang', 'ja'), ('#text', 'test_series') ])) - ])), + ])), ('notes', OrderedDict([ ('#text','test_text'), ('@sub_text','sub_test_text') ])), ('verStmt', OrderedDict([ ('version', OrderedDict([ - ('@date', '2023-03-07'), - ('@xml:lang', 'ja'), + ('@date', '2023-03-07'), + ('@xml:lang', 'ja'), ('#text', '1.2') ])) - ])), + ])), ('biblCit', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test.input.content') - ])), + ])), ('holdings', [ OrderedDict([ ('@URI', 'https://192.168.56.103/records/75') @@ -1301,217 +1301,217 @@ def test_ddi_harvest_processing(self,db_itemtype): ('#text','http://other_prefix') ]) ]) - ])), + ])), ('stdyInfo', [ OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('subject', OrderedDict([ ('topcClas', [ OrderedDict([ - ('@vocab', 'test_topic_vocab'), - ('@vocabURI', 'http://test.topic.vocab'), - ('@xml:lang', 'ja'), + ('@vocab', 'test_topic_vocab'), + ('@vocabURI', 'http://test.topic.vocab'), + ('@xml:lang', 'ja'), ('#text', 'Test Topic') - ]), + ]), OrderedDict([ - ('@vocab', 'CESSDA Topic Classification'), - ('@vocabURI', 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification'), - ('@xml:lang', 'ja'), + ('@vocab', 'CESSDA Topic Classification'), + ('@vocabURI', 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification'), + ('@xml:lang', 'ja'), ('#text', '人口') ]), "test_str_value" ]) - ])), + ])), ('sumDscr', OrderedDict([ ('timePrd', [ OrderedDict([ - ('@event', 'start'), + ('@event', 'start'), ('#text', '2023-03-01') - ]), + ]), OrderedDict([ - ('@event', 'end'), + ('@event', 'end'), ('#text', '2023-03-03') ]) - ]), + ]), ('collDate', [ OrderedDict([ - ('@event', 'start'), + ('@event', 'start'), ('#text', '2023-03-01') - ]), + ]), OrderedDict([ - ('@event', 'end'), + ('@event', 'end'), ('#text', '2023-03-06') ]) - ]), + ]), ('geogCover', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test_geographic_coverage') - ])), + ])), ('anlyUnit', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', '個人') - ])), + ])), ('universe', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test parent set') - ])), + ])), ('dataKind', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', '量的調査') ])) ])) - ]), + ]), OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('subject', OrderedDict([ ('topcClas', OrderedDict([ - ('@vocab', 'CESSDA Topic Classification'), - ('@vocabURI', 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification'), - ('@xml:lang', 'en'), + ('@vocab', 'CESSDA Topic Classification'), + ('@vocabURI', 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification'), + ('@xml:lang', 'en'), ('#text', 'Demography') ])) - ])), + ])), ('abstract', [ OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('@subvalue','this is description for ddi item.\nthis is description for ddi item.'), ('#text', '

this is description for ddi item.

\n

this is description for ddi item.

') ]), - ]), + ]), ('sumDscr', OrderedDict([ ('timePrd', [ OrderedDict([ - ('@event', 'start'), + ('@event', 'start'), ('#text', '2023-03-01') - ]), + ]), OrderedDict([ - ('@event', 'end'), + ('@event', 'end'), ('#text', '2023-03-03') ]) - ]), + ]), ('collDate', [ OrderedDict([ - ('@event', 'start'), + ('@event', 'start'), ('#text', '2023-03-01') - ]), + ]), OrderedDict([ - ('@event', 'end'), + ('@event', 'end'), ('#text', '2023-03-06') ]) - ]), + ]), ('anlyUnit', [ OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'test_unit_of_analysis') - ]), + ]), OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'Individual') ]) - ]), + ]), ('dataKind', OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'quantatitive research') ])) ])) ]) - ]), + ]), ('method', [ OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('dataColl', OrderedDict([ ('sampProc', [ OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test sampling procedure') - ]), + ]), OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', '母集団/ 全数調査') ]) - ]), + ]), ('collMode', [ OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test collection method') - ]), + ]), OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'インタビュー') ]) ]) - ])), + ])), ('anlyInfo', OrderedDict([ ('respRate', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('#text', 'test sampling procedure_sampling_rate') ])) ])) - ]), + ]), OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('dataColl', OrderedDict([ ('sampProc', OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'Total universe/Complete enumeration') - ])), + ])), ('collMode', OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'Interview') ])) - ])), + ])), ('anlyInfo', None) ]) - ]), + ]), ('dataAccs', [ OrderedDict([ - ('@xml:lang', 'jp'), + ('@xml:lang', 'jp'), ('setAvail', OrderedDict([ ('avlStatus', OrderedDict([ - ('@xml:lang', 'jp'), + ('@xml:lang', 'jp'), ('#text', 'オープンアクセス') ])) - ])), + ])), ('notes', 'jpn') - ]), + ]), OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('setAvail', OrderedDict([ ('avlStatus', OrderedDict([ - ('@xml:lang', 'en'), + ('@xml:lang', 'en'), ('#text', 'open access') ])) - ])), + ])), ('notes', 'jpn') ]) ]), ('othrStdyMat', OrderedDict([ - ('@xml:lang', 'ja'), + ('@xml:lang', 'ja'), ('relStdy', OrderedDict([ - ('@ID', 'test_related_study_identifier'), - ('@xml:lang', 'ja'), + ('@ID', 'test_related_study_identifier'), + ('@xml:lang', 'ja'), ('#text', 'test_related_study_title') ])) #('relPubl', OrderedDict([ - # ('@ID', 'test_related_publication_identifier'), - # ('@xml:lang', 'ja'), + # ('@ID', 'test_related_publication_identifier'), + # ('@xml:lang', 'ja'), # ('#text', 'test_related_publication_title') #])) ])) ])), ('stdyDscr.othrStdyMat.relStdy',OrderedDict([ - ('@ID', 'test_related_study_identifier_out1'), - ('@xml:lang', 'ja'), + ('@ID', 'test_related_study_identifier_out1'), + ('@xml:lang', 'ja'), ('#text', 'test_related_study_title') ])), ('stdyDscr.othrStdyMat.relPubl',OrderedDict([ - ('@ID', 'test_related_publication_identifier_out1'), - ('@xml:lang', 'ja'), + ('@ID', 'test_related_publication_identifier_out1'), + ('@xml:lang', 'ja'), ('#text', 'test_related_publication_title_out')])), ('stdyDscr.citation.holdings',[OrderedDict([ ('#text', 'test_url') ])]) - + ]) xml_str = '2023-03-02T04:08:53Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000752023-03-02T04:06:46Z1557820086539
test ddi full itemother ddi titletest_study_idテスト, 太郎テスト, 太郎test_publishertest_rightstest_founder_nametest_grant_noTest Distributor Nametest_seriestest_text1.2test.input.contenthttp://doi.org/test_doihttp://hdl.handle.net/test_doihttp://other_prefixTest Topic人口2023-03-012023-03-032023-03-012023-03-06test_geographic_coverage個人test parent set量的調査Demographythis is description for ddi item. this is description for ddi item.2023-03-012023-03-032023-03-012023-03-06test_unit_of_analysisIndividualquantatitive researchtest sampling procedure母集団/ 全数調査test collection methodインタビューtest sampling procedure_sampling_rateTotal universe/Complete enumerationInterviewオープンアクセスjpnopen accessjpntest_related_study_titletest_related_publication_title' tree = etree.fromstring(xml_str) @@ -1519,23 +1519,25 @@ def test_ddi_harvest_processing(self,db_itemtype): xml = etree.tostring(record,encoding="utf-8").decode() mapper = DDIMapper(xml) mapper.map_itemtype('codeBook') - test = {'$schema': 11, 'pubdate': str(mapper.datestamp()), 'item_1586157591881': [{'subitem_1586156939407': 'titlSmt_top1'}, {'subitem_1586156939407': 'titlSmt_top2'}, {'subitem_1586156939407': 'test_study_id', 'subitem_1591256665864': 'test_id_agency', 'subitem_1586311767281': 'ja'}], 'item_1551264308487': [{'subitem_1551255647225': 'test ddi full item', 'subitem_1551255648112': 'ja'}], 'item_1551264326373': [{'subitem_1551255720400': 'other ddi title', 'subitem_1551255721061': 'ja'}], 'item_1593074267803': [{'creatorNames': [{'creatorName': 'テスト, 太郎', 'creatorNameLang': 'ja'}], 'nameIdentifiers': [{'nameIdentifier': '4'}], 'creatorAffiliations': [{'affiliationNames': [{'affiliationName': 'author.affiliation'}]}]}], 'item_1551264917614': [{'subitem_1551255702686': 'test_publisher', 'subitem_1551255710277': 'ja'}], 'item_1551264629907': [{'subitem_1602213569986': {'subitem_1602213569987': 'test_rights'}, 'subitem_1602213570623': 'ja', 'subitem_1602213569989': {'subitem_1602213569990': {'subitem_1602213569988': 'this is rights description.'}}, 'subitem_1602213569991': {'subitem_1602213569992': 'today'}}], 'item_1602145817646': [{'subitem_1602142814330': 'test_founder_name', 'subitem_1602142815328': 'ja'}], 'item_1602145850035': [{'subitem_1602142123771': 'test_grant_no'}], 'item_1592405734122': [{'subitem_1592369405220': 'Test Distributor Name', 'subitem_1591320914113': 'https://test.distributor.affiliation', 'subitem_1591320889728': 'TDN', 'subitem_1592369407829': 'ja', 'subitem_1591320890384': 'Test Distributor Affiliation'}], 'item_1588254290498': [{'subitem_1587462181884': 'test_series', 'subitem_1587462183075': 'ja'}], 'item_1645678901234': [{'interim': 'test_text', 'subitem_165678901234567': 'sub_test_text'}], 'item_1551265075370': [{'subitem_1591254914934': '1.2', 'subitem_1591254915862': '2023-03-07', 'subitem_1591254915406': 'ja'}], 'item_1592880868902': [{'subitem_1586228465211': 'test.input.content', 'subitem_1586228490356': 'ja'}], 'item_1612345678910': [{'subitem_1623456789123': 'http://doi.org/test_doi'}, {'subitem_1623456789123': 'http://hdl.handle.net/test_doi'}, {'subitem_1623456789123': 'http://other_prefix'}], 'item_1551264822581': [{'subitem_1592472785169': 'Test Topic', 'subitem_1592472786088': 'test_topic_vocab', 'subitem_1592472786560': 'http://test.topic.vocab', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': '人口', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': 'test_str_value'}, {'subitem_1592472785169': 'Demography', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'en'}], 'item_1602145192334': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-03', 'subitem_1602144587621': 'end'}], 'item_1586253152753': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-06', 'subitem_1602144587621': 'end'}], 'item_1570068313185': [{'subitem_1586419454219': 'test_geographic_coverage', 'subitem_1586419462229': 'ja'}], 'item_1586253224033': [{'subitem_1596608607860': '個人', 'subitem_1596608609366': 'ja'}, {'subitem_1596608607860': 'test_unit_of_analysis', 'subitem_1596608609366': 'en'}, {'subitem_1596608607860': 'Individual', 'subitem_1596608609366': 'en'}], 'item_1586253249552': [{'subitem_1596608974429': 'test parent set', 'subitem_1596608975087': 'ja'}], 'item_1588260046718': [{'subitem_1591178807921': '量的調査', 'subitem_1591178808409': 'ja'}, {'subitem_1591178807921': 'quantatitive research', 'subitem_1591178808409': 'en'}], 'item_1551264846237': [{'subitem_1551255577890': 'this is description for ddi item.\nthis is description for ddi item.', 'subitem_1551255592625': 'en'}], 'item_1586253334588': [{'subitem_1596609826487': 'test sampling procedure', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': '母集団/ 全数調査', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': 'Total universe/Complete enumeration', 'subitem_1596609827068': 'en'}], 'item_1586253349308': [{'subitem_1596610500817': 'test collection method', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'インタビュー', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'Interview', 'subitem_1596610501381': 'en'}], 'item_1586253589529': [{'subitem_1596609826487': 'test sampling procedure_sampling_rate', 'subitem_1596609827068': 'ja'}], 'item_1588260178185': [{'subitem_1522650727486': 'オープンアクセス', 'subitem_1522650717957': 'jp'}, {'subitem_1522650727486': 'open access', 'subitem_1522650717957': 'en'}], 'item_1551265002099': [{'subitem_1551255818386': 'jpn'}], 'item_1592405736602': [{'subitem_1602215239359': 'test_related_study_title', 'subitem_1602215240520': 'test_related_study_identifier', 'subitem_1602215239925': 'ja'}, {'subitem_1602215239359': 'test_related_study_title', 'subitem_1602215240520': 'test_related_study_identifier_out1', 'subitem_1602215239925': 'ja'}], 'item_1592405735401': [{'subitem_1602214558730': 'test_related_publication_title_out', 'subitem_1602214560358': 'test_related_publication_identifier_out1', 'subitem_1602214559588': 'ja'}]} + test = {'$schema': 11, 'pubdate': str(mapper.datestamp()), 'item_1551264326373': [{'subitem_1551255720400': 'titlSmt_top1'}, {'subitem_1551255720400': 'titlSmt_top2'}, {'subitem_1551255720400': 'other ddi title', 'subitem_1551255721061': 'ja'}], 'item_1551264308487': [{'subitem_1551255647225': 'test ddi full item', 'subitem_1551255648112': 'ja'}], 'item_1586157591881': [{'subitem_1586156939407': 'test_study_id', 'subitem_1591256665864': 'test_id_agency', 'subitem_1586311767281': 'ja'}], 'item_1593074267803': [{'creatorNames': [{'creatorName': 'テスト, 太郎', 'creatorNameLang': 'ja'}], 'nameIdentifiers': [{'nameIdentifier': '4'}], 'creatorAffiliations': [{'affiliationNames': [{'affiliationName': 'author.affiliation'}]}]}], 'item_1551264917614': [{'subitem_1551255702686': 'test_publisher', 'subitem_1551255710277': 'ja'}], 'item_1551264629907': [{'subitem_1602213569986': {'subitem_1602213569987': 'test_rights'}, 'subitem_1602213569991': {'subitem_1602213569992': 'today'}, 'subitem_1602213570623': 'ja', 'subitem_1602213569989': {'subitem_1602213569990': {'subitem_1602213569988': 'this is rights description.'}}}], 'item_1602145817646': [{'subitem_1602142814330': 'test_founder_name', 'subitem_1602142815328': 'ja'}], 'item_1602145850035': [{'subitem_1602142123771': 'test_grant_no'}], 'item_1592405734122': [{'subitem_1592369405220': 'Test Distributor Name', 'subitem_1591320914113': 'https://test.distributor.affiliation', 'subitem_1591320889728': 'TDN', 'subitem_1592369407829': 'ja', 'subitem_1591320890384': 'Test Distributor Affiliation'}], 'item_1588254290498': [{'subitem_1587462181884': 'test_series', 'subitem_1587462183075': 'ja'}], 'item_1551265075370': [{'subitem_1591254914934': '1.2', 'subitem_1591254915862': '2023-03-07', 'subitem_1591254915406': 'ja'}], 'item_1592880868902': [{'subitem_1586228465211': 'test.input.content', 'subitem_1586228490356': 'ja'}], 'item_1602145007095': [{'subitem_1602144759036': 'http://doi.org/test_doi'}, {'subitem_1602144759036': 'http://hdl.handle.net/test_doi'}, {'subitem_1602144759036': 'http://other_prefix'}], 'item_1551264822581': [{'subitem_1592472785169': 'Test Topic', 'subitem_1592472786088': 'test_topic_vocab', 'subitem_1592472786560': 'http://test.topic.vocab', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': '人口', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': 'test_str_value'}, {'subitem_1592472785169': 'Demography', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'en'}], 'item_1602145192334': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-03', 'subitem_1602144587621': 'end'}], 'item_1586253152753': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-06', 'subitem_1602144587621': 'end'}], 'item_1570068313185': [{'subitem_1586419454219': 'test_geographic_coverage', 'subitem_1586419462229': 'ja'}], 'item_1586253224033': [{'subitem_1596608607860': '個人', 'subitem_1596608609366': 'ja'}, {'subitem_1596608607860': 'test_unit_of_analysis', 'subitem_1596608609366': 'en'}, {'subitem_1596608607860': 'Individual', 'subitem_1596608609366': 'en'}], 'item_1586253249552': [{'subitem_1596608974429': 'test parent set', 'subitem_1596608975087': 'ja'}], 'item_1588260046718': [{'subitem_1591178807921': '量的調査', 'subitem_1591178808409': 'ja'}, {'subitem_1591178807921': 'quantatitive research', 'subitem_1591178808409': 'en'}], 'item_1551264846237': [{'subitem_1551255577890': 'this is description for ddi item.\nthis is description for ddi item.', 'subitem_1551255592625': 'en'}], 'item_1586253334588': [{'subitem_1596609826487': 'test sampling procedure', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': '母集団/ 全数調査', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': 'Total universe/Complete enumeration', 'subitem_1596609827068': 'en'}], 'item_1586253349308': [{'subitem_1596610500817': 'test collection method', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'インタビュー', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'Interview', 'subitem_1596610501381': 'en'}], 'item_1586253589529': [{'subitem_1596609826487': 'test sampling procedure_sampling_rate', 'subitem_1596609827068': 'ja'}], 'item_1588260178185': [{'subitem_1522650727486': 'オープンアクセス', 'subitem_1522650717957': 'jp'}, {'subitem_1522650727486': 'open access', 'subitem_1522650717957': 'en'}], 'item_1551265002099': [{'subitem_1551255818386': 'jpn'}], 'item_1592405736602': [{'subitem_1602215239359': 'test_related_study_title', 'subitem_1602215240520': 'test_related_study_identifier', 'subitem_1602215239925': 'ja'}, {'subitem_1602215239359': 'test_related_study_title', 'subitem_1602215240520': 'test_related_study_identifier_out1', 'subitem_1602215239925': 'ja'}], 'item_1592405735401': [{'subitem_1602214558730': 'test_related_publication_title_out', 'subitem_1602214560358': 'test_related_publication_identifier_out1', 'subitem_1602214559588': 'ja'}]} res = {"$schema":mapper.itemtype.id,"pubdate":str(mapper.datestamp())} mapper.ddi_harvest_processing(data,res) assert res == test - + data = OrderedDict([ - ('@xmlns:dc', 'http://purl.org/dc/terms/'), - ('@xmlns:fn', 'http://www.w3.org/2005/xpath-functions'), - ('@xmlns:saxon', 'http://xml.apache.org/xslt'), - ('@xmlns:xhtml', 'http://www.w3.org/1999/xhtml'), - ('@xmlns:xs', 'http://www.w3.org/2001/XMLSchema'), - ('@xmlns', 'ddi:codebook:2_5'), - ('@xsi:schemaLocation', 'https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd'), + ('@xmlns:dc', 'http://purl.org/dc/terms/'), + ('@xmlns:fn', 'http://www.w3.org/2005/xpath-functions'), + ('@xmlns:saxon', 'http://xml.apache.org/xslt'), + ('@xmlns:xhtml', 'http://www.w3.org/1999/xhtml'), + ('@xmlns:xs', 'http://www.w3.org/2001/XMLSchema'), + ('@xmlns', 'ddi:codebook:2_5'), + ('@xsi:schemaLocation', 'https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd'), ('stdyDscr', OrderedDict([ ('citation', OrderedDict([ - ('raise_error',OrderedDict([ - ('#text','error_value') + ('titlStmt', OrderedDict([ + ('titl', OrderedDict([ + ('#text', 123) + ])) ])) ])) ])) @@ -1550,7 +1552,7 @@ def test_ddi_harvest_processing(self,db_itemtype): with pytest.raises(Exception): mapper.ddi_harvest_processing(data,res) assert res == {"$schema":mapper.itemtype.id,"pubdate":str(mapper.datestamp())} - + # def get_mapping_ddi(): # def merge_dict(result, target, val, keys): # def merge_data_by_mapping_keys(parent_key, data_mapping): @@ -1573,7 +1575,7 @@ def test_map(self, db_itemtype): mapper = DDIMapper(xml) result = mapper.map() assert result == {} - + # json["record"]["metadata"]["codeBook"] is none without_codebook_xml = '2023-03-02T04:08:53Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000752023-03-02T04:06:46Z1557820086539
' tree = etree.fromstring(without_codebook_xml) @@ -1582,8 +1584,8 @@ def test_map(self, db_itemtype): mapper = DDIMapper(xml) result = mapper.map() assert result == None - - + + xml_str = '2023-03-02T04:08:53Zhttps://192.168.56.103/oai
oai:weko3.example.org:000000752023-03-02T04:06:46Z1557820086539
test ddi full itemother ddi titletest_study_idテスト, 太郎test_publishertest_rightstest_founder_nametest_grant_noTest Distributor Nametest_series1.2test.input.contentTest Topic人口2023-03-012023-03-032023-03-012023-03-06test_geographic_coverage個人test parent set量的調査Demographythis is description for ddi item. this is description for ddi item.2023-03-012023-03-032023-03-012023-03-06test_unit_of_analysisIndividualquantatitive researchtest sampling procedure母集団/ 全数調査test collection methodインタビューtest sampling procedure_sampling_rateTotal universe/Complete enumerationInterviewオープンアクセスjpnopen accessjpntest_related_study_titletest_related_publication_title' tree = etree.fromstring(xml_str) record = tree.findall("./GetRecord/record",namespaces=tree.nsmap)[0] @@ -1591,6 +1593,6 @@ def test_map(self, db_itemtype): mapper = DDIMapper(xml) test = {'$schema': 11, 'pubdate': '2023-03-02', 'item_1551264308487': [{'subitem_1551255647225': 'test ddi full item', 'subitem_1551255648112': 'ja'}], 'item_1551264326373': [{'subitem_1551255720400': 'other ddi title', 'subitem_1551255721061': 'ja'}], 'item_1586157591881': [{'subitem_1586156939407': 'test_study_id', 'subitem_1591256665864': 'test_id_agency', 'subitem_1586311767281': 'ja'}], 'item_1593074267803': [{'creatorNames': [{'creatorName': 'テスト, 太郎', 'creatorNameLang': 'ja'}], 'nameIdentifiers': [{'nameIdentifier': '4'}], 'creatorAffiliations': [{'affiliationNames': [{'affiliationName': 'author.affiliation'}]}]}], 'item_1551264917614': [{'subitem_1551255702686': 'test_publisher', 'subitem_1551255710277': 'ja'}], 'item_1551264629907': [{'subitem_1602213569986': {'subitem_1602213569987': 'test_rights'}, 'subitem_1602213570623': 'ja'}], 'item_1602145817646': [{'subitem_1602142814330': 'test_founder_name', 'subitem_1602142815328': 'ja'}], 'item_1602145850035': [{'subitem_1602142123771': 'test_grant_no'}], 'item_1592405734122': [{'subitem_1592369405220': 'Test Distributor Name', 'subitem_1591320914113': 'https://test.distributor.affiliation', 'subitem_1591320889728': 'TDN', 'subitem_1592369407829': 'ja', 'subitem_1591320890384': 'Test Distributor Affiliation'}], 'item_1588254290498': [{'subitem_1587462181884': 'test_series', 'subitem_1587462183075': 'ja'}], 'item_1551265075370': [{'subitem_1591254914934': '1.2', 'subitem_1591254915862': '2023-03-07', 'subitem_1591254915406': 'ja'}], 'item_1592880868902': [{'subitem_1586228465211': 'test.input.content', 'subitem_1586228490356': 'ja'}], 'item_1551264822581': [{'subitem_1592472785169': 'Test Topic', 'subitem_1592472786088': 'test_topic_vocab', 'subitem_1592472786560': 'http://test.topic.vocab', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': '人口', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'ja'}, {'subitem_1592472785169': 'Demography', 'subitem_1592472786088': 'CESSDA Topic Classification', 'subitem_1592472786560': 'https://vocabularies.cessda.eu/urn/urn:ddi:int.cessda.cv:TopicClassification', 'subitem_1592472785698': 'en'}], 'item_1602145192334': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-03', 'subitem_1602144587621': 'end'}], 'item_1586253152753': [{'subitem_1602144573160': '2023-03-01', 'subitem_1602144587621': 'start'}, {'subitem_1602144573160': '2023-03-06', 'subitem_1602144587621': 'end'}], 'item_1570068313185': [{'subitem_1586419454219': 'test_geographic_coverage', 'subitem_1586419462229': 'ja'}], 'item_1586253224033': [{'subitem_1596608607860': '個人', 'subitem_1596608609366': 'ja'}, {'subitem_1596608607860': 'test_unit_of_analysis', 'subitem_1596608609366': 'en'}, {'subitem_1596608607860': 'Individual', 'subitem_1596608609366': 'en'}], 'item_1586253249552': [{'subitem_1596608974429': 'test parent set', 'subitem_1596608975087': 'ja'}], 'item_1588260046718': [{'subitem_1591178807921': '量的調査', 'subitem_1591178808409': 'ja'}, {'subitem_1591178807921': 'quantatitive research', 'subitem_1591178808409': 'en'}], 'item_1551264846237': [{'subitem_1551255577890': 'this is description for ddi item. this is description for ddi item.', 'subitem_1551255592625': 'en'}], 'item_1586253334588': [{'subitem_1596609826487': 'test sampling procedure', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': '母集団/ 全数調査', 'subitem_1596609827068': 'ja'}, {'subitem_1596609826487': 'Total universe/Complete enumeration', 'subitem_1596609827068': 'en'}], 'item_1586253349308': [{'subitem_1596610500817': 'test collection method', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'インタビュー', 'subitem_1596610501381': 'ja'}, {'subitem_1596610500817': 'Interview', 'subitem_1596610501381': 'en'}], 'item_1586253589529': [{'subitem_1596609826487': 'test sampling procedure_sampling_rate', 'subitem_1596609827068': 'ja'}], 'item_1588260178185': [{'subitem_1522650727486': 'オープンアクセス', 'subitem_1522650717957': 'jp'}, {'subitem_1522650727486': 'open access', 'subitem_1522650717957': 'en'}], 'item_1551265002099': [{'subitem_1551255818386': 'jpn'}], 'item_1592405736602': [{'subitem_1602215239359': 'test_related_study_title', 'subitem_1602215240520': 'test_related_study_identifier', 'subitem_1602215239925': 'ja'}], 'item_1592405735401': [{'subitem_1602214558730': 'test_related_publication_title', 'subitem_1602214560358': 'test_related_publication_identifier', 'subitem_1602214559588': 'ja'}], 'title': 'test ddi full item', 'type': [{'resourcetype': 'dataset', 'resourceuri': 'http://purl.org/coar/resource_type/c_ddb1'}]} result = mapper.map() - + assert result == test diff --git a/modules/invenio-oaiserver/invenio_oaiserver/response.py b/modules/invenio-oaiserver/invenio_oaiserver/response.py index 830459def2..327d8df25e 100644 --- a/modules/invenio-oaiserver/invenio_oaiserver/response.py +++ b/modules/invenio-oaiserver/invenio_oaiserver/response.py @@ -711,13 +711,11 @@ def check_correct_system_props_mapping(object_uuid, system_mapping_config): Correct mapping mean item map have the 2 field same with config """ - from weko_records.api import ItemsMetadata, Mapping + from weko_records.api import ItemsMetadata from weko_records.serializers.utils import get_mapping item_type = ItemsMetadata.get_by_object_id(object_uuid) - item_type_id = item_type.item_type_id - type_mapping = Mapping.get_record(item_type_id) - item_map = get_mapping(type_mapping, "jpcoar_mapping") + item_map = get_mapping(item_type.item_type_id, "jpcoar_mapping") if system_mapping_config: for key in system_mapping_config: @@ -734,49 +732,49 @@ def combine_record_file_urls(record, object_uuid, meta_prefix): Get file property information by item_mapping and put to metadata. """ - from weko_records.api import ItemsMetadata, Mapping + from weko_records.api import ItemsMetadata from weko_records.serializers.utils import get_mapping from weko_schema_ui.schema import get_oai_metadata_formats metadata_formats = get_oai_metadata_formats(current_app) item_type = ItemsMetadata.get_by_object_id(object_uuid) - item_type_id = item_type.item_type_id - type_mapping = Mapping.get_record(item_type_id) mapping_type = metadata_formats[meta_prefix]['serializer'][1]['schema_type'] - item_map = get_mapping(type_mapping, + item_map = get_mapping(item_type.item_type_id, "{}_mapping".format(mapping_type)) - file_keys = None + file_keys_str = None if item_map: file_props = current_app.config["OAISERVER_FILE_PROPS_MAPPING"] if mapping_type in file_props: - file_keys = item_map.get(file_props[mapping_type]) + file_keys_str = item_map.get(file_props[mapping_type]) else: - file_keys = None + file_keys_str = None - if not file_keys: + if not file_keys_str: return record else: - file_keys = file_keys.split('.') - - if len(file_keys) == 3 and record.get(file_keys[0]): - attr_mlt = record[file_keys[0]]["attribute_value_mlt"] - if isinstance(attr_mlt, list): - for attr in attr_mlt: - if attr.get('filename'): - if not attr.get(file_keys[1]): - attr[file_keys[1]] = {} - attr[file_keys[1]][file_keys[2]] = create_files_url( - request.url_root, - record.get('recid'), - attr.get('filename')) - elif isinstance(attr_mlt, dict) and \ - attr_mlt.get('filename'): - if not attr_mlt.get(file_keys[1]): - attr_mlt[file_keys[1]] = {} - attr_mlt[file_keys[1]][file_keys[2]] = create_files_url( - request.url_root, - record.get('recid'), - attr_mlt.get('filename')) + file_keys = file_keys_str.split(',') + + for file_key in file_keys: + key = file_key.split('.') + if len(key) == 3 and record.get(key[0]): + attr_mlt = record[key[0]]["attribute_value_mlt"] + if isinstance(attr_mlt, list): + for attr in attr_mlt: + if attr.get('filename'): + if not attr.get(key[1]): + attr[key[1]] = {} + attr[key[1]][key[2]] = create_files_url( + request.url_root, + record.get('recid'), + attr.get('filename')) + elif isinstance(attr_mlt, dict) and \ + attr_mlt.get('filename'): + if not attr_mlt.get(key[1]): + attr_mlt[key[1]] = {} + attr_mlt[key[1]][key[2]] = create_files_url( + request.url_root, + record.get('recid'), + attr_mlt.get('filename')) return record diff --git a/modules/invenio-records-rest/invenio_records_rest/serializers/base.py b/modules/invenio-records-rest/invenio_records_rest/serializers/base.py index 6aa2ac17e0..786e90b476 100644 --- a/modules/invenio-records-rest/invenio_records_rest/serializers/base.py +++ b/modules/invenio-records-rest/invenio_records_rest/serializers/base.py @@ -14,6 +14,9 @@ import pickle import pytz +import traceback + +from flask import current_app from weko_records.api import Mapping from invenio_records_rest.config import RECORDS_REST_DEFAULT_MAPPING_DICT @@ -155,51 +158,76 @@ def get_keys(arr): def get_mapping(item_type_id): """Get keys of metadata record by mapping.""" # Get default mapping key and lang from config (defaults are None). + from weko_items_ui.utils import get_options_and_order_list, get_hide_list_by_schema_form + from weko_records.api import ItemTypes + mapping_dict = RECORDS_REST_DEFAULT_MAPPING_DICT # Get mapping of this record. - mapping = Mapping.get_record(item_type_id) - if not mapping: + meta_option, item_type_mapping = get_options_and_order_list(item_type_id) + item_type = ItemTypes.get_by_id(item_type_id) + item_type_list = None + if item_type: + item_type_list = item_type.render.get('table_row') + hide_list = get_hide_list_by_schema_form(item_type_id) + if not item_type_mapping or not item_type_list: return mapping_dict # Update default mapping key and lang by mapping of this record. identifier = 'system_identifier' - for k, v in mapping.items(): - if not type(v.get('jpcoar_mapping')) is dict: - continue - for k1, v1 in v.get('jpcoar_mapping').items(): - for k2, v2 in mapping_dict.items(): - if k1 != k2.split(':')[1] or not type(v1) is dict: + for k in item_type_list: + if k in item_type_mapping: + v = item_type_mapping.get(k) + prop_hidden = meta_option.get(k, {}).get('option', {}).get('hidden', False) + if not type(v.get('jpcoar_mapping')) is dict \ + or prop_hidden: + continue + for k1, v1 in v.get('jpcoar_mapping').items(): + skip_flag = False + for h in hide_list: + if h.startswith(k) and \ + (('@value' in v1.keys() and + h.endswith(v1.get('@value').split('.')[-1])) or \ + ('creatorName' in v1.keys() and + h.endswith(v1.get('creatorName', {}).get('@value').split('.')[-1]))): + skip_flag = True + if skip_flag: continue - key = identifier if identifier in k else k - key_arr = ['metadata', key, 'attribute_value_mlt', 0] - lang_arr = key_arr.copy() - if k1 == 'creator': - name = v1.get('creatorName') - # Set all key for __lang - attr = name.get('@attributes', {}) - xml_lang = attr.get('xml:lang', '').split('.') - lang_arr.extend(get_keys(xml_lang)) - # Set all key for key - name_arr = name.get('@value').split('.') - key_arr.extend(get_keys(name_arr)) - elif '.' in v1.get('@value', ''): - # Set key for __lang - attr = v1.get('@attributes', {}) - xml_lang = attr.get('xml:lang', '').split('.') - lang_arr.extend(get_keys(xml_lang)) - # Set all key for key - name_arr = v1.get('@value').split('.') - key_arr.extend(get_keys(name_arr)) - else: - # Set key for __lang - attr = v1.get('@attributes', {}) - lang_arr.append(attr.get('xml:lang')) - # Set all key for key - key_arr.append(v1.get('@value')) - mapping_dict[k2] = key_arr - mapping_dict['{}__lang'.format(k2)] = lang_arr + for k2, v2 in mapping_dict.items(): + if k1 != k2.split(':')[1] or \ + not type(v1) is dict or \ + mapping_dict[k2]: + continue + key = identifier if identifier in k else k + key_arr = ['metadata', key, 'attribute_value_mlt', 0] + lang_arr = key_arr.copy() + if k1 == 'creator': + name = v1.get('creatorName') + # Set all key for __lang + attr = name.get('@attributes', {}) + xml_lang = attr.get('xml:lang', '').split('.') + lang_arr.extend(get_keys(xml_lang)) + # Set all key for key + name_arr = name.get('@value').split('.') + key_arr.extend(get_keys(name_arr)) + elif '.' in v1.get('@value', ''): + # Set key for __lang + attr = v1.get('@attributes', {}) + xml_lang = attr.get('xml:lang', '').split('.') + lang_arr.extend(get_keys(xml_lang)) + # Set all key for key + name_arr = v1.get('@value').split('.') + key_arr.extend(get_keys(name_arr)) + else: + # Set key for __lang + attr = v1.get('@attributes', {}) + lang_arr.append(attr.get('xml:lang')) + # Set all key for key + key_arr.append(v1.get('@value')) + mapping_dict[k2] = key_arr + mapping_dict['{}__lang'.format(k2)] = lang_arr return mapping_dict links_factory = links_factory or (lambda x, record=None, **k: dict()) + metadata = pickle.loads(pickle.dumps(record.replace_refs(), -1)) if self.replace_refs \ else record.dumps() # Get keys of metadata record by mapping. diff --git a/modules/invenio-records-rest/invenio_records_rest/views.py b/modules/invenio-records-rest/invenio_records_rest/views.py index c309bb16d8..58b8ae85ed 100644 --- a/modules/invenio-records-rest/invenio_records_rest/views.py +++ b/modules/invenio-records-rest/invenio_records_rest/views.py @@ -54,6 +54,7 @@ from flask_security import current_user from flask import session from invenio_accounts.models import User +from invenio_records_rest.sorter import parse_sort_field from weko_redis.redis import RedisConnection from .config import RECORDS_REST_DEFAULT_TTL_VALUE @@ -157,7 +158,7 @@ def create_blueprint(endpoints): __name__, url_prefix='', ) - + @blueprint.teardown_request def dbsession_clean(exception): current_app.logger.debug("invenio_records_rest dbsession_clean: {}".format(exception)) @@ -586,7 +587,7 @@ def get(self, **kwargs): type=int) size = RecordsListResource.adjust_list_view_num(size) formats = request.values.getlist('format') - if (not formats or 'html' in formats) and request.values.get('q') == None: + if (not formats or formats == [''] or 'html' in formats) and request.values.get('q') == None: return redirect_to_search(page, size) # if page * size >= self.max_result_window: @@ -622,7 +623,7 @@ def get_item_control_number(search_result): # This exception is for when sort type "relevance" is used sort_element = "control_number" relevance_sort_is_used = True - + if relevance_sort_is_used: sort_key = sort_element sort_key_arrangement = "desc" @@ -639,7 +640,7 @@ def get_item_control_number(search_result): cache_name = User.query.get(current_user.id).email else: cache_name = "anonymous_user" - + # For saving a specific page for search_after use as cache key cache_key = str(page) @@ -675,7 +676,7 @@ def url_args_check(): RECORDS_REST_DEFAULT_TTL_VALUE ) ) - + url_args_check() next_items_sort_value = "" @@ -724,7 +725,7 @@ def url_args_check(): next_search_after_set._extra.update(search_after_size) next_search_after_set._extra.update({"search_after": [orjson.loads(sessionstorage.get(cache_name)).get(cache_key).get("control_number")]}) next_items_sort_value = next_search_after_set.execute()["hits"]["hits"][-1]["_source"]["control_number"] - + else: page_list = [] cache_name_stored_data = orjson.loads(sessionstorage.get(cache_name)) @@ -746,7 +747,7 @@ def url_args_check(): next_search_after_set = search next_search_after_set._extra.update(search_after_size) next_search_after_set._extra.update({"search_after": [orjson.loads(sessionstorage.get(cache_name)).get(str(sorted_page_list[-1])).get("control_number")]}) - + try: if cache_data.get("control_number") is None: next_items_sort_value = last_sort_value @@ -800,6 +801,25 @@ def url_args_check(): # Search after trigger set to true use_search_after = True + idx = request.values.get("idx","") + idx = [int(i) for i in idx.split(",") if i and i.isdecimal()] + index_id = request.values.get("index_id") + if index_id and index_id.isdecimal(): + target_index = set(idx + [int(index_id)]) + else: + target_index = set(idx) + recursive = request.values.get("recursive", 0) + from weko_index_tree.api import Indexes + if recursive == "1": + for i in idx: + if Indexes.get_index(i): + target_index.update( + [int(cid) for cid in Indexes.get_child_list_recursive(str(i))] + ) + request_sort = request.values.get('sort','', str) + key, is_asc = parse_sort_field(request_sort) + is_custom_sort = key == "custom_sort" and target_index + if use_search_after: search = search[0:size] else: @@ -809,8 +829,30 @@ def url_args_check(): if query: urlkwargs['q'] = query + if is_custom_sort : + search = self._override_params_for_customsort(search, is_asc) + # Execute search search_result = search.execute() + search_result_dict = search_result.to_dict() + + if is_custom_sort: + if search_result.hits.total < self.max_result_window: + self._do_custom_sort(search_result_dict,target_index, is_asc, page, size) + else: + start = (page - 1) * size + end = page * size + search_result_dict["hits"]["hits"] = search_result_dict["hits"]["hits"][start:end] + # The order is reversed by the `prepend` method in + # `weko_records.serializers.feed:WekoFeedGenerator.add_entry`. + # As a temporary workaround, set to “prepend,” reverse the order. + # When it changes to “append,” please remove the flag and the if statement. + flag = "prepend" + if (flag == "prepend" + and any(i in formats for i in ("rss", "atom", "jpcoar")) + and request.values.get('q') is None + ): + search_result_dict["hits"]["hits"].reverse() if not sessionstorage.redis.exists(cache_name) and size * math.floor(self.max_result_window/size) <= self.max_result_window: json_data = orjson.dumps({cache_key: {"control_number": [next_items_sort_value]}}) @@ -839,7 +881,7 @@ def url_args_check(): return self.make_response( pid_fetcher=self.pid_fetcher, - search_result=search_result.to_dict(), + search_result=search_result_dict, links=links, item_links_factory=self.item_links_factory, ) @@ -864,6 +906,107 @@ def adjust_list_view_num(cls, size: int) -> int: return 100 + def _override_params_for_customsort(self, search, is_asc): + """ Set the sort clause for custom sort + + Override the sort conditions of the search object for custom sorting. + + Args: + search(invenio_search.api.RecordsSearch): search query + is_asc(boolean): Whether to sort in ascending or descending order. + + Returns: + search(invenio_search.api.RecordsSearch): search query with added sort + + """ + search = search[0:self.max_result_window] + search._sort = [] + + if is_asc: + order = "asc" + mode = "min" + else: + order = "desc" + mode = "max" + + path_sort = {"path": {"order": order, "mode": mode}} + default_sort = {"_created": {"order": order, "unmapped_type": "long"}} + + search._sort.append(path_sort) + search._sort.append(default_sort) + return search + + @classmethod + def _do_custom_sort(cls, search_result_dict, target_index, is_asc, page, size): + """ do custom_sort + + For each hit, taking target_index into account, + generate a priority key and sort the results. + + Args: + search_result_dict(dict): search result + is_asc(boolean): Whether to sort in ascending or descending order. + target_index(set): search index + page(int): page + size(int): size + + """ + + custom_sort = {} + + sorted_result = sorted((( + cls._customsort_priority(hit, target_index, is_asc, custom_sort), hit) + for hit in search_result_dict["hits"]["hits"] + ), reverse=not is_asc + ) + + formats = request.values.getlist('format') + # The order is reversed by the `prepend` method in + # `weko_records.serializers.feed:WekoFeedGenerator.add_entry`. + # As a temporary workaround, set to “prepend,” reverse the order. + # When it changes to “append,” please remove the flag and the if statement. + flag = "prepend" + start, end = (page - 1) * size, page * size + sorted_hits = [hit for _, hit in sorted_result[start:end]] + if (flag == "prepend" + and any(i in formats for i in ("rss", "atom", "jpcoar")) + and request.values.get('q') is None + ): + sorted_hits.reverse() + + search_result_dict["hits"]["hits"] = sorted_hits + + @classmethod + def _customsort_priority(cls, hit, target_index, is_asc, custom_sort): + """ Generate a priority key for custom sorting for each hit in the search results. + + From the index paths associated with `hit`, select a single path based on `target_index`, + and return a sort key that combines the custom sort definition for each index, + the creation date and time, and the `control_number`. + + Args: + hit(dict): item in the search results that is subject to sorting + target_index(set): search index + is_asc(boolean): Whether to sort in ascending or descending order. + custom_sort(dict): cache that retains the custom sort settings for each index + + """ + from weko_index_tree.api import Indexes + paths = {int(p) for p in hit["_source"]["path"]} + paths = paths.intersection(target_index) + path = min(paths) if is_asc else max(paths) + if path not in custom_sort: + index_custom_sort = Indexes.get_item_sort(path) + if index_custom_sort: + custom_sort[path] = index_custom_sort + + cn = hit["_source"]["control_number"] + v = custom_sort.get(path, {}).get(cn) + created = hit["_source"].get("_created") + + priority = 0 if v is not None else 1 + v = int(v) if v is not None else None + return (path, priority, v, created, int(cn)) @need_record_permission('create_permission_factory') def post(self, **kwargs): @@ -1175,4 +1318,4 @@ def get(self, **kwargs): for field, val, opts in completions: result[field] = response[field] - return make_response(jsonify(result)) \ No newline at end of file + return make_response(jsonify(result)) diff --git a/modules/invenio-records-rest/tests/conftest.py b/modules/invenio-records-rest/tests/conftest.py index 4853bc2140..df3420c8a8 100644 --- a/modules/invenio-records-rest/tests/conftest.py +++ b/modules/invenio-records-rest/tests/conftest.py @@ -16,6 +16,8 @@ import shutil import sys import tempfile +import random +import copy from os.path import dirname, join import pytest @@ -126,7 +128,7 @@ def flush(self,index="",wait_if_ongoing=""): pass def delete_alias(self, index="", name="",ignore=""): pass - + # def search(self,index="",doc_type="",body={},**kwargs): # pass class MockCluster(): @@ -414,3 +416,81 @@ def i18n_app(app): app.extensions['invenio-i18n'] = MagicMock() app.extensions['invenio-i18n'].language = "ja" yield app + +@pytest.fixture() +def prepare_search_result(): + search_result_dict, target = make_search_result() + + start_key = 2000001 + return_value = { + str(start_key + i): i + 1 + for i in range(100) + } + + return search_result_dict, target, return_value + +@pytest.fixture() +def prepare_search_result2(): + search_result_dict, target = make_search_result() + start_key = 2000001 + random.seed(0) + numbers = list(range(0, 100)) + random.shuffle(numbers) + + return_value = { + str(start_key + i): v + for i, v in zip(range(100), numbers) + } + + return search_result_dict, target, return_value + +def make_search_result(): + search_result_dict = { + "took": 1, + "timed_out": False, + "_shards": { + "total": 100, + "successful": 100, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 100, + "max_score": None, + "hits": [ + { + "_index": "tenant1-weko-item-v1.0.0", + "_type": "item-v1.0.0", + "_id": "5b2393bf-e43e-4785-a32d-d1acfe300461", + "_version": 4, + "_score": None, + "_source": { + "control_number": "2000001", + "item_title": "test", + "publish_date": "2026-03-30", + "publish_status": "0", + "_created": "2026-03-30T04:08:30.080849+00:00", + "_updated": "2026-03-30T04:09:21.867115+00:00", + "type": ["journal article"], + "path": ["1623632832836"] + }, + "sort": ["1623632832836"] + } + ] + } + } + + base_hit = search_result_dict["hits"]["hits"][0] + new_hits = [] + + start_key = 2000001 + for i in range(100): + new_hit = copy.deepcopy(base_hit) + new_hit["_source"]["control_number"] = str(start_key + i) + new_hits.append(new_hit) + + search_result_dict["hits"]["hits"] = new_hits + + target = {1623632832836} + + return search_result_dict, target diff --git a/modules/invenio-records-rest/tests/helpers.py b/modules/invenio-records-rest/tests/helpers.py index 4a3b6db498..3cce6fbeaa 100644 --- a/modules/invenio-records-rest/tests/helpers.py +++ b/modules/invenio-records-rest/tests/helpers.py @@ -12,6 +12,7 @@ import json import uuid +from os.path import join, dirname from flask import url_for from invenio_db import db from invenio_pidstore import current_pidstore @@ -82,3 +83,8 @@ def record_url(pid): def _mock_validate_fail(self): """Simulate a validation fail.""" raise ValidationError("") + + +def json_data(filename): + with open(join(dirname(__file__),filename), "r") as f: + return json.load(f) \ No newline at end of file diff --git a/modules/invenio-records-rest/tests/test_serializer_base.py b/modules/invenio-records-rest/tests/test_serializer_base.py index b386779acc..de08d8ab3a 100644 --- a/modules/invenio-records-rest/tests/test_serializer_base.py +++ b/modules/invenio-records-rest/tests/test_serializer_base.py @@ -11,12 +11,14 @@ from __future__ import absolute_import, print_function from datetime import datetime +from mock import patch, MagicMock -from tests.helpers import create_record +from tests.helpers import create_record, json_data from invenio_pidstore.models import PersistentIdentifier from invenio_records import Record from invenio_records_rest.serializers.base import PreprocessorMixin +from weko_records.api import ItemType keys = ['pid', 'metadata', 'links', 'revision', 'created', 'updated'] @@ -41,11 +43,18 @@ def test_preprocessor_mixin_record(app, db,item_type_mapping): assert data['created'] is None assert data['updated'] is None assert data['metadata']['aref'] == 'test2' - + pid, record = create_record({'title': 'test3', 'aref':{'$ref':'#/title'},'item_type_id':1}) record.model.created = datetime(2015, 10, 1, 11, 11, 11, 1) db.session.commit() - data = PreprocessorMixin().preprocess_record(pid, record) + ret = {'pubdate': {'title': 'PubDate', 'option': {'crtf': False, 'hidden': False, 'multiple': False, 'required': True, 'showlist': False}, 'input_type': 'datetime', 'title_i18n': {'en': 'PubDate', 'ja': '公開日'}, 'input_value': ''}, 'item_1617186331708': {'title': 'Title', 'option': {'crtf': True, 'hidden': False, 'oneline': False, 'multiple': True, 'required': True, 'showlist': True}, 'input_type': 'cus_67', 'title_i18n': {'en': 'Title', 'ja': 'タイトル'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186385884': {'title': 'Alternative Title', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_69', 'title_i18n': {'en': 'Alternative Title', 'ja': 'その他のタイトル'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186419668': {'title': 'Creator', 'option': {'crtf': True, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_60', 'title_i18n': {'en': 'Creator', 'ja': '作成者'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186476635': {'title': 'Access Rights', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': False}, 'input_type': 'cus_4', 'title_i18n': {'en': 'Access Rights', 'ja': 'アクセス権'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186499011': {'title': 'Rights', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_14', 'title_i18n': {'en': 'Rights', 'ja': '権利情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186609386': {'title': 'Subject', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_6', 'title_i18n': {'en': 'Subject', 'ja': '主題'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186626617': {'title': 'Description', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_17', 'title_i18n': {'en': 'Description', 'ja': '内容記述'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186643794': {'title': 'Publisher', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_5', 'title_i18n': {'en': 'Publisher', 'ja': '出版者'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186660861': {'title': 'Date', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_11', 'title_i18n': {'en': 'Date', 'ja': '日付'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186702042': {'title': 'Language', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_71', 'title_i18n': {'en': 'Language', 'ja': '言語'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186783814': {'title': 'Identifier', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_176', 'title_i18n': {'en': 'Identifier', 'ja': '識別子'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186819068': {'title': 'Identifier Registration', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': False}, 'input_type': 'cus_16', 'title_i18n': {'en': 'Identifier Registration', 'ja': 'ID登録'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186859717': {'title': 'Temporal', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_18', 'title_i18n': {'en': 'Temporal', 'ja': '時間的範囲'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186882738': {'title': 'Geo Location', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_19', 'title_i18n': {'en': 'Geo Location', 'ja': '位置情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186901218': {'title': 'Funding Reference', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_21', 'title_i18n': {'en': 'Funding Reference', 'ja': '助成情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186920753': {'title': 'Source Identifier', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_10', 'title_i18n': {'en': 'Source Identifier', 'ja': '収録物識別子'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186941041': {'title': 'Source Title', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_13', 'title_i18n': {'en': 'Source Title', 'ja': '収録物名'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186959569': {'title': 'Volume Number', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_88', 'title_i18n': {'en': 'Volume Number', 'ja': '巻'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186981471': {'title': 'Issue Number', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_87', 'title_i18n': {'en': 'Issue Number', 'ja': '号'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617186994930': {'title': 'Number of Pages', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_85', 'title_i18n': {'en': 'Number of Pages', 'ja': 'ページ数'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187024783': {'title': 'Page Start', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_84', 'title_i18n': {'en': 'Page Start', 'ja': '開始ページ'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187045071': {'title': 'Page End', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_83', 'title_i18n': {'en': 'Page End', 'ja': '終了ページ'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187056579': {'title': 'Bibliographic Information', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_102', 'title_i18n': {'en': 'Bibliographic Information', 'ja': '書誌情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187087799': {'title': 'Dissertation Number', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_82', 'title_i18n': {'en': 'Dissertation Number', 'ja': '学位授与番号'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187112279': {'title': 'Degree Name', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_80', 'title_i18n': {'en': 'Degree Name', 'ja': '学位名'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187136212': {'title': 'Date Granted', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': True}, 'input_type': 'cus_79', 'title_i18n': {'en': 'Date Granted', 'ja': '学位授与年月日'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617187187528': {'title': 'Conference', 'option': {'crtf': True, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_75', 'title_i18n': {'en': 'Conference', 'ja': '会議記述'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617258105262': {'title': 'Resource Type', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': True, 'showlist': False}, 'input_type': 'cus_8', 'title_i18n': {'en': 'Resource Type', 'ja': '資源タイプ'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617265215918': {'title': 'Version Type', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': False}, 'input_type': 'cus_9', 'title_i18n': {'en': 'Version Type', 'ja': '出版タイプ'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617349709064': {'title': 'Contributor', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_62', 'title_i18n': {'en': 'Contributor', 'ja': '寄与者'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617349808926': {'title': 'Version', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': False}, 'input_type': 'cus_28', 'title_i18n': {'en': 'Version', 'ja': 'バージョン情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617351524846': {'title': 'APC', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': False, 'required': False, 'showlist': False}, 'input_type': 'cus_27', 'title_i18n': {'en': 'APC', 'ja': 'APC'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617353299429': {'title': 'Relation', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_12', 'title_i18n': {'en': 'Relation', 'ja': '関連情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617605131499': {'title': 'File', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_65', 'title_i18n': {'en': 'File', 'ja': 'ファイル情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617610673286': {'title': 'Rights Holder', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_3', 'title_i18n': {'en': 'Rights Holder', 'ja': '権利者情報'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617620223087': {'title': 'Heading', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': False}, 'input_type': 'cus_119', 'title_i18n': {'en': 'Heading', 'ja': '見出し'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1617944105607': {'title': 'Degree Grantor', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_78', 'title_i18n': {'en': 'Degree Grantor', 'ja': '学位授与機関'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}, 'item_1662046377046': {'title': 'サムネイル', 'option': {'crtf': False, 'hidden': False, 'oneline': False, 'multiple': True, 'required': False, 'showlist': True}, 'input_type': 'cus_1037', 'title_i18n': {'en': 'thumbnail', 'ja': 'サムネイル'}, 'input_value': '', 'input_maxItems': '9999', 'input_minItems': '1'}} + item_type_mapping = json_data("data/item_type_with_hide/mapping.json") + item_type = MagicMock() + item_type.render.get.return_value = json_data("data/item_type_with_hide/render.json").get('table_row') + with patch("weko_items_ui.utils.get_options_and_order_list", return_value=(ret, item_type_mapping)), \ + patch("weko_items_ui.utils.get_hide_list_by_schema_form", return_value=["description.@value"]), \ + patch("weko_records.api.ItemTypes.get_by_id", return_value=item_type): + data = PreprocessorMixin().preprocess_record(pid, record) assert data['metadata']['title'] == 'test3' assert data['created'] == '2015-10-01T11:11:11.000001+00:00' assert data['revision'] == 1 diff --git a/modules/invenio-records-rest/tests/test_views_list_post.py b/modules/invenio-records-rest/tests/test_views_list_post.py index 46512cef46..32d06a4f97 100644 --- a/modules/invenio-records-rest/tests/test_views_list_post.py +++ b/modules/invenio-records-rest/tests/test_views_list_post.py @@ -21,6 +21,8 @@ from sqlalchemy.exc import SQLAlchemyError from weko_redis.redis import RedisConnection from elasticsearch_dsl import response, Search +from invenio_search import RecordsSearch +from invenio_records_rest.views import RecordsListResource @pytest.mark.parametrize('content_type', [ 'application/json', 'application/json;charset=utf-8' @@ -162,8 +164,15 @@ def test_jsonschema_validation_error(app, db, search_url, content_type): data = get_json(res) assert data['message'] +#.tox/c1/bin/pytest --cov=invenio_records_rest tests/test_views_list_post.py::test_RecordsListResource_get -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/invenio-records-rest/.tox/c1/tmp def test_RecordsListResource_get(app, i18n_app, db, es, test_data, search_url, search_class): """Test VALID record creation request (POST .../records/).""" + + app.config.update( + CACHE_REDIS_HOST='redis', + REDIS_PORT='6379' + ) + json_data = { "hits": { "total": 1, @@ -186,38 +195,167 @@ def test_RecordsListResource_get(app, i18n_app, db, es, test_data, search_url, s mock_user = MagicMock() mock_user.is_authenticated = False - with app.test_client() as client: - with patch('weko_admin.utils.get_facet_search_query', return_value=MagicMock()): - with patch('weko_search_ui.permissions.search_permission.can', return_value=MagicMock()): - with patch("flask_login.utils._get_user", return_value=mock_user): - with patch("elasticsearch_dsl.Search.execute", return_value=search_result): - redis_connection = RedisConnection() - sessionstore = redis_connection.connection(db=app.config['ACCOUNTS_SESSION_REDIS_DB_NO'], kv = True) - cache_key = f"{cache_name}_url_args" - cache_data = {'page': '1'} - sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - res = client.get(search_url, query_string=dict(page=1, size=2)) - assert res.status_code == 200 + with app.test_client() as client, \ + patch('weko_admin.utils.get_facet_search_query', return_value=MagicMock()), \ + patch('weko_search_ui.permissions.search_permission.can', return_value=MagicMock()), \ + patch('flask_login.utils._get_user', return_value=mock_user), \ + patch("elasticsearch_dsl.Search.execute", return_value=search_result) as mock_execute: + redis_connection = RedisConnection() + sessionstore = redis_connection.connection(db=app.config['ACCOUNTS_SESSION_REDIS_DB_NO'], kv = True) + cache_key = f"{cache_name}_url_args" + cache_data = {'page': '1', 'q':''} + sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - cache_key = f"{cache_name}_url_args" - sessionstore.delete(cache_key) - res = client.get(search_url, query_string=dict(page=1, size=2)) - assert res.status_code == 200 + res = client.get(search_url, query_string=dict(page=1, size=2, q="")) + assert res.status_code == 200 - cache_key = f"{cache_name}_url_args" - cache_data = {'page': '1'} - sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - cache_key = cache_name - cache_data = {"10000": {"control_number": 1}} - sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - res = client.get(search_url, query_string=dict(page=10000)) - assert res.status_code == 200 + cache_key = f"{cache_name}_url_args" + sessionstore.delete(cache_key) + res = client.get(search_url, query_string=dict(page=1, size=2, q="")) + assert res.status_code == 200 - cache_key = f"{cache_name}_url_args" - cache_data = {'page': '1'} - sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - cache_key = cache_name - cache_data = {"1": {"control_number": [1]}, "2": {"control_number": 1}} - sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) - res = client.get(search_url, query_string=dict(page=10000)) + cache_key = f"{cache_name}_url_args" + cache_data = {'page': '1', 'q':''} + sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) + cache_key = cache_name + cache_data = {"10000": {"control_number": 1}} + sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) + res = client.get(search_url, query_string=dict(page=10000,q="")) + assert res.status_code == 200 + + cache_key = f"{cache_name}_url_args" + cache_data = {'page': '1', 'q':''} + sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) + cache_key = cache_name + cache_data = {"1": {"control_number": [1]}, "2": {"control_number": 1}} + sessionstore.put(cache_key, (json.dumps(cache_data)).encode('utf-8')) + res = client.get(search_url, query_string=dict(page=10000, q="")) + assert res.status_code == 200 + with patch("weko_index_tree.api.Indexes.get_child_list_recursive",return_value = [1235]) as recursive, \ + patch("invenio_records_rest.views.RecordsListResource._do_custom_sort") as do: + res = client.get(search_url, query_string=dict(page=1, size=2, format="rss", recursive=1, idx=[1234,1235], sort="custom_sort")) + assert res.status_code == 200 + assert do.assert_called + assert recursive.assert_not_called + res = client.get(search_url, query_string=dict(page=1, size=2, format="rss", recursive=1, index_id=[1234,1235], sort="custom_sort")) + assert res.status_code == 200 + assert do.assert_not_called + with patch("weko_index_tree.api.Indexes.get_index",return_value=[1235]): + search_result.hits.total = 10001 + res = client.get(search_url, query_string=dict(page=1, size=2, format="rss", recursive=1, idx=[1234,1235], sort="custom_sort")) assert res.status_code == 200 + assert do.assert_not_called + assert recursive.assert_called + res = client.get(search_url, query_string=dict(page=1, size=2, recursive=1, idx=[1234,1235], sort="custom_sort")) + assert res.status_code == 302 + res = client.get(search_url, query_string=dict(page=1, size=2, format='', recursive=1, idx=[1234,1235], sort="custom_sort")) + assert res.status_code == 302 + res = client.get(search_url, query_string=dict(page=1, size=2, format="html", recursive=1, idx=[1234,1235], sort="custom_sort")) + assert res.status_code == 302 + + + +#.tox/c1/bin/pytest --cov=invenio_records_rest tests/test_views_list_post.py::test__override_params_for_customsort -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/invenio-records-rest/.tox/c1/tmp +@pytest.mark.parametrize( + "is_asc, expect", + [ + (True, [{'path': {'mode': 'min', 'order': 'asc'}}, {'_created': {'order': 'asc', 'unmapped_type': 'long'}}]), + (False, [{'path': {'mode': 'max', 'order': 'desc'}}, {'_created': {'order': 'desc', 'unmapped_type': 'long'}}]), + ] +) +def test__override_params_for_customsort(app, is_asc, expect): + test_class = RecordsSearch() + test_class2 = RecordsListResource.__new__(RecordsListResource) + test_class2.max_result_window = 10000 + result=test_class2._override_params_for_customsort(test_class,is_asc) + assert result._sort == expect + +#.tox/c1/bin/pytest --cov=invenio_records_rest tests/test_views_list_post.py::test__do_custom_sort -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/invenio-records-rest/.tox/c1/tmp +@pytest.mark.parametrize( + "is_asc, size, page, format, q, expect_len, expect_start, expect_end", + [ + (True, 20, 1, "rss", None, 20, "2000020", "2000001"), + (True, 20, 1, "rss", "test", 20, "2000001", "2000020"), + (True, 20, 1, "atom", None, 20, "2000020", "2000001"), + (True, 20, 1, "atom", "test", 20, "2000001", "2000020"), + (True, 20, 1, "jpcoar", None, 20, "2000020", "2000001"), + (True, 20, 1, "jpcoar", "test", 20, "2000001", "2000020"), + (True, 20, 1, "html", None, 20, "2000001", "2000020"), + (True, 20, 1, "html", "test", 20, "2000001", "2000020"), + (True, 30, 3, "rss", None, 30, "2000090", "2000061"), + (True, 30, 3, "rss", "test", 30, "2000061", "2000090"), + (True, 30, 3, "atom", None, 30, "2000090", "2000061"), + (True, 30, 3, "atom", "test", 30, "2000061", "2000090"), + (True, 30, 3, "jpcoar", None, 30, "2000090", "2000061"), + (True, 30, 3, "jpcoar", "test", 30, "2000061", "2000090"), + (True, 30, 3, "html", None, 30, "2000061", "2000090"), + (True, 30, 3, "html", "test", 30, "2000061", "2000090"), + (False, 20, 1, "rss", None, 20, "2000081", "2000100"), + (False, 20, 1, "rss", "test", 20, "2000100", "2000081"), + (False, 20, 1, "atom", None, 20, "2000081", "2000100"), + (False, 20, 1, "atom", "test", 20, "2000100", "2000081"), + (False, 20, 1, "jpcoar", None, 20, "2000081", "2000100"), + (False, 20, 1, "jpcoar", "test", 20, "2000100", "2000081"), + (False, 20, 1, "html", None, 20, "2000100", "2000081"), + (False, 20, 1, "html", "test", 20, "2000100", "2000081"), + (False, 30, 3, "rss", None, 30, "2000011", "2000040"), + (False, 30, 3, "rss", "test", 30, "2000040", "2000011"), + (False, 30, 3, "atom", None, 30, "2000011", "2000040"), + (False, 30, 3, "atom", "test", 30, "2000040", "2000011"), + (False, 30, 3, "jpcoar", None, 30, "2000011", "2000040"), + (False, 30, 3, "jpcoar", "test", 30, "2000040", "2000011"), + (False, 30, 3, "html", None, 30, "2000040", "2000011"), + (False, 30, 3, "html", "test", 30, "2000040", "2000011"), + (True, 100, 100, "rss", None, 0, "", ""), + (True, 20, 1, "json", None, 20, "2000001", "2000020") + ], +) +def test__do_custom_sort(app, is_asc, size, page, format, q, expect_len, + expect_start, expect_end, prepare_search_result): + search_result_dict, target, return_value = prepare_search_result + query_string = f"/?format={format}" + if q is not None: + query_string += f"&q={q}" + with app.test_request_context(query_string), \ + patch("weko_index_tree.api.Indexes.get_item_sort", + return_value=return_value): + RecordsListResource._do_custom_sort( + search_result_dict,target, is_asc, page, size) + if expect_len > 0: + assert len(search_result_dict["hits"]["hits"]) == expect_len + assert search_result_dict[ + "hits"]["hits"][0]["_source"]["control_number"] == expect_start + assert search_result_dict[ + "hits"]["hits"][size-1]["_source"]["control_number"] == expect_end + else: + assert search_result_dict["hits"]["hits"] == [] + +#.tox/c1/bin/pytest --cov=invenio_records_rest tests/test_views_list_post.py::test__customsort_priority -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/invenio-records-rest/.tox/c1/tmp +def test__customsort_priority(prepare_search_result2): + search_result_dict, target, return_value = prepare_search_result2 + custom_sort = {} + hit = next( + h for h in search_result_dict["hits"]["hits"] + if h["_source"]["control_number"] == "2000035" + ) + + hit2 = next( + h for h in search_result_dict["hits"]["hits"] + if h["_source"]["control_number"] == "2000070" + ) + with patch("weko_index_tree.api.Indexes.get_item_sort",return_value = return_value) as get_item_sort: + result = RecordsListResource._customsort_priority(hit,target,True,custom_sort) + expect = (1623632832836, 0, return_value.get("2000035"), '2026-03-30T04:08:30.080849+00:00', 2000035) + assert result == expect + + result = RecordsListResource._customsort_priority(hit2,target,True,custom_sort) + expect = (1623632832836, 0, return_value.get("2000070"), '2026-03-30T04:08:30.080849+00:00', 2000070) + assert result == expect + assert get_item_sort.assert_called_once + + custom_sort = {} + with patch("weko_index_tree.api.Indexes.get_item_sort",return_value = None): + result = RecordsListResource._customsort_priority(hit,target,True,custom_sort) + expect = (1623632832836, 1, None, '2026-03-30T04:08:30.080849+00:00', 2000035) + assert result == expect + diff --git a/modules/invenio-stats/invenio_stats/utils.py b/modules/invenio-stats/invenio_stats/utils.py index 611b9ceec1..3a95d4a6b3 100644 --- a/modules/invenio-stats/invenio_stats/utils.py +++ b/modules/invenio-stats/invenio_stats/utils.py @@ -15,6 +15,7 @@ import os import re import pickle +import traceback from base64 import b64encode from datetime import datetime, timedelta from math import ceil @@ -42,8 +43,6 @@ from weko_accounts.utils import get_remote_addr from werkzeug.utils import import_string -from weko_index_tree.models import Index - from . import config from .models import StatsAggregation, StatsBookmark, StatsEvents from .proxies import current_stats @@ -275,6 +274,8 @@ def calc_file_stats_reports(cls, res, data_list, all_groups): def calc_billing_file_stats_reports(cls, res, data_list, all_groups): """Create response object for billing_file_download.""" # Get all role names. + from weko_index_tree.models import Index + roles = Role.query.all() role_name_list = ['guest'] + [role.name for role in roles] @@ -412,21 +413,30 @@ def get_file_stats_report(cls, **kwargs): event = kwargs.get('event') year = kwargs.get('year') month = kwargs.get('month') + start_date = kwargs.get('start_date') + end_date = kwargs.get('end_date') try: - query_month = str(year) + '-' + str(month).zfill(2) - _, lastday = calendar.monthrange(year, month) - all_params = {'start_date': query_month + '-01', - 'end_date': - query_month + '-' + str(lastday).zfill(2) - + 'T23:59:59', - "should":[{"bool":{"must_not":{"term":{"is_open_access":True}}}}, - {"bool":{"must_not":{"exists":{"field":"is_open_access"}}}}]} - params = {'start_date': query_month + '-01', - 'end_date': - query_month + '-' + str(lastday).zfill(2) - + 'T23:59:59', - 'is_open_access': True} + if not start_date or not end_date: + query_month = str(year) + '-' + str(month).zfill(2) + _, lastday = calendar.monthrange(year, month) + start_date = query_month + '-01' + end_date = query_month + '-' + str(lastday).zfill(2) + else: + query_month = f"{start_date}-{end_date}" + all_params = { + 'start_date': start_date, + 'end_date': end_date + 'T23:59:59', + "should":[ + {"bool": {"must_not": {"term": {"is_open_access": True}}}}, + {"bool": {"must_not": {"exists": {"field": "is_open_access"}}}} + ] + } + params = { + 'start_date': start_date, + 'end_date': end_date + 'T23:59:59', + 'is_open_access': True + } all_query_name = '' open_access_query_name = '' @@ -456,6 +466,7 @@ def get_file_stats_report(cls, **kwargs): open_access_res = open_access.run(**params) cls.Calculation(open_access_res, open_access_list, event=event) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_month @@ -474,14 +485,22 @@ def get_file_per_using_report(cls, **kwargs): event = kwargs.get('event') year = kwargs.get('year') month = kwargs.get('month') + start_date = kwargs.get('start_date') + end_date = kwargs.get('end_date') try: - query_month = str(year) + '-' + str(month).zfill(2) - _, lastday = calendar.monthrange(year, month) - params = {'start_date': query_month + '-01', - 'end_date': query_month + '-' + str(lastday).zfill(2) - + 'T23:59:59'} + if not start_date or not end_date: + query_month = str(year) + '-' + str(month).zfill(2) + _, lastday = calendar.monthrange(year, month) + start_date = query_month + '-01' + end_date = query_month + '-' + str(lastday).zfill(2) + else: + query_month = f"{start_date}-{end_date}" + params = { + 'start_date': start_date, + 'end_date': end_date + 'T23:59:59' + } all_query_name = ['get-file-download-per-user-report', 'get-file-preview-per-user-report'] for query in all_query_name: @@ -492,6 +511,7 @@ def get_file_per_using_report(cls, **kwargs): cls.Calculation(all_res, all_list, event) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_month @@ -539,6 +559,8 @@ def get(cls, **kwargs): if not start_date or not end_date: start_date, end_date = get_start_end_date(year, month) result['date'] = str(year) + '-' + str(month).zfill(2) + else: + result['date'] = f"{start_date}-{end_date}" params = {'start_date': start_date, 'end_date': end_date + 'T23:59:59', 'agg_size': kwargs.get('agg_size', 0), @@ -556,13 +578,15 @@ def get(cls, **kwargs): current_report['search_key'] = report['search_key'] current_report['count'] = report['count'] all.append(current_report) - all = sorted(all, key=lambda x:x['count'], reverse=True) + all = sorted(all, key=lambda x:x['count'], reverse=True) result['all'] = all except es_exceptions.NotFoundError as e: + traceback.print_exc() current_app.logger.debug( "Indexes do not exist yet:" + str(e.info['error'])) result['all'] = [] except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['all'] = [] @@ -575,26 +599,21 @@ class QueryCommonReportsHelper(object): @classmethod def get_common_params(cls, **kwargs): """Get common params.""" - year = kwargs.get('year') - month = kwargs.get('month') start_date = kwargs.get('start_date') end_date = kwargs.get('end_date') if not start_date or not end_date: - if month > 0 and month <= 12: - query_date = str(year) + '-' + str(month).zfill(2) - _, lastday = calendar.monthrange(year, month) - params = {'start_date': query_date + '-01', - 'end_date': query_date + '-' - + str(lastday).zfill(2) + 'T23:59:59'} - else: - query_date = 'all' - params = {'interval': 'day'} + year = kwargs.get('year') + month = kwargs.get('month') + query_date = str(year) + '-' + str(month).zfill(2) + _, lastday = calendar.monthrange(year, month) + start_date = query_date + '-01' + end_date = query_date + '-' + str(lastday).zfill(2) else: query_date = start_date + '-' + end_date - params = {'start_date': start_date, - 'end_date': end_date + 'T23:59:59', - 'agg_size': kwargs.get('agg_size', 0), - 'agg_sort': kwargs.get('agg_sort', {'_term': 'desc'})} + params = { + 'start_date': start_date, + 'end_date': end_date + 'T23:59:59', + } return query_date, params @classmethod @@ -655,6 +674,7 @@ def Calculation(res, data_list): Calculation(all_res, all_list) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_month @@ -714,6 +734,7 @@ def Calculation(query_list, res, site_license_list, other_list, institution_name_list) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_month @@ -749,6 +770,7 @@ def Calculation(res, result): res = query.run(**params) Calculation(res, data_list) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_date @@ -762,12 +784,12 @@ def get_common_params(cls, **kwargs): """Get common params. Args: **kwargs (dict): start date,end_date. - + Returns: string: Aggregation date. dict: Param by Elaticsearch Aggregation. list: List Aggregation date by month. - + """ start_date = kwargs.get('start_date') end_date = kwargs.get('end_date') @@ -802,10 +824,12 @@ def get_site_license_report(cls, **kwargs): """Get site license download report. Args: **kwargs (dict): start date,end_date. - + Returns: dict: Dict calculation data. """ + from weko_index_tree.models import Index + def Calculation(query_list, all_res, datelist): """Calculation. @@ -1047,12 +1071,18 @@ def get(cls, **kwargs): result = {} year = kwargs.get('year') month = kwargs.get('month') + start_date = kwargs.get('start_date') + end_date = kwargs.get('end_date') try: - query_month = str(year) + '-' + str(month).zfill(2) - _, lastday = calendar.monthrange(year, month) - start_date = query_month + '-01' - end_date = query_month + '-' + str(lastday).zfill(2) + 'T23:59:59' + if not start_date or not end_date: + query_month = str(year) + '-' + str(month).zfill(2) + _, lastday = calendar.monthrange(year, month) + start_date = query_month + '-01' + end_date = query_month + '-' + str(lastday).zfill(2) + 'T23:59:59' + else: + query_month = f"{start_date}-{end_date}" + result = {'date': query_month, 'all': [], 'total': 0} first_search = True after_key = None @@ -1074,6 +1104,7 @@ def get(cls, **kwargs): count += cls.parse_bucket_response(aggs, result) except Exception as e: + traceback.print_exc() current_app.logger.debug(e) return {} @@ -1086,8 +1117,8 @@ class QueryRecordViewReportHelper(object): @classmethod def Calculation(cls, res, data_list): """Create response object.""" - for item in res['buckets']: - data = { + for item in res['buckets']: + data = { 'record_id': item['record_id'], 'record_name': item['record_name'], 'index_names': item['record_index_names'], @@ -1174,7 +1205,7 @@ def get(cls, **kwargs): _, lastday = calendar.monthrange(year, month) start_date = query_date + '-01' end_date = query_date + '-' + str(lastday).zfill(2) - query_date = start_date + '-' + end_date + query_date = start_date + '-' + end_date params = {'start_date': start_date, 'end_date': end_date + 'T23:59:59'} if not kwargs.get('ranking', False): @@ -1187,9 +1218,11 @@ def get(cls, **kwargs): cls.Calculation(all_res, all_list) except es_exceptions.NotFoundError as e: + traceback.print_exc() current_app.logger.debug(e) result['all'] = [] except Exception as e: + traceback.print_exc() current_app.logger.debug(e) result['date'] = query_date @@ -1564,8 +1597,8 @@ class QueryRankingHelper(object): @classmethod def Calculation(cls, res, data_list): """Create response object.""" - for item in res['aggregations']['my_buckets']['buckets']: - data = { + for item in res['aggregations']['my_buckets']['buckets']: + data = { 'key': item['key'], 'count': int(item['my_sum']['value']) } diff --git a/modules/invenio-stats/tests/test_utils.py b/modules/invenio-stats/tests/test_utils.py index bb3378154f..5d91620977 100644 --- a/modules/invenio-stats/tests/test_utils.py +++ b/modules/invenio-stats/tests/test_utils.py @@ -386,6 +386,15 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): res = QueryFileReportsHelper.get(year=2022, month=10, event='test') assert res==[] + res = QueryFileReportsHelper.get(start_date='2022-10-01', end_date='2022-10-31', event='file_download') + assert res=={'all': [], 'all_groups': [], 'date': '2022-10-01-2022-10-31', 'open_access': []} + res = QueryFileReportsHelper.get(start_date='2022-10-01', end_date='2022-10-31', event='billing_file_download') + assert res=={'all': [], 'all_groups': [], 'date': '2022-10-01-2022-10-31', 'open_access': []} + res = QueryFileReportsHelper.get(start_date='2022-10-01', end_date='2022-10-31', event='file_using_per_user') + assert res=={'all': {}, 'date': '2022-10-01-2022-10-31'} + res = QueryFileReportsHelper.get(start_date='2022-10-01', end_date='2022-10-31', event='test') + assert res==[] + def test_query_file_reports_helper_error(app): # get res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download') @@ -452,12 +461,12 @@ def test_query_search_report_helper(app, es): with patch('invenio_stats.queries.ESWekoTermsQuery.run', return_value=_raw_res1): res = QuerySearchReportHelper.get( year=2022, month=10, start_date='2022-10-01', end_date='2022-10-31') - assert res=={'all': []} + assert res=={'all': [], 'date': '2022-10-01-2022-10-31'} with patch('invenio_stats.queries.ESWekoTermsQuery.run', return_value=_raw_res2): res = QuerySearchReportHelper.get( - year=2022, month=10, start_date='2022-10-01', end_date='2022-10-31') - assert res=={'all': [{'search_key': 'key2', 'count': 7}, {'search_key': 'key1', 'count': 4}]} + year=2022, month=10) + assert res=={'all': [{'search_key': 'key2', 'count': 7}, {'search_key': 'key1', 'count': 4}], 'date': '2022-10'} def test_query_search_report_helper_error(app): res = QuerySearchReportHelper.get( @@ -494,17 +503,10 @@ def test_query_common_reports_helper(app, es): with patch('invenio_stats.queries.ESTermsQuery.run', return_value=_res): res = QueryCommonReportsHelper.get(event='top_page_access', year=2022, month=10, start_date='2022-10-01', end_date='2022-10-10') assert res=={'date': '2022-10-01-2022-10-10', 'all': {'localhost': {'host': 'name2', 'ip': 'localhost', 'count': 2}}} + with patch('invenio_stats.queries.ESTermsQuery.run', return_value=_res): + res = QueryCommonReportsHelper.get(event='top_page_access', year=2022, month=10) + assert res=={'date': '2022-10', 'all': {'localhost': {'host': 'name2', 'ip': 'localhost', 'count': 2}}} - _res = { - 'buckets': [ - { - 'value': 2 - } - ] - } - with patch('invenio_stats.queries.ESDateHistogramQuery.run', return_value=_res): - res = QueryCommonReportsHelper.get(event='top_page_access', year=2022, month=-1) - assert res=={'date': 'all', 'all': {'count': 2}} _res = { 'buckets': [ @@ -522,29 +524,6 @@ def test_query_common_reports_helper(app, es): res = QueryCommonReportsHelper.get(event='site_access', year=2022, month=10) assert res=={'date': '2022-10', 'site_license': [{'top_view': 2, 'search': 2, 'record_view': 2, 'file_download': 2, 'file_preview': 2}], 'other': [{'top_view': 1, 'search': 1, 'record_view': 1, 'file_download': 1, 'file_preview': 1}], 'institution_name': [{'name': 'name1', 'top_view': 2, 'search': 2, 'record_view': 2, 'file_download': 2, 'file_preview': 2}]} - _res = { - 'buckets': [ - { - 'key': 1640995200, - 'buckets': [ - { - 'key': 'key1.1', - }, - { - 'key': 'key1.2', - 'buckets': [ - { - 'key': 'key1.2.1' - } - ] - } - ] - } - ] - } - with patch('invenio_stats.queries.ESWekoTermsQuery.run', return_value=_res): - res = QueryCommonReportsHelper.get(event='item_create', year=2022, month=-1) - assert res=={'date': 'all', 'all': [{'create_date': 1640995.2, 'pid_value': 'key1.1', 'record_name': ''}, {'create_date': 1640995.2, 'pid_value': 'key1.2', 'record_name': 'key1.2.1'}]} res = QueryCommonReportsHelper.get(event='') assert res==[] @@ -684,6 +663,9 @@ def test_query_record_view_per_index_report_helper(app, es): res = QueryRecordViewPerIndexReportHelper.get(year=2022, month=10) assert res=={'all': [], 'date': '2022-10', 'total': 0} + res = QueryRecordViewPerIndexReportHelper.get(start_date='2022-10-01', end_date='2022-10-31') + assert res=={'all': [], 'date': '2022-10-01-2022-10-31', 'total': 0} + def test_query_record_view_per_index_report_helper_error(app): # get res = QueryRecordViewPerIndexReportHelper.get(year=2022, month=10) @@ -722,9 +704,9 @@ def test_query_record_view_report_helper(app, es, db, records): } _data_list = [] # Calculation - with pytest.raises(Exception) as e: - QueryRecordViewReportHelper.Calculation(_res, _data_list) - assert e.type==UnsupportedCompilationError + QueryRecordViewReportHelper.Calculation(_res, _data_list) + assert _data_list + # correct_record_title _res = [['2', ['name2old']]] @@ -761,13 +743,15 @@ def test_query_record_view_report_helper(app, es, db, records): # get res = QueryRecordViewReportHelper.get(year=2022, month=9) assert res=={'all': [], 'date': '2022-09-01-2022-09-30'} + res = QueryRecordViewReportHelper.get(start_date='2022-09-01', end_date='2022-09-30') + assert res=={'all': [], 'date': '2022-09-01-2022-09-30'} # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_query_record_view_report_helper_error -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp def test_query_record_view_report_helper_error(app, db): # get res = QueryRecordViewReportHelper.get(start_date='2022-09-01', end_date='2022-09-30', ranking=True) - assert res=={'all': [], 'date': ''} + assert res=={'all': [], 'date': '2022-09-01-2022-09-30'} res = QueryRecordViewReportHelper.get() assert res=={'all': [], 'date': 'None-None'} diff --git a/modules/weko-accounts/tests/test_api.py b/modules/weko-accounts/tests/test_api.py index fae82cc68e..bfe6c1b8f5 100644 --- a/modules/weko-accounts/tests/test_api.py +++ b/modules/weko-accounts/tests/test_api.py @@ -13,6 +13,7 @@ #class ShibUser(object): class TestShibUser: # def __init__(self, shib_attr=None): +# .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_init -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_init(self,db,users): user = users[0]["obj"] attr = { @@ -22,16 +23,28 @@ def test_init(self,db,users): assert shibuser.shib_attr == attr assert shibuser.user == None assert shibuser.shib_user == None - - + + attr = { + "shib_handle":"test_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletes" + } + shibuser = ShibUser(attr) + assert shibuser.shib_attr['shib_handle'] == "test_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handletest_handlete" + + attr = { + "shib_handle":"テストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハ" + } + shibuser = ShibUser(attr) + assert shibuser.shib_attr['shib_handle'] == "テストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテストハンドルテスト" + + # def _set_weko_user_role(self, roles): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_set_weko_user_role -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_set_weko_user_role(self,app,db,users): - + role_sysadmin = Role.query.filter_by(name='System Administrator').first() role_repoadmin = Role.query.filter_by(name='Repository Administrator').first() role_original = Role.query.filter_by(name='Original Role').first() - + user = users[6]["obj"] attr = { "shib_eppn":"test_eppn" @@ -40,16 +53,16 @@ def test_set_weko_user_role(self,app,db,users): db.session.add(s_user) s_user.shib_roles.append(role_original) db.session.commit() - + shibuser = ShibUser(attr) shibuser.shib_user = s_user shibuser.user=user - + roles = ['System Administrator','Repository Administrator'] result = shibuser._set_weko_user_role(roles) assert shibuser.user.roles == [role_repoadmin,role_sysadmin] assert shibuser.shib_user.shib_roles == [role_sysadmin] - + # raise Exception error = Exception("test_error") with patch("weko_accounts.api.db.session.begin_nested",side_effect=error): @@ -85,13 +98,13 @@ def test__create_unknown_roles(self, app, users, mocker): app.config.update(WEKO_ACCOUNTS_SHIB_ALLOW_CREATE_GROUP_ROLE=True) shibuser._create_unknown_roles(role_names) assert before_roles == Role.query.all() - + # Occurred exception role_names = ['System Administrator', 'new_role_1', 'new_role_2', ''] with patch("weko_accounts.api.db.session.commit",side_effect=Exception): shibuser._create_unknown_roles(role_names) assert before_roles == Role.query.all() - + # Success shibuser._create_unknown_roles(role_names) assert Role.query.filter_by(name='new_role_1').count() == 1 @@ -99,101 +112,225 @@ def test__create_unknown_roles(self, app, users, mocker): assert Role.query.filter_by(name='').count() == 0 # def get_relation_info(self): -# .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_get_relation_info -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp - def test_get_relation_info(self,app,db,users): - - user1 = users[0]["obj"] - user2 = users[1]["obj"] +# .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_get_relation_info -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-accounts/.tox/c1/tmp + def test_get_relation_info(self, app, db, users, mocker): + # 1. shib_eppnでヒット、weko_userあり、全属性更新 attr = { - "shib_eppn": "test_eppn", + "shib_eppn": "eppn1", + "shib_mail": "mail1@example.com", + "shib_user_name": "user_name1", + "shib_role_authority_name": "role1", + "shib_page_name": "page1", + "shib_active_flag": "TRUE", + "shib_ip_range_flag": "TRUE", + "shib_organization": "org1", + "shib_handle": "handle1" + } + user = users[0]["obj"] + s_user = ShibbolethUser(weko_uid=user.id, weko_user=user, **attr) + db.session.add(s_user) + db.session.commit() + shibuser = ShibUser(attr) + result = shibuser.get_relation_info() + assert result.shib_mail == "mail1@example.com" + assert result.shib_user_name == "user_name1" + assert result.shib_role_authority_name == "role1" + assert result.shib_page_name == "page1" + assert result.shib_active_flag == "TRUE" + assert result.shib_ip_range_flag == "TRUE" + assert result.shib_organization == "org1" + assert result.shib_handle == "handle1" + assert result.weko_user.email == "mail1@example.com" + + # 2. shib_eppnでヒット、weko_userなし + attr2 = { + "shib_eppn": "eppn2", + "shib_mail": "mail2@example.com", + "shib_user_name": "user_name2" + } + s_user2 = ShibbolethUser(weko_uid=None, weko_user=None, **attr2) + db.session.add(s_user2) + db.session.commit() + shibuser2 = ShibUser(attr2) + result2 = shibuser2.get_relation_info() + assert result2 is None + + # 3. shib_eppnでヒットせず、shib_user_nameでヒット（設定ON） + app.config.update(WEKO_ACCOUNTS_SHIB_ALLOW_USERNAME_INST_EPPN=True) + attr3 = { + "shib_eppn": "", + "shib_user_name": "user_name3", + "shib_mail": "mail3@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None + } + user3 = users[1]["obj"] + s_user3 = ShibbolethUser(weko_uid=user3.id, weko_user=user3, shib_eppn="dummy_eppn3", shib_user_name="user_name3") + db.session.add(s_user3) + db.session.commit() + shibuser3 = ShibUser(attr3) + result3 = shibuser3.get_relation_info() + assert result3.shib_mail == "mail3@example.com" + assert result3.shib_user_name == "user_name3" + assert result3.weko_user.email == "mail3@example.com" + + # 4. shib_eppnでヒットせず、shib_user_nameでヒット（設定OFF） + app.config.update(WEKO_ACCOUNTS_SHIB_ALLOW_USERNAME_INST_EPPN=False) + attr4 = { + "shib_eppn": "", + "shib_user_name": "user_name3", + "shib_mail": "mail4@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None + } + shibuser4 = ShibUser(attr4) + result4 = shibuser4.get_relation_info() + assert result4 is None + + # 5. shib_eppnもshib_user_nameも一致しない場合 + attr5 = { + "shib_eppn": "not_exist_eppn3", + "shib_user_name": "not_exist_name3", + "shib_mail": "mail4@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None + } + shibuser5 = ShibUser(attr5) + result5 = shibuser5.get_relation_info() + assert result5 is None + + # 6. shib_eppnでヒット、shib_mailのみ更新 + attr6 = { + "shib_eppn": "eppn4", + "shib_user_name": "user_name6", + "shib_mail": "mail6@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None + } + user6 = users[2]["obj"] + s_user6 = ShibbolethUser(weko_uid=user6.id, weko_user=user6, shib_eppn="eppn4", shib_user_name="user_name6") + db.session.add(s_user6) + db.session.commit() + shibuser6 = ShibUser(attr6) + result6 = shibuser6.get_relation_info() + assert result6.shib_mail == "mail6@example.com" + assert result6.weko_user.email == "mail6@example.com" + + # 7. shib_eppnでヒット、shib_user_nameのみ更新 + attr7 = { + "shib_eppn": "eppn5", + "shib_user_name": "user_name7", "shib_mail": None, - "shib_user_name": None, "shib_role_authority_name": None, "shib_page_name": None, "shib_active_flag": None, "shib_ip_range_flag": None, - "shib_organization": None + "shib_organization": None, + "shib_handle": None } - s_user1 = ShibbolethUser(weko_uid=user1.id,weko_user=user1,**attr) - db.session.add(s_user1) + user7 = users[3]["obj"] + s_user7 = ShibbolethUser(weko_uid=user7.id, weko_user=user7, shib_eppn="eppn5", shib_user_name="old_name7") + db.session.add(s_user7) db.session.commit() + shibuser7 = ShibUser(attr7) + result7 = shibuser7.get_relation_info() + assert result7.shib_user_name == "user_name7" - # exist shib_eppn,exist shib_user.weko_user,not exist self.user - # attribute does not exist - shibuser = ShibUser(attr) - result = shibuser.get_relation_info() - assert result.shib_mail == None - assert result.shib_user_name == None - assert result.shib_role_authority_name == None - assert result.shib_page_name == None - assert result.shib_active_flag == None - assert result.shib_ip_range_flag == None - assert result.shib_organization == None - - # attribute exists - attr = { - "shib_eppn":"test_eppn", - "shib_mail":"shib.user@test.org", - "shib_user_name":"shib name1", - "shib_role_authority_name":"shib auth", - "shib_page_name":"shib page", - "shib_active_flag":"TRUE", - "shib_ip_range_flag":"TRUE", - "shib_organization":"shib org" + # 8. shib_user_nameがNoneの場合のテスト + attr_none_name = { + "shib_eppn": "eppn_none_name", + "shib_mail": "mail_none_name@example.com", + "shib_user_name": None, + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None } - shibuser = ShibUser(attr) - result = shibuser.get_relation_info() - assert result.shib_mail == "shib.user@test.org" - assert result.shib_user_name == "shib name1" - assert result.shib_role_authority_name == "shib auth" - assert result.shib_page_name == "shib page" - assert result.shib_active_flag == "TRUE" - assert result.shib_ip_range_flag == "TRUE" - assert result.shib_organization == "shib org" + user_none_name = users[6]["obj"] + s_user_none_name = ShibbolethUser(weko_uid=user_none_name.id, weko_user=user_none_name, **attr_none_name) + db.session.add(s_user_none_name) + db.session.commit() + shibuser_none_name = ShibUser(attr_none_name) + result_none_name = shibuser_none_name.get_relation_info() + assert result_none_name is not None + assert result_none_name.shib_user_name is None - # not exist shib_eppn,not exist shib_user.weko_user - attr = { - "shib_eppn":"", - "shib_user_name":"shib name2", - "shib_mail":None, - "shib_role_authority_name":None, - "shib_page_name":None, - "shib_active_flag":None, - "shib_ip_range_flag":None, - "shib_organization":None + # 9. 事前にself.userがセットされている場合のテスト + attr9 = { + "shib_eppn": "eppn9", + "shib_user_name": "user_name9", + "shib_mail": "mail9@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None } - s_user2 = ShibbolethUser(**attr) - db.session.add(s_user2) + user9 = users[5]["obj"] + s_user9 = ShibbolethUser(weko_uid=user9.id, weko_user=user9, shib_eppn="eppn9", shib_user_name="user_name9") + db.session.add(s_user9) db.session.commit() - shibuser = ShibUser(attr) - result = shibuser.get_relation_info() - assert result == None - - # not exist shib_eppn, exist shib_user.weko_user,exist self.user, raise SQLAlchemyError - s_user2.weko_user = user2 - s_user2.weko_uid = user2.id - db.session.merge(s_user2) + shibuser9 = ShibUser(attr9) + shibuser9.user = user9 # ここがポイント: 事前にuserをセット + result9 = shibuser9.get_relation_info() + assert result9 == s_user9 + assert shibuser9.user == user9 + + # 10. DB更新時に例外発生（ロールバックされること） + attr8 = { + "shib_eppn": "eppn8", + "shib_user_name": "user_name8", + "shib_mail": "mail8@example.com", + "shib_role_authority_name": None, + "shib_page_name": None, + "shib_active_flag": None, + "shib_ip_range_flag": None, + "shib_organization": None, + "shib_handle": None + } + user8 = users[4]["obj"] + s_user8 = ShibbolethUser(weko_uid=user8.id, weko_user=user8, shib_eppn="eppn8", shib_user_name="user_name8") + db.session.add(s_user8) db.session.commit() - shibuser.user = user2 - with patch("weko_accounts.api.db.session.commit",side_effect=SQLAlchemyError): - with pytest.raises(SQLAlchemyError): - shibuser.get_relation_info() + shibuser8 = ShibUser(attr8) + mocker.patch("weko_accounts.api.db.session.commit", side_effect=SQLAlchemyError) + with pytest.raises(SQLAlchemyError): + shibuser8.get_relation_info() # def check_weko_user(self, account, pwd): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_check_weko_user -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_check_weko_user(self,app,users): user = users[0]["obj"] password = user.password_plaintext - + # exist wkeo_user, correct password shibuser = ShibUser({}) result = shibuser.check_weko_user(user.email,password) assert result == True - + # not exist weko_user result = shibuser.check_weko_user("not.exist.user@test.org",password) assert result == False - + # exist weko_user, not correct password result = shibuser.check_weko_user(user.email,"wrong passwd") assert result == False @@ -212,7 +349,7 @@ def test_bind_relation_info(self,app,users): assert users[0]["obj"].email == "new.sysadmin_mail@test.org" assert shibuser.shib_attr["shib_eppn"] == "shib name" assert result == ShibbolethUser.query.filter_by(shib_eppn="shib name").one_or_none() - + # exist shib_eppn, raise Exception user = users[1]["email"] attr = { @@ -230,7 +367,7 @@ def test_new_relation_info(self,users,mocker): today = datetime(2022,10,6,1,2,3,4) datetime_mock.utcnow.return_value=today mocker.patch("weko_accounts.api.ShibUser.new_shib_profile") - + # exist user user = users[0]["obj"] attr = { @@ -241,7 +378,7 @@ def test_new_relation_info(self,users,mocker): result = shibuser.new_relation_info() assert result.shib_eppn == "test_eppn1" assert result.weko_uid == user.id - + # not exist user attr = { "shib_mail":"newuser@test.org", @@ -264,11 +401,11 @@ def test_new_shib_profile(self,db,users): shibuser = ShibUser(attr) shibuser.shib_user = s_user shibuser.user=user - + result = shibuser.new_shib_profile() profile = UserProfile.query.filter_by(user_id=user.id).one_or_none() assert result==profile - + # def shib_user_login(self): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_shib_user_login -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_shib_user_login(self,request_context,users,mocker): @@ -283,13 +420,13 @@ def test_shib_user_login(self,request_context,users,mocker): # def assign_user_role(self): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::TestShibUser::test_assign_user_role -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_assign_user_role(self,app, users,mocker): - + # not exist self.user shibuser = ShibUser({}) flg, ret = shibuser.assign_user_role() assert flg == False assert ret == "Can't get relation Weko User." - + # exist self.user, issubset, ret is None attr = { "shib_role_authority_name":"管理者;図書館員" @@ -325,7 +462,7 @@ def test_assign_user_role(self,app, users,mocker): mock_set_role.assert_called_with(['System Administrator','Repository Administrator','IPSJ:学会員','AL:会員','SLDM:会員','']) assert flg == False assert ret == error - + # not issubset attr = { "shib_role_authority_name":"異常役員" @@ -385,4 +522,4 @@ def test_shib_user_logout(self,request_context,users,mocker): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_api.py::test_get_user_info_by_role_name -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp def test_get_user_info_by_role_name(users): result = get_user_info_by_role_name('Repository Administrator') - assert result == [users[1]["obj"],users[6]["obj"]] \ No newline at end of file + assert result == [users[1]["obj"],users[6]["obj"]] diff --git a/modules/weko-accounts/tests/test_views.py b/modules/weko-accounts/tests/test_views.py index a7590955ec..2c408ae0a4 100644 --- a/modules/weko-accounts/tests/test_views.py +++ b/modules/weko-accounts/tests/test_views.py @@ -45,22 +45,22 @@ def test_redirect_method(app,mocker): mock_render = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) _redirect_method(False) mock_render.assert_called_with(url_for('security.login')) - + mock_render = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) _redirect_method(True) mock_render.assert_called_with(url_for('security.login',next=url)) - + current_app.config.update( WEKO_ACCOUNTS_SHIB_LOGIN_ENABLED=True ) mock_render = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) _redirect_method(False) mock_render.assert_called_with("http://TEST_SERVER.localdomain/secure/login.php") - + mock_render = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) _redirect_method(True) mock_render.assert_called_with("http://TEST_SERVER.localdomain/secure/login.php?next="+url) - + #def index(): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_views.py::test_index -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workflow/.tox/c1/tmp def test_index(client,mocker): @@ -77,15 +77,15 @@ def test_shib_auto_login(client,redis_connect,mocker): mock_redirect_ = mocker.patch("weko_accounts.views._redirect_method",return_value=make_response()) client.get(url) mock_redirect_.assert_called_once() - - + + mocker.patch("weko_accounts.views.RedisConnection.connection",return_value=redis_connect) # not exist cache mock_redirect_ = mocker.patch("weko_accounts.views._redirect_method",return_value=make_response()) client.get(url+"?SHIB_ATTR_SESSION_ID=2222") mock_redirect_.assert_called_once() - - + + redis_connect.put("Shib-Session-1111",bytes("","utf-8")) # not cache_val @@ -93,11 +93,11 @@ def test_shib_auto_login(client,redis_connect,mocker): client.get(url+"?SHIB_ATTR_SESSION_ID=1111") mock_redirect_.assert_called_once() assert redis_connect.redis.exists("Shib-Session-1111") == False - + mock_get_relation = mocker.patch("weko_accounts.views.ShibUser.get_relation_info") mock_new_relation = mocker.patch("weko_accounts.views.ShibUser.new_relation_info") mock_shib_login = mocker.patch("weko_accounts.views.ShibUser.shib_user_login") - + redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn"}',"utf-8")) # is_auto_bind is false, check_in is error mock_redirect_ = mocker.patch("weko_accounts.views._redirect_method",return_value=make_response()) @@ -106,19 +106,19 @@ def test_shib_auto_login(client,redis_connect,mocker): mock_get_relation.assert_called_once() mock_redirect_.assert_called_once() assert redis_connect.redis.exists("Shib-Session-1111") == False - + redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn"}',"utf-8")) - + set_session(client,{"shib_session_id":"1111"}) with patch("weko_accounts.views.ShibUser.check_in",return_value=None): # is_auto_bind is true,shib_user is None - mock_redirect = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) + mock_redirect = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) client.get(url) mock_new_relation.assert_called_once() mock_shib_login.assert_not_called() mock_redirect.assert_called_with("/") assert redis_connect.redis.exists("Shib-Session-1111") == False - + # is_auto_bind is true,shib_user exis redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn"}',"utf-8")) set_session(client,{"shib_session_id":"1111","next":"/next_page"}) @@ -141,26 +141,26 @@ def test_confirm_user(client,redis_connect,mocker): mocker.patch("weko_accounts.views.RedisConnection.connection",return_value=redis_connect) mocker.patch("weko_accounts.views.ShibUser.shib_user_login") url = url_for("weko_accounts.confirm_user") - + # not correct csrf_random set_session(client,{"csrf_random":"xxxx"}) form = {"csrf_random":"test_csrf"} mock_flash = mocker.patch("weko_accounts.views.flash") client.post(url,data=form) mock_flash.assert_called_with("csrf_random",category="error") - + # not exist shib_session_id set_session(client,{"csrf_random":"test_csrf","shib_session_id":None}) mock_flash = mocker.patch("weko_accounts.views.flash") client.post(url,data=form) mock_flash.assert_called_with("shib_session_id",category="error") - + # not exist cache_key set_session(client,{"csrf_random":"test_csrf","shib_session_id":"2222"}) mock_flash = mocker.patch("weko_accounts.views.flash") client.post(url,data=form) mock_flash.assert_called_with("cache_key",category="error") - + set_session(client,{"csrf_random":"test_csrf","shib_session_id":"1111"}) # not exist cache_value redis_connect.put("Shib-Session-1111",bytes("","utf-8")) @@ -168,7 +168,7 @@ def test_confirm_user(client,redis_connect,mocker): client.post(url,data=form) mock_flash.assert_called_with("cache_val",category="error") assert redis_connect.redis.exists("Shib-Session-1111") is False - + # shib_user.check_weko_user is false redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn"}',"utf-8")) with patch("weko_accounts.views.ShibUser.check_weko_user",return_value=False): @@ -199,7 +199,7 @@ def test_confirm_user(client,redis_connect,mocker): client.post(url,data=form) mock_redirect.assert_called_with("/") assert redis_connect.redis.exists("Shib-Session-1111") is False - + # exist ShibUser.shib_user set_session(client,{"csrf_random":"test_csrf","shib_session_id":"1111","next":"/next_page"}) redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn"}',"utf-8")) @@ -212,7 +212,7 @@ def test_confirm_user(client,redis_connect,mocker): client.post(url,data=form) mock_redirect.assert_called_with("/next_page") assert redis_connect.redis.exists("Shib-Session-1111") is False - + # raise BaseException with patch("weko_accounts.views._redirect_method",side_effect=BaseException("test_error")): res = client.post(url,data=form) @@ -224,19 +224,19 @@ def test_shib_login(client,redis_connect,users,mocker): mocker.patch("weko_accounts.views.RedisConnection.connection",return_value=redis_connect) mocker.patch("weko_accounts.views.generate_random_str",return_value="asdfghjkl") url_base = url_for("weko_accounts.shib_login") - + # not shib_session_id mock_flash = mocker.patch("weko_accounts.views.flash") client.get(url_base) mock_flash.assert_called_with("Missing SHIB_ATTR_SESSION_ID!",category="error") - + url = url_base+"?SHIB_ATTR_SESSION_ID=2222" - + # not exist cache_key mock_flash = mocker.patch("weko_accounts.views.flash") client.get(url) mock_flash.assert_called_with("Missing SHIB_CACHE_PREFIX!",category="error") - + url = url_base+"?SHIB_ATTR_SESSION_ID=1111" # not cache_val redis_connect.put("Shib-Session-1111",bytes('',"utf-8")) @@ -244,19 +244,19 @@ def test_shib_login(client,redis_connect,users,mocker): client.get(url) mock_flash.assert_called_with("Missing SHIB_ATTR!",category="error") assert redis_connect.redis.exists("Shib-Session-1111") is False - + # exist user redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn","shib_mail":"user@test.org"}',"utf-8")) mock_render = mocker.patch("weko_accounts.views.render_template",return_value=make_response()) client.get(url) mock_render.assert_called_with('weko_accounts/confirm_user.html',csrf_random="asdfghjkl",email="user@test.org") - + # not exist user redis_connect.put("Shib-Session-1111",bytes('{"shib_eppn":"test_eppn","shib_mail":"not_exist_user@test.org"}',"utf-8")) mock_render = mocker.patch("weko_accounts.views.render_template",return_value=make_response()) client.get(url) mock_render.assert_called_with('weko_accounts/confirm_user.html',csrf_random="asdfghjkl",email="") - + # raise BaseException with patch("weko_accounts.views.flash",side_effect=BaseException("test_error")): res = client.get(url_base) @@ -266,21 +266,21 @@ def test_shib_login(client,redis_connect,users,mocker): def test_shib_sp_login(client, redis_connect,mocker): mocker.patch("weko_accounts.views.RedisConnection.connection",return_value=redis_connect) url = url_for("weko_accounts.shib_sp_login") - + # not shib_session_id mock_flash = mocker.patch("weko_accounts.views.flash") mock_redirect = mocker.patch("weko_accounts.views.redirect",return_value=make_response()) client.post(url,data={}) mock_flash.assert_called_with("Missing SHIB_ATTR_SESSION_ID!",category="error") mock_redirect.assert_called_with(url_for("security.login")) - + current_app.config.update( WEKO_ACCOUNTS_SHIB_LOGIN_ENABLED=True ) form = { "SHIB_ATTR_SESSION_ID":"1111" } - + # parse_attribute is error with patch("weko_accounts.views.parse_attributes",return_value=("attr",True)): mock_flash = mocker.patch("weko_accounts.views.flash") @@ -361,22 +361,22 @@ def test_shib_sp_login(client, redis_connect,mocker): res = client.post(url,data=form) assert res.status_code == 200 #assert res == "/weko/auto/login?SHIB_ATTR_SESSION_ID=1111&_method=GET" - + # next is / with patch("weko_accounts.views.ShibUser.get_relation_info", return_value=None): res = client.post(url + "?next=/", data=form) assert res.status_code == 200 - + # next is /next_page with patch("weko_accounts.views.ShibUser.get_relation_info", return_value=None): res = client.post(url + "?next=/next_page", data=form) assert res.status_code == 200 - + # next is None with patch("weko_accounts.views.ShibUser.get_relation_info", return_value=None): res = client.post(url, data=form) assert res.status_code == 200 - + # raise BaseException with patch("weko_accounts.views.flash",side_effect=BaseException("test_error")): mock_redirect_ = mocker.patch("weko_accounts.views._redirect_method",return_value=make_response()) @@ -386,11 +386,11 @@ def test_shib_sp_login(client, redis_connect,mocker): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_views.py::test_shib_stub_login -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workflow/.tox/c1/tmp def test_shib_stub_login(client,mocker): url = url_for("weko_accounts.shib_stub_login") - + # WEKO_ACCOUNTS_SHIB_LOGIN_ENABLED is false res = client.get(url) assert res.status_code == 403 - + current_app.config.update( WEKO_ACCOUNTS_SHIB_LOGIN_ENABLED=True ) @@ -407,7 +407,7 @@ def test_shib_stub_login(client,mocker): # next is None res = client.get(url) mock_redirect.assert_called_with("http://test_server.localdomain/secure/login.php?next=/") - + current_app.config.update( WEKO_ACCOUNTS_SHIB_IDP_LOGIN_ENABLED=False ) @@ -415,10 +415,10 @@ def test_shib_stub_login(client,mocker): mock_render_template = mocker.patch("weko_accounts.views.render_template",return_value=make_response()) res = client.get(url+"?next=/next_page") mock_render_template.assert_called_with('weko_accounts/login_shibuser_pattern_1.html',module_name="WEKO-Accounts") - + #def shib_logout(): # .tox/c1/bin/pytest --cov=weko_accounts tests/test_views.py::test_shib_logout -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workflow/.tox/c1/tmp def test_shib_logout(client, mocker): mocker.patch("weko_accounts.views.ShibUser.shib_user_logout") res = client.get(url_for("weko_accounts.shib_logout")) - assert res.data == bytes("logout success","utf-8") \ No newline at end of file + assert res.data == bytes("logout success","utf-8") diff --git a/modules/weko-accounts/weko_accounts/api.py b/modules/weko-accounts/weko_accounts/api.py index cf2cfaf54e..ddc696edb2 100644 --- a/modules/weko-accounts/weko_accounts/api.py +++ b/modules/weko-accounts/weko_accounts/api.py @@ -35,6 +35,8 @@ def __init__(self, shib_attr=None): :param shib_attr: passed attribute for shibboleth user """ self.shib_attr = shib_attr + if self.shib_attr.get('shib_handle'): + self.shib_attr['shib_handle']=self.shib_attr['shib_handle'][:255] self.user = None """The :class:`invenio_accounts.models.User` instance.""" self.shib_user = None @@ -149,6 +151,8 @@ def get_relation_info(self): shib_user.shib_ip_range_flag = self.shib_attr['shib_ip_range_flag'] if self.shib_attr['shib_organization']: shib_user.shib_organization = self.shib_attr['shib_organization'] + if self.shib_attr['shib_handle']: + shib_user.shib_handle = self.shib_attr['shib_handle'] db.session.commit() except SQLAlchemyError as ex: current_app.logger.error("SQLAlchemyError: {}".format(ex)) @@ -190,7 +194,7 @@ def bind_relation_info(self, account): shib_user_count = ShibbolethUser.query.filter_by(weko_uid=self.user.id).count() if shib_user_count > 0: raise SQLAlchemyError("User already exists. (weko_uid={}, shib_eppn={})".format(self.user.id, self.shib_attr.get('shib_eppn'))) - + self.user.email = self.shib_attr['shib_mail'] self.shib_user = ShibbolethUser.create( self.user, diff --git a/modules/weko-accounts/weko_accounts/config.py b/modules/weko-accounts/weko_accounts/config.py index 286b60495a..1e234a0f62 100644 --- a/modules/weko-accounts/weko_accounts/config.py +++ b/modules/weko-accounts/weko_accounts/config.py @@ -64,7 +64,7 @@ WEKO_ACCOUNTS_SSO_ATTRIBUTE_MAP = { 'SHIB_ATTR_EPPN': (False, 'shib_eppn'), # "SHIB_ATTR_LOGIN_ID": (False, 'shib_uid'), - # "SHIB_ATTR_HANDLE": (False, 'shib_handle'), + 'SHIB_ATTR_HANDLE': (False, 'shib_handle'), 'SHIB_ATTR_ROLE_AUTHORITY_NAME': (False, 'shib_role_authority_name'), 'SHIB_ATTR_PAGE_NAME': (False, 'shib_page_name'), 'SHIB_ATTR_ACTIVE_FLAG': (False, 'shib_active_flag'), diff --git a/modules/weko-admin/tests/conftest.py b/modules/weko-admin/tests/conftest.py index efc386c571..69682745bb 100644 --- a/modules/weko-admin/tests/conftest.py +++ b/modules/weko-admin/tests/conftest.py @@ -381,7 +381,7 @@ def teardown(): app.kvsession_store.redis.flushall() request.addfinalizer(teardown) - return a + return app @pytest.yield_fixture() def api(app): diff --git a/modules/weko-admin/tests/test_tasks.py b/modules/weko-admin/tests/test_tasks.py index 7f1811b222..198e299cc2 100644 --- a/modules/weko-admin/tests/test_tasks.py +++ b/modules/weko-admin/tests/test_tasks.py @@ -1,14 +1,20 @@ import os -from datetime import datetime, timedelta +import calendar +from datetime import datetime, timedelta, timezone +from unittest.mock import ANY + +import pytest from flask import current_app from weko_admin.models import AdminSettings from weko_admin.tasks import ( send_all_reports, + _get_start_end_date, check_send_all_reports, send_feedback_mail, _due_to_run, + _is_end_of_month, check_send_site_access_report, clean_temp_info ) @@ -24,33 +30,113 @@ def test_send_all_reports(app, users, statistic_email_addrs,mocker): class MockZip(): def getvalue(self): return "test_value" + + mocker.patch("weko_admin.tasks.render_template",return_value="test_html") + now = datetime.now() - subject = "{} Log report.".format(now.strftime("%Y-%m")) + + # report_type is None, frequency is daily + start_date = now - timedelta(days=2) + end_date = now - timedelta(days=2) + subject = "{} Log report.".format(start_date.strftime("%Y-%m-%d")) target_email=[email.email_address for email in statistic_email_addrs] - mocker.patch("weko_admin.tasks.package_reports", return_value=MockZip()) - mocker.patch("weko_admin.tasks.render_template",return_value="test_html") + schedule = {"details": "", "enabled": True, "frequency": "daily"} + with patch("weko_admin.tasks.get_reports") as mock_get_reports, \ + patch("weko_admin.tasks.package_reports", return_value=MockZip()) as mock_package_reports, \ + patch("weko_admin.tasks.send_mail") as mock_mail: + + send_all_reports(schedule=schedule) - # report_type is None - mock_mail = mocker.patch("weko_admin.tasks.send_mail") - send_all_reports() + args, kwargs = mock_get_reports.call_args + assert args == ("all",) + assert kwargs["start_date"].strftime("%Y-%m-%d") == start_date.strftime("%Y-%m-%d") + assert kwargs["end_date"].strftime("%Y-%m-%d") == end_date.strftime("%Y-%m-%d") + args, kwargs = mock_package_reports.call_args + assert args == (ANY,) + assert kwargs["report_date"] == start_date.strftime("%Y-%m-%d") args, kwargs = mock_mail.call_args assert args == (subject, target_email) assert kwargs["html"] == "test_html" - - # report_type is not None - mock_mail = mocker.patch("weko_admin.tasks.send_mail") - send_all_reports("fiile_download") + + # report_type is not None, frequency is weekly + start_date = now - timedelta(days=8) + end_date = now - timedelta(days=2) + subject = "{} Log report.".format(start_date.strftime("%Y-%m-%d") + "_" + end_date.strftime("%Y-%m-%d")) + schedule = {"details": str(now.weekday()), "enabled": True, "frequency": "weekly"} + with patch("weko_admin.tasks.get_reports") as mock_get_reports, \ + patch("weko_admin.tasks.package_reports", return_value=MockZip()) as mock_package_reports, \ + patch("weko_admin.tasks.send_mail") as mock_mail: + + send_all_reports("fiile_download", schedule=schedule) + + args, kwargs = mock_get_reports.call_args + assert args == ("fiile_download",) + assert kwargs["start_date"].strftime("%Y-%m-%d") == start_date.strftime("%Y-%m-%d") + assert kwargs["end_date"].strftime("%Y-%m-%d") == end_date.strftime("%Y-%m-%d") + args, kwargs = mock_package_reports.call_args + assert args == (ANY,) + assert kwargs["report_date"] == f'{start_date.strftime("%Y-%m-%d")}_{end_date.strftime("%Y-%m-%d")}' + args, kwargs = mock_mail.call_args + assert args == (subject, target_email) + assert kwargs["html"] == "test_html" + + # frequency is monthly + end_date = now.replace(day=1) - timedelta(days=1) + start_date = end_date.replace(day=1) + subject = "{} Log report.".format(start_date.strftime("%Y-%m")) + schedule = {"details": "2", "enabled": True, "frequency": "monthly"} + with patch("weko_admin.tasks.get_reports") as mock_get_reports, \ + patch("weko_admin.tasks.package_reports", return_value=MockZip()) as mock_package_reports, \ + patch("weko_admin.tasks.send_mail") as mock_mail: + + send_all_reports("fiile_download", schedule=schedule) + + args, kwargs = mock_get_reports.call_args + assert args == ("fiile_download",) + assert kwargs["start_date"].strftime("%Y-%m-%d") == start_date.strftime("%Y-%m-%d") + assert kwargs["end_date"].strftime("%Y-%m-%d") == end_date.strftime("%Y-%m-%d") + args, kwargs = mock_package_reports.call_args + assert args == (ANY, start_date.year, start_date.month) + assert not kwargs args, kwargs = mock_mail.call_args assert args == (subject, target_email) assert kwargs["html"] == "test_html" # raise Exception mock_mail = mocker.patch("weko_admin.tasks.send_mail", side_effect=Exception("test_error")) - send_all_reports("file_download") + mocker.patch("weko_admin.tasks.package_reports", return_value=MockZip()) + send_all_reports("file_download", schedule=schedule) args, kwargs = mock_mail.call_args assert args == (subject, target_email) assert kwargs["html"] == "test_html" - + +# def _get_start_end_date(dt, frequency): +# .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test__get_start_end_date -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp +def test__get_start_end_date(): + dt = datetime(year=2024, month=2, day=15, tzinfo=timezone.utc) + + # frequency is daily, start_date and end_date are 2 days ago + start_date, end_date = _get_start_end_date(dt, 'daily') + assert start_date == datetime(year=2024, month=2, day=13, tzinfo=timezone.utc) + assert end_date == datetime(year=2024, month=2, day=13, tzinfo=timezone.utc) + + # frequency is weekly, a week ago ending 2 days ago + start_date, end_date = _get_start_end_date(dt, 'weekly') + assert start_date == datetime(year=2024, month=2, day=7, tzinfo=timezone.utc) + assert end_date == datetime(year=2024, month=2, day=13, tzinfo=timezone.utc) + + # frequency is monthly, last month + start_date, end_date = _get_start_end_date(dt, 'monthly') + assert start_date == datetime(year=2024, month=1, day=1, tzinfo=timezone.utc) + assert end_date == datetime(year=2024, month=1, day=31, tzinfo=timezone.utc) + + # leap year, frequency is monthly + dt = datetime(year=2024, month=3, day=15, tzinfo=timezone.utc) + start_date, end_date = _get_start_end_date(dt, 'monthly') + assert start_date == datetime(year=2024, month=2, day=1, tzinfo=timezone.utc) + assert end_date == datetime(year=2024, month=2, day=29, tzinfo=timezone.utc) + + # def check_send_all_reports(): # .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_check_send_all_reports -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp def test_check_send_all_reports(app, admin_settings, mocker): @@ -74,7 +160,7 @@ def test_send_feedback_mail(app, mocker): # def _due_to_run(schedule): # .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_due_to_run -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp -def test_due_to_run(): +def test_due_to_run(mocker): schedule = {"details":"","enabled":False,"frequency":"daily"} result = _due_to_run(schedule) assert result == False @@ -83,6 +169,70 @@ def test_due_to_run(): result = _due_to_run(schedule) assert result == True + today = datetime.now(tz=timezone.utc) + weekday = today.weekday() + + schedule = {"details":f"{weekday}","enabled":False,"frequency":"weekly"} + result = _due_to_run(schedule) + assert result == False + + schedule = {"details":f"{weekday}","enabled":True,"frequency":"weekly"} + result = _due_to_run(schedule) + assert result == True + + schedule = {"details":f"{weekday + 1}","enabled":True,"frequency":"weekly"} + result = _due_to_run(schedule) + assert result == False + + day = today.day + + schedule = {"details":f"{day}","enabled":False,"frequency":"monthly"} + result = _due_to_run(schedule) + assert result == False + + schedule = {"details":f"{day}","enabled":True,"frequency":"monthly"} + result = _due_to_run(schedule) + assert result == True + + schedule = {"details":f"{day + 1}","enabled":True,"frequency":"monthly"} + result = _due_to_run(schedule) + assert result == False + + mocker.patch("weko_admin.tasks._is_end_of_month", return_value=True) + schedule = {"details":"-1","enabled":True,"frequency":"monthly"} + result = _due_to_run(schedule) + assert result == True + + mocker.patch("weko_admin.tasks._is_end_of_month", return_value=False) + schedule = {"details":"-1","enabled":True,"frequency":"monthly"} + result = _due_to_run(schedule) + assert result == False + + schedule = {"details":"1","enabled":True,"frequency":"unknown"} + result = _due_to_run(schedule) + assert result == False + +# def _is_end_of_month(dt): +# .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_is_end_of_month_match -vv -s --cov-branch --cov-report=term +@pytest.mark.parametrize("month", [i for i in range(1, 13)], ids=list(calendar.month_abbr)[1:]) +def test_is_end_of_month_match(month): + dt = datetime(year=2025, month=month, day=calendar.mdays[month], tzinfo=timezone.utc) + assert _is_end_of_month(dt) == True + +# .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_is_end_of_month_not_match -vv -s --cov-branch --cov-report=term +@pytest.mark.parametrize("month", [i for i in range(1, 13)], ids=list(calendar.month_abbr)[1:]) +def test_is_end_of_month_not_match(month): + dt = datetime(year=2025, month=month, day=(calendar.mdays[month] - 1), tzinfo=timezone.utc) + assert _is_end_of_month(dt) == False + +# .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_is_end_of_month_leap_year -vv -s --cov-branch --cov-report=term +def test_is_end_of_month_leap_year(): + dt = datetime(year=2024, month=2, day=28, tzinfo=timezone.utc) + assert _is_end_of_month(dt) == False + + dt = datetime(year=2024, month=2, day=29, tzinfo=timezone.utc) + assert _is_end_of_month(dt) == True + # def check_send_site_access_report(): # .tox/c1/bin/pytest --cov=weko_admin tests/test_tasks.py::test_check_send_site_access_report -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp def test_check_send_site_access_report(client, admin_settings, mocker): diff --git a/modules/weko-admin/tests/test_utils.py b/modules/weko-admin/tests/test_utils.py index 7f276255b6..3b06bdd432 100644 --- a/modules/weko-admin/tests/test_utils.py +++ b/modules/weko-admin/tests/test_utils.py @@ -271,7 +271,7 @@ def test_get_user_report_data(users): # def get_reports(type, year, month): # .tox/c1/bin/pytest --cov=weko_admin tests/test_utils.py::test_get_reports -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp -def test_get_reports(client, mocker): +def test_get_reports(app, mocker): file_download_result = { "all": [ {'admin': 0, 'file_key': 'test1.pdf', 'group_counts': {'test1': 1, 'test2': 1}, 'index_list': 'index1', 'login': 0, 'no_login': 1, 'reg': 0, 'site_license': 1, 'total': 1}, @@ -411,19 +411,22 @@ def common_reports_helper(**args): else: return [] - mocker.patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) - mocker.patch('weko_admin.utils.QueryRecordViewReportHelper.get', return_value=detail_view_result) - mocker.patch('weko_admin.utils.QueryRecordViewPerIndexReportHelper.get', return_value=index_access_result) - mocker.patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) - mocker.patch('weko_admin.utils.QuerySearchReportHelper.get', return_value=search_count_result) - mocker.patch('weko_admin.utils.get_user_report_data', return_value=user_roles_result) - year = 2024 month = 5 + start_date = datetime(year, month, 1) + start_date_str = start_date.strftime("%Y-%m-%d") + end_date = datetime(year, month, 31) + end_date_str = end_date.strftime("%Y-%m-%d") # type is all - type = 'all' - result = get_reports(type, year, month) + report_type = 'all' + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_file_helper, \ + patch('weko_admin.utils.QueryRecordViewReportHelper.get', return_value=detail_view_result) as mock_record_view_helper, \ + patch('weko_admin.utils.QueryRecordViewPerIndexReportHelper.get', return_value=index_access_result) as mock_record_view_per_index_helper, \ + patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) as mock_common_helper, \ + patch('weko_admin.utils.QuerySearchReportHelper.get', return_value=search_count_result) as mock_search_helper, \ + patch('weko_admin.utils.get_user_report_data', return_value=user_roles_result) as mock_user_report_helper: + result = get_reports(report_type, year, month) assert result == { 'file_download': file_download_result, 'file_preview': file_preview_result, @@ -436,80 +439,175 @@ def common_reports_helper(**args): 'user_roles': user_roles_result, 'site_access': site_access_result } + mock_file_helper.assert_any_call(event='file_download', year=year, month=month) + mock_file_helper.assert_any_call(event='file_preview', year=year, month=month) + mock_file_helper.assert_any_call(event='billing_file_download', year=year, month=month) + mock_file_helper.assert_any_call(event='file_using_per_user', year=year, month=month) + mock_record_view_helper.assert_any_call(event='detail_view', year=year, month=month) + mock_record_view_per_index_helper.assert_any_call(event='index_access', year=year, month=month) + mock_common_helper.assert_any_call(event='top_page_access', year=year, month=month) + mock_common_helper.assert_any_call(event='site_access', year=year, month=month) + mock_search_helper.assert_any_call(event='search_count', year=year, month=month) + mock_user_report_helper.assert_called_once_with() # type is file_download - type = 'file_download' - result = get_reports(type, year, month) + report_type = 'file_download' + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'file_download': file_download_result + } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'file_download': file_download_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is file_preview - type = 'file_preview' - result = get_reports(type, year, month) + report_type = 'file_preview' + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'file_preview': file_preview_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) + assert result == { + 'file_preview': file_preview_result + } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is billing_file_download - type = 'billing_file_download' - result = get_reports(type, year, month) + report_type = 'billing_file_download' + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'billing_file_download': billing_file_download_result + } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'billing_file_download': billing_file_download_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is detail_view - type = 'detail_view' - result = get_reports(type, year, month) + report_type = 'detail_view' + with patch('weko_admin.utils.QueryRecordViewReportHelper.get', return_value=detail_view_result) as mock_helper: + result = get_reports(report_type, year, month) assert result == { 'detail_view': detail_view_result } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryRecordViewReportHelper.get', return_value=detail_view_result) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) + assert result == { + 'detail_view': detail_view_result + } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is index_access - type = 'index_access' - result = get_reports(type, year, month) + report_type = 'index_access' + with patch('weko_admin.utils.QueryRecordViewPerIndexReportHelper.get', return_value=index_access_result) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'index_access': index_access_result + } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryRecordViewPerIndexReportHelper.get', return_value=index_access_result) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'index_access': index_access_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is file_using_per_user - type = 'file_using_per_user' - result = get_reports(type, year, month) + report_type = 'file_using_per_user' + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, year, month) assert result == { 'file_using_per_user': file_using_per_user_result } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryFileReportsHelper.get', side_effect=file_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) + assert result == { + 'file_using_per_user': file_using_per_user_result + } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is top_page_access - type = 'top_page_access' - result = get_reports(type, year, month) + report_type = 'top_page_access' + with patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'top_page_access': top_page_access_result + } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'top_page_access': top_page_access_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is search_count - type = 'search_count' - result = get_reports(type, year, month) + report_type = 'search_count' + with patch('weko_admin.utils.QuerySearchReportHelper.get', return_value=search_count_result) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'search_count': search_count_result + } + mock_helper.assert_called_once_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QuerySearchReportHelper.get', return_value=search_count_result) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'search_count': search_count_result } + mock_helper.assert_called_once_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is user_roles - type = 'user_roles' - result = get_reports(type, year, month) + report_type = 'user_roles' + with patch('weko_admin.utils.get_user_report_data', return_value=user_roles_result) as mock_helper: + result = get_reports(report_type, year, month) assert result == { 'user_roles': user_roles_result } + mock_helper.assert_called_once_with() + with patch('weko_admin.utils.get_user_report_data', return_value=user_roles_result) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) + assert result == { + 'user_roles': user_roles_result + } + mock_helper.assert_called_once_with() # type is site_access - type = 'site_access' - result = get_reports(type, year, month) + report_type = 'site_access' + with patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) as mock_helper: + result = get_reports(report_type, year, month) + assert result == { + 'site_access': site_access_result + } + mock_helper.assert_called_with(event=report_type, year=year, month=month) + with patch('weko_admin.utils.QueryCommonReportsHelper.get', side_effect=common_reports_helper) as mock_helper: + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'site_access': site_access_result } + mock_helper.assert_called_with(event=report_type, start_date=start_date_str, end_date=end_date_str) # type is test - type = 'test' - result = get_reports(type, year, month) + report_type = 'test' + result = get_reports(report_type, year, month) + assert result == { + 'test': {} + } + result = get_reports(report_type, range=True, start_date=start_date, end_date=end_date) assert result == { 'test': {} } @@ -517,14 +615,20 @@ def common_reports_helper(**args): # def package_reports(all_stats, year, month): # .tox/c1/bin/pytest --cov=weko_admin tests/test_utils.py::test_package_reports -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp -def test_package_reports(client,mocker): +def test_package_reports(client, mocker): mock_stream = StringIO() mock_stream.write("test") - mocker.patch("weko_admin.utils.make_stats_file",return_value=mock_stream) - all_stats = { - "file_download":"test_stats" - } - result = package_reports(all_stats,"2022","10") + all_stats = {"file_download":"test_stats"} + + with patch("weko_admin.utils.make_stats_file", return_value=mock_stream) as mock_make_stats_file: + result = package_reports(all_stats, "2022", "10") + assert b"logReport_FileDownload_2022-10.tsv" in result.getvalue() + mock_make_stats_file.assert_called_once_with("test_stats", "file_download", "2022-10", False) + + with patch("weko_admin.utils.make_stats_file", return_value=mock_stream) as mock_make_stats_file: + result = package_reports(all_stats, report_date="2022-10-01_2022-10-07") + assert b"logReport_FileDownload_2022-10-01_2022-10-07.tsv" in result.getvalue() + mock_make_stats_file.assert_called_once_with("test_stats", "file_download", "2022-10-01_2022-10-07", True) # raise Exception mocker.patch("weko_admin.utils.make_stats_file",side_effect=Exception("test_error")) @@ -538,8 +642,7 @@ def test_make_stats_file(client,mocker, users): mocker.patch("weko_admin.utils.write_report_file_rows") raw_stats="" file_type = "" - year = "2022" - month = "10" + report_date = "2022-10" # filetype = index_access file_type = "index_access" @@ -551,7 +654,7 @@ def test_make_stats_file(client,mocker, users): 'Detail Views Per Index\n'\ 'Index,No. Of Views\n'\ 'Total Detail Views,10\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, False) assert result.getvalue() == test # filetype = billing_file_download @@ -563,7 +666,7 @@ def test_make_stats_file(client,mocker, users): '""\n'\ 'No. Of Paid File Downloads\n'\ 'File Name,Registered Index Name,No. Of Times Downloaded,Non-Logged in User,System Administrator,Repository Administrator,Contributor,Community Administrator,General,Original Role,Student,Site License,Admin,Registrar\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, False) assert result.getvalue() == test # filetype = site_access @@ -578,21 +681,23 @@ def test_make_stats_file(client,mocker, users): '""\n'\ 'Access Number Breakdown By Site License\n'\ ',WEKO Top Page Access Count,Number Of Searches,Number Of Views,Number Of File download,Number Of File Regeneration\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, False) assert result.getvalue() == test + + report_date = "2022-10-01_2022-10-07" ## institution_name in raw_stats file_type = "site_access" raw_stats={"institution_name":"test_institution_name"} test = \ 'Access Count By Site License\n'\ - 'Aggregation Month,2022-10\n'\ + 'Aggregation Period,2022-10-01_2022-10-07\n'\ '""\n'\ ',WEKO Top Page Access Count,Number Of Searches,Number Of Views,Number Of File download,Number Of File Regeneration\n'\ '""\n'\ 'Access Number Breakdown By Site License\n'\ 'Institution Name,WEKO Top Page Access Count,Number Of Searches,Number Of Views,Number Of File download,Number Of File Regeneration\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, True) assert result.getvalue() == test ## open_access not in raw_stats,institution_name not in raw_stats @@ -600,12 +705,12 @@ def test_make_stats_file(client,mocker, users): raw_stats={"other_raw":"test_institution_name"} test = \ 'Access Count By Site License\n'\ - 'Aggregation Month,2022-10\n'\ + 'Aggregation Period,2022-10-01_2022-10-07\n'\ '""\n'\ ',WEKO Top Page Access Count,Number Of Searches,Number Of Views,Number Of File download,Number Of File Regeneration\n'\ '""\n'\ 'Access Number Breakdown By Site License\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, True) assert result.getvalue() == test # filetype = other @@ -613,11 +718,11 @@ def test_make_stats_file(client,mocker, users): raw_stats={} test = \ 'Detail Views Count\n'\ - 'Aggregation Month,2022-10\n'\ + 'Aggregation Period,2022-10-01_2022-10-07\n'\ '""\n'\ 'Detail Views Count\n'\ 'Title,Registered Index Name,View Count,Non-logged-in User\n' - result = make_stats_file(raw_stats,file_type,year,month) + result = make_stats_file(raw_stats, file_type, report_date, True) assert result.getvalue() == test diff --git a/modules/weko-admin/weko_admin/admin.py b/modules/weko-admin/weko_admin/admin.py index 70dc7db54d..bbf0f92111 100644 --- a/modules/weko-admin/weko_admin/admin.py +++ b/modules/weko-admin/weko_admin/admin.py @@ -519,7 +519,7 @@ def set_email_schedule(self): try: AdminSettings.update('report_email_schedule_settings', schedule) - flash(_('Successfully Changed Schedule.'), 'error') + flash(_('Successfully Changed Schedule.')) except Exception: flash(_('Could Not Save Changes.'), 'error') return redirect(url_for('report.index')) diff --git a/modules/weko-admin/weko_admin/config.py b/modules/weko-admin/weko_admin/config.py index 1c9744ce1b..32c3af4a02 100644 --- a/modules/weko-admin/weko_admin/config.py +++ b/modules/weko-admin/weko_admin/config.py @@ -23,7 +23,7 @@ from flask_babelex import gettext as __ from flask_babelex import lazy_gettext as _ -WEKO_ADMIN_DEFAULT_AGGREGATION_MONTH = 2 +WEKO_ADMIN_DEFAULT_AGGREGATION_MONTH = 1 """default aggregation month for site license mail.""" WEKO_ADMIN_DEFAULT_LIFETIME = 60 diff --git a/modules/weko-admin/weko_admin/static/js/weko_admin/email_schedule.js b/modules/weko-admin/weko_admin/static/js/weko_admin/email_schedule.js index ab55fbcc7b..353d718819 100644 --- a/modules/weko-admin/weko_admin/static/js/weko_admin/email_schedule.js +++ b/modules/weko-admin/weko_admin/static/js/weko_admin/email_schedule.js @@ -17,11 +17,13 @@ $(document).ready(function () { switch(frequency) { case 'monthly': $('#email_sched_details_weekly').addClass('hidden'); + $('#sched_details_label').removeClass('hidden'); $('.weekly-option').prop('selected', false); $('#email_sched_details_monthly').removeClass('hidden'); break; case 'weekly': $('#email_sched_details_monthly').addClass('hidden'); + $('#sched_details_label').removeClass('hidden'); $('.monthly-option').prop('selected', false); $('#email_sched_details_weekly').removeClass('hidden'); break; diff --git a/modules/weko-admin/weko_admin/tasks.py b/modules/weko-admin/weko_admin/tasks.py index 73701f33da..e5b10d1407 100644 --- a/modules/weko-admin/weko_admin/tasks.py +++ b/modules/weko-admin/weko_admin/tasks.py @@ -22,7 +22,9 @@ import os import shutil -from datetime import datetime, timedelta +import calendar +from datetime import datetime, timedelta, timezone +import traceback from celery import shared_task from celery.utils.log import get_task_logger @@ -30,14 +32,13 @@ from flask_babelex import gettext as _ from flask_mail import Attachment from invenio_mail.api import send_mail -from invenio_stats.utils import QueryCommonReportsHelper, \ - QueryFileReportsHelper, QueryRecordViewPerIndexReportHelper, \ - QueryRecordViewReportHelper, QuerySearchReportHelper from weko_admin.api import TempDirInfo from .models import AdminSettings, StatisticsEmail -from .utils import StatisticMail, get_user_report_data, package_reports ,elasticsearch_reindex +from .utils import ( + StatisticMail, package_reports, elasticsearch_reindex, get_reports +) from .views import handle_site_license_mail from celery.task.control import inspect from weko_search_ui.tasks import check_celery_is_run @@ -112,42 +113,32 @@ def is_reindex_running(): return False @shared_task(ignore_results=True) -def send_all_reports(report_type=None, year=None, month=None): +def send_all_reports(report_type="all", *, schedule): """Query elasticsearch for each type of stats report.""" # By default get the current month and year - now = datetime.now() - month = month or now.month - year = year or now.year - all_results = { - 'file_download': QueryFileReportsHelper.get( - year=year, month=month, event='file_download'), - 'file_preview': QueryFileReportsHelper.get( - year=year, month=month, event='file_preview'), - 'index_access': QueryRecordViewPerIndexReportHelper.get( - year=year, month=month), - 'detail_view': QueryRecordViewReportHelper.get( - year=year, month=month), - 'file_using_per_user': QueryFileReportsHelper.get( - year=year, month=month, event='file_using_per_user'), - 'top_page_access': QueryCommonReportsHelper.get( - year=year, month=month, event='top_page_access'), - 'search_count': QuerySearchReportHelper.get( - year=year, month=month), - 'user_roles': get_user_report_data(), - 'site_access': QueryCommonReportsHelper.get( - year=year, month=month, event='site_access') - } + frequency = schedule['frequency'] # daily, weekly, monthly + now = datetime.now(tz=timezone.utc) + + start_date, end_date = _get_start_end_date(now, frequency) + + reports = get_reports( + report_type, range=True, + start_date=start_date, end_date=end_date, + ) + with current_app.app_context(): # Allow for this to be used to get specific emails as well - reports = {} - if report_type is not None and report_type in all_results: - reports[report_type] = all_results[report_type] + if frequency == 'monthly': + zip_date = start_date.strftime('%Y-%m') + zip_stream = package_reports(reports, start_date.year, start_date.month) else: - reports = all_results - - zip_date = str(year) + '-' + str(month).zfill(2) + zip_date = start_date.strftime('%Y-%m-%d') + if frequency == 'weekly': + zip_date += '_' + end_date.strftime('%Y-%m-%d') + zip_stream = package_reports( + reports, report_date=zip_date, + ) zip_name = 'logReport_' + zip_date + '.zip' - zip_stream = package_reports(reports, year, month) recepients = StatisticsEmail.get_all_emails() attachments = [Attachment(zip_name, @@ -163,9 +154,35 @@ def send_all_reports(report_type=None, year=None, month=None): attachments=attachments) current_app.logger.info('[{0}] [{1}] '.format(0, 'Sent email')) except Exception as e: + traceback.print_exc() current_app.logger.info('[{0}] [{1}] '.format(1, 'Could not send')) +def _get_start_end_date(dt, frequency): + """Get start date and end date. + + Args: + dt (datetime): + The date to calculate from. + frequency (str): + The frequency of the report ('daily', 'weekly', 'monthly'). + """ + if frequency == 'daily': + # on 2 days ago + start_date = end_date = dt - timedelta(days=2) + elif frequency == 'weekly': + # from 8 days ago to 2 days ago + start_date = dt - timedelta(days=8) + end_date = dt - timedelta(days=2) + else: + # monthly, + # from first day of previous month to last day of previous month + end_date = dt.replace(day=1) - timedelta(days=1) + start_date = end_date.replace(day=1) + + return start_date, end_date + + @shared_task(ignore_results=True) # Set for timedelta(days=1) def check_send_all_reports(): """Check Redis periodically for when to run a task.""" @@ -175,7 +192,7 @@ def check_send_all_reports(): dict_to_object=False) schedule = schedule if schedule else None if schedule and _due_to_run(schedule): - send_all_reports.delay() + send_all_reports.delay(schedule=schedule) @shared_task(ignore_results=True) @@ -189,12 +206,24 @@ def _due_to_run(schedule): """Check if a task needs to be ran.""" if not schedule['enabled']: return False - now = datetime.now() - return (schedule['frequency'] == 'daily') or \ - (schedule['frequency'] == 'weekly' - and int(schedule['details']) == now.weekday()) or \ - (schedule['frequency'] == 'monthly' - and int(schedule['details']) == now.day) + now = datetime.now(tz=timezone.utc) + + if schedule['frequency'] == 'daily': + return True + if schedule['frequency'] == 'weekly': + return int(schedule['details']) == now.weekday() + if schedule['frequency'] == 'monthly': + if int(schedule['details']) == now.day: + return True + if int(schedule['details']) == -1: + return _is_end_of_month(now) + return False + + +def _is_end_of_month(dt): + """Check if the date is end of month.""" + _, last_day = calendar.monthrange(dt.year, dt.month) + return dt.day == last_day @shared_task(ignore_results=True) diff --git a/modules/weko-admin/weko_admin/templates/weko_admin/partials/email_schedule.html b/modules/weko-admin/weko_admin/templates/weko_admin/partials/email_schedule.html index 7b24cf22eb..0abda11066 100644 --- a/modules/weko-admin/weko_admin/templates/weko_admin/partials/email_schedule.html +++ b/modules/weko-admin/weko_admin/templates/weko_admin/partials/email_schedule.html @@ -20,7 +20,7 @@

{{_('Report Email Schedule')}}

{{_('Details')}}

+ id="sched_details_label">{{_('Send Date')}}

@@ -43,7 +43,7 @@

{{_('Report Email Schedule')}}

-
\ No newline at end of file + diff --git a/modules/weko-records-ui/weko_records_ui/utils.py b/modules/weko-records-ui/weko_records_ui/utils.py index 8666a6cc2a..7b01def741 100644 --- a/modules/weko-records-ui/weko_records_ui/utils.py +++ b/modules/weko-records-ui/weko_records_ui/utils.py @@ -22,6 +22,7 @@ import base64 import orjson +import pickle from datetime import datetime as dt from datetime import timedelta from decimal import Decimal @@ -29,11 +30,12 @@ from urllib.parse import quote from elasticsearch_dsl import Q -from flask import abort, current_app, request +from flask import abort, current_app, request, session from flask_babelex import gettext as _ from flask_babelex import to_utc +from flask_login import login_user from flask_security import current_user -from invenio_accounts.models import Role +from invenio_accounts.models import Role, User from invenio_cache import current_cache from invenio_db import db from invenio_i18n.ext import current_i18n @@ -52,6 +54,7 @@ from weko_records.serializers.utils import get_mapping from weko_records.utils import replace_fqdn from weko_records_ui.models import InstitutionName +from weko_redis.redis import RedisConnection from weko_schema_ui.models import PublishStatus from weko_workflow.api import WorkActivity, WorkFlow @@ -92,9 +95,9 @@ def check_items_settings(settings=None): if isinstance(settings,dict): if 'items_display_email' in settings: current_app.config['EMAIL_DISPLAY_FLG'] = settings['items_display_email'] - if 'items_search_author' in settings: + if 'items_search_author' in settings: current_app.config['ITEM_SEARCH_FLG'] = settings['items_search_author'] - if 'item_display_open_date' in settings: + if 'item_display_open_date' in settings: current_app.config['OPEN_DATE_DISPLAY_FLG'] = \ settings['item_display_open_date'] else: @@ -232,10 +235,10 @@ def is_open_access(record: Dict, file_name: str) -> bool: Args: record (dict): item's meta data file_name (str): target file name - + Returns: bool: open access item or not - + """ from weko_records_ui.permissions import check_publish_status target_index_list = record['path'] @@ -256,7 +259,7 @@ def is_open_access(record: Dict, file_name: str) -> bool: return True if access_role == 'open_date': if file.get('date') and file.get('date')[0] and file.get('date')[0].get('dateValue'): - if dt.strptime(file.get('date')[0].get('dateValue'), + if dt.strptime(file.get('date')[0].get('dateValue'), '%Y-%m-%d').date() <= dt.now().date(): return True return False @@ -433,7 +436,7 @@ def get_license_pdf(license, item_metadata_json, pdf, file_item_id, footer_w, # current_app.logger.debug("footer_h:{}".format(footer_h)) # current_app.logger.debug("cc_logo_xposition:{}".format(cc_logo_xposition)) # current_app.logger.debug("item:{}".format(item)) - + from .views import blueprint license_icon_pdf_location = \ current_app.config['WEKO_RECORDS_UI_LICENSE_ICON_PDF_LOCATION'] @@ -469,7 +472,7 @@ def get_pair_value(name_keys, lang_keys, datas): current_app.logger.debug("name_keys:{}".format(name_keys)) current_app.logger.debug("lang_keys:{}".format(lang_keys)) current_app.logger.debug("datas:{}".format(datas)) - + if len(name_keys) == 1 and len(lang_keys) == 1: if isinstance(datas, list): for data in datas: @@ -509,7 +512,7 @@ def hide_item_metadata(record, settings=None, item_type_mapping=None, record['item_type_id'], item_type_mapping, item_type_data ) record = hide_by_itemtype(record, list_hidden) - + hide_email = hide_meta_data_for_role(record) if hide_email: # Hidden owners_ext.email @@ -518,7 +521,7 @@ def hide_item_metadata(record, settings=None, item_type_mapping=None, del record['_deposit']['owners_ext']['email'] if hide_email and not current_app.config['EMAIL_DISPLAY_FLG']: - record = hide_by_email(record) + record = hide_by_email(record, True) record = hide_by_file(record) @@ -539,7 +542,7 @@ def hide_item_metadata_email_only(record): check_items_settings() record['weko_creator_id'] = record.get('owner') - + hide_email = hide_meta_data_for_role(record) if hide_email: # Hidden owners_ext.email @@ -548,7 +551,7 @@ def hide_item_metadata_email_only(record): del record['_deposit']['owners_ext']['email'] if hide_email and not current_app.config['EMAIL_DISPLAY_FLG']: - record = hide_by_email(record) + record = hide_by_email(record, True) return True record.pop('weko_creator_id') @@ -579,6 +582,7 @@ def hide_by_email(item_metadata, force_flag=False): """Hiding emails. :param item_metadata: + :param force_flag: force to hide :return: """ from weko_items_ui.utils import get_options_and_order_list, get_hide_list_by_schema_form @@ -586,15 +590,13 @@ def hide_by_email(item_metadata, force_flag=False): subitem_keys = current_app.config['WEKO_RECORDS_UI_EMAIL_ITEM_KEYS'] item_type_id = item_metadata.get('item_type_id') - if item_type_id: meta_options, type_mapping = get_options_and_order_list(item_type_id) hide_list = get_hide_list_by_schema_form(item_type_id) - # Hidden owners_ext info if item_metadata.get('_deposit') and item_metadata['_deposit'].get('owners_ext'): - del item_metadata['_deposit']['owners_ext'] - + if force_flag or not show_email_flag: + del item_metadata['_deposit']['owners_ext'] for item in item_metadata: _item = item_metadata[item] prop_hidden = meta_options.get(item, {}).get('option', {}).get('hidden', False) @@ -656,44 +658,6 @@ def item_setting_show_email(): is_display = False return is_display -def is_show_email_of_creator(item_type_id): - """Check setting show/hide email for 'Detail' and 'PDF Cover Page' screen. - - :param item_type_id: item type id of current record. - :return: True/False, True: show, False: hide. - """ - def get_creator_id(item_type_id): - type_mapping = Mapping.get_record(item_type_id) - item_map = get_mapping(type_mapping, "jpcoar_mapping") - creator = 'creator.creatorName.@value' - creator_id = None - if creator in item_map: - creator_id = item_map[creator].split('.')[0] - return creator_id - - def item_type_show_email(item_type_id): - # Get flag of creator's email hide from item type. - creator_id = get_creator_id(item_type_id) - if not creator_id: - return None - item_type = ItemTypes.get_by_id(item_type_id) - schema_editor = item_type.render.get('schemaeditor', {}) - schema = schema_editor.get('schema', {}) - creator = schema.get(creator_id) - if not creator: - return None - properties = creator.get('properties', {}) - creator_mails = properties.get('creatorMails', {}) - items = creator_mails.get('items', {}) - properties = items.get('properties', {}) - creator_mail = properties.get('creatorMail', {}) - is_hide = creator_mail.get('isHide', None) - return is_hide - - is_hide = item_type_show_email(item_type_id) - is_display = item_setting_show_email() - - return not is_hide and is_display def replace_license_free(record_metadata, is_change_label=True): """Change the item name 'licensefree' to 'license_note'. @@ -813,16 +777,16 @@ def get_data_by_key_array_json(key, array_json, get_key): for item in array_json: if str(item.get('id')) == str(key): return item.get(get_key) - + def is_price_highlight(p_file): """"Rerturn True if price is highlighted - + Args: p_file (dict): all metadata of a record. Returns: bool: rerturn True if price is highlighted. - + """ min_price = p_file['min_price'] user_flag = True @@ -851,7 +815,7 @@ def is_price_highlight(p_file): priceinfo['has_role'] and min_price == priceinfo['price']: is_highlight = True priceinfo['is_highlight'] = is_highlight - + def is_display_purchased(p_file): """"Remove purchased from roles that do not have min_price Args: @@ -1138,7 +1102,7 @@ def generate_one_time_download_url( :param record_id: File Version ID :param guest_mail: guest email :return: - """ + """ secret_key = current_app.config['WEKO_RECORDS_UI_SECRET_KEY'] download_pattern = current_app.config[ 'WEKO_RECORDS_UI_ONETIME_DOWNLOAD_PATTERN'] @@ -1539,7 +1503,7 @@ def get_google_detaset_meta(record,record_tree=None): # Required property check min_length = current_app.config.get('WEKO_RECORDS_UI_GOOGLE_DATASET_DESCRIPTION_MIN',WEKO_RECORDS_UI_GOOGLE_DATASET_DESCRIPTION_MIN) max_length = current_app.config.get('WEKO_RECORDS_UI_GOOGLE_DATASET_DESCRIPTION_MAX',WEKO_RECORDS_UI_GOOGLE_DATASET_DESCRIPTION_MAX) - + for title in mtdata.findall('dc:title', namespaces=mtdata.nsmap): res_data['name'] = title.text for description in mtdata.findall('datacite:description', namespaces=mtdata.nsmap): @@ -1746,3 +1710,21 @@ def get_billing_role(record: Dict) -> Tuple[str, str]: min_role = Role.query.get(int(min_price_info.get(billing_role_key))) return min_role.name, min_price_info.get(billing_price_key, '') + +def restore_session_info(session_id: str, redis_connection: RedisConnection) -> None: + """Restore session info. + + Args: + session_id (str): Session ID + redis_connection (RedisConnection): Redis connection + """ + # Get session data + session_store = redis_connection.connection( + db=current_app.config['ACCOUNTS_SESSION_REDIS_DB_NO'], kv=True) + session_data = pickle.loads(session_store.redis.get(session_id)) + for k, v in session_data.items(): + session[k] = v + + # Login user + user = User.query.filter_by(id=session['user_id']).first() + login_user(user) diff --git a/modules/weko-records-ui/weko_records_ui/views.py b/modules/weko-records-ui/weko_records_ui/views.py index 07dc0055b4..a5a7043db7 100644 --- a/modules/weko-records-ui/weko_records_ui/views.py +++ b/modules/weko-records-ui/weko_records_ui/views.py @@ -45,6 +45,7 @@ from invenio_files_rest.signals import file_downloaded from invenio_records_ui.utils import obj_or_import_string from lxml import etree +from urllib.parse import urljoin from weko_accounts.views import _redirect_method from weko_admin.models import AdminSettings from weko_admin.utils import get_search_setting @@ -58,6 +59,7 @@ from weko_records.serializers.utils import get_mapping from weko_records.utils import custom_record_medata_for_export, \ remove_weko2_special_character, selected_value_by_language +from weko_redis.redis import RedisConnection from weko_schema_ui.models import PublishStatus from weko_workflow.api import WorkFlow @@ -69,11 +71,11 @@ from .permissions import check_content_clickable, check_created_id, \ check_file_download_permission, check_original_pdf_download_permission, \ check_permission_period, file_permission_factory, get_permission, \ - check_charge, create_charge, close_charge + check_charge, create_charge, close_charge, secure_charge from .utils import get_billing_file_download_permission, \ get_google_detaset_meta, get_google_scholar_meta, get_groups_price, \ get_min_price_billing_file_download, get_record_permalink, hide_by_email, \ - is_show_email_of_creator,hide_by_itemtype + hide_by_itemtype, restore_session_info from .utils import restore as restore_imp from .utils import soft_delete as soft_delete_imp @@ -283,8 +285,8 @@ def check_file_permission(record, fjson): Args: record (weko_deposit.api.WekoRecord): _description_ fjson (dict): _description_ - - """ + + """ return check_file_download_permission(record, fjson) @@ -368,21 +370,22 @@ def default_view_method(pid, record, filename=None, template=None, **kwargs): :param kwargs: Additional view arguments based on URL rule. :returns: The rendered template. """ - def _get_rights_title(result, rights_key, rights_values, current_lang, meta_options): + def _get_rights_title(result, rights_key_str, rights_values, current_lang, meta_options): """Get multi-lang rights title.""" - item_key = rights_key.split('.')[0] - if item_key in meta_options: - if meta_options[item_key].get('title'): - item_title = meta_options[item_key]['title'] - if meta_options[item_key]['title_i18n'].get(current_lang, None): - item_title = meta_options[item_key]['title_i18n'][current_lang] - elif meta_options[item_key]['title_i18n'].get('en', None): - item_title = meta_options[item_key]['title_i18n']['en'] - if rights_values: - result[item_key] = { - 'item_title': item_title, - 'item_values': rights_values - } + for rights_key in rights_key_str.split(','): + item_key = rights_key.split('.')[0] + if item_key in meta_options: + if meta_options[item_key].get('title'): + item_title = meta_options[item_key]['title'] + if meta_options[item_key]['title_i18n'].get(current_lang, None): + item_title = meta_options[item_key]['title_i18n'][current_lang] + elif meta_options[item_key]['title_i18n'].get('en', None): + item_title = meta_options[item_key]['title_i18n']['en'] + if rights_values: + result[item_key] = { + 'item_title': item_title, + 'item_values': rights_values + } # Check file permision if request is File Information page. file_order = int(request.args.get("file_order", -1)) @@ -449,7 +452,7 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti if hasattr(current_user, 'site_license_flag') else False send_info['site_license_name'] = current_user.site_license_name \ if hasattr(current_user, 'site_license_name') else '' - + record_viewed.send( current_app._get_current_object(), pid=pid, @@ -478,7 +481,7 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti record["relation"] = res else: record["relation"] = {} - + recstr = etree.tostring( getrecord( identifier=record['_oai'].get('id'), @@ -489,48 +492,81 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti et=etree.fromstring(recstr) google_scholar_meta = get_google_scholar_meta(record,record_tree=et) google_dataset_meta = get_google_detaset_meta(record,record_tree=et) - + current_lang = current_i18n.language \ if hasattr(current_i18n, 'language') else None # get title name - from weko_records.utils import get_options_and_order_list from weko_search_ui.utils import get_data_by_property - from weko_records.utils import get_options_and_order_list + from weko_items_ui.utils import get_options_and_order_list, get_hide_list_by_schema_form + from weko_workflow.utils import get_sub_item_value + title_name = '' rights_values = {} accessRight = '' - solst, meta_options = get_options_and_order_list( + meta_options, item_type_mapping = get_options_and_order_list( record.get('item_type_id')) - item_type_mapping = Mapping.get_record(record.get('item_type_id')) - item_map = get_mapping(item_type_mapping, 'jpcoar_mapping') - suffixes = '.@attributes.xml:lang' - for key in item_map: - prefix = key.replace(suffixes, '') - if prefix == 'title' and key.find(suffixes) != -1: + hide_list = get_hide_list_by_schema_form(record.get('item_type_id')) + item_map = get_mapping(record.get('item_type_id'), 'jpcoar_mapping') + + # get title info + title_value_key = 'title.@value' + title_lang_key = 'title.@attributes.xml:lang' + if title_value_key in item_map: + title_languages = [] + _title_key_str = '' + if title_lang_key in item_map: # get language - title_languages, _title_key = get_data_by_property( - record, item_map, key) - # get value - title_values, _title_key1 = get_data_by_property( - record, item_map, prefix + '.@value') - title_name = selected_value_by_language( - title_languages, - title_values, - _title_key, - _title_key1, - current_lang, - record) - elif key == 'rights.@value': - _rights_values, _rights_key = get_data_by_property( - record, item_map, key) - if _rights_key: - _get_rights_title(rights_values, _rights_key, - _rights_values, current_lang, meta_options) - elif key == 'accessRights.@value': - accessRights, _access_rights_key = get_data_by_property( - record, item_map, key) - if accessRights and len(accessRights) > 0: - accessRight = accessRights[0] + title_languages, _title_key_str = get_data_by_property( + record, item_map, title_lang_key) + # get value + title_values, _title_key1_str = get_data_by_property( + record, item_map, title_value_key) + title_name = selected_value_by_language( + title_languages, + title_values, + _title_key_str, + _title_key1_str, + current_lang, + record, + meta_options, + hide_list) + # get rights info + rights_value_key = 'rights.@value' + if rights_value_key in item_map: + key_list = item_map.get(rights_value_key) + for k in key_list.split(","): + subkey_list = k.split('.') + _rights_values = [] + attribute = record.get(subkey_list[0]) + if attribute: + data_result = get_sub_item_value(attribute, subkey_list[-1]) + if data_result: + if isinstance(data_result, list): + for value in data_result: + _rights_values.append(value) + elif isinstance(data_result, str): + _rights_values.append(data_result) + prop_hidden = meta_options.get(subkey_list[0], {}).get('option', {}).get('hidden', False) + if not prop_hidden and (subkey_list[0] not in hide_list or subkey_list[-1] not in hide_list): + _get_rights_title(rights_values, k, _rights_values, + current_lang, meta_options) + # get accessRights info + accessRights_value_key = 'accessRights.@value' + if accessRights_value_key in item_map: + key_list = item_map.get(accessRights_value_key) + for k in key_list.split(","): + subkey_list = k.split('.') + prop_hidden = meta_options.get(subkey_list[0], {}).get('option', {}).get('hidden', False) + attribute = record.get(subkey_list[0]) + if attribute and not prop_hidden and (subkey_list[0] not in hide_list or subkey_list[-1] not in hide_list): + data_result = get_sub_item_value(attribute, subkey_list[-1]) + if data_result: + if isinstance(data_result, list) and len(data_result) > 0: + accessRight = data_result[0] + break + elif isinstance(data_result, str): + accessRight = data_result + break pdfcoverpage_set_rec = PDFCoverPageSettings.find(1) # Check if user has the permission to download original pdf file @@ -563,7 +599,7 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti display_stats = display_setting.get('display_stats') else: display_stats = True - + items_display_settings = AdminSettings.get(name='items_display_settings', dict_to_object=False) if items_display_settings: @@ -605,12 +641,9 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti # Hide email of creator in pdf cover page if record.get('item_type_id'): item_type_id = record['item_type_id'] - is_show_email = is_show_email_of_creator(item_type_id) - if not is_show_email: - # list_hidden = get_ignore_item(record['item_type_id']) - # record = hide_by_itemtype(record, list_hidden) - record = hide_by_email(record) - + + record = hide_by_email(record, False) + # Remove hide item from weko_items_ui.utils import get_ignore_item list_hidden = get_ignore_item(record['item_type_id']) @@ -687,7 +720,7 @@ def _get_rights_title(result, rights_key, rights_values, current_lang, meta_opti flg_display_resourcetype = current_app.config.get('WEKO_RECORDS_UI_DISPLAY_RESOURCE_TYPE') , search_author_flg=search_author_flg, billing_settings=billing_settings, - + **ctx, **kwargs ) @@ -786,7 +819,7 @@ def handle_over_max_file_size(error): flash(_('PDF cover page settings have been updated.'), category='success') return redirect('/admin/pdfcoverpage') - + return redirect('/admin/pdfcoverpage') @@ -960,14 +993,14 @@ def get_uri(): """_summary_ --- post: - description: + description: requestBody: required: true content: application/json: {"uri":"https://localhost/record/1/files/001.jpg","pid_value":"1","accessrole":"1"} responses: 200: - """ + """ data = request.get_json() uri = data.get('uri') pid_value = data.get('pid_value') @@ -1011,18 +1044,35 @@ def charge(): title : タイトル price : 支払い金額 - Response parameter(json): - status : - success : 課金成功 - already : 課金済み - error : 課金失敗 - credit_error : 課金失敗(クレジットカード情報の不備) + Response parameter + json: + status: + already : 課金済み + error : 課金失敗 + credit_error : 課金失敗(クレジットカード情報の不備) + redirect: + カード会社の3DS2.0画面へのリダイレクトURL ''' + # Get session_id + session_id = request.cookies.get('session') + item_id = request.values.get('item_id') file_name = request.values.get('file_name') title = request.values.get('title') price = request.values.get('price') file_url = current_app.config['THEME_SITEURL'] + f'/record/{item_id}/files/{file_name}' + ret_url = urljoin( + current_app.config['THEME_SITEURL'], + url_for('weko_records_ui.charge_secure', session_id=session_id.split('.')[0]), + ) + + # 課金中のアイテムIDをキャッシュから取得 + redis_connection = RedisConnection() + datastore = redis_connection.connection(db=current_app.config['CACHE_REDIS_DB'], kv=True) + cache_key = f'charge_{current_user.id}' + if datastore.redis.exists(cache_key): + # 課金中だったら課金しない + return jsonify({'status': 'error'}) # 課金チェック try: @@ -1037,14 +1087,14 @@ def charge(): # 課金予約 try: - trade_id = create_charge(current_user.id, int(item_id), price, title, file_url) - if trade_id in ['connection_error', 'api_error']: + redirect_url = create_charge(current_user.id, int(item_id), price, title, file_url, ret_url) + if redirect_url in ['connection_error', 'api_error']: # 課金失敗 return jsonify({'status': 'error'}) - if trade_id == 'credit_error': + if redirect_url == 'credit_error': # 課金失敗(クレジットカード情報の不備) return jsonify({'status': 'credit_error'}) - if trade_id == 'already': + if redirect_url == 'already': # 課金済みだったら課金しない return jsonify({'status': 'already'}) except Exception as e: @@ -1052,14 +1102,88 @@ def charge(): current_app.logger.error(e) return abort(500) + if not redirect_url: + # 課金失敗 + return jsonify({'status': 'error'}) + + # 課金中のアイテムIDをキャッシュに保存 + datastore.put(cache_key, str(item_id).encode('utf-8'), ttl_secs=300) + + return jsonify({'redirect_url': redirect_url}) + +@blueprint.route('/charge/secure/', methods=['POST']) +def charge_secure(session_id): + """3DS2.0認証後の課金処理を行う。 + + Request parameter: + session_id : セッションID(URLパラメータ) + AccessID : 課金予約時に取得したaccess_id(POSTパラメータ) + + Response parameter + json: + status: + success : 課金成功 + error : 課金失敗 + """ + access_id = request.values.get('AccessID') + redis_connection = RedisConnection() + + if not current_user.is_authenticated: + restore_session_info(session_id, redis_connection) + + # 課金中のアイテムIDをキャッシュから取得 + datastore = redis_connection.connection(db=current_app.config['CACHE_REDIS_DB'], kv=True) + cache_key = f'charge_{current_user.id}' + if not datastore.redis.exists(cache_key): + return redirect('/') + + item_id = datastore.redis.get(cache_key).decode('utf-8') + redirect_url = '/records/{}'.format(item_id) + datastore.delete(cache_key) + + # 3DS2.0認証後決済 + try: + trade_id = secure_charge(current_user.id, access_id) + if trade_id in ['connection_error', 'api_error']: + # 課金失敗 + return redirect(redirect_url) + except Exception as e: + current_app.logger.error(f'Error in secure_charge: user: {current_user.id}, access_id: {access_id}') + current_app.logger.error(e) + return abort(500) + # 課金確定 - try: - charge_result = close_charge(current_user.id, trade_id) - if charge_result: + try: + close_charge(current_user.id, trade_id) + return redirect(redirect_url) + except Exception as e: + current_app.logger.error(f'Error in close_charge: user: {current_user.id}, trade_id: {trade_id}') + current_app.logger.error(e) + return abort(500) + +@blueprint.route('/charge/show', methods=['GET']) +def charge_show(): + """課金済みかどうかを確認する。 + + Request parameter: + item_id : アイテムID + + Response parameter + json: + status: + success : 課金済み + error : 課金失敗 + """ + item_id = request.values.get('item_id') + # 課金チェック + try: + charge_result = check_charge(current_user.id, int(item_id)) + if charge_result == 'already': + # 課金済みの場合、課金成功 return jsonify({'status': 'success'}) else: return jsonify({'status': 'error'}) except Exception as e: - current_app.logger.error(f'Error in close_charge: user: {current_user.id}, trade_id: {trade_id}') + current_app.logger.error(f'Error in check_charge: user: {current_user.id}, item_id: {item_id}') current_app.logger.error(e) return abort(500) diff --git a/modules/weko-records/tests/data/get_mapping.json b/modules/weko-records/tests/data/get_mapping.json index 98130aca07..526ae7c3f7 100644 --- a/modules/weko-records/tests/data/get_mapping.json +++ b/modules/weko-records/tests/data/get_mapping.json @@ -8,8 +8,8 @@ "system_file.extent.@value": "system_file.subitem_systemfile_size", "system_file.version.@value": "system_file.subitem_systemfile_version", "system_file.mimeType.@value": "system_file.subitem_systemfile_mimetype", - "title.@value": "item_15512643084872.subitem_15512556472252", - "title.@attributes.xml:lang": "item_15512643084872.subitem_1551255648112", + "title.@value": "item_1551264308487.subitem_1551255647225,item_15512643084872.subitem_15512556472252", + "title.@attributes.xml:lang": "item_1551264308487.subitem_1551255648112,item_15512643084872.subitem_1551255648112", "alternative.@value": "item_1551264326373.subitem_1551255720400", "alternative.@attributes.xml:lang": "item_1551264326373.subitem_1551255721061", "creator.givenName.@value": "item_1551264340087.subitem_1551255991424.subitem_1551256006332", diff --git a/modules/weko-records/tests/data/item_map.json b/modules/weko-records/tests/data/item_map.json new file mode 100644 index 0000000000..667dc7619e --- /dev/null +++ b/modules/weko-records/tests/data/item_map.json @@ -0,0 +1,35 @@ +{ + "test1.@value": "item_test1_5.test_value1", + "test1.@attributes.xml:lang": "item_test1_5.test_item_lang1", + "test1_2.@value": "item_test1_2", + "test1_2.@attributes.xml:lang": "item_test1_2.test_item_lang1", + "test1_3.@value": "item_test1_3.test_value1", + "test1_3.@attributes.test": "item_test1_3", + "test1_4.@value": "item_test1_4.test_value1", + "test1_4.@attributes.test": "item_test1_4", + "title.@value": "item_test2_title.test_title", + "title.@attributes.xml:lang": "item_test2_title.test_title_lang", + "file.URI.@value": "item_test3_file.url.url", + "file.URI.@attributes.label": "item_test3_file.url.label", + "file.URI.@attributes.objectType": "item_test3_file.url.objectType", + "file.mimeType.@value": "item_test3_file.format", + "type.@value": "item_resource_type.resourcetype", + "type.@attributes.rdf:resource": "item_resource_type.resourceuri", + "description.@value": "item_description.subitem_description_value", + "description.@attributes.xml:lang": "item_description.subitem_description_lang", + "date.@value": "item_date.subitem_date_value", + "date.@attributes.dateType": "item_date.subitem_date_type", + "publisher.@value": "item_publisher.subitem_publisher", + "publisher.@attributes.xml:lang": "item_publisher.subitem_publisherLang", + "creator.creatorName.@value": "item_test5_creator.creatorNames.creatorName", + "creator.creatorName.@attributes.xml:lang": "item_test5_creator.creatorName.creatorNameLang", + "issue": "item_test6_bibliographic.bibliographic_issue", + "volume": "item_test6_bibliographic.bibliographic_volume", + "pageStart": "item_test6_bibliographic.bibliographic_pageStart", + "pageEnd": "item_test6_bibliographic.bibliographic_pageEnd", + "sourceTitle.@value": "item_test6_bibliographic.bibliographic_titles.bibliographic_title", + "sourceTitle.@attributes.xml:lang": "item_test6_bibliographic.bibliographic_titles.bibliographic_titleLang", + "system_file.URI.@value": "system_file.subitem_systemfile_filename_uri", + "system_file.URI.@attributes.label": "system_file.subitem_systemfile_filename_label", + "system_file.URI.@attributes.objectType": "system_file.subitem_systemfile_filename_type" +} diff --git a/modules/weko-records/tests/data/meta00.json b/modules/weko-records/tests/data/meta00.json index 83a1b2db24..dce1a98b51 100644 --- a/modules/weko-records/tests/data/meta00.json +++ b/modules/weko-records/tests/data/meta00.json @@ -25,7 +25,11 @@ { "subitem_1551255905565": "\u4f5c\u8005", "subitem_1551255907416": "ja" - } + }, + { + "subitem_1551255905565": "Creator", + "subitem_1551255907416": "en" + } ] } ] @@ -183,4 +187,4 @@ "item_type_id": "12", "control_number": "870" } -] \ No newline at end of file +] diff --git a/modules/weko-records/tests/test_api.py b/modules/weko-records/tests/test_api.py index 25805ef46d..16fff0afa7 100644 --- a/modules/weko-records/tests/test_api.py +++ b/modules/weko-records/tests/test_api.py @@ -76,14 +76,14 @@ def __init__(self, id, version_id, created, updated): assert result["created"] == "yesterday" assert result["updated"] == "now" assert record.validate(validator=Draft4Validator)==None - + schema = { 'type': 'object', 'properties': { 'id': { 'type': 'integer' }, 'version_id': { 'type': 'integer' }, 'created': {'type': 'string' }, - 'updated': { 'type': 'string' }, + 'updated': { 'type': 'string' }, }, 'required': ['id'] } @@ -98,7 +98,7 @@ def __init__(self, id, version_id, created, updated): test_model.__getitem__.side_effect = data.__getitem__ record = RecordBase(data) record.model = test_model - assert record.validate(validator=Draft4Validator) == None + assert record.validate(validator=Draft4Validator) == None assert record.replace_refs()=={'id': 1, 'version_id': 10, 'created': 'yesterday', 'updated': 'now', '$schema': {'type': 'object', 'properties': {'id': {'type': 'integer'}, 'version_id': {'type': 'integer'}, 'created': {'type': 'string'}, 'updated': {'type': 'string'}}, 'required': ['id']}} # class ItemTypeNames(RecordBase): @@ -362,7 +362,7 @@ def all(): filter_magicmock.all = all return filter_magicmock - + def myfilter_2(item): def all_2(): all2_magicmock = MagicMock() @@ -401,7 +401,7 @@ def all_2(): result=item_type, schema=_schema ) != None - + with patch("weko_records.api.db.session.query", return_value=data1): assert test.update_item_type( form=_form, @@ -851,7 +851,7 @@ def test_itemtypes_restore(app, db): # def revisions(self): def test_revision_ItemTypes(app): test = ItemTypes(data={}) - + # Exception coverage try: test.revisions() @@ -859,7 +859,7 @@ def test_revision_ItemTypes(app): pass test.model = True - + with patch('weko_records.api.RevisionsIterator', return_value=MagicMock()): assert test.revisions() != None @@ -1275,7 +1275,7 @@ def all_func(): all_magicmock = MagicMock() all_magicmock.id = 1 return [all_magicmock] - + data1.query = MagicMock() data1.query.filter_by = MagicMock() data1.query.filter_by.all = all_func @@ -1391,7 +1391,7 @@ def test_item_metadata_revert(app, db): # def revisions(self): def test_revision_ItemsMetadata(app): test = ItemsMetadata(data={}) - + # Exception coverage try: test.revisions() @@ -1427,7 +1427,7 @@ def test_files_metadata_create(app, db): def test_files_metadata_get_record(app, db): FilesMetadata.create(data={'data': 'test'}, pid=1, con=bytes('test content', 'utf-8')) FilesMetadata.create(data={}) - + record = FilesMetadata.get_record(1) assert record.id==1 assert record.model.pid==1 @@ -1469,7 +1469,7 @@ def test_files_metadata_get_records(app, db): # def patch(self, patch): def test_patch_FilesMetadata(app): test = FilesMetadata(data={}) - + with patch("weko_records.api.apply_patch", return_value=""): test.patch(patch="test") @@ -1555,7 +1555,7 @@ def test_files_metadata_revert(app, db): # def revisions(self): def test_revision_FilesMetadata(app): test = FilesMetadata(data={}) - + # Exception coverage try: test.revisions() @@ -1563,7 +1563,7 @@ def test_revision_FilesMetadata(app): pass test.model = True - + with patch('weko_records.api.RevisionsIterator', return_value=MagicMock()): assert test.revisions() != None @@ -1668,7 +1668,7 @@ def get_func(item1, item2): # def depid(self): def test_depid_WekoRecord(app): test = WekoRecord(data={}) - + with patch('weko_records.api.PersistentIdentifier', return_value=True): assert test.depid() != None diff --git a/modules/weko-records/tests/test_serializers_dc.py b/modules/weko-records/tests/test_serializers_dc.py index 70fd8a517b..da6285a221 100644 --- a/modules/weko-records/tests/test_serializers_dc.py +++ b/modules/weko-records/tests/test_serializers_dc.py @@ -1,10 +1,42 @@ import pytest +from lxml import etree from mock import patch, MagicMock -from weko_records.serializers.dc import DcWekoEntryExtension +from weko_records.serializers.dc import DcWekoBaseExtension, DcWekoEntryExtension +# .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_dc.py::test_dc_creator -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp +# class DcWekoBaseExtension(JSONSerializer): +# def dc_creator +def test_dc_creator(app): + test = DcWekoBaseExtension() + # creator: None, lang: None + assert test.dc_creator() is None -# class DcWekoEntryExtension(JSONSerializer): + # creator: str, lang: None + result = test.dc_creator(creator="creator1") + assert result == ["creator1"] + + # creator: list, lang: None + result = test.dc_creator(creator=["creator2", "creator3"], replace=True) + assert result == ["creator2", "creator3"] + + # creator: str, lang: str + result = test.dc_creator(creator="creator4", lang="ja", replace=True) + assert result == ["creator4"] + assert test._dcelem_creator_lang == ["ja"] + + # creator: str, lang: list + result = test.dc_creator(creator="creator5", lang=["en"], replace=True) + assert result == ["creator5"] + assert test._dcelem_creator_lang == ["en"] + + # creator: list, lang: list, replace=False（既存値に追加されることを確認） + result = test.dc_creator(creator=["creator6", "creator7"], lang=["de", "it"], replace=False) + assert result == ["creator5", "creator6", "creator7"] + assert test._dcelem_creator_lang == ["en", "de", "it"] + + +# class DcWekoEntryExtension(JSONSerializer): # def extend_atom def test_extend_atom(app): test = DcWekoEntryExtension() @@ -30,4 +62,35 @@ def _extend_xml(item): assert test.extend_jpcoar( item=item - ) != None \ No newline at end of file + ) != None + +# .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_dc.py::test__extend_xml -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp +# class DcWekoEntryExtension(JSONSerializer): +# def _extend_xml +def test__extend_xml(app): + ext = DcWekoBaseExtension() + ext._dcelem_title = ["title1"] + ext._dcelem_creator_lang = ["ja"] + ext._dcelem_creator = ["creator1", "creator2"] + ext._dcelem_publisher = ["publisher1"] + del ext._dcelem_rights + del ext._dcelem_publisher_lang + + item = etree.Element('item') + ext._extend_xml(item) + + titles = item.findall("{http://purl.org/dc/elements/1.1/}title") + assert len(titles) == 1 + assert titles[0].text == "title1" + + creators = item.findall("{http://purl.org/dc/elements/1.1/}creator") + assert len(creators) == 2 + assert creators[0].text == "creator1" + assert creators[0].get("{http://www.w3.org/XML/1998/namespace}lang") == "ja" + assert creators[1].text == "creator2" + assert creators[1].get("{http://www.w3.org/XML/1998/namespace}lang") is None + + publishers = item.findall("{http://purl.org/dc/elements/1.1/}publisher") + assert len(publishers) == 1 + assert publishers[0].text == "publisher1" + assert publishers[0].get("{http://www.w3.org/XML/1998/namespace}lang") is None diff --git a/modules/weko-records/tests/test_serializers_utils.py b/modules/weko-records/tests/test_serializers_utils.py index 1d3a7ee655..1e4c11ab53 100644 --- a/modules/weko-records/tests/test_serializers_utils.py +++ b/modules/weko-records/tests/test_serializers_utils.py @@ -1,8 +1,7 @@ import pytest -import uuid -import json -import base64 import copy +from flask import request +from urllib.parse import urlencode from lxml import etree from tests.helpers import json_data from mock import patch, MagicMock @@ -25,11 +24,17 @@ # def get_mapping(item_type_mapping, mapping_type): # .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test_get_mapping -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp -def test_get_mapping(): +def test_get_mapping(app): mapping = json_data("data/item_type_mapping.json") - result = get_mapping(mapping, 'jpcoar_mapping') - data = json_data("data/get_mapping.json") - assert result == data + item_type_list = list(mapping.keys()) + item_type_list.append("dummy") + mock_return = MagicMock() + mock_return.render.get.return_value = item_type_list + with patch("weko_records.api.Mapping.get_record", return_value=mapping), \ + patch("weko_records.api.ItemTypes.get_by_id", return_value=mock_return): + result = get_mapping("item_type_id", "jpcoar_mapping") + data = json_data("data/get_mapping.json") + assert result == data # def get_full_mapping(item_type_mapping, mapping_type): # .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test_get_full_mapping -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp @@ -160,121 +165,123 @@ def test_get_wekolog_2(db, record2): True)] @pytest.mark.parametrize("render, form, mapping, hit, licence", params) def test_open_search_detail_data(app, db, db_index, record1, render, form, mapping, hit, licence): - def fetcher(obj_uuid, data): - assert obj_uuid=="1" - return PersistentIdentifier(pid_type='recid', pid_value=data['pid']) - _item_type_name=ItemTypeName(name="test") - _item_type = ItemTypes.create( - name="test", - item_type_name=_item_type_name, - schema=json_data("data/item_type/item_type_schema.json"), - render=json_data(render), - form=json_data(form), - tag=1 - ) - Mapping.create( - item_type_id=_item_type.id, - mapping=json_data(mapping) - ) - _data = { - 'lang': 'en', - 'log_term': '2021-01' - } - hit_data = json_data(hit) - hit_data['_id'] = record1[0].object_uuid - _search_result = {'hits': {'total': 1, 'hits': [hit_data]}} - detail = OpenSearchDetailData(fetcher, _search_result, 'rss') - with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): - res = detail.output_open_search_detail_data() - res = res.decode('utf-8') - cnt = 1 if res.find('wekolog:terms') else 0 - cnt += 1 if res.find('wekolog:view') else 0 - cnt += 1 if res.find('wekolog:download') else 0 - assert True if cnt == 3 else False - - _data = { - 'lang': 'en', - 'index_id': "1" - } - hit_data1 = copy.deepcopy(hit_data) - hit_data1['_source']['_item_metadata']['pubdate']['attribute_value'] = "" - _search_result = {'hits': {'total': 1, 'hits': [hit_data1]}} - detail = OpenSearchDetailData(fetcher, _search_result, 'rss') - with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): - res = detail.output_open_search_detail_data() - res = res.decode('utf-8') - cnt = 1 if res.find('wekolog:terms') == -1 else 0 - cnt += 1 if res.find('wekolog:view') == -1 else 0 - cnt += 1 if res.find('wekolog:download') == -1 else 0 - assert True if cnt == 3 else False - - _search_result = {'hits': {'total': 0, 'hits': []}} - data = OpenSearchDetailData(fetcher, _search_result, 'rss') - with app.test_request_context(): - assert data.output_open_search_detail_data() - - # test for atom - _data = { - 'lang': 'en', - 'index_id': 1, - } - _search_result = {'hits': {'total': 2, 'hits': [hit_data, hit_data]}} - data = OpenSearchDetailData(fetcher, _search_result, 'atom') - with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): - assert data.output_open_search_detail_data() - with patch("weko_records.api.Mapping.get_record", return_value=dict()): - assert data.output_open_search_detail_data() - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=None): - assert data.output_open_search_detail_data() + with patch("weko_records.serializers.utils.get_mapping", return_value=json_data("data/item_map.json")): + def fetcher(obj_uuid, data): + assert obj_uuid=="1" + return PersistentIdentifier(pid_type='recid', pid_value=data['pid']) + _item_type_name=ItemTypeName(name="test") + _item_type = ItemTypes.create( + name="test", + item_type_name=_item_type_name, + schema=json_data("data/item_type/item_type_schema.json"), + render=json_data(render), + form=json_data(form), + tag=1 + ) + Mapping.create( + item_type_id=_item_type.id, + mapping=json_data(mapping) + ) + _data = { + 'lang': 'en', + 'log_term': '2021-01' + } - hit_data2 = json_data("data/record_hit/record_hit1_2.json") - _search_result = {'hits': {'total': 1, 'hits': [hit_data2]}} - data = OpenSearchDetailData(fetcher, _search_result, 'atom') - with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): - assert data.output_open_search_detail_data() - - hit_data3 = copy.deepcopy(hit_data) - hit_data3['_source']['_item_metadata'] = { - "item_type_id":"1", - "control_number": "1", - "item_title":"Test title", - } - _search_result = {'hits': {'total': 1, 'hits': [hit_data3]}} - data = OpenSearchDetailData(fetcher, _search_result, 'atom') - with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): - assert data.output_open_search_detail_data() - - _data = { - 'lang': 'ja', - 'index_id': "aaa", - 'idx': "bbb", - } - hit_data4 = copy.deepcopy(hit_data) - hit_data4['_source']['_item_metadata']['path'] = ["9999"] - _search_result = {'hits': {'total': 1, 'hits': [hit_data4]}} - data = OpenSearchDetailData(fetcher, _search_result, 'atom') - with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): - assert data.output_open_search_detail_data() - - # test for atom with Yhandle - _data = { - 'lang': 'ja', - 'index_id': "1", - 'idx': "bbb", - } - _search_result = {'hits': {'total': 1, 'hits': [hit_data]}} - data = OpenSearchDetailData(fetcher, _search_result, 'atom') - with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): - with patch("invenio_pidstore.models.PersistentIdentifier.get_by_object", return_value=PersistentIdentifier(pid_type='yhdl', pid_value="http://test.com/1000/00000001/")): - res = data.output_open_search_detail_data() + hit_data = json_data(hit) + hit_data['_id'] = record1[0].object_uuid + _search_result = {'hits': {'total': 1, 'hits': [hit_data]}} + detail = OpenSearchDetailData(fetcher, _search_result, 'rss') + with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): + res = detail.output_open_search_detail_data() res = res.decode('utf-8') - assert res.find('') > 0 - _data['index_id'] = "aaa" - with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): - with patch("invenio_pidstore.models.PersistentIdentifier.get_by_object", return_value=PersistentIdentifier(pid_type='yhdl', pid_value="http://test.com/1000/00000001")): - res = data.output_open_search_detail_data() + cnt = 1 if res.find('wekolog:terms') else 0 + cnt += 1 if res.find('wekolog:view') else 0 + cnt += 1 if res.find('wekolog:download') else 0 + assert True if cnt == 3 else False + + _data = { + 'lang': 'en', + 'index_id': "1" + } + hit_data1 = copy.deepcopy(hit_data) + hit_data1['_source']['_item_metadata']['pubdate']['attribute_value'] = "" + _search_result = {'hits': {'total': 1, 'hits': [hit_data1]}} + detail = OpenSearchDetailData(fetcher, _search_result, 'rss') + with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): + res = detail.output_open_search_detail_data() res = res.decode('utf-8') - assert res.find('') > 0 + cnt = 1 if res.find('wekolog:terms') == -1 else 0 + cnt += 1 if res.find('wekolog:view') == -1 else 0 + cnt += 1 if res.find('wekolog:download') == -1 else 0 + assert True if cnt == 3 else False + + _search_result = {'hits': {'total': 0, 'hits': []}} + data = OpenSearchDetailData(fetcher, _search_result, 'rss') + with app.test_request_context(): + assert data.output_open_search_detail_data() + + # test for atom + _data = { + 'lang': 'en', + 'index_id': 1, + } + _search_result = {'hits': {'total': 2, 'hits': [hit_data, hit_data]}} + data = OpenSearchDetailData(fetcher, _search_result, 'atom') + with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): + assert data.output_open_search_detail_data() + with patch("weko_records.api.Mapping.get_record", return_value=dict()): + assert data.output_open_search_detail_data() + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=None): + assert data.output_open_search_detail_data() + + hit_data2 = json_data("data/record_hit/record_hit1_2.json") + _search_result = {'hits': {'total': 1, 'hits': [hit_data2]}} + data = OpenSearchDetailData(fetcher, _search_result, 'atom') + with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): + assert data.output_open_search_detail_data() + + hit_data3 = copy.deepcopy(hit_data) + hit_data3['_source']['_item_metadata'] = { + "item_type_id":"1", + "control_number": "1", + "item_title":"Test title", + } + _search_result = {'hits': {'total': 1, 'hits': [hit_data3]}} + data = OpenSearchDetailData(fetcher, _search_result, 'atom') + with app.test_request_context(headers=[('Accept-Language','en')], query_string=_data): + assert data.output_open_search_detail_data() + + _data = { + 'lang': 'ja', + 'index_id': "aaa", + 'idx': "bbb", + } + hit_data4 = copy.deepcopy(hit_data) + hit_data4['_source']['_item_metadata']['path'] = ["9999"] + _search_result = {'hits': {'total': 1, 'hits': [hit_data4]}} + data = OpenSearchDetailData(fetcher, _search_result, 'atom') + with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): + assert data.output_open_search_detail_data() + + # test for atom with Yhandle + _data = { + 'lang': 'ja', + 'index_id': "1", + 'idx': "bbb", + } + _search_result = {'hits': {'total': 1, 'hits': [hit_data]}} + data = OpenSearchDetailData(fetcher, _search_result, 'atom') + with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): + with patch("invenio_pidstore.models.PersistentIdentifier.get_by_object", return_value=PersistentIdentifier(pid_type='yhdl', pid_value="http://test.com/1000/00000001/")): + res = data.output_open_search_detail_data() + res = res.decode('utf-8') + assert res.find('') > 0 + _data['index_id'] = "aaa" + with app.test_request_context(headers=[('Accept-Language','ja')], query_string=_data): + with patch("invenio_pidstore.models.PersistentIdentifier.get_by_object", return_value=PersistentIdentifier(pid_type='yhdl', pid_value="http://test.com/1000/00000001")): + res = data.output_open_search_detail_data() + res = res.decode('utf-8') + assert res.find('') > 0 sample = OpenSearchDetailData( @@ -287,7 +294,7 @@ def fetcher(obj_uuid, data): ) # class OpenSearchDetailData: -# def output_open_search_detail_data(self): +# def output_open_search_detail_data(self): # .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test_output_open_search_detail_data -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp def test_output_open_search_detail_data(app): item_map = { @@ -421,10 +428,10 @@ def test__set_publication_date(app): with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=["date"]): sample_copy._set_publication_date(fe=fe, item_map=item_map, item_metadata=item_metadata) - + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value={"date.@value": "date.@value"}): sample_copy._set_publication_date(fe=fe, item_map=item_map, item_metadata=item_metadata) - + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value={"date.@value": ["date.@value"]}): sample_copy._set_publication_date(fe=fe, item_map=item_map, item_metadata=item_metadata) @@ -446,7 +453,7 @@ def test__set_publication_date(app): with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=data2): sample_copy._set_publication_date(fe=fe, item_map=item_map, item_metadata=item_metadata) - + # def _set_source_identifier(self, fe, item_map, item_metadata): def test__set_source_identifier(app): @@ -456,7 +463,7 @@ def test__set_source_identifier(app): def issn(item): return item - + fe.prism.issn = issn item_map = { @@ -485,14 +492,14 @@ def issn(item): with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None - + item_metadata["sourceIdentifier"] = [ "sourceIdentifier" ] with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None - + sample_copy.output_type = "not_atom" item_metadata["sourceIdentifier"] = "ISSN" @@ -503,8 +510,8 @@ def issn(item): item_metadata["sourceIdentifier"] = ["ISSN"] assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None - +# .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test__set_author_info -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp # def _set_author_info(self, fe, item_map, item_metadata, request_lang): def test__set_author_info(app): sample_copy = copy.deepcopy(sample) @@ -540,8 +547,156 @@ def test__set_author_info(app): assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang=request_lang) == None + # output_typeがRSSでget_pair_valueが複数値を返し、request_langが一致する場合 + sample_copy = copy.deepcopy(sample) + sample_copy.output_type = sample_copy.OUTPUT_RSS + fe = MagicMock() + fe.dc = MagicMock() + item_map = {"creator.creatorName.@value": "creator.creatorName"} + item_metadata = {"creator": {"attribute_value_mlt": []}} + # get_pair_valueが2件返し、request_langが'en'の場合 + with patch("weko_records_ui.utils.get_pair_value", return_value=[("Taro", "en"), ("Jiro", "ja")]): + sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") + fe.dc.dc_creator.assert_called_once_with("Taro", "en") + # 'ja'は呼ばれない + calls = [call[0] for call in fe.dc.dc_creator.call_args_list] + assert ("Jiro", "ja") not in calls + + # 異常系テスト + + # 1. item_mapに'creator.creatorName.@value'がない場合 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + item_map = {} + item_metadata = {} + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") is None + fe.dc.dc_creator.assert_not_called() + fe.author.assert_not_called() + + # 2. item_idがitem_metadataにない場合 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + item_map = {"creator.creatorName.@value": "notfound.creatorName"} + item_metadata = {} + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") is None + fe.dc.dc_creator.assert_not_called() + fe.author.assert_not_called() + + # 3. RSS分岐: get_pair_valueが空リスト + fe = MagicMock() + fe.dc = MagicMock() + sample_copy.output_type = sample_copy.OUTPUT_RSS + item_map = {"creator.creatorName.@value": "creator.creatorName"} + item_metadata = {"creator": {"attribute_value_mlt": []}} + with patch("weko_records_ui.utils.get_pair_value", return_value=[]): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") is None + fe.dc.dc_creator.assert_not_called() + + # 4. RSS形式でdc_creator_dataに言語情報が無い場合 + fe = MagicMock() + fe.dc = MagicMock() + sample_copy.output_type = sample_copy.OUTPUT_RSS + item_map = {"creator.creatorName.@value": "creator.creatorName"} + item_metadata = {"creator": {"attribute_value_mlt": []}} + # get_pair_valueが言語情報なし(None, 空文字)を返す場合 + with patch("weko_records_ui.utils.get_pair_value", return_value=[("Taro", None), ("Jiro", "")]): + sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") + fe.dc.dc_creator.assert_not_called() + + # 5. RSS分岐: get_pair_valueが値あり、request_langなし + fe = MagicMock() + fe.dc = MagicMock() + with patch("weko_records_ui.utils.get_pair_value", return_value=[("Taro", "ja")]): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang=None) is None + fe.dc.dc_creator.assert_called_with("Taro", "ja") + + # 6. ATOM分岐: get_metadata_from_mapがNone + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + sample_copy.output_type = sample_copy.OUTPUT_ATOM + item_map = {"creator.creatorName.@value": "creator.creatorName"} + item_metadata = {"creator": {}} + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=None): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") is None + fe.author.assert_not_called() + + # 7. ATOM分岐: creator_metadata.get(create_name_key)がNone + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value={}): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata=item_metadata, request_lang="en") is None + fe.author.assert_not_called() + + # 8. ATOM分岐: creator_name_langsがリストでrequest_lang一致 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + creator_metadata = { + "creator.creatorName": ["Taro", "Jiro"], + "creator.creatorNames.creatorNameLang": ["en", "ja"] + } + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=creator_metadata): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata={"creator": {}}, request_lang="en") is None + fe.author.assert_any_call({'name': "Taro", 'lang': "en"}) + + # 9. ATOM分岐: creator_name_langsが単一値 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + creator_metadata = { + "creator.creatorName": "Taro", + "creator.creatorNames.creatorNameLang": "en" + } + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=creator_metadata): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata={"creator": {}}, request_lang="en") is None + fe.author.assert_any_call({'name': "Taro", 'lang': "en"}) + + # 10. ATOM分岐: creator_name_langsが単一値でrequest_lang一致しない場合 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + creator_metadata = { + "creator.creatorName": "Taro", + "creator.creatorNames.creatorNameLang": "en" + } + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=creator_metadata): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata={"creator": {}}, request_lang="ja") is None + fe.author.assert_not_called() + + # 11. ATOM分岐: creator_name_langsがリストでrequest_langが無い場合 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + creator_metadata = { + "creator.creatorName": ["Taro", "Jiro"], + "creator.creatorNames.creatorNameLang": ["en", "ja"] + } + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=creator_metadata): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata={"creator": {}}, request_lang=None) is None + fe.author.assert_any_call({'name': "Taro", 'lang': "en"}) + fe.author.assert_any_call({'name': "Jiro", 'lang': "ja"}) + assert fe.author.call_count == 2 + + # 12. ATOM分岐: creator_name_langsが単一値でrequest_langが無い場合 + fe = MagicMock() + fe.dc = MagicMock() + fe.author = MagicMock() + creator_metadata = { + "creator.creatorName": "Taro", + "creator.creatorNames.creatorNameLang": "en" + } + with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=creator_metadata): + assert sample_copy._set_author_info(fe=fe, item_map=item_map, item_metadata={"creator": {}}, request_lang=None) is None + fe.author.assert_any_call({'name': "Taro", 'lang': "en"}) + assert fe.author.call_count == 1 + + -# def _set_publisher(self, fe, item_map, item_metadata, request_lang): +# def _set_publisher(self, fe, item_map, item_metadata, request_lang): def test__set_publisher(app): sample_copy = copy.deepcopy(sample) fe = MagicMock() diff --git a/modules/weko-records/tests/test_utils.py b/modules/weko-records/tests/test_utils.py index 3244c6eac1..8c014f0866 100644 --- a/modules/weko-records/tests/test_utils.py +++ b/modules/weko-records/tests/test_utils.py @@ -443,8 +443,8 @@ def test_get_all_items(admin_settings,roles): # 55 role_name = 'role_name' _nlst2 = [ - {'billing': ['billing_file'], - 'priceinfo': [{'tax': 'include_tax', 'price': '110', 'billingrole': role_name, 'has_role': True}], + {'billing': ['billing_file'], + 'priceinfo': [{'tax': 'include_tax', 'price': '110', 'billingrole': role_name, 'has_role': True}], } ] _klst2 = [ @@ -458,8 +458,8 @@ def test_get_all_items(admin_settings,roles): role_name = '6' _nlst2 = [ - {'billing': ['billing_file'], - 'priceinfo': [{'tax': 'include_tax', 'price': '110', 'billingrole': role_name, 'has_role': True}], + {'billing': ['billing_file'], + 'priceinfo': [{'tax': 'include_tax', 'price': '110', 'billingrole': role_name, 'has_role': True}], } ] res2 = get_all_items(_nlst2, _klst2) @@ -684,7 +684,7 @@ async def test_sort_meta_data_by_options_sample_1(i18n_app, db, admin_settings): mapping={} ) settings = AdminSettings.get("items_display_settings") - + data1 = { "key1.@attributes.xml:lang": "test", } @@ -710,7 +710,7 @@ async def test_sort_meta_data_by_options_sample_1(i18n_app, db, admin_settings): "meta_list_9999": "meta_list_9999", } } - + with patch("weko_records.serializers.utils.get_mapping", return_value=data1): with patch("weko_search_ui.utils.get_data_by_property", return_value=("1", "2")): await sort_meta_data_by_options(record_hit,settings,item_type_mapping,item_type) @@ -832,7 +832,7 @@ async def test_sort_meta_data_by_options_sample_1(i18n_app, db, admin_settings): "meta_list": { "meta_list_9999": "meta_list_9999", } - } + } await sort_meta_data_by_options( record_hit=record_hit_2, @@ -1147,17 +1147,45 @@ def test_selected_value_by_language(app, meta): _val_id = 'item_1551264308487.subitem_1551255647225' res = selected_value_by_language(['ja', 'en'], ['タイトル日本語', 'Title'], _lang_id, _val_id, 'en', meta[0]) assert res=='Title' + app.config['WEKO_RECORDS_UI_LANG_DISP_FLG'] = False _lang_id = 'item_1551264340087.subitem_1551255898956.subitem_1551255907416' _val_id = 'item_1551264340087.subitem_1551255898956.subitem_1551255905565' res = selected_value_by_language(['ja'], ['作者'], _lang_id, _val_id, 'en', meta[0]) assert res=='作者' + app.config['WEKO_RECORDS_UI_LANG_DISP_FLG'] = True res = selected_value_by_language(['en'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) assert res=='Creator' + res = selected_value_by_language(['en'], ['Creator'], _lang_id, _val_id, 'en', meta[0],{},['item_1551264340087.subitem_1551255898956.subitem_1551255905565','dummy']) + assert res == None + res = selected_value_by_language([], ['Creator'], _lang_id, _val_id, 'ja', meta[0]) + assert res == '作者' + with patch("weko_records.utils.check_info_in_metadata", return_value="en"): res = selected_value_by_language(["ja-Latn"], ['ja-Latn'], _lang_id, _val_id, 'en', meta[0]) assert res=='en' + with patch("weko_records.utils.check_info_in_metadata", side_effect=[None, "expected"]): + res = selected_value_by_language(['en'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res=="expected" + with patch("weko_records.utils.check_info_in_metadata", side_effect=[None, "expected"]): + res = selected_value_by_language(['ja-Latn'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res=="expected" + with patch("weko_records.utils.check_info_in_metadata", side_effect=[None, None, None, None, None]): + res = selected_value_by_language(['en'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res==None + with patch("weko_records.utils.check_info_in_metadata", side_effect=[None, None]): + res = selected_value_by_language(['ja'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res==None + with patch("weko_records.utils.check_info_in_metadata", side_effect=[None, None]): + res = selected_value_by_language([''], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res==None + + _lang_id = 'item_1551264308487.subitem_1551255648112' + _val_id = 'item_1551264340087.subitem_1551255898956.subitem_1551255905565' + res = selected_value_by_language(['en'], ['Creator'], _lang_id, _val_id, 'en', meta[0]) + assert res == '作者' + # .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_selected_value_by_language_2 -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp def test_selected_value_by_language_2(app, meta): @@ -1176,20 +1204,11 @@ def test_selected_value_by_language_3(app): res = selected_value_by_language(["en"], ['title_with_nolang','title_en'], _lang_id, _val_id, 'ja', meta) assert res!='title_en' assert res=='title_with_nolang' - with patch("weko_records.utils.check_info_in_metadata", return_value="en"): res = selected_value_by_language(["ja-Latn"], ['ja-Latn'], _lang_id, _val_id, 'en', meta) assert res=='en' -def test_selected_value_by_language_2(app, meta): - _lang_id = 'item_1551264308487.subitem_15512556481122' - _val_id = 'item_1551264308487.subitem_15512556472252' - - with patch("weko_records.utils.check_info_in_metadata", return_value="en"): - res = selected_value_by_language(["en"], ['ja'], _lang_id, _val_id, 'ja', meta[0]) - assert res=='en' - # def check_info_in_metadata(str_key_lang, str_key_val, str_lang, metadata): # .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_check_info_in_metadata -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp def test_check_info_in_metadata(app, meta): @@ -1206,6 +1225,23 @@ def test_check_info_in_metadata(app, meta): _str_key_val = 'item_1551264418667.subitem_1551257245638.subitem_1551257276108' res = check_info_in_metadata(_str_key_lang, _str_key_val, 'en', meta[0]) assert res=='Contributor' + _str_key_lang = 'item_30002_title0.subitem_title_language' + _str_key_val = 'item_30002_title0.subitem_title' + _lang = 'ja' + _meta = {'path': ['1623632832836'], 'pubdate': '2024-12-26', 'item_30002_title0': {'subitem_title': 'title', 'subitem_title_language': 'ja'}, 'item_30002_resource_type13': {'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}, 'item_30002_file35': [{'filename': '日本語.png'}], 'item_1735223788720': {'subitem_thumbnail': [{'thumbnail_label': 'jdcat.jsps.go.jp_about.png', 'thumbnail_url': '/api/files/d88a107f-c3a7-4ad6-81ca-4c34691682d1/jdcat.jsps.go.jp_about.png?versionId=37ad1676-9099-4f82-ba7a-35ef4f325b39'}]}} + res = check_info_in_metadata(_str_key_lang, _str_key_val, _lang, _meta) + assert res is not None + assert res=='title' + _str_key_lang = 'item_30002_title0.subitem_title_language' + _str_key_val = 'item_30002_title0.subitem_title' + _lang = 'ja' + _meta = {'path': ['1623632832836'], 'pubdate': '2024-12-26', 'item_30002_title0': [{'subitem_title': 'title', 'subitem_title_language': 'ja'}], 'item_30002_resource_type13': {'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}, 'item_30002_file35': [{'filename': '日本語.png'}], 'item_1735223788720': {'subitem_thumbnail': [{'thumbnail_label': 'jdcat.jsps.go.jp_about.png', 'thumbnail_url': '/api/files/d88a107f-c3a7-4ad6-81ca-4c34691682d1/jdcat.jsps.go.jp_about.png?versionId=37ad1676-9099-4f82-ba7a-35ef4f325b39'}]}} + res = check_info_in_metadata(_str_key_lang, _str_key_val, _lang, _meta) + assert res=='title' + + + + # def get_value_and_lang_by_key(key, data_json, data_result, stt_key): # .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_get_value_and_lang_by_key -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp @@ -1299,7 +1335,7 @@ def test_get_value_by_selected_lang(app): res = get_value_by_selected_lang(_source_title3, 'en') assert res=='no_lang_test' - + # def get_show_list_author(solst_dict_array, hide_email_flag, author_key, creates): # .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_get_show_list_author -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp @@ -1443,7 +1479,7 @@ def test_format_creates(i18n_app): }] }] _hide_creator_keys = ['familyNames'] - + res = format_creates(_creates, _hide_creator_keys) assert res==[{'creatorName': ['en_name'], 'affiliationName': ['en_af'], 'creatorAlternative': ['en_al']}] @@ -1566,13 +1602,13 @@ def test_get_author_has_language(app): ] res = get_author_has_language(_create, {}, 'en', ['affiliationName', 'af_lang']) assert res=={'affiliationName': ['en_af']} - + res = get_author_has_language(_create, {}, 'fr', ['affiliationName', 'af_lang']) assert res=={'affiliationName': ['en_af']} - + res = get_author_has_language(_create, {}, 'en', ['affiliationName', 'af_lang2']) assert res=={'affiliationName': ['en_af']} - + res = get_author_has_language(_create, {}, 'en', ['affiliationName2', 'af_lang2']) assert res=={} @@ -1586,7 +1622,7 @@ def test_add_author(app): _data = {'creatorName': 'test1'} _s1 = {'key': 'item_1.names[].creatorName'} _s2 = {'key': 'item_1.ids[].nameIdentifier'} - + stt_key, data_result, newline_array = add_author(_data, [], [], _s1, 'value1', {}, False, False, True) assert stt_key==['creatorName'] assert data_result=={'creatorName': {'creatorName': {'value': 'test1'}}} @@ -1635,7 +1671,7 @@ def test_convert_bibliographic(): assert res2=='key name1, key name2, key name3' -# def add_biographic(sys_bibliographic, bibliographic_key, s, stt_key, data_result, +# def add_biographic(sys_bibliographic, bibliographic_key, s, stt_key, data_result, def test_add_biographic(app): def get_bibliographic_list(x): return x @@ -1704,7 +1740,7 @@ def test_replace_fqdn_of_file_metadata(app): 'version_id': '1' } ] - + _file_url = [] replace_fqdn_of_file_metadata(_file_metadata_list1) assert _file_metadata_list1==[{'url': {'url': 'http://test/a'}}, {'url': {'url': 'http://test/b'}}] @@ -1712,4 +1748,3 @@ def test_replace_fqdn_of_file_metadata(app): assert _file_url==['http://test/a', 'http://test/b'] replace_fqdn_of_file_metadata(_file_metadata_list2) assert _file_metadata_list2==[{'url': {'url': 'https://localhost/a'}, 'version_id': '1'}, {'url': {'url': 'https://localhost/b'}, 'version_id': '1'}] - \ No newline at end of file diff --git a/modules/weko-records/weko_records/api.py b/modules/weko-records/weko_records/api.py index 8e696ec46e..eba50da64a 100644 --- a/modules/weko-records/weko_records/api.py +++ b/modules/weko-records/weko_records/api.py @@ -465,7 +465,7 @@ def __get_delete_mapping_key(item_type_mapping, _delete_list): result = [] for key in _delete_list: prop_mapping = item_type_mapping.get(key, {}).get("jpcoar_mapping", {}) - if prop_mapping: + if prop_mapping and isinstance(prop_mapping, dict): result.extend(list(prop_mapping.keys())) return result diff --git a/modules/weko-records/weko_records/serializers/dc.py b/modules/weko-records/weko_records/serializers/dc.py index dc4f8b5699..1eba5e2346 100644 --- a/modules/weko-records/weko_records/serializers/dc.py +++ b/modules/weko-records/weko_records/serializers/dc.py @@ -35,6 +35,8 @@ def __init__(self): """__init__.""" self._dcelem_publisher = None self._dcelem_publisher_lang = None + self._dcelem_creator = None + self._dcelem_creator_lang = None super().__init__() def _extend_xml(self, xml_elem): @@ -56,7 +58,7 @@ def _extend_xml(self, xml_elem): node = etree.SubElement(xml_elem, '{%s}%s' % (DCELEMENTS_NS, elem)) node.text = val - if elem == 'publisher': + if elem == 'publisher' or elem == 'creator': if hasattr(self, '_dcelem_%s_lang' % elem): langs = getattr( self, '_dcelem_%s_lang' % @@ -90,7 +92,34 @@ def dc_publisher(self, publisher=None, lang=None, replace=False): self._dcelem_publisher_lang += lang return self._dcelem_publisher + + def dc_creator(self, creator=None, lang=None, replace=False): + """Get or set the dc:creator. + Which is an entity responsible for + making the resource available. + For more information see: + http://dublincore.org/documents/dcmi-terms + :param creator: Creator or list of creators. + :param replace: Replace alredy set creators (deault: False). + :returns: List of creators. + """ + if creator is not None: + if not isinstance(creator, list): + creator = [creator] + if replace or not self._dcelem_creator: + self._dcelem_creator = [] + self._dcelem_creator += creator + + if lang is not None: + if not isinstance(lang, list): + lang = [lang] + if replace or not self._dcelem_creator_lang: + self._dcelem_creator_lang = [] + self._dcelem_creator_lang += lang + + return self._dcelem_creator + def extend_jpcoar(self, jpcoar_feed): """Extend a JPCOAR feed with the set DC fields. diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index 4fe1d81504..cc3a459061 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -24,7 +24,7 @@ from datetime import datetime import pytz -from flask import request +from flask import request, current_app from invenio_db import db from invenio_pidstore.models import PersistentIdentifier from invenio_records.models import RecordMetadata @@ -32,7 +32,7 @@ from invenio_stats.views import QueryRecordViewCount, QueryFileStatsCount from weko_index_tree.api import Index -from weko_records.api import Mapping +from weko_records.api import Mapping, ItemTypes from weko_records.models import ItemType, ItemTypeName, ItemTypeProperty from .dc import DcWekoBaseExtension, DcWekoEntryExtension @@ -42,11 +42,11 @@ from .wekolog import WekologEntryExtension, WekologExtension -def get_mapping(item_type_mapping, mapping_type): +def get_mapping(item_type_id, mapping_type): """Format itemtype mapping data. [Key:Schema, Value:ItemId] - :param item_type_mapping: + :param item_type_id: :param mapping_type: :return: """ @@ -63,12 +63,18 @@ def get_schema_key_info(schema, parent_key, schema_json={}): return schema_json schema = {} - for item_id, maps in item_type_mapping.items(): - if mapping_type in maps.keys() and isinstance(maps[mapping_type], dict): - item_schema = get_schema_key_info(maps[mapping_type], '', {}) - for k, v in item_schema.items(): - item_schema[k] = item_id + '.' + v if v else item_id - schema.update(item_schema) + item_type_mapping = Mapping.get_record(item_type_id) + item_type_list = ItemTypes.get_by_id(item_type_id).render.get('table_row') + for item_id in item_type_list: + if item_id in item_type_mapping: + maps = item_type_mapping.get(item_id) + if isinstance(maps,dict) and mapping_type in maps.keys() and isinstance(maps[mapping_type], dict): + item_schema = get_schema_key_info(maps[mapping_type], '', {}) + for k, v in item_schema.items(): + if k in schema: + schema[k] += ',' + item_id + '.' + v if v else ',' + item_id + else: + schema[k] = item_id + '.' + v if v else item_id return schema @@ -94,6 +100,7 @@ def get_schema_key_info(schema, parent_key, schema_json={}): return schema_json schema = {} + for item_id, maps in item_type_mapping.items(): if mapping_type in maps.keys() \ and isinstance(maps[mapping_type], dict): @@ -396,12 +403,11 @@ def output_open_search_detail_data(self): item_metadata = hit['_source']['_item_metadata'] item_type_id = item_metadata['item_type_id'] - type_mapping = Mapping.get_record(item_type_id) if item_type_id in jpcoar_map: item_map = jpcoar_map[item_type_id] else: - item_map = get_mapping(type_mapping, 'jpcoar_mapping') + item_map = get_mapping(item_type_id, 'jpcoar_mapping') jpcoar_map[item_type_id] = item_map fe = fg.add_entry() @@ -790,99 +796,130 @@ def _set_publisher(self, fe, item_map, item_metadata, request_lang): publisher_lang) def _set_source_identifier(self, fe, item_map, item_metadata): - if self.output_type == self.OUTPUT_ATOM: - _source_identifier_value = 'sourceIdentifier.@value' - if _source_identifier_value in item_map: - source_identifier_key = item_map[_source_identifier_value] - item_id = source_identifier_key.split('.')[0] + """Retrieve the source identifier from + the item's metadata and set it in the feedentry. - # Get item data - if item_id in item_metadata: - source_identifier_metadata = get_metadata_from_map( - item_metadata[item_id], item_id) - - if not isinstance(source_identifier_metadata, dict) \ - or source_identifier_metadata.get( - source_identifier_key) is None: - return - source_identifiers = source_identifier_metadata.get( - source_identifier_key) - - if source_identifiers: - if isinstance(source_identifiers, list): - for source_identifier in source_identifiers: - fe.prism.issn(source_identifier) + Args: + fe(WekoFeedGenerator): feed entry + item_map(dict): itemtype mapping + item_metadata(dict): item metadata + """ + _source_identifier_value = 'sourceIdentifier.@value' + _source_identifier_attr_type = \ + 'sourceIdentifier.@attributes.identifierType' + try: + if _source_identifier_value in item_map \ + and _source_identifier_attr_type in item_map: + source_identifier_value_keys = \ + item_map[_source_identifier_value].split(',') + source_identifier_attr_type_keys = \ + item_map[_source_identifier_attr_type].split(',') + for source_identifier_value_key, source_identifier_attr_type_key in zip( + source_identifier_value_keys, + source_identifier_attr_type_keys + ): + item_id = source_identifier_value_key.split('.')[0] + # Get item data + if item_id in item_metadata: + source_identifier_metadata = get_metadata_from_map( + item_metadata[item_id], item_id) + if self.output_type == self.OUTPUT_ATOM: + source_identifiers = source_identifier_metadata.get( + source_identifier_value_key) + + if source_identifiers: + if isinstance(source_identifiers, list): + for source_identifier in source_identifiers: + fe.prism.issn(source_identifier) + else: + fe.prism.issn(source_identifiers) else: - fe.prism.issn(source_identifiers) - else: - _source_identifier_attr_type = \ - 'sourceIdentifier.@attributes.identifierType' - _source_identifier_value = 'sourceIdentifier.@value' - if _source_identifier_value in item_map: - item_id = item_map[_source_identifier_value].split('.')[0] - - # Get item data - if item_id in item_metadata: - source_identifier_metadata = get_metadata_from_map( - item_metadata[item_id], item_id) - - source_identifiers = source_identifier_metadata[ - item_map[_source_identifier_attr_type]] - - source_identifier_types = source_identifier_metadata[ - item_map[_source_identifier_value]] - - if source_identifiers: - if isinstance(source_identifiers, list): - for i in range(len(source_identifiers)): - source_identifier_type = \ - source_identifier_types[i] - if source_identifier_type \ - and source_identifier_type == 'ISSN': - fe.prism.issn(source_identifiers[i]) - - elif source_identifier_types \ - and source_identifier_types == 'ISSN': - fe.prism.issn(source_identifiers) + source_identifiers = source_identifier_metadata.get( + source_identifier_value_key) + source_identifier_types = source_identifier_metadata.get( + source_identifier_attr_type_key) + if source_identifiers and source_identifier_types: + if isinstance(source_identifiers, str): + source_identifiers = [source_identifiers] + if isinstance(source_identifier_types, str): + source_identifier_types = [source_identifier_types] + if len(source_identifiers) != len(source_identifier_types): + attr = item_metadata.get(item_id).get("attribute_value_mlt") + if not attr: + continue + for a in attr: + if a.get(source_identifier_attr_type_key.split('.')[1]) \ + and a.get(source_identifier_value_key.split('.')[1]): + continue + value = next(iter(a.values())) + if value in source_identifiers: + source_identifiers.remove(value) + if value in source_identifier_types: + source_identifier_types.remove(value) + for source_identifier_type, source_identifier in zip( + source_identifier_types, source_identifiers + ): + if source_identifier_type == 'ISSN': + fe.prism.issn(source_identifier) + except Exception as ex: + current_app.logger.error(ex) def _set_author_info(self, fe, item_map, item_metadata, request_lang): + from weko_records_ui.utils import get_pair_value _creator_name_value = 'creator.creatorName.@value' if _creator_name_value in item_map: - item_id = item_map[_creator_name_value].split('.')[0] - - # Get item data - if item_id in item_metadata: - creator_metadata = get_metadata_from_map( - item_metadata[item_id], item_id) - - create_name_key = item_map[_creator_name_value] - if not isinstance(creator_metadata, dict) \ - or creator_metadata.get(create_name_key) is None: - return - - creator_names = creator_metadata[create_name_key] - - _creator_name_attr_lang = item_id + '.' + 'creatorNameLang' - creator_name_langs = creator_metadata[ - _creator_name_attr_lang] \ - if _creator_name_attr_lang in creator_metadata else None - - if creator_name_langs: - if isinstance(creator_name_langs, list): - for i in range(len(creator_name_langs)): - creator_name_lang = creator_name_langs[i] + create_name_keys = item_map[_creator_name_value].split(',') + for create_name_key in create_name_keys: + item_id = create_name_key.split('.')[0] + if item_id not in item_metadata: + continue + + _creator_name_attr_lang = item_id + '.creatorNames.creatorNameLang' + + # RSS dc:creator + if self.output_type == self.OUTPUT_RSS: + dc_creator_data = get_pair_value( + create_name_key.split('.')[1:], + _creator_name_attr_lang.split('.')[1:], + item_metadata[item_id]['attribute_value_mlt'] + ) + + for dc_creator_name, dc_creator_name_lang in dc_creator_data: + if dc_creator_name and dc_creator_name_lang: if request_lang: - if creator_name_lang == request_lang: + if dc_creator_name_lang == request_lang: + fe.dc.dc_creator(dc_creator_name, dc_creator_name_lang) + else: + fe.dc.dc_creator(dc_creator_name, dc_creator_name_lang) + else: + # Atom author + creator_metadata = get_metadata_from_map( + item_metadata[item_id], item_id) + if not isinstance(creator_metadata, dict) \ + or creator_metadata.get(create_name_key) is None: + continue + + creator_names = creator_metadata[create_name_key] + creator_name_langs = creator_metadata[ + _creator_name_attr_lang] \ + if _creator_name_attr_lang in creator_metadata else None + + if creator_name_langs: + if isinstance(creator_name_langs, list): + for i in range(len(creator_name_langs)): + creator_name_lang = creator_name_langs[i] + if request_lang: + if creator_name_lang == request_lang: + fe.author({'name': creator_names[i], + 'lang': creator_name_lang}) + else: fe.author({'name': creator_names[i], - 'lang': creator_name_lang}) + 'lang': creator_name_lang}) + else: + if request_lang: + if creator_name_langs == request_lang: + fe.author({'name': creator_names, + 'lang': creator_name_langs}) else: - fe.author({'name': creator_names[i], - 'lang': creator_name_lang}) - else: - if request_lang: - if creator_name_langs == request_lang: fe.author({'name': creator_names, - 'lang': creator_name_langs}) - else: - fe.author({'name': creator_names, - 'lang': creator_name_langs}) + 'lang': creator_name_langs}) diff --git a/modules/weko-records/weko_records/utils.py b/modules/weko-records/weko_records/utils.py index 156c4d912f..7b1a3e060b 100644 --- a/modules/weko-records/weko_records/utils.py +++ b/modules/weko-records/weko_records/utils.py @@ -1059,7 +1059,7 @@ def get_option_value(option_type, parent_option, child_option): ) return result - + def get_value_by_selected_language(values,lang_key,current_lang): dict = convert_array_to_dict(values,lang_key) if dict.get(current_lang): @@ -1068,7 +1068,7 @@ def get_value_by_selected_language(values,lang_key,current_lang): return dict.get("None") elif dict.get("en"): return dict.get("en") - + def get_creator_comments(key,meta_options,creators,is_hide_email): """ TODO: affiliationは未実装 @@ -1081,108 +1081,108 @@ def get_creator_comments(key,meta_options,creators,is_hide_email): if creator.get("creatorMails"): opt = dict["{}.{}".format(key,"creatorMails")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')|is_hide_email): creator.pop("creatorMails") opt = dict["{}.{}.{}".format(key,"creatorMails","creatorMail")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')|is_hide_email): if creator.get("creatorMails"): creator.pop("creatorMails") - + if creator.get("familyNames"): opt = dict["{}.{}".format(key,"familyNames")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("familyNames") else: creator["familyNames"] = get_value_by_selected_language(creator["familyNames"],"familyNameLang",current_lang) opt = dict["{}.{}.{}".format(key,"familyNames","familyName")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("familyNames"): creator.pop("familyNames") - + if creator.get("creatorNames"): opt = dict["{}.{}".format(key,"creatorNames")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("creatorNames") else: creator["creatorNames"] = get_value_by_selected_language(creator["creatorNames"],"creatorNameLang",current_lang) opt = dict["{}.{}.{}".format(key,"creatorNames","creatorName")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("creatorNames"): creator.pop("creatorNames") - + if creator.get("givenNames"): opt = dict["{}.{}".format(key,"givenNames")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("givenNames") else: creator["givenNames"] = get_value_by_selected_language(creator["givenNames"],"givenNameLang",current_lang) opt = dict["{}.{}.{}".format(key,"givenNames","givenName")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("givenNames"): creator.pop("givenNames") - + if creator.get("nameIdentifiers"): opt = dict["{}.{}".format(key,"nameIdentifiers")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("nameIdentifiers") opt = dict["{}.{}.{}".format(key,"nameIdentifiers","nameIdentifierScheme")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("nameIdentifiers"): creator.pop("nameIdentifiers") - + opt = dict["{}.{}.{}".format(key,"nameIdentifiers","nameIdentifierURI")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("nameIdentifiers"): creator.pop("nameIdentifiers") - + opt = dict["{}.{}.{}".format(key,"nameIdentifiers","nameIdentifier")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): if creator.get("nameIdentifiers"): - creator.pop("nameIdentifiers") + creator.pop("nameIdentifiers") if creator.get("creatorAffiliations"): opt = dict["{}.{}".format(key,"creatorAffiliations")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("creatorAffiliations") - + if creator.get("affiliationNameIdentifiers"): opt = dict["{}.{}".format(key,"affiliationNameIdentifiers")] if opt.get('option'): - _opt = opt.get('option') + _opt = opt.get('option') if (_opt.get('hide') | _opt.get('non_display')): creator.pop("affiliationNameIdentifiers") - + if creator.get("creatorNames") or creator.get("familyNames") or creator.get("givenNames"): ret.append(creator) - + return ret - - + + def get_file_comments(record, files): """Check and get file info.""" @@ -1325,56 +1325,45 @@ def get_title_option(solst_dict_array): item_type_id = record_hit["_source"].get("item_type_id") or src.get( "item_type_id" ) - item_map = get_mapping(item_type_mapping, "jpcoar_mapping") - language_dict = {} - suffixes = ".@attributes.xml:lang" - for key in item_map: - if key.find(suffixes) != -1: - # get language - title_languages, _title_key = get_data_by_property(src, item_map, key) - # get value - prefix = key.replace(suffixes, "") - title_values, _title_key1 = get_data_by_property( - src, item_map, prefix + ".@value" - ) - language_dict.update( - { - prefix: { - "lang": title_languages, - "lang-id": _title_key, - "val": title_values, - "val-id": _title_key1, - } - } - ) + item_map = get_mapping(item_type_id, "jpcoar_mapping") + # selected title - title_obj = language_dict.get("title") - if title_obj is not None: - lang_arr = title_obj.get("lang") - val_arr = title_obj.get("val") - lang_id = title_obj.get("lang-id") - val_id = title_obj.get("val-id") - if lang_arr and len(lang_arr) > 0 and lang_arr != "null": - result = selected_value_by_language( - lang_arr, val_arr, lang_id, val_id, web_screen_lang, _item_metadata - ) - if result is not None: - for idx, val in enumerate(record_hit["_source"]["title"]): - if val == result: - arr = [] - record_hit["_source"]["title"][idx] = record_hit["_source"][ - "title" - ][0] - record_hit["_source"]["title"][0] = result - arr.append(result) - record_hit["_source"]["_comment"] = arr - break - elif val_arr and len(val_arr) > 0: - record_hit["_source"]["_comment"] = [val_arr[0]] + from weko_items_ui.utils import get_hide_list_by_schema_form + solst, meta_options = get_options_and_order_list(item_type_id, item_type_data) + hide_list = get_hide_list_by_schema_form(item_type_id) + title_value_key = 'title.@value' + title_lang_key = 'title.@attributes.xml:lang' + title_languages = [] + title_values = [] + _title_key_str = '' + _title_key1_str = '' + if title_value_key in item_map: + if title_lang_key in item_map: + # get language + title_languages, _title_key_str = get_data_by_property( + src, item_map, title_lang_key) + # get value + title_values, _title_key1_str = get_data_by_property( + src, item_map, title_value_key) + if title_languages and len(title_languages) > 0: + result = selected_value_by_language( + title_languages, title_values, _title_key_str, _title_key1_str, web_screen_lang, _item_metadata, meta_options, hide_list + ) + if result is not None: + for idx, val in enumerate(record_hit["_source"]["title"]): + if val == result: + arr = [] + record_hit["_source"]["title"][idx] = record_hit["_source"]["title"][0] + record_hit["_source"]["title"][0] = result + arr.append(result) + record_hit["_source"]["_comment"] = arr + break + elif title_values and len(title_values) > 0: + record_hit["_source"]["_comment"] = [title_values[0]] + record_hit["_source"]["title"][0] = title_values[0] if not item_type_id: return - solst, meta_options = get_options_and_order_list(item_type_id, item_type_data) solst_dict_array = convert_data_to_dict(solst) files_info = [] creator_info = {} @@ -1493,7 +1482,7 @@ def convert_array_to_dict(solst_dict_array,key): dict['None']['creatorName'] = creator_names idx=idx+1 return dict - + def get_keywords_data_load(str): """Get a json of item type info. @@ -1966,7 +1955,7 @@ def __remove_special_character(_s_str: str): def selected_value_by_language( - lang_array, value_array, lang_id, val_id, lang_selected, _item_metadata + lang_array, value_array, lang_key_str, val_key_str, lang_selected, _item_metadata, meta_option={}, hide_list=[] ): """Select value by language. @@ -1976,76 +1965,89 @@ def selected_value_by_language( @param val_id: @param lang_selected: @param _item_metadata: + @param meta_option: + @param hide_list: @return: """ - if ( - (lang_array is not None) - and (value_array is not None) - and isinstance(lang_selected, str) - ): - if len(value_array) < 1: - return None - else: - if len(lang_array) > 0: - for idx, lang in enumerate(lang_array): - lang_array[idx] = lang.strip() - if lang_selected in lang_array: # Web screen display language - value = check_info_in_metadata( - lang_id, val_id, lang_selected, _item_metadata - ) - if value is not None: - return value - - if len(value_array)>len(lang_array): # First title without language code - value0 = value_array[0] - tmp = copy.copy(value_array) - for lang in lang_array: - value = check_info_in_metadata( - lang_id, val_id, lang, _item_metadata - ) - if value and value in tmp: - tmp.remove(value) - if len(tmp)>0 and tmp[0]==value0: - return tmp[0] - - if "ja-Latn" in lang_array: # ja_Latn - value = check_info_in_metadata( - lang_id, val_id, "ja-Latn", _item_metadata - ) - if value is not None: - return value - if "en" in lang_array and ( - lang_selected != "ja" - or not current_app.config.get("WEKO_RECORDS_UI_LANG_DISP_FLG", False) - ): # English - value = check_info_in_metadata(lang_id, val_id, "en", _item_metadata) - if value is not None: - return value - # 1st language when registering items - if len(lang_array) > 0: - noreturn = False - for idx, lg in enumerate(lang_array): - if current_app.config.get( - "WEKO_RECORDS_UI_LANG_DISP_FLG", False - ) and ( - (lg == "ja" and lang_selected == "en") - or (lg == "en" and lang_selected == "ja") - ): - noreturn = True - break - if len(lg) > 0: + result = None + lang_key_list = lang_key_str.split(",") + val_key_list = val_key_str.split(",") + for val_key in val_key_list: + val_parent_key = val_key.split(".")[0] + val_sub_key = val_key.split(".")[-1] + prop_hidden = meta_option.get(val_parent_key, {}).get('option', {}).get('hidden', False) + for h in hide_list: + if h.startswith(val_parent_key) and h.endswith(val_sub_key): + prop_hidden = True + for lang_key in lang_key_list: + if val_parent_key == lang_key.split(".")[0]: + if ( + lang_array is not None + and (value_array is not None and len(value_array) > 0) + and isinstance(lang_selected, str) + and not prop_hidden + ): + if len(lang_array) > 0: + for idx, lang in enumerate(lang_array): + lang_array[idx] = lang.strip() + if lang_selected in lang_array: # Web screen display language value = check_info_in_metadata( - lang_id, val_id, lg, _item_metadata + lang_key, val_key, lang_selected, _item_metadata ) if value is not None: - return value - if noreturn: - return None - # 1st value when registering without language - if len(value_array) > 0: - return value_array[0] - else: - return None + result = value + + if len(value_array)>len(lang_array): # First title without language code + result = check_info_in_metadata(lang_key, val_key, None, _item_metadata) + + if not result and "ja-Latn" in lang_array: # ja_Latn + value = check_info_in_metadata( + lang_key, val_key, "ja-Latn", _item_metadata + ) + if value is not None: + result = value + if not result and "en" in lang_array and ( + lang_selected != "ja" + or not current_app.config.get("WEKO_RECORDS_UI_LANG_DISP_FLG", False) + ): # English + value = check_info_in_metadata(lang_key, val_key, "en", _item_metadata) + if value is not None: + result = value + # 1st language when registering items + if not result and len(lang_array) > 0: + noreturn = False + for idx, lg in enumerate(lang_array): + if current_app.config.get( + "WEKO_RECORDS_UI_LANG_DISP_FLG", False + ) and ( + (lg == "ja" and lang_selected == "en") + or (lg == "en" and lang_selected == "ja") + ): + noreturn = True + break + if len(lg) > 0: + value = check_info_in_metadata( + lang_key, val_key, lg, _item_metadata + ) + if value is not None: + result = value + if noreturn: + result = None + # 1st value when registering without language + if not result and len(value_array) > 0: + result = check_info_in_metadata(lang_key, val_key, None, _item_metadata) + if not result: + break + if not result: + if ( + (value_array is not None and len(value_array) > 0) + and isinstance(lang_selected, str) + and not prop_hidden + ): + result = check_info_in_metadata('', val_key, None, _item_metadata) + if result: + break + return result def check_info_in_metadata(str_key_lang, str_key_val, str_lang, metadata): @@ -2058,8 +2060,7 @@ def check_info_in_metadata(str_key_lang, str_key_val, str_lang, metadata): @return """ if ( - len(str_key_lang) > 0 - and (len(str_lang) > 0 or str_lang is None) + (str_lang is None or len(str_lang) > 0) and len(metadata) > 0 and str_key_val is not None and len(str_key_val) > 0 @@ -2071,34 +2072,49 @@ def check_info_in_metadata(str_key_lang, str_key_val, str_lang, metadata): metadata = ( metadata.get("_item_metadata") if "_item_metadata" in metadata else metadata ) - if str_key_lang[0] in metadata: - obj = metadata.get(str_key_lang[0]).get("attribute_value_mlt") + if str_key_val[0] in metadata: + obj = metadata.get(str_key_val[0]) + if not isinstance(obj,list): + obj = obj.get("attribute_value_mlt",obj) save = obj - for ob in str_key_lang: + for ob in str_key_val: if ( - ob not in str_key_lang[0] - and ob not in str_key_lang[len(str_key_lang) - 1] + ob not in str_key_val[0] + and ob not in str_key_val[len(str_key_val) - 1] ): for x in save: if x.get(ob): save = x.get(ob) - for s in save: - if str_lang is None: - value = s.get(str_key_val[len(str_key_val) - 1]).strip() - if len(value) > 0: - return value - if ( - s - and isinstance(s, dict) - and s.get(str_key_lang[-1]) - and s.get(str_key_val[-1]) - ): + + if isinstance(save, list): + for s in save: + if s is not None and str_lang is None: + value = s + if isinstance(s,dict): + value = s.get(str_key_val[len(str_key_val) - 1]).strip() + if len(value) > 0: + return value + if ( - s.get(str_key_lang[-1]).strip() == str_lang.strip() - and str_key_val[-1] in s - and len(s.get(str_key_val[-1]).strip()) > 0 + s + and isinstance(s, dict) + and s.get(str_key_lang[-1]) + and s.get(str_key_val[-1]) ): - return s.get(str_key_val[-1]) + if ( + s.get(str_key_lang[-1]).strip() == str_lang.strip() + and str_key_val[-1] in s + and len(s.get(str_key_val[-1]).strip()) > 0 + ): + return s.get(str_key_val[-1]) + elif isinstance(save, dict): + if ( + save.get(str_key_lang[-1]) + and save.get(str_key_val[-1]) + and save.get(str_key_lang[-1]).strip() == str_lang.strip() + ): + return save.get(str_key_val[-1]) + return None @@ -2187,10 +2203,10 @@ def get_value_by_selected_lang(source_title, current_lang): if len(title_data_langs_none)>0: source = list(source_title.values())[0] - target = list(title_data_langs_none[0].values())[0] + target = list(title_data_langs_none[0].values())[0] if source==target: return target - + if value_latn: return value_latn diff --git a/modules/weko-search-ui/tests/test_utils.py b/modules/weko-search-ui/tests/test_utils.py index 2854a89508..eae8e1294d 100644 --- a/modules/weko-search-ui/tests/test_utils.py +++ b/modules/weko-search-ui/tests/test_utils.py @@ -180,7 +180,7 @@ def test_DefaultOrderDict_deepcopy(): class MockSearchPerm: def __init__(self): pass - + def can(self): return True # def get_tree_items(index_tree_id): ERROR ~ AttributeError: '_AppCtxGlobals' object has no attribute 'identity' @@ -1442,232 +1442,51 @@ def test_handle_fill_system_item(app, test_list_records, mocker): assert items == items_result -# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id1 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp # doi2, doi_ra2 は自動補完が原則 @pytest.mark.parametrize( - "item_id, before_doi,after_doi,warnings,errors,is_change_identifier,is_register_cnri", + "before_doi,after_doi,warnings,errors,is_change_identifier", [ - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False,False), - (1,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (1,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (1,{"doi": None,"doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (1,{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},[],[],True,False), - (1,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (1,{"doi": "xyz.jalc/0000000001","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (1,{"doi": None,"doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": "","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), - - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False,False), - (2,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (2,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (2,{"doi": None,"doi_ra":"Crossref","doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (2,{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},[],[],True,False), - (2,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (2,{"doi": "xyz.crossref/0000000002","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (2,{"doi": None,"doi_ra":"Crossref","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"Crossref","doi2": "","doi_ra2":"Crossref"},[],['Please specify DOI prefix/suffix.'],True,False), - - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False,False), - (3,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000004","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (3,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (3,{"doi": None,"doi_ra":"DataCite","doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000004","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (3,{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},[],[],True,False), - (3,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (3,{"doi": "xyz.datacite/0000000003","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (3,{"doi": None,"doi_ra":"DataCite","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"DataCite","doi2": "","doi_ra2":"DataCite"},[],['Please specify DOI prefix/suffix.'],True,False), - - - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False,False), - (4,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000005","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - (4,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (4,{"doi": None,"doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False,False), - - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000005","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (4,{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},[],[],True,False), - (4,{"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (4,{"doi": "xyz.ndl/0000000004","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (4,{"doi": None,"doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"NDL JaLC","doi2": "","doi_ra2":"NDL JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), - - (5,{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False,False), - (5,{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (5,{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],['Specified Prefix of DOI is incorrect.'],False,False), - (5,{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],['Specified Prefix of DOI is incorrect.'],False,False), - # 更新によるDOI付与 - (5,{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (5,{"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"JaLC","doi2":"","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), - (5,{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":"xyz.jalc","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":"xyz.jalc/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.jalc/xyz","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/xyz","doi_ra":"JaLC","doi2":"xyz.jalc/xyz","doi_ra2":"JaLC"},[],[],True,False), - (5,{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"Crossref","doi2":"","doi_ra2":"Crossref"},[],['Please specify DOI prefix/suffix.'],True,False), - (5,{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":"xyz.crossref","doi_ra2":"Crossref"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":"xyz.crossref/","doi_ra2":"Crossref"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"DataCite","doi2":"","doi_ra2":"DataCite"},[],['Please specify DOI prefix/suffix.'],True,False), - (5,{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":"xyz.datacite","doi_ra2":"DataCite"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":"xyz.datacite/","doi_ra2":"DataCite"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"NDL JaLC","doi2":"","doi_ra2":"NDL JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), - (5,{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":"xyz.ndl","doi_ra2":"NDL JaLC"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":"xyz.ndl/","doi_ra2":"NDL JaLC"},[],['Please specify DOI suffix.'],True,False), - (5,{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (5,{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":"xyz.ndl","doi_ra2":"JaLC"},[],['Please specify DOI suffix.', 'Specified Prefix of DOI is incorrect.'],True,False), - (5,{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":"xyz.ndl/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.', 'Specified Prefix of DOI is incorrect.'],True,False), - # 新規登録時DOI付与 - (None,{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],[],False,False), - (None,{"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), - (None,{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), - (None,{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), - (None,{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (None,{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (None,{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), - (None,{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],['Do not specify DOI suffix.'],False,False), - # 新規登録時DOI付与 - (None,{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), - (None,{"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": "","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), - (None,{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": "xyz.jalc","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), - (None,{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": "xyz.jalc/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), - (None,{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (None,{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (None,{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), - (None,{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,False), - # cnri登録可能 - (None,{"cnri":"","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"cnri":"", "doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,True), - (None,{"cnri":"","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"cnri":"","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,True), - (None,{"cnri":"test_cnri","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"cnri":"test_cnri","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,True), - (None,{"cnri":"test_cnri","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"cnri":"test_cnri","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],[],True,True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": None,"doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":""},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},[],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000001","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "xyz.jalc/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":"JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.jalc/0000000001","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},[],[],True), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": "xyz.jalc/0000000001","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},{"doi": "xyz.jalc/0000000002","doi_ra":"JaLC","doi2": "xyz.jalc/0000000002","doi_ra2":"JaLC"},[],[],True), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.jalc/0000000001","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/0000000001","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": None,"doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": "","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True), ]) -def test_handle_fill_system_item3(app,doi_records,item_id,before_doi,after_doi,warnings,errors,is_change_identifier,is_register_cnri): +def test_handle_fill_system_item3_id1(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier): + item_id = 1 app.config.update( - WEKO_HANDLE_ALLOW_REGISTER_CRNI=is_register_cnri + WEKO_HANDLE_ALLOW_REGISTER_CRNI=False ) before = { "metadata": { @@ -1698,29 +1517,28 @@ def test_handle_fill_system_item3(app,doi_records,item_id,before_doi,after_doi,w "item_type_id": 1, "$schema": "https://localhost:8443/items/jsonschema/1", } - - if item_id: - before["id"] = "{}".format(item_id) - before["uri"] = "https://localhost:8443/records/{}".format(item_id) - before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + before["id"] = "{}".format(item_id) + before["uri"] = "https://localhost:8443/records/{}".format(item_id) + before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} if before_doi["doi_ra2"] is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) - + if before_doi["doi2"] is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) if before_doi['doi_ra'] is not None: before["doi_ra"]="{}".format(before_doi['doi_ra']) - + if before_doi['doi'] is not None: before["doi"]="{}".format(before_doi['doi']) - + if "cnri" in before_doi and before_doi["cnri"] is not None: before["cnri"] = "{}".format(before_doi["cnri"]) - + before_list = [before] after = { "metadata": { @@ -1755,30 +1573,29 @@ def test_handle_fill_system_item3(app,doi_records,item_id,before_doi,after_doi,w "warnings": warnings, } - if item_id: - after["id"] = "{}".format(item_id) - after["uri"] = "https://localhost:8443/records/{}".format(item_id) - after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} - + after["id"] = "{}".format(item_id) + after["uri"] = "https://localhost:8443/records/{}".format(item_id) + after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + if after_doi["doi_ra"] is not None: after["doi_ra"]="{}".format(after_doi['doi_ra']) - + if after_doi["doi"] is not None: - after["doi"]="{}".format(after_doi["doi"]) - + after["doi"]="{}".format(after_doi["doi"]) + if after_doi["doi_ra2"] is not None: after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) - + if after_doi["doi2"] is not None: after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) if "cnri" in after_doi and after_doi["cnri"] is not None: after["cnri"] = after_doi["cnri"] - + after_list = [after] - + if is_change_identifier: before_list[0]['is_change_identifier']=True after_list[0]['is_change_identifier']=True @@ -1788,116 +1605,925 @@ def test_handle_fill_system_item3(app,doi_records,item_id,before_doi,after_doi,w handle_fill_system_item(before_list) assert after_list == before_list +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id2 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# doi2, doi_ra2 は自動補完が原則 +@pytest.mark.parametrize( + "before_doi,after_doi,warnings,errors,is_change_identifier", + [ + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.crossref/0000000002","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": None,"doi_ra":"Crossref","doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":""},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},[],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000002","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "xyz.crossref/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":"Crossref", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.crossref/0000000002","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "","doi_ra2":""},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},[],[],True), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True), + ({"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": "xyz.crossref/0000000002","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},{"doi": "xyz.crossref/0000000003","doi_ra":"Crossref","doi2": "xyz.crossref/0000000003","doi_ra2":"Crossref"},[],[],True), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.crossref/0000000002","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.crossref/0000000002","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": None,"doi_ra":"Crossref","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"Crossref","doi2": "","doi_ra2":"Crossref"},[],['Please specify DOI prefix/suffix.'],True), + ]) +def test_handle_fill_system_item3_id2(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier): + item_id = 2 + app.config.update( + WEKO_HANDLE_ALLOW_REGISTER_CRNI=False + ) + before = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + } + + before["id"] = "{}".format(item_id) + before["uri"] = "https://localhost:8443/records/{}".format(item_id) + before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} -# def get_thumbnail_key(item_type_id=0): -def test_get_thumbnail_key(i18n_app, db_itemtype, db_workflow): - assert get_thumbnail_key(item_type_id=1) + if before_doi["doi_ra2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) + if before_doi["doi2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) -# def handle_check_thumbnail_file_type(thumbnail_paths): -def test_handle_check_thumbnail_file_type(i18n_app): - assert handle_check_thumbnail_file_type(["/"]) + if before_doi['doi_ra'] is not None: + before["doi_ra"]="{}".format(before_doi['doi_ra']) + if before_doi['doi'] is not None: + before["doi"]="{}".format(before_doi['doi']) -# def handle_check_metadata_not_existed(str_keys, item_type_id=0): *** not yet done -def test_handle_check_metadata_not_existed(i18n_app, db_itemtype): - # Test 1 - assert not handle_check_metadata_not_existed( - ".metadata", db_itemtype["item_type"].id - ) + if "cnri" in before_doi and before_doi["cnri"] is not None: + before["cnri"] = "{}".format(before_doi["cnri"]) + + before_list = [before] + after = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + "identifier_key": "item_1617186819068", + "errors": errors, + "warnings": warnings, + } + after["id"] = "{}".format(item_id) + after["uri"] = "https://localhost:8443/records/{}".format(item_id) + after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} -# def handle_get_all_sub_id_and_name(items, root_id=None, root_name=None, form=[]): + if after_doi["doi_ra"] is not None: + after["doi_ra"]="{}".format(after_doi['doi_ra']) + + if after_doi["doi"] is not None: + after["doi"]="{}".format(after_doi["doi"]) + + if after_doi["doi_ra2"] is not None: + after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) + + if after_doi["doi2"] is not None: + after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) + + if "cnri" in after_doi and after_doi["cnri"] is not None: + after["cnri"] = after_doi["cnri"] + + after_list = [after] + + if is_change_identifier: + before_list[0]['is_change_identifier']=True + after_list[0]['is_change_identifier']=True + + with app.test_request_context(): + assert before_list != after_list + handle_fill_system_item(before_list) + assert after_list == before_list + +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id3 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# doi2, doi_ra2 は自動補完が原則 @pytest.mark.parametrize( - "items,root_id,root_name,form,ids,names", + "before_doi,after_doi,warnings,errors,is_change_identifier", [ - pytest.param( - {"interim": {"type": "string"}}, - ".metadata.item_1657196790737[0]", - "text[0]", - [{"key": "item_1657196790737[].interim", "type": "text", "notitle": True}], - [".metadata.item_1657196790737[0].interim"], - ["text[0].None"], - ), - pytest.param( - { - "interim": { - "enum": [None, "op1", "op2", "op3", "op4"], - "type": ["null", "string"], - "title": "list", - "title_i18n": {"en": "", "ja": ""}, - } - }, - ".metadata.item_1657204077414[0]", - "list[0]", - [ - { - "key": "item_1657204077414[].interim", - "type": "select", - "title": "list", - "notitle": True, - "titleMap": [ - {"name": "op1", "value": "op1"}, - {"name": "op2", "value": "op2"}, - {"name": "op3", "value": "op3"}, - {"name": "op4", "value": "op4"}, - ], - "title_i18n": {"en": "", "ja": ""}, - } - ], - [".metadata.item_1657204026946.interim[0]"], - ["check.check[0]"], - ), - pytest.param( - { - "interim": { - "enum": [None, "op1", "op2", "op3", "op4"], - "type": ["null", "string"], - "title": "list", - "format": "select", - } - }, - ".metadata.item_1657204070640", - "list", - [ - { - "key": "item_1657204070640.interim", - "type": "select", - "title": "list", - "titleMap": [ - {"name": "op1", "value": "op1"}, - {"name": "op2", "value": "op2"}, - {"name": "op3", "value": "op3"}, - {"name": "op4", "value": "op4"}, - ], - "title_i18n": {"en": "", "ja": ""}, - } - ], - [".metadata.item_1657204036771[0].interim[0]"], - ["checjk[0].checjk[0]"], - ), - pytest.param( - { - "interim": { - "type": "array", - "items": {"enum": ["op1", "op2", "op3", "op4"], "type": "string"}, - "title": "check", - "format": "checkboxes", - "title_i18n": {"en": "", "ja": ""}, - } + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000004","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.datacite/0000000003","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": None,"doi_ra":"DataCite","doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":""},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},[],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000003","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "xyz.datacite/0000000004","doi_ra2":"JaLC"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":"DataCite", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.datacite/0000000003","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "","doi_ra2":""},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},[],[],True), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True), + ({"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": "xyz.datacite/0000000003","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.datacite/0000000004","doi_ra":"DataCite","doi2": "xyz.datacite/0000000004","doi_ra2":"DataCite"},[],[],True), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.datacite/0000000003","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.datacite/0000000003","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": None,"doi_ra":"DataCite","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"DataCite","doi2": "","doi_ra2":"DataCite"},[],['Please specify DOI prefix/suffix.'],True), + ]) +def test_handle_fill_system_item3_id3(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier): + item_id = 3 + app.config.update( + WEKO_HANDLE_ALLOW_REGISTER_CRNI=False + ) + before = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], }, - ".metadata.item_1657204026946", - "check", - [ - { - "key": "item_1657204026946.interim", - "type": "template", - "title": "check", - "titleMap": [ - {"name": "op1", "value": "op1"}, - {"name": "op2", "value": "op2"}, - {"name": "op3", "value": "op3"}, - {"name": "op4", "value": "op4"}, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + } + + before["id"] = "{}".format(item_id) + before["uri"] = "https://localhost:8443/records/{}".format(item_id) + before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if before_doi["doi_ra2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) + + if before_doi["doi2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) + + if before_doi['doi_ra'] is not None: + before["doi_ra"]="{}".format(before_doi['doi_ra']) + + if before_doi['doi'] is not None: + before["doi"]="{}".format(before_doi['doi']) + + if "cnri" in before_doi and before_doi["cnri"] is not None: + before["cnri"] = "{}".format(before_doi["cnri"]) + + before_list = [before] + after = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + "identifier_key": "item_1617186819068", + "errors": errors, + "warnings": warnings, + } + + after["id"] = "{}".format(item_id) + after["uri"] = "https://localhost:8443/records/{}".format(item_id) + after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if after_doi["doi_ra"] is not None: + after["doi_ra"]="{}".format(after_doi['doi_ra']) + + if after_doi["doi"] is not None: + after["doi"]="{}".format(after_doi["doi"]) + + if after_doi["doi_ra2"] is not None: + after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) + + if after_doi["doi2"] is not None: + after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) + + if "cnri" in after_doi and after_doi["cnri"] is not None: + after["cnri"] = after_doi["cnri"] + + after_list = [after] + + if is_change_identifier: + before_list[0]['is_change_identifier']=True + after_list[0]['is_change_identifier']=True + + with app.test_request_context(): + assert before_list != after_list + handle_fill_system_item(before_list) + assert after_list == before_list + +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id4 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# doi2, doi_ra2 は自動補完が原則 +@pytest.mark.parametrize( + "before_doi,after_doi,warnings,errors,is_change_identifier", + [ + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000005","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.','The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],False), + ({"doi": "xyz.ndl/0000000004","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi": None,"doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],False), + + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": "","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "","doi_ra":"","doi2": "","doi_ra2":""},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":""},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000004","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "xyz.ndl/0000000005","doi_ra2":"DataCite"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.', 'The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC", "doi2": "","doi_ra2":"JaLC2"},{"doi": "xyz.ndl/0000000004","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},['The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.'],[],True), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "","doi_ra2":""},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},['The specified DOI is wrong and fixed with the registered DOI.'],[],True), + ({"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": "xyz.ndl/0000000004","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},{"doi": "xyz.ndl/0000000005","doi_ra":"NDL JaLC","doi2": "xyz.ndl/0000000005","doi_ra2":"NDL JaLC"},[],[],True), + ({"doi": None,"doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True), + ({"doi": "xyz.ndl/0000000004","doi_ra":None,"doi2": None,"doi_ra2":None},{"doi": "xyz.ndl/0000000004","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi": None,"doi_ra":"NDL JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"NDL JaLC","doi2": "","doi_ra2":"NDL JaLC"},[],['Please specify DOI prefix/suffix.'],True), + ]) +def test_handle_fill_system_item3_id4(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier): + item_id = 4 + app.config.update( + WEKO_HANDLE_ALLOW_REGISTER_CRNI=False + ) + before = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + } + + before["id"] = "{}".format(item_id) + before["uri"] = "https://localhost:8443/records/{}".format(item_id) + before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if before_doi["doi_ra2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) + + if before_doi["doi2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) + + if before_doi['doi_ra'] is not None: + before["doi_ra"]="{}".format(before_doi['doi_ra']) + + if before_doi['doi'] is not None: + before["doi"]="{}".format(before_doi['doi']) + + if "cnri" in before_doi and before_doi["cnri"] is not None: + before["cnri"] = "{}".format(before_doi["cnri"]) + + before_list = [before] + after = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + "identifier_key": "item_1617186819068", + "errors": errors, + "warnings": warnings, + } + + after["id"] = "{}".format(item_id) + after["uri"] = "https://localhost:8443/records/{}".format(item_id) + after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if after_doi["doi_ra"] is not None: + after["doi_ra"]="{}".format(after_doi['doi_ra']) + + if after_doi["doi"] is not None: + after["doi"]="{}".format(after_doi["doi"]) + + if after_doi["doi_ra2"] is not None: + after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) + + if after_doi["doi2"] is not None: + after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) + + if "cnri" in after_doi and after_doi["cnri"] is not None: + after["cnri"] = after_doi["cnri"] + + after_list = [after] + + if is_change_identifier: + before_list[0]['is_change_identifier']=True + after_list[0]['is_change_identifier']=True + + with app.test_request_context(): + assert before_list != after_list + handle_fill_system_item(before_list) + assert after_list == before_list + +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id5 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# doi2, doi_ra2 は自動補完が原則 +@pytest.mark.parametrize( + "before_doi,after_doi,warnings,errors,is_change_identifier", + [ + ({"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},[],[],False), + ({"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False), + ({"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],['Specified Prefix of DOI is incorrect.'],False), + ({"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},[],['Specified Prefix of DOI is incorrect.'],False), + #更新によるDOI付与 + ({"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True), + ({"doi":"","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"JaLC","doi2":"","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True), + ({"doi":"xyz.jalc","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"JaLC","doi2":"xyz.jalc","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"JaLC","doi2":"xyz.jalc/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.jalc/xyz","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/xyz","doi_ra":"JaLC","doi2":"xyz.jalc/xyz","doi_ra2":"JaLC"},[],[],True), + ({"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.jalc/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"Crossref","doi2":"","doi_ra2":"Crossref"},[],['Please specify DOI prefix/suffix.'],True), + ({"doi":"xyz.crossref","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"Crossref","doi2":"xyz.crossref","doi_ra2":"Crossref"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"Crossref","doi2":"xyz.crossref/","doi_ra2":"Crossref"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.crossref/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"DataCite","doi2":"","doi_ra2":"DataCite"},[],['Please specify DOI prefix/suffix.'],True), + ({"doi":"xyz.datacite","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"DataCite","doi2":"xyz.datacite","doi_ra2":"DataCite"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"DataCite","doi2":"xyz.datacite/","doi_ra2":"DataCite"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.datacite/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"","doi_ra":"NDL JaLC","doi2":"","doi_ra2":"NDL JaLC"},[],['Please specify DOI prefix/suffix.'],True), + ({"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"NDL JaLC","doi2":"xyz.ndl","doi_ra2":"NDL JaLC"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"NDL JaLC","doi2":"xyz.ndl/","doi_ra2":"NDL JaLC"},[],['Please specify DOI suffix.'],True), + ({"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"","doi2":None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True), + ({"doi":"xyz.ndl","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl","doi_ra":"JaLC","doi2":"xyz.ndl","doi_ra2":"JaLC"},[],['Please specify DOI suffix.', 'Specified Prefix of DOI is incorrect.'],True), + ({"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":None,"doi_ra2":None},{"doi":"xyz.ndl/","doi_ra":"JaLC","doi2":"xyz.ndl/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.', 'Specified Prefix of DOI is incorrect.'],True), + ]) +def test_handle_fill_system_item3_id5(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier): + item_id = 5 + app.config.update( + WEKO_HANDLE_ALLOW_REGISTER_CRNI=False + ) + before = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + } + + before["id"] = "{}".format(item_id) + before["uri"] = "https://localhost:8443/records/{}".format(item_id) + before["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if before_doi["doi_ra2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) + + if before_doi["doi2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) + + if before_doi['doi_ra'] is not None: + before["doi_ra"]="{}".format(before_doi['doi_ra']) + + if before_doi['doi'] is not None: + before["doi"]="{}".format(before_doi['doi']) + + if "cnri" in before_doi and before_doi["cnri"] is not None: + before["cnri"] = "{}".format(before_doi["cnri"]) + + before_list = [before] + after = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + "identifier_key": "item_1617186819068", + "errors": errors, + "warnings": warnings, + } + + after["id"] = "{}".format(item_id) + after["uri"] = "https://localhost:8443/records/{}".format(item_id) + after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} + + if after_doi["doi_ra"] is not None: + after["doi_ra"]="{}".format(after_doi['doi_ra']) + + if after_doi["doi"] is not None: + after["doi"]="{}".format(after_doi["doi"]) + + if after_doi["doi_ra2"] is not None: + after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) + + if after_doi["doi2"] is not None: + after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) + + if "cnri" in after_doi and after_doi["cnri"] is not None: + after["cnri"] = after_doi["cnri"] + + after_list = [after] + + if is_change_identifier: + before_list[0]['is_change_identifier']=True + after_list[0]['is_change_identifier']=True + + with app.test_request_context(): + assert before_list != after_list + handle_fill_system_item(before_list) + assert after_list == before_list + +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item3_id_none -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# doi2, doi_ra2 は自動補完が原則 +@pytest.mark.parametrize( + "before_doi,after_doi,warnings,errors,is_change_identifier,is_register_cnri", + [ + # 新規登録時DOI付与 + ({"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],[],False,False), + ({"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), + ({"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), + ({"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],[],False,False), + ({"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), + ({"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), + ({"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],False,False), + ({"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},[],['Do not specify DOI suffix.'],False,False), + # 新規登録時DOI付与 + ({"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,False), + ({"doi": "","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC","doi2": "","doi_ra2":"JaLC"},[],['Please specify DOI prefix/suffix.'],True,False), + ({"doi": "xyz.jalc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"JaLC","doi2": "xyz.jalc","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), + ({"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"JaLC","doi2": "xyz.jalc/","doi_ra2":"JaLC"},[],['Please specify DOI suffix.'],True,False), + ({"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), + ({"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), + ({"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},{"doi": "","doi_ra":"JaLC2","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.', 'DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.'],True,False), + ({"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,False), + # cnri登録可能 + ({"cnri":"","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"cnri":"", "doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,True), + ({"cnri":"","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"cnri":"","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],['Please specify DOI prefix/suffix.'],True,True), + ({"cnri":"test_cnri","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": None,"doi_ra2":None},{"cnri":"test_cnri","doi": "xyz.jalc/abc","doi_ra":"JaLC","doi2": "xyz.jalc/abc","doi_ra2":"JaLC"},[],[],True,True), + ({"cnri":"test_cnri","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},{"cnri":"test_cnri","doi": "","doi_ra":"","doi2": None,"doi_ra2":None},[],[],True,True), + ]) +def test_handle_fill_system_item3_id_none(app,doi_records,before_doi,after_doi,warnings,errors,is_change_identifier,is_register_cnri): + item_id = None + app.config.update( + WEKO_HANDLE_ALLOW_REGISTER_CRNI=is_register_cnri + ) + before = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + } + + if before_doi["doi_ra2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) + + if before_doi["doi2"] is not None: + before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) + before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) + + if before_doi['doi_ra'] is not None: + before["doi_ra"]="{}".format(before_doi['doi_ra']) + + if before_doi['doi'] is not None: + before["doi"]="{}".format(before_doi['doi']) + + if "cnri" in before_doi and before_doi["cnri"] is not None: + before["cnri"] = "{}".format(before_doi["cnri"]) + + before_list = [before] + after = { + "metadata": { + "path": ["1667004052852"], + "pubdate": "2022-10-29", + "item_1617186331708": [ + {"subitem_1551255647225": "title", "subitem_1551255648112": "en"} + ], + "item_1617258105262": { + "resourcetype": "conference paper", + "resourceuri": "http://purl.org/coar/resource_type/c_5794", + }, + "item_1617605131499": [ + { + "accessrole": "open_access", + "date": [{"dateType": "Available", "dateValue": "2022-10-29"}], + "filename": "a.zip", + "filesize": [{"value": "82 KB"}], + "format": "application/zip", + } + ], + }, + "pos_index": ["IndexA"], + "publish_status": "public", + "edit_mode": "Keep", + "file_path": [""], + "item_type_name": "デフォルトアイテムタイプ（フル）", + "item_type_id": 1, + "$schema": "https://localhost:8443/items/jsonschema/1", + "identifier_key": "item_1617186819068", + "errors": errors, + "warnings": warnings, + } + + if after_doi["doi_ra"] is not None: + after["doi_ra"]="{}".format(after_doi['doi_ra']) + + if after_doi["doi"] is not None: + after["doi"]="{}".format(after_doi["doi"]) + + if after_doi["doi_ra2"] is not None: + after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) + + if after_doi["doi2"] is not None: + after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) + after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) + + if "cnri" in after_doi and after_doi["cnri"] is not None: + after["cnri"] = after_doi["cnri"] + + after_list = [after] + + if is_change_identifier: + before_list[0]['is_change_identifier']=True + after_list[0]['is_change_identifier']=True + + with app.test_request_context(): + assert before_list != after_list + handle_fill_system_item(before_list) + assert after_list == before_list + + +# def get_thumbnail_key(item_type_id=0): +def test_get_thumbnail_key(i18n_app, db_itemtype, db_workflow): + assert get_thumbnail_key(item_type_id=1) + + +# def handle_check_thumbnail_file_type(thumbnail_paths): +def test_handle_check_thumbnail_file_type(i18n_app): + assert handle_check_thumbnail_file_type(["/"]) + + +# def handle_check_metadata_not_existed(str_keys, item_type_id=0): *** not yet done +def test_handle_check_metadata_not_existed(i18n_app, db_itemtype): + # Test 1 + assert not handle_check_metadata_not_existed( + ".metadata", db_itemtype["item_type"].id + ) + + +# def handle_get_all_sub_id_and_name(items, root_id=None, root_name=None, form=[]): +@pytest.mark.parametrize( + "items,root_id,root_name,form,ids,names", + [ + pytest.param( + {"interim": {"type": "string"}}, + ".metadata.item_1657196790737[0]", + "text[0]", + [{"key": "item_1657196790737[].interim", "type": "text", "notitle": True}], + [".metadata.item_1657196790737[0].interim"], + ["text[0].None"], + ), + pytest.param( + { + "interim": { + "enum": [None, "op1", "op2", "op3", "op4"], + "type": ["null", "string"], + "title": "list", + "title_i18n": {"en": "", "ja": ""}, + } + }, + ".metadata.item_1657204077414[0]", + "list[0]", + [ + { + "key": "item_1657204077414[].interim", + "type": "select", + "title": "list", + "notitle": True, + "titleMap": [ + {"name": "op1", "value": "op1"}, + {"name": "op2", "value": "op2"}, + {"name": "op3", "value": "op3"}, + {"name": "op4", "value": "op4"}, + ], + "title_i18n": {"en": "", "ja": ""}, + } + ], + [".metadata.item_1657204026946.interim[0]"], + ["check.check[0]"], + ), + pytest.param( + { + "interim": { + "enum": [None, "op1", "op2", "op3", "op4"], + "type": ["null", "string"], + "title": "list", + "format": "select", + } + }, + ".metadata.item_1657204070640", + "list", + [ + { + "key": "item_1657204070640.interim", + "type": "select", + "title": "list", + "titleMap": [ + {"name": "op1", "value": "op1"}, + {"name": "op2", "value": "op2"}, + {"name": "op3", "value": "op3"}, + {"name": "op4", "value": "op4"}, + ], + "title_i18n": {"en": "", "ja": ""}, + } + ], + [".metadata.item_1657204036771[0].interim[0]"], + ["checjk[0].checjk[0]"], + ), + pytest.param( + { + "interim": { + "type": "array", + "items": {"enum": ["op1", "op2", "op3", "op4"], "type": "string"}, + "title": "check", + "format": "checkboxes", + "title_i18n": {"en": "", "ja": ""}, + } + }, + ".metadata.item_1657204026946", + "check", + [ + { + "key": "item_1657204026946.interim", + "type": "template", + "title": "check", + "titleMap": [ + {"name": "op1", "value": "op1"}, + {"name": "op2", "value": "op2"}, + {"name": "op3", "value": "op3"}, + {"name": "op4", "value": "op4"}, ], "title_i18n": {"en": "", "ja": ""}, "templateUrl": "/static/templates/weko_deposit/checkboxes.html", @@ -2100,7 +2726,7 @@ def get_record_by_uuid(uuid): assert msg.decode() == "Export failed. Please check item id range." # raise Exception - with patch("weko_search_ui.utils.os.makedirs", side_effect=Exception("test_error")): + with patch("weko_search_ui.utils.os.makedirs", side_effect=[None, Exception("test_error")]): export_all(root_url, user_id, data, start_time_str) msg = datastore.get(msg_key) assert msg.decode() == "Export failed." @@ -2139,7 +2765,7 @@ def test_write_files(db_activity, i18n_app, users, redis_connect, mocker, item_t mocker.patch("weko_search_ui.utils.os.makedirs") mocker.patch('builtins.open', side_effect=unittest.mock.mock_open()) assert write_files(item_datas, "tests/data/write_files", current_user.get_id(), 0) - + # result is False with patch("weko_items_ui.utils.make_stats_file_with_permission", side_effect=SQLAlchemyError("test_error")): assert not write_files(item_datas, "tests/data/write_files", current_user.get_id(), 0) @@ -2187,7 +2813,7 @@ def test_cancel_export_all(i18n_app, users, redis_connect, mocker): assert json.loads(ds_file_json) == file_result_json mock_revoke.assert_called_with("test_task_key",terminate=True) mock_delete_id.assert_called_with(args=("test_task_key","admin_cache_KEY_EXPORT_ALL_5"),countdown=60) - + # export_status is False with patch("weko_search_ui.utils.get_export_status", return_value=(False,None,None,None,None,None,None)): datastore.put(file_cache_key, json.dumps(file_json).encode('utf-8'), ttl_secs=30) @@ -2199,7 +2825,7 @@ def test_cancel_export_all(i18n_app, users, redis_connect, mocker): assert json.loads(ds_file_json) == file_json mock_revoke.assert_not_called() mock_delete_id.assert_not_called() - + # raise Exception with patch("weko_search_ui.utils.get_export_status",side_effect=Exception("test_error")): result = cancel_export_all() @@ -2219,7 +2845,7 @@ def successful(self): return self.state == "SUCCESS" def failed(self): return self.state == "FAILED" - + start_time_str = '2024/05/21 14:23:46' def create_file_json(status): return { @@ -2249,7 +2875,7 @@ def create_file_json(status): name='RUN_MSG_EXPORT_ALL_FILE_CREATE', user_id=current_user.get_id() ) datastore = redis_connect - + datastore.put(cache_uri, "test_uri".encode("utf-8"), ttl_secs=30) datastore.put(cache_msg, "test_msg".encode("utf-8"), ttl_secs=30) datastore.put(run_msg, "test_run_msg".encode("utf-8"), ttl_secs=30) @@ -2293,7 +2919,7 @@ def create_file_json(status): datastore.put(run_msg, "test_run_msg".encode("utf-8"), ttl_secs=30) result=get_export_status() assert result == (False, uri.decode(), "test_msg", "test_run_msg", "SUCCESS", start_time_str, "") - + # raise Exception with patch("weko_search_ui.utils.AsyncResult",side_effect=Exception("test_error")): datastore.delete(cache_uri) @@ -2304,7 +2930,7 @@ def create_file_json(status): datastore.put(run_msg, "test_run_msg".encode("utf-8"), ttl_secs=30) result=get_export_status() assert result == (False, "test_uri", "test_msg", "test_run_msg", "", "", "") - + # write_file_status is canceled datastore.delete(file_msg) datastore.put(file_msg, json.dumps(create_file_json('canceled')).encode('utf-8'), ttl_secs=30) @@ -2437,11 +3063,17 @@ def test_get_data_by_property(i18n_app): assert get_data_by_property(item_metadata, item_map, mapping_key) assert get_data_by_property(item_metadata, {}, mapping_key) + item_metadata = {"test": "test"} with patch( - "weko_workflow.utils.get_sub_item_value", return_value=[True, ["value"]] + "weko_search_ui.utils.get_sub_item_value", return_value=[True, ["value"]] ): assert get_data_by_property(item_metadata, item_map, mapping_key) + with patch( + "weko_search_ui.utils.get_sub_item_value", return_value=None + ): + + assert get_data_by_property(item_metadata, item_map, mapping_key) # def get_filenames_from_metadata(metadata): def test_get_filenames_from_metadata(i18n_app, record_with_metadata): @@ -2466,8 +3098,8 @@ def test_get_billinginfo_from_metadata(i18n_app, record_with_metadata,record_wit metadata = { "item_1712310680597": [{ - "accessrole": "open_login", - "billing": ["billing_file"], + "accessrole": "open_login", + "billing": ["billing_file"], "displaytype": "detail", "priceinfo": [{"billingrole": "10005", "price": "100", "tax": ["include_tax"]}] }] @@ -2500,6 +3132,7 @@ def test_handle_check_filename_consistence(i18n_app): assert handle_check_filename_consistence(file_paths, meta_filenames) +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_function_issue34520 -v -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp @pytest.mark.parametrize( "item_id, before_doi,after_doi,warnings,errors,is_change_identifier", [ @@ -2543,7 +3176,7 @@ def test_function_issue34520(app, doi_records, item_id, before_doi, after_doi, w "item_type_id": 1, "$schema": "https://localhost/items/jsonschema/1", } - + if item_id: before["id"] = "{}".format(item_id) before["uri"] = "https://localhost/records/{}".format(item_id) @@ -2552,17 +3185,17 @@ def test_function_issue34520(app, doi_records, item_id, before_doi, after_doi, w if before_doi["doi_ra2"] is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) - + if before_doi["doi2"] is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) if before_doi['doi_ra'] is not None: before["doi_ra"]="{}".format(before_doi['doi_ra']) - + if before_doi['doi'] is not None: - before["doi"]="{}".format(before_doi['doi']) - + before["doi"]="{}".format(before_doi['doi']) + before_list = [before] after = { "metadata": { @@ -2601,23 +3234,23 @@ def test_function_issue34520(app, doi_records, item_id, before_doi, after_doi, w after["id"] = "{}".format(item_id) after["uri"] = "https://localhost/records/{}".format(item_id) after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} - + if after_doi["doi_ra"] is not None: after["doi_ra"]="{}".format(after_doi['doi_ra']) - + if after_doi["doi"] is not None: - after["doi"]="{}".format(after_doi["doi"]) - + after["doi"]="{}".format(after_doi["doi"]) + if after_doi["doi_ra2"] is not None: after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) - + if after_doi["doi2"] is not None: after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) after_list = [after] - + if is_change_identifier: before_list[0]['is_change_identifier']=True after_list[0]['is_change_identifier']=True @@ -2626,7 +3259,7 @@ def test_function_issue34520(app, doi_records, item_id, before_doi, after_doi, w assert before_list != after_list handle_fill_system_item(before_list) assert after_list == before_list - + # .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_function_issue34535 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search_ui/.tox/c1/tmp def test_function_issue34535(db,db_index,db_itemtype,location,db_oaischema,mocker): mocker.patch("weko_search_ui.utils.find_and_update_location_size") @@ -2661,7 +3294,7 @@ def test_function_issue34535(db,db_index,db_itemtype,location,db_oaischema,mocke # new item root_path = os.path.dirname(os.path.abspath(__file__)) new_item = {'$schema': 'https://192.168.56.103/items/jsonschema/1', 'edit_mode': 'Keep', 'errors': None, 'file_path': [''], 'filenames': [{'filename': '', 'id': '.metadata.item_1617605131499[0].filename'}], 'id': '4', 'identifier_key': 'item_1617186819068', 'is_change_identifier': False, 'item_title': 'test item in br', 'item_type_id': 1, 'item_type_name': 'デフォルトアイテムタイプ（フル）', 'metadata': {'item_1617186331708': [{'subitem_1551255647225': 'test item in br', 'subitem_1551255648112': 'ja'}], 'item_1617186626617': [{'subitem_description': 'this is line1.
this is line2.', 'subitem_description_language': 'en', 'subitem_description_type': 'Abstract'}], 'item_1617258105262': {'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}, 'path': [1], 'pubdate': '2022-11-21'}, 'pos_index': ['Faculty of Humanities and Social Sciences'], 'publish_status': 'public', 'status': 'keep', 'uri': 'https://192.168.56.103/records/4', 'warnings': [], 'root_path': root_path} - + register_item_metadata(new_item,root_path,True) record = WekoDeposit.get_record(recid.object_uuid) assert record == {'_oai': {'id': 'oai:weko3.example.org:00000004', 'sets': ['1']}, 'path': ['1'], 'owner': '1', 'recid': '4', 'title': ['test item in br'], 'pubdate': {'attribute_name': 'PubDate', 'attribute_value': '2022-11-21'}, '_buckets': {'deposit': '0796e490-6dcf-4e7d-b241-d7201c3de83a'}, '_deposit': {'id': '4', 'pid': {'type': 'depid', 'value': '4', 'revision_id': 0}, 'owner': '1', 'owners': [1], 'status': 'draft', 'created_by': 1}, 'item_title': 'test item in br', 'author_link': [], 'item_type_id': '1', 'publish_date': '2022-11-21', 'publish_status': '0', 'weko_shared_id': -1, 'item_1617186331708': {'attribute_name': 'Title', 'attribute_value_mlt': [{'subitem_1551255647225': 'test item in br', 'subitem_1551255648112': 'ja'}]}, 'item_1617186626617': {'attribute_name': 'Description', 'attribute_value_mlt': [{'subitem_description': 'this is line1.\nthis is line2.', 'subitem_description_language': 'en', 'subitem_description_type': 'Abstract'}]}, 'item_1617258105262': {'attribute_name': 'Resource Type', 'attribute_value_mlt': [{'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}]}, 'relation_version_is_last': True, 'control_number': '4'} @@ -2682,14 +3315,14 @@ def test_function_issue34958(app, make_itemtype): os.path.dirname(os.path.realpath(__file__)), "data/import_file/34958" ) csv_path = "issue34958_import.csv" - - + + test = [{"publish_status": "public", "pos_index": ["Index A"], "metadata": {"pubdate": "2022-11-11", "item_1671508244520": {"subitem_1551255647225": "Extra items test", "subitem_1551255648112": "en"}, "item_1671508260839": {"resourcetype": "conference paper", "resourceuri": "http://purl.org/coar/resource_type/c_5794"}, "item_1671508308460": [{"interim": "test_text_value0"}, {"interim": "test_text_value1"}], "item_1671606815997": "test_text", "item_1617186626617": {"subitem_description": "test_description"}}, "edit_mode": "keep", "item_type_name": "test_import", "item_type_id": 34958, "$schema": "http://TEST_SERVER/items/jsonschema/34958", "warnings": ["The following items are not registered because they do not exist in the specified item type. item_1617186626617.subitem_description"], "is_change_identifier": False, "errors": None}] result = unpackage_import_file(data_path,csv_path,"csv",False,False) - + assert result == test -# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_function_issue34520 -v -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_handle_fill_system_item_issue34520 -v -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp @pytest.mark.parametrize( "item_id, before_doi,after_doi,warnings,errors,is_change_identifier", [ @@ -2742,7 +3375,7 @@ def test_handle_fill_system_item_issue34520(app, doi_records, item_id, before_do "item_type_id": 1, "$schema": "https://localhost/items/jsonschema/1", } - + if item_id: before["id"] = "{}".format(item_id) before["uri"] = "https://localhost/records/{}".format(item_id) @@ -2751,17 +3384,17 @@ def test_handle_fill_system_item_issue34520(app, doi_records, item_id, before_do if before_doi.get("doi_ra2") is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(before_doi['doi_ra2']) - + if before_doi.get("doi2") is not None: before["metadata"]["item_1617186819068"]=before["metadata"].get("item_1617186819068",{}) before["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(before_doi['doi2']) if before_doi.get('doi_ra') is not None: before["doi_ra"]="{}".format(before_doi['doi_ra']) - + if before_doi.get('doi') is not None: - before["doi"]="{}".format(before_doi['doi']) - + before["doi"]="{}".format(before_doi['doi']) + before_list = [before] after = { "metadata": { @@ -2800,17 +3433,17 @@ def test_handle_fill_system_item_issue34520(app, doi_records, item_id, before_do after["id"] = "{}".format(item_id) after["uri"] = "https://localhost/records/{}".format(item_id) after["metadata"]["item_1617605131499"][0]["url"] = {"url": "https://weko3.example.org/record/{}/files/a.zip".format(item_id)} - + if after_doi.get("doi_ra") is not None: after["doi_ra"]="{}".format(after_doi['doi_ra']) - + if after_doi.get("doi") is not None: - after["doi"]="{}".format(after_doi["doi"]) - + after["doi"]="{}".format(after_doi["doi"]) + if after_doi.get("doi_ra2") is not None: after["metadata"]["item_1617186819068"]=after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_type"]= "{}".format(after_doi['doi_ra2']) - + if after_doi.get("doi2") is not None: after["metadata"]["item_1617186819068"]= after["metadata"].get("item_1617186819068",{}) after["metadata"]["item_1617186819068"]["subitem_identifier_reg_text"]= "{}".format(after_doi['doi2']) @@ -2863,7 +3496,7 @@ def test_handle_check_exist_record_issue35315(app, doi_records, id, uri, warning if uri is not None: before["uri"] = uri before_list = [before] - + after = { "metadata": { "path": ["1667004052852"], @@ -2890,9 +3523,9 @@ def test_handle_check_exist_record_issue35315(app, doi_records, id, uri, warning if uri is not None: after["uri"] = uri after["status"] = status - + after_list = [after] - + with app.test_request_context(): assert before_list != after_list result = handle_check_exist_record(before_list) diff --git a/modules/weko-search-ui/weko_search_ui/utils.py b/modules/weko-search-ui/weko_search_ui/utils.py index 5b57fb4c37..c01c62b783 100644 --- a/modules/weko-search-ui/weko_search_ui/utils.py +++ b/modules/weko-search-ui/weko_search_ui/utils.py @@ -33,6 +33,7 @@ import traceback import uuid import zipfile +import chardet import gc from collections import Callable, OrderedDict from datetime import datetime @@ -249,7 +250,7 @@ def delete_records(index_tree_id, ignore_items): paths.remove(index_id) break - + indexer = WekoIndexer() if not del_flag: @@ -547,13 +548,15 @@ def check_import_items(file, is_change_identifier: bool, is_gakuninrdm=False, try: # Create temp dir for import data os.mkdir(data_path) - with zipfile.ZipFile(file) as z: for info in z.infolist(): try: - info.filename = info.orig_filename.encode("cp437").decode("cp932") - if os.sep != "/" and os.sep in info.filename: - info.filename = info.filename.replace(os.sep, "/") + info.filename = info.orig_filename + inf = chardet.detect(info.orig_filename) + if inf['encoding'] is not None and inf['encoding'] == 'cp437': + info.filename = info.orig_filename.encode("cp437").decode("cp932") + if os.sep != "/" and os.sep in info.filename: + info.filename = info.filename.replace(os.sep, "/") except Exception: current_app.logger.warning("-" * 60) traceback.print_exc(file=sys.stdout) @@ -666,7 +669,7 @@ def unpackage_import_file(data_path: str, file_name: str, file_format: str, forc for record in list_record: record["id"] = None record["uri"] = None - + current_app.logger.debug('list_record2: {}'.format(list_record)) # [{'pos_index': ['Index A'], 'publish_status': 'public', 'feedback_mail': ['wekosoftware@nii.ac.jp'], 'edit_mode': 'Keep', 'metadata': {'pubdate': '2021-03-19', 'item_1617186331708': [{'subitem_1551255647225': 'ja_conference paperITEM00000001(public_open_access_open_access_simple)', 'subitem_1551255648112': 'ja'}, {'subitem_1551255647225': 'en_conference paperITEM00000001(public_open_access_simple)', 'subitem_1551255648112': 'en'}], 'item_1617186385884': [{'subitem_1551255720400': 'Alternative Title', 'subitem_1551255721061': 'en'}, {'subitem_1551255720400': 'Alternative Title', 'subitem_1551255721061': 'ja'}], 'item_1617186419668': [{'creatorAffiliations': [{'affiliationNameIdentifiers': [{'affiliationNameIdentifier': '0000000121691048', 'affiliationNameIdentifierScheme': 'ISNI', 'affiliationNameIdentifierURI': 'http://isni.org/isni/0000000121691048'}], 'affiliationNames': [{'affiliationName': 'University', 'affiliationNameLang': 'en'}]}], 'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': '4', 'nameIdentifierScheme': 'WEKO'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}, {'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}, {'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}], 'item_1617349709064': [{'contributorMails': [{'contributorMail': 'wekosoftware@nii.ac.jp'}], 'contributorNames': [{'contributorName': '情報, 太郎', 'lang': 'ja'}, {'contributorName': 'ジョウホウ, タロウ', 'lang': 'ja-Kana'}, {'contributorName': 'Joho, Taro', 'lang': 'en'}], 'contributorType': 'ContactPerson', 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}], 'item_1617186476635': {'subitem_1522299639480': 'open access', 'subitem_1600958577026': 'http://purl.org/coar/access_right/c_abf2'}, 'item_1617351524846': {'subitem_1523260933860': 'Unknown'}, 'item_1617186499011': [{'subitem_1522650717957': 'ja', 'subitem_1522650727486': 'http://localhost', 'subitem_1522651041219': 'Rights Information'}], 'item_1617610673286': [{'nameIdentifiers': [{'nameIdentifier': 'xxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}], 'rightHolderNames': [{'rightHolderLanguage': 'ja', 'rightHolderName': 'Right Holder Name'}]}], 'item_1617186609386': [{'subitem_1522299896455': 'ja', 'subitem_1522300014469': 'Other', 'subitem_1522300048512': 'http://localhost/', 'subitem_1523261968819': 'Sibject1'}], 'item_1617186626617': [{'subitem_description': 'Description\nDescription
Description', 'subitem_description_language': 'en', 'subitem_description_type': 'Abstract'}, {'subitem_description': '概要\n概要\n概要\n概要', 'subitem_description_language': 'ja', 'subitem_description_type': 'Abstract'}], 'item_1617186643794': [{'subitem_1522300295150': 'en', 'subitem_1522300316516': 'Publisher'}], 'item_1617186660861': [{'subitem_1522300695726': 'Available', 'subitem_1522300722591': '2021-06-30'}], 'item_1617186702042': [{'subitem_1551255818386': 'jpn'}], 'item_1617258105262': {'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}, 'item_1617349808926': {'subitem_1523263171732': 'Version'}, 'item_1617265215918': {'subitem_1522305645492': 'AO', 'subitem_1600292170262': 'http://purl.org/coar/version/c_b1a7d7d4d402bcce'}, 'item_1617186783814': [{'subitem_identifier_type': 'URI', 'subitem_identifier_uri': 'http://localhost'}], 'item_1617353299429': [{'subitem_1522306207484': 'isVersionOf', 'subitem_1522306287251': {'subitem_1522306382014': 'arXiv', 'subitem_1522306436033': 'xxxxx'}, 'subitem_1523320863692': [{'subitem_1523320867455': 'en', 'subitem_1523320909613': 'Related Title'}]}], 'item_1617186859717': [{'subitem_1522658018441': 'en', 'subitem_1522658031721': 'Temporal'}], 'item_1617186882738': [{'subitem_geolocation_place': [{'subitem_geolocation_place_text': 'Japan'}]}], 'item_1617186901218': [{'subitem_1522399143519': {'subitem_1522399281603': 'ISNI', 'subitem_1522399333375': 'http://xxx'}, 'subitem_1522399412622': [{'subitem_1522399416691': 'en', 'subitem_1522737543681': 'Funder Name'}], 'subitem_1522399571623': {'subitem_1522399585738': 'Award URI', 'subitem_1522399628911': 'Award Number'}, 'subitem_1522399651758': [{'subitem_1522721910626': 'en', 'subitem_1522721929892': 'Award Title'}]}], 'item_1617186920753': [{'subitem_1522646500366': 'ISSN', 'subitem_1522646572813': 'xxxx-xxxx-xxxx'}], 'item_1617186941041': [{'subitem_1522650068558': 'en', 'subitem_1522650091861': 'Source Title'}], 'item_1617186959569': {'subitem_1551256328147': '1'}, 'item_1617186981471': {'subitem_1551256294723': '111'}, 'item_1617186994930': {'subitem_1551256248092': '12'}, 'item_1617187024783': {'subitem_1551256198917': '1'}, 'item_1617187045071': {'subitem_1551256185532': '3'}, 'item_1617187112279': [{'subitem_1551256126428': 'Degree Name', 'subitem_1551256129013': 'en'}], 'item_1617187136212': {'subitem_1551256096004': '2021-06-30'}, 'item_1617944105607': [{'subitem_1551256015892': [{'subitem_1551256027296': 'xxxxxx', 'subitem_1551256029891': 'kakenhi'}], 'subitem_1551256037922': [{'subitem_1551256042287': 'Degree Grantor Name', 'subitem_1551256047619': 'en'}]}], 'item_1617187187528': [{'subitem_1599711633003': [{'subitem_1599711636923': 'Conference Name', 'subitem_1599711645590': 'ja'}], 'subitem_1599711655652': '1', 'subitem_1599711660052': [{'subitem_1599711680082': 'Sponsor', 'subitem_1599711686511': 'ja'}], 'subitem_1599711699392': {'subitem_1599711704251': '2020/12/11', 'subitem_1599711712451': '1', 'subitem_1599711727603': '12', 'subitem_1599711731891': '2000', 'subitem_1599711735410': '1', 'subitem_1599711739022': '12', 'subitem_1599711743722': '2020', 'subitem_1599711745532': 'ja'}, 'subitem_1599711758470': [{'subitem_1599711769260': 'Conference Venue', 'subitem_1599711775943': 'ja'}], 'subitem_1599711788485': [{'subitem_1599711798761': 'Conference Place', 'subitem_1599711803382': 'ja'}], 'subitem_1599711813532': 'JPN'}], 'item_1617605131499': [{'accessrole': 'open_access', 'date': [{'dateType': 'Available', 'dateValue': '2021-07-12'}], 'displaytype': 'simple', 'filename': '1KB.pdf', 'filesize': [{'value': '1 KB'}], 'format': 'text/plain'}, {'filename': ''}], 'item_1617620223087': [{'subitem_1565671149650': 'ja', 'subitem_1565671169640': 'Banner Headline', 'subitem_1565671178623': 'Subheading'}, {'subitem_1565671149650': 'en', 'subitem_1565671169640': 'Banner Headline', 'subitem_1565671178623': 'Subheding'}]}, 'file_path': ['file00000001/1KB.pdf', ''], 'item_type_name': 'デフォルトアイテムタイプ（フル）', 'item_type_id': 15, '$schema': 'https://localhost:8443/items/jsonschema/15'}] @@ -681,7 +684,7 @@ def unpackage_import_file(data_path: str, file_name: str, file_format: str, forc # current_app.logger.debug('list_record4: {}'.format(list_record)) # [{'pos_index': ['Index A'], 'publish_status': 'public', 'feedback_mail': ['wekosoftware@nii.ac.jp'], 'edit_mode': 'Keep', 'metadata': {'pubdate': '2021-03-19', 'item_1617186331708': [{'subitem_1551255647225': 'ja_conference paperITEM00000001(public_open_access_open_access_simple)', 'subitem_1551255648112': 'ja'}, {'subitem_1551255647225': 'en_conference paperITEM00000001(public_open_access_simple)', 'subitem_1551255648112': 'en'}], 'item_1617186385884': [{'subitem_1551255720400': 'Alternative Title', 'subitem_1551255721061': 'en'}, {'subitem_1551255720400': 'Alternative Title', 'subitem_1551255721061': 'ja'}], 'item_1617186419668': [{'creatorAffiliations': [{'affiliationNameIdentifiers': [{'affiliationNameIdentifier': '0000000121691048', 'affiliationNameIdentifierScheme': 'ISNI', 'affiliationNameIdentifierURI': 'http://isni.org/isni/0000000121691048'}], 'affiliationNames': [{'affiliationName': 'University', 'affiliationNameLang': 'en'}]}], 'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': '4', 'nameIdentifierScheme': 'WEKO'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}, {'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}, {'creatorMails': [{'creatorMail': 'wekosoftware@nii.ac.jp'}], 'creatorNames': [{'creatorName': '情報, 太郎', 'creatorNameLang': 'ja'}, {'creatorName': 'ジョウホウ, タロウ', 'creatorNameLang': 'ja-Kana'}, {'creatorName': 'Joho, Taro', 'creatorNameLang': 'en'}], 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'zzzzzzz', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}], 'item_1617349709064': [{'contributorMails': [{'contributorMail': 'wekosoftware@nii.ac.jp'}], 'contributorNames': [{'contributorName': '情報, 太郎', 'lang': 'ja'}, {'contributorName': 'ジョウホウ, タロウ', 'lang': 'ja-Kana'}, {'contributorName': 'Joho, Taro', 'lang': 'en'}], 'contributorType': 'ContactPerson', 'familyNames': [{'familyName': '情報', 'familyNameLang': 'ja'}, {'familyName': 'ジョウホウ', 'familyNameLang': 'ja-Kana'}, {'familyName': 'Joho', 'familyNameLang': 'en'}], 'givenNames': [{'givenName': '太郎', 'givenNameLang': 'ja'}, {'givenName': 'タロウ', 'givenNameLang': 'ja-Kana'}, {'givenName': 'Taro', 'givenNameLang': 'en'}], 'nameIdentifiers': [{'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'CiNii', 'nameIdentifierURI': 'https://ci.nii.ac.jp/'}, {'nameIdentifier': 'xxxxxxx', 'nameIdentifierScheme': 'KAKEN2', 'nameIdentifierURI': 'https://kaken.nii.ac.jp/'}]}], 'item_1617186476635': {'subitem_1522299639480': 'open access', 'subitem_1600958577026': 'http://purl.org/coar/access_right/c_abf2'}, 'item_1617351524846': {'subitem_1523260933860': 'Unknown'}, 'item_1617186499011': [{'subitem_1522650717957': 'ja', 'subitem_1522650727486': 'http://localhost', 'subitem_1522651041219': 'Rights Information'}], 'item_1617610673286': [{'nameIdentifiers': [{'nameIdentifier': 'xxxxxx', 'nameIdentifierScheme': 'ORCID', 'nameIdentifierURI': 'https://orcid.org/'}], 'rightHolderNames': [{'rightHolderLanguage': 'ja', 'rightHolderName': 'Right Holder Name'}]}], 'item_1617186609386': [{'subitem_1522299896455': 'ja', 'subitem_1522300014469': 'Other', 'subitem_1522300048512': 'http://localhost/', 'subitem_1523261968819': 'Sibject1'}], 'item_1617186626617': [{'subitem_description': 'Description\nDescription
Description', 'subitem_description_language': 'en', 'subitem_description_type': 'Abstract'}, {'subitem_description': '概要\n概要\n概要\n概要', 'subitem_description_language': 'ja', 'subitem_description_type': 'Abstract'}], 'item_1617186643794': [{'subitem_1522300295150': 'en', 'subitem_1522300316516': 'Publisher'}], 'item_1617186660861': [{'subitem_1522300695726': 'Available', 'subitem_1522300722591': '2021-06-30'}], 'item_1617186702042': [{'subitem_1551255818386': 'jpn'}], 'item_1617258105262': {'resourcetype': 'conference paper', 'resourceuri': 'http://purl.org/coar/resource_type/c_5794'}, 'item_1617349808926': {'subitem_1523263171732': 'Version'}, 'item_1617265215918': {'subitem_1522305645492': 'AO', 'subitem_1600292170262': 'http://purl.org/coar/version/c_b1a7d7d4d402bcce'}, 'item_1617186783814': [{'subitem_identifier_type': 'URI', 'subitem_identifier_uri': 'http://localhost'}], 'item_1617353299429': [{'subitem_1522306207484': 'isVersionOf', 'subitem_1522306287251': {'subitem_1522306382014': 'arXiv', 'subitem_1522306436033': 'xxxxx'}, 'subitem_1523320863692': [{'subitem_1523320867455': 'en', 'subitem_1523320909613': 'Related Title'}]}], 'item_1617186859717': [{'subitem_1522658018441': 'en', 'subitem_1522658031721': 'Temporal'}], 'item_1617186882738': [{'subitem_geolocation_place': [{'subitem_geolocation_place_text': 'Japan'}]}], 'item_1617186901218': [{'subitem_1522399143519': {'subitem_1522399281603': 'ISNI', 'subitem_1522399333375': 'http://xxx'}, 'subitem_1522399412622': [{'subitem_1522399416691': 'en', 'subitem_1522737543681': 'Funder Name'}], 'subitem_1522399571623': {'subitem_1522399585738': 'Award URI', 'subitem_1522399628911': 'Award Number'}, 'subitem_1522399651758': [{'subitem_1522721910626': 'en', 'subitem_1522721929892': 'Award Title'}]}], 'item_1617186920753': [{'subitem_1522646500366': 'ISSN', 'subitem_1522646572813': 'xxxx-xxxx-xxxx'}], 'item_1617186941041': [{'subitem_1522650068558': 'en', 'subitem_1522650091861': 'Source Title'}], 'item_1617186959569': {'subitem_1551256328147': '1'}, 'item_1617186981471': {'subitem_1551256294723': '111'}, 'item_1617186994930': {'subitem_1551256248092': '12'}, 'item_1617187024783': {'subitem_1551256198917': '1'}, 'item_1617187045071': {'subitem_1551256185532': '3'}, 'item_1617187112279': [{'subitem_1551256126428': 'Degree Name', 'subitem_1551256129013': 'en'}], 'item_1617187136212': {'subitem_1551256096004': '2021-06-30'}, 'item_1617944105607': [{'subitem_1551256015892': [{'subitem_1551256027296': 'xxxxxx', 'subitem_1551256029891': 'kakenhi'}], 'subitem_1551256037922': [{'subitem_1551256042287': 'Degree Grantor Name', 'subitem_1551256047619': 'en'}]}], 'item_1617187187528': [{'subitem_1599711633003': [{'subitem_1599711636923': 'Conference Name', 'subitem_1599711645590': 'ja'}], 'subitem_1599711655652': '1', 'subitem_1599711660052': [{'subitem_1599711680082': 'Sponsor', 'subitem_1599711686511': 'ja'}], 'subitem_1599711699392': {'subitem_1599711704251': '2020/12/11', 'subitem_1599711712451': '1', 'subitem_1599711727603': '12', 'subitem_1599711731891': '2000', 'subitem_1599711735410': '1', 'subitem_1599711739022': '12', 'subitem_1599711743722': '2020', 'subitem_1599711745532': 'ja'}, 'subitem_1599711758470': [{'subitem_1599711769260': 'Conference Venue', 'subitem_1599711775943': 'ja'}], 'subitem_1599711788485': [{'subitem_1599711798761': 'Conference Place', 'subitem_1599711803382': 'ja'}], 'subitem_1599711813532': 'JPN'}], 'item_1617605131499': [{'accessrole': 'open_access', 'date': [{'dateType': 'Available', 'dateValue': '2021-07-12'}], 'displaytype': 'simple', 'filename': '1KB.pdf', 'filesize': [{'value': '1 KB'}], 'format': 'text/plain'}, {'filename': ''}], 'item_1617620223087': [{'subitem_1565671149650': 'ja', 'subitem_1565671169640': 'Banner Headline', 'subitem_1565671178623': 'Subheading'}, {'subitem_1565671149650': 'en', 'subitem_1565671169640': 'Banner Headline', 'subitem_1565671178623': 'Subheding'}]}, 'file_path': ['file00000001/1KB.pdf', ''], 'item_type_name': 'デフォルトアイテムタイプ（フル）', 'item_type_id': 15, '$schema': 'https://localhost:8443/items/jsonschema/15', 'identifier_key': 'item_1617186819068', 'errors': None, 'status': 'new', 'id': None, 'item_title': 'ja_conference paperITEM00000001(public_open_access_open_access_simple)'}] - + return list_record @@ -1044,7 +1047,7 @@ def handle_check_exist_record(list_record) -> list: exist_url = ( request.host_url + "records/" + str(item_exist.get("recid")) ) - + if item.get("uri") == exist_url: _edit_mode = item.get("edit_mode") if not _edit_mode or _edit_mode.lower() not in [ @@ -1237,7 +1240,7 @@ def clean_file_metadata(item_type_id, data): # clear metadata of file information is_cleaned = True file_key = None - item_map = get_mapping(Mapping.get_record(item_type_id), "jpcoar_mapping") + item_map = get_mapping(item_type_id, "jpcoar_mapping") key = item_map.get("file.URI.@value") if key: file_key = key.split(".")[0] @@ -1279,11 +1282,11 @@ def autofill_thumbnail_metadata(item_type_id, data): def escape_newline(data): """Replace
in metadata with \n. - + {"key1":["test
test"]} -> {"key1":["test\ntest"]} :argument data -- {obj} escape target - :return + :return obj -- Obj after escaping """ if isinstance(data,list): @@ -1733,21 +1736,53 @@ def handle_item_title(list_record): :return """ + from weko_items_ui.utils import get_options_and_order_list, get_hide_list_by_schema_form + from weko_records.utils import check_info_in_metadata + for item in list_record: error = None - item_type_mapping = Mapping.get_record(item["item_type_id"]) - item_map = get_mapping(item_type_mapping, "jpcoar_mapping") + meta_option, item_type_mapping = get_options_and_order_list(item["item_type_id"]) + hide_list = get_hide_list_by_schema_form(item["item_type_id"]) + item_map = get_mapping(item["item_type_id"], "jpcoar_mapping") # current_app.logger.debug("item_type_mapping: {}".format(item_type_mapping)) # current_app.logger.debug("item_map: {}".format(item_map)) title_data, _title_key = get_data_by_property( item["metadata"], item_map, "title.@value" ) - if not title_data: - error = _("Title is required item.") - else: - item["item_title"] = title_data[0] + title_lang_data, _title_lang_key = get_data_by_property( + item["metadata"], item_map, "title.@attributes.xml:lang" + ) + title_val = None + lang_key_list = _title_lang_key.split(",") + val_key_list = _title_key.split(",") + for val_key in val_key_list: + val_parent_key = val_key.split(".")[0] + val_sub_key = val_key.split(".")[-1] + for lang_key in lang_key_list: + if val_parent_key == lang_key.split(".")[0]: + prop_hidden = meta_option.get(val_parent_key, {}).get('option', {}).get('hidden', False) + for h in hide_list: + if h.startswith(val_parent_key) and h.endswith(val_sub_key): + prop_hidden = True + if ( + title_lang_data is not None + and title_data is not None + and title_lang_data is not None + and len(title_data) > 0 + and len(title_lang_data) > 0 + and not prop_hidden + ): + title_val = check_info_in_metadata( + lang_key, val_key, title_lang_data[0], item["metadata"] + ) + item["item_title"] = title_val + if title_val: + break + if title_val: + break - if error: + if not title_val: + error = _("Title is required item.") item["errors"] = item["errors"] + [error] if item.get("errors") else [error] @@ -1833,7 +1868,7 @@ def check(index_id, index_name_path): info.name.replace( '-/-', current_app.config['WEKO_ITEMS_UI_INDEX_PATH_SPLIT']): index_info = info - + if not index_info: # index does not exist by index path if index_id: errors.append(msg_not_exist.format("IndexID, POS_INDEX")) @@ -2318,7 +2353,7 @@ def prepare_doi_link(item_id): Returns: _type_: _description_ {'identifier_grant_jalc_doi_link': 'https://doi.org/aaa.bbb/0000000006', 'identifier_grant_jalc_cr_doi_link': 'https://doi.org/ccc.ddd/0000000006', 'identifier_grant_jalc_dc_doi_link': 'https://doi.org/eee.fff/0000000006', 'identifier_grant_ndl_jalc_doi_link': 'https://doi.org/ggg.hhh/0000000006'} - """ + """ item_id = "%010d" % int(item_id) identifier_setting = prepare_doi_setting() suffix = identifier_setting.suffix or "" @@ -2458,7 +2493,7 @@ def register_item_update_publish_status(item, status): def register_item_custom_sort_order(item): """ register item custom sort order - + Args: item (dict): item data """ @@ -2908,9 +2943,8 @@ def recursive_sub(keys, node, uri_key, current_type): warnings = item.get('warnings') or [] if item_type_id != item["item_type_id"]: item_type_id = item["item_type_id"] - record = Mapping.get_record(item_type_id) # current_app.logger.debug("record_hoge: {}".format(record)) - item_map = get_mapping(record, "jpcoar_mapping") + item_map = get_mapping(item_type_id, "jpcoar_mapping") # Resource Type resourcetype_key = item_map.get("type.@value") @@ -2954,27 +2988,27 @@ def recursive_sub(keys, node, uri_key, current_type): "identifierRegistration.@attributes.identifierType", "" ) identifierRegistration_key = identifierRegistration_key.split(".")[0] - + item_doi = item.get("doi","") item_doi_prefix = "" item_doi_suffix = "" item_cnri = item.get("cnri", "") - + if item_doi and "/" in item_doi: item_doi_prefix, item_doi_suffix = item_doi.split("/") else: item_doi_prefix = item_doi - + item_doi_ra = item.get("doi_ra","") item_id = item.get('id',"") checked_registerd_doi_ra = False existed_doi = False - + if identifierRegistration_key: item["identifier_key"] = identifierRegistration_key is_change_identifier = item.get("is_change_identifier", False) doi_setting = prepare_doi_setting() - + pid_doi = None if item_id: try: @@ -2983,12 +3017,12 @@ def recursive_sub(keys, node, uri_key, current_type): pid_doi = rec.pid_doi except PIDDoesNotExistError: pid_doi=None - + registerd_doi = None registerd_doi_prefix = None registerd_doi_suffix = None registerd_doi_ra = None - + if pid_doi and doi_setting: doi_value = pid_doi.pid_value registerd_doi = doi_value.replace("https://doi.org/","") @@ -3007,7 +3041,7 @@ def recursive_sub(keys, node, uri_key, current_type): checked_registerd_doi_ra = False else: existed_doi = False - + checked_item_doi_ra = False if item_doi_prefix is not "" and doi_setting: if doi_setting.jalc_doi == item_doi_prefix: @@ -3023,7 +3057,7 @@ def recursive_sub(keys, node, uri_key, current_type): fixed_doi = False fixed_doi_ra = False - + if is_change_identifier: item["doi"] = item_doi elif is_change_identifier == False: @@ -3031,7 +3065,7 @@ def recursive_sub(keys, node, uri_key, current_type): item["doi"] = registerd_doi if registerd_doi != item_doi: fixed_doi = True - + if is_change_identifier == True: item["doi_ra"] = item_doi_ra elif is_change_identifier == False: @@ -3039,9 +3073,9 @@ def recursive_sub(keys, node, uri_key, current_type): item["doi_ra"] = registerd_doi_ra if registerd_doi_ra != item_doi_ra: fixed_doi_ra = True - + if identifierRegistration_key in item["metadata"]: - if existed_doi and checked_registerd_doi_ra and checked_item_doi_ra: + if existed_doi and checked_registerd_doi_ra and checked_item_doi_ra: if 'subitem_identifier_reg_type' in item["metadata"][identifierRegistration_key]: _doi_ra = item["metadata"][identifierRegistration_key]['subitem_identifier_reg_type'] if item.get("is_change_identifier", False) == True: @@ -3062,7 +3096,7 @@ def recursive_sub(keys, node, uri_key, current_type): if registerd_doi_ra != item_doi_ra: fixed_doi_ra = True - if 'subitem_identifier_reg_text' in item["metadata"][identifierRegistration_key]: + if 'subitem_identifier_reg_text' in item["metadata"][identifierRegistration_key]: _doi = item["metadata"][identifierRegistration_key]['subitem_identifier_reg_text'] if item.get("is_change_identifier", False) == True: item["metadata"][identifierRegistration_key]['subitem_identifier_reg_text'] = item_doi @@ -3078,7 +3112,7 @@ def recursive_sub(keys, node, uri_key, current_type): if item.get("is_change_identifier", False) == False: item["metadata"][identifierRegistration_key]['subitem_identifier_reg_text'] = registerd_doi if registerd_doi != item_doi: - fixed_doi = True + fixed_doi = True else: del item["metadata"][identifierRegistration_key] elif item.get("is_change_identifier", False): @@ -3089,10 +3123,10 @@ def recursive_sub(keys, node, uri_key, current_type): if fixed_doi: warnings.append(_('The specified DOI is wrong and fixed with the registered DOI.')) - + if fixed_doi_ra: warnings.append(_('The specified DOI RA is wrong and fixed with the correct DOI RA of the registered DOI.')) - + if is_change_identifier: if not (current_app.config["WEKO_HANDLE_ALLOW_REGISTER_CNRI"] and item_cnri): if item_doi is "": @@ -3107,8 +3141,8 @@ def recursive_sub(keys, node, uri_key, current_type): if item_doi_ra not in ("JaLC","Crossref","DataCite","NDL JaLC"): errors.append(_('DOI_RA should be set by one of JaLC, Crossref, DataCite, NDL JaLC.')) elif item_doi_prefix: - errors.append(_('Specified Prefix of DOI is incorrect.')) - + errors.append(_('Specified Prefix of DOI is incorrect.')) + item['errors'] = errors item['warnings'] = warnings @@ -3441,10 +3475,10 @@ def _get_export_data(export_path, item_types, retrys, fromid="", toid="", retry_ item_types.remove(it) continue - record_ids = [(recid.pid_value, recid.object_uuid) + record_ids = [(recid.pid_value, recid.object_uuid) for recid in recids if recid.json and 'publish_status' in recid.json \ and recid.json['publish_status'] in [PublishStatus.PUBLIC.value, PublishStatus.PRIVATE.value]] - + # recidsを削除 del recids gc.collect() @@ -3458,6 +3492,11 @@ def _get_export_data(export_path, item_types, retrys, fromid="", toid="", retry_ _file_create_key, orjson.dumps(write_file_json).decode() ) + + if len(record_ids) == 0: + item_types.remove(it) + continue + for recid, uuid in record_ids: if counter % current_app.config["WEKO_SEARCH_UI_BULK_EXPORT_LIMIT"] == 0 and item_datas: # Create export info file @@ -3511,7 +3550,7 @@ def _get_export_data(export_path, item_types, retrys, fromid="", toid="", retry_ with open(pickle_file_name, 'wb') as f: pickle.dump(item_datas, f) - + del item_datas gc.collect() @@ -3571,7 +3610,7 @@ def _get_export_data(export_path, item_types, retrys, fromid="", toid="", retry_ else: fromid = item_id_range toid = item_id_range - + result = None if not fromid or not toid or (fromid and toid and int(fromid) <= int(toid)): result = _get_export_data(export_path, item_types, 0, fromid, toid) @@ -3600,7 +3639,7 @@ def write_files(item_datas, export_path, user_id, retrys): export_path (str): file creation destination user_id (int): performing user id retrys (int): retry time - + Returns: bool: task is success or failure. """ @@ -3784,7 +3823,7 @@ def _check_write_file_info(json): status = "" start_time = "" finish_time = "" - + try: task_id = get_redis_cache(cache_key) download_uri = get_redis_cache(cache_uri) @@ -3961,7 +4000,7 @@ def handle_check_file_metadata(list_record, data_path): def handle_check_custom_sort_order(list_record): """Check CustomSortOrder. - + Args: list_record (list): List record import. """ @@ -4130,20 +4169,21 @@ def get_data_by_property(item_metadata, item_map, mapping_key): :param mapping_key: Mapping key. :return: Property key and values. """ - key = item_map.get(mapping_key) + key_list = item_map.get(mapping_key) data = [] - if not key: + if not key_list: current_app.logger.error(str(mapping_key) + " jpcoar:mapping " "is not correct") return None, None - attribute = item_metadata.get(key.split(".")[0]) - if not attribute: - return None, key - else: - data_result = get_sub_item_value(attribute, key.split(".")[-1]) - if data_result: - for value in data_result: - data.append(value) - return data, key + for key in key_list.split(","): + attribute = item_metadata.get(key.split(".")[0]) + if not attribute: + return None, key_list + else: + data_result = get_sub_item_value(attribute, key.split(".")[-1]) + if data_result: + for value in data_result: + data.append(value) + return data, key_list def get_filenames_from_metadata(metadata): @@ -4195,7 +4235,7 @@ def get_billinginfo_from_metadata(metadata): Args: metadata (dict): record metadata. - + Returns: billing_info (dict): Dict billing info from metadata. """ @@ -4235,7 +4275,7 @@ def get_billinginfo_from_metadata(metadata): } priceinfo.append(priceinfo_data) second_count += 1 - + accessrole_info.append(accessrole_data) displaytype_info.append(displaytype_data) billing_file.append(billing_data) diff --git a/modules/weko-sitemap/weko_sitemap/config.py b/modules/weko-sitemap/weko_sitemap/config.py index f7a5df0cda..4233c932a4 100644 --- a/modules/weko-sitemap/weko_sitemap/config.py +++ b/modules/weko-sitemap/weko_sitemap/config.py @@ -36,6 +36,4 @@ SITEMAP_MAX_URL_COUNT = 10000 WEKO_SITEMAP__ROBOT_TXT = """ -User-agent: Bingbot -Crawl-delay: 30 """ diff --git a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl index 015caae79d..a9b098d2c5 100644 --- a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl +++ b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl @@ -1,6 +1,27 @@ User-agent: * Disallow: /admin/ Disallow: /api/ +Disallow: /oai +Disallow: /search +Disallow: /items/ranking +Disallow: /items/export +Disallow: /login/ +Disallow: /shibauthorizer +Disallow: /Shibboleth.sso +Disallow: /shibboleth-sp +Disallow: /secure/ +Disallow: /weko/shib +Disallow: /ping +Disallow: /static +Disallow: /data +Disallow: /records/*/export/json +Disallow: /records/*/export/bibtex + +User-agent: Bingbot +Crawl-delay: 30 + +User-agent: PetalBot +Disallow: / {{ robot_txt }} diff --git a/modules/weko-theme/weko_theme/templates/weko_theme/admin_header.html b/modules/weko-theme/weko_theme/templates/weko_theme/admin_header.html index bee7b9359f..c769954d22 100644 --- a/modules/weko-theme/weko_theme/templates/weko_theme/admin_header.html +++ b/modules/weko-theme/weko_theme/templates/weko_theme/admin_header.html @@ -53,9 +53,11 @@
- - {{ username or current_user.email }} - +
+ + {{ username or current_user.email }} + +
{%- if username and username != current_user.email %}

{{ current_user.email }}

{%- endif %} diff --git a/modules/weko-theme/weko_theme/templates/weko_theme/header_login.html b/modules/weko-theme/weko_theme/templates/weko_theme/header_login.html index feb8c8f3d9..eafdb8ec5c 100644 --- a/modules/weko-theme/weko_theme/templates/weko_theme/header_login.html +++ b/modules/weko-theme/weko_theme/templates/weko_theme/header_login.html @@ -44,8 +44,11 @@
+ {%- set username = current_user.shib_weko_user[0].shib_handle if current_user.shib_weko_user and current_user.shib_weko_user|length > 0 and current_user.shib_weko_user[0].shib_handle else current_user.email %} - {{current_user.email}} +
+ {{username}} +