From 5f35c3fe0661f563c80ef15f2d7dfd70a7931797 Mon Sep 17 00:00:00 2001 From: uxairibrar Date: Sun, 11 May 2025 18:58:24 +0200 Subject: [PATCH 1/3] Export data to wikidata --- optimap/settings.py | 4 + publications/admin.py | 21 ++++- publications/wikidata.py | 190 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 publications/wikidata.py diff --git a/optimap/settings.py b/optimap/settings.py index e21323f..48ce1c4 100644 --- a/optimap/settings.py +++ b/optimap/settings.py @@ -222,6 +222,10 @@ EMAIL_SEND_DELAY = 2 DATA_DUMP_INTERVAL_HOURS = 6 OPENALEX_MAILTO = "login@optimap.science" +WIKIBASE_API_URL = env("WIKIBASE_API_URL") +WIKIBASE_USERNAME = env("WIKIBASE_USERNAME") +WIKIBASE_PASSWORD = env("WIKIBASE_PASSWORD") + MIDDLEWARE = [ 'django.middleware.cache.UpdateCacheMiddleware', diff --git a/publications/admin.py b/publications/admin.py index a94f1b3..65c1e4a 100644 --- a/publications/admin.py +++ b/publications/admin.py @@ -17,6 +17,24 @@ from publications.tasks import regenerate_geopackage_cache from django.test import Client from django.http import HttpResponse +from publications.wikidata import export_publications_to_wikidata + +@admin.action(description="Create new Wikidata items for selected publications") +def export_to_wikidata(modeladmin, request, queryset): + created_count, updated_count, error_records = export_publications_to_wikidata(queryset) + + # Success messages + if created_count: + messages.success(request, f"{created_count} new Wikidata item(s) created.") + if updated_count: + messages.success(request, f"{updated_count} existing Wikidata item(s) updated.") + + # Warnings and errors + for publication, error_message in error_records: + if error_message == "no publicationDate": + messages.warning(request, f"Skipping “{publication.title}”: no publication date") + else: + messages.error(request, f"Failed to export “{publication.title}”: {error_message}") @admin.action(description="Mark selected publications as published") def make_public(modeladmin, request, queryset): @@ -155,7 +173,7 @@ class PublicationAdmin(LeafletGeoAdmin, ImportExportModelAdmin): "openalex_ids", "openalex_open_access_status") readonly_fields = ("created_by", "updated_by", "openalex_link") actions = ["make_public", "make_draft", "regenerate_all_exports", - "export_permalinks_csv", "email_permalinks_preview"] + "export_permalinks_csv", "email_permalinks_preview", "export_to_wikidata"] @admin.display(boolean=True, description="Has DOI") def has_permalink(self, obj): @@ -261,3 +279,4 @@ class UserAdmin(admin.ModelAdmin): @admin.register(GlobalRegion) class GlobalRegionAdmin(admin.ModelAdmin): """GlobalRegion Admin.""" + diff --git a/publications/wikidata.py b/publications/wikidata.py new file mode 100644 index 0000000..cf74afa --- /dev/null +++ b/publications/wikidata.py @@ -0,0 +1,190 @@ +import os +import requests +from datetime import datetime +from django.conf import settings + +from wikibaseintegrator.wbi_exceptions import ModificationFailed +from wikibaseintegrator import WikibaseIntegrator +from wikibaseintegrator.wbi_login import Login +from wikibaseintegrator.datatypes import ( + MonolingualText, + Time, + String, + ExternalID, + GlobeCoordinate +) +try: + from wikibaseintegrator.datatypes import Url +except ImportError: + from wikibaseintegrator.datatypes import URL as Url + +# Our instance’s SPARQL endpoint (for local lookups by DOI) +if "www.wikidata.org/w/api.php" in settings.WIKIBASE_API_URL: + SPARQL_ENDPOINT = "https://query.wikidata.org/sparql" +else: + SPARQL_ENDPOINT = settings.WIKIBASE_API_URL.replace("/w/api.php", "/query/sparql") + +# constant for all dates +CALENDAR_MODEL = "http://www.wikidata.org/entity/Q1985727" + +# Wikidata property IDs mapping +P_TITLE = "P1476" # title (monolingual text) +P_ABSTRACT = "P1810" # abstract +P_URL = "P856" # official website / URL +P_PUBLICATION_DATE = "P577" # publication date +P_PERIOD_START = "P580" # start time +P_PERIOD_END = "P582" # end time +P_DOI = "P356" # DOI as External ID +P_AUTHOR_STRING = "P2093" # author name string +P_JOURNAL_NAME = "P1448" # journal name (monolingual text) +P_GEOMETRY = "P625" # coordinate location + +def normalize_date_and_precision(date_str): + parts = date_str.split("-") + if len(parts) == 1 and parts[0].isdigit(): + # "YYYY" + return f"{parts[0]}-01-01", 9 + if len(parts) == 2 and all(p.isdigit() for p in parts): + # "YYYY-MM" + return f"{parts[0]}-{parts[1]}-01", 10 + # assume full "YYYY-MM-DD" + return date_str, 11 + +def add_time_claims(dates, prop_nr, statements): + for ds in dates: + iso, prec = normalize_date_and_precision(ds) + timestamp = f"+{iso}T00:00:00Z" + statements.append(Time( + prop_nr=prop_nr, + time=timestamp, + timezone=0, + before=0, + after=0, + precision=prec, + calendarmodel=CALENDAR_MODEL + )) + + +def find_local_item_by_doi(doi): + """ + Return the Q-ID of an existing item in our Wikibase instance for the given DOI, + or None if no match is found. + """ + sparql_query = f''' + SELECT ?item WHERE {{ + ?item wdt:{P_DOI} "{doi}" . + }} LIMIT 1 + ''' + response = requests.get( + SPARQL_ENDPOINT, + params={"query": sparql_query, "format": "json"}, + headers={"Accept": "application/json"} + ) + response.raise_for_status() + + data = response.json() + bindings = data.get("results", {}).get("bindings", []) + if not bindings: + return None + + item_uri = bindings[0]["item"]["value"] + return item_uri.rsplit("/", 1)[-1] + +def upsert_publication(publication, wikibase_integrator): + """ + Create or update a single Publication on Wikibase. + Returns a tuple (action, qid): + - action is "created", "updated", or "skipped" + - qid is the Wikibase item ID (or None if skipped) + """ + # 1) Build statements + iso_date = publication.publicationDate.isoformat() + publication_timestamp = f"+{iso_date}T00:00:00Z" + + statements = [ + MonolingualText(prop_nr=P_TITLE, text=publication.title, language="en"), + Time(prop_nr=P_PUBLICATION_DATE, time=publication_timestamp, timezone=0, before=0, after=0, precision=11, calendarmodel=CALENDAR_MODEL), + String(prop_nr=P_AUTHOR_STRING, value=(publication.created_by.username if publication.created_by else "Unknown author")), + ] + + if publication.abstract: + statements.append(String(prop_nr=P_ABSTRACT, value=publication.abstract)) + + if publication.url: + statements.append(Url(prop_nr=P_URL, value=publication.url)) + + if publication.timeperiod_startdate: + add_time_claims(publication.timeperiod_startdate, P_PERIOD_START, statements) + + if publication.timeperiod_enddate: + add_time_claims(publication.timeperiod_enddate, P_PERIOD_END, statements) + + if publication.source: + statements.append(MonolingualText(prop_nr=P_JOURNAL_NAME, text=publication.source, language="en")) + + if publication.doi: + statements.append( ExternalID(prop_nr=P_DOI, value=publication.doi)) + + if publication.geometry: + geometries = getattr(publication.geometry, "geoms", [publication.geometry]) + for geom in geometries: + if getattr(geom, "geom_type", None) != "Point": + geom = geom.centroid + statements.append(GlobeCoordinate(prop_nr=P_GEOMETRY, latitude=geom.y, longitude=geom.x, precision=0.0001)) + + # 7) Check for existing item by DOI + existing_qid = find_local_item_by_doi(publication.doi) if publication.doi else None + + if existing_qid: + # Update existing item + entity = wikibase_integrator.item.get(entity_id=existing_qid) + entity.claims.add(statements) + try: + entity.write(summary="Update publication via OptimapBot") + return "updated", existing_qid + except ModificationFailed as e: + if "already has label" in str(e): + return "skipped", existing_qid + raise + else: + # Create new item + entity = wikibase_integrator.item.new() + entity.labels.set("en", publication.title) + entity.descriptions.set("en", "Publication imported from Optimap") + entity.claims.add(statements) + try: + write_result = entity.write(summary="Create publication via OptimapBot") + created_qid = write_result.get("entity", {}).get("id") + return "created", created_qid + except ModificationFailed as e: + if "already has label" in str(e): + return "skipped", None + raise + +def export_publications_to_wikidata(publications): + login_session = Login( + user=settings.WIKIBASE_USERNAME, + password=settings.WIKIBASE_PASSWORD, + mediawiki_api_url=settings.WIKIBASE_API_URL, + ) + wikibase_client = WikibaseIntegrator(login=login_session) + + created_count = 0 + updated_count = 0 + error_records = [] + + for publication in publications: + if not publication.publicationDate: + error_records.append((publication, "no publicationDate")) + continue + + try: + action, entity_id = upsert_publication(publication, wikibase_client) + if action == "created": + created_count += 1 + elif action == "updated": + updated_count += 1 + except Exception as err: + error_records.append((publication, str(err))) + + return created_count, updated_count, error_records From 39706207c169ca2bad4426e77c85a24c3c283f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20N=C3=BCst?= Date: Sat, 25 Oct 2025 19:58:54 +0200 Subject: [PATCH 2/3] Adds fixes for Wikibase export for publications Implements functionality to export publication metadata to a Wikibase instance, including Wikidata --- .claude/settings.local.json | 11 +- README.md | 7 +- optimap/.env.example | 7 + optimap/settings.py | 25 +- publications/admin.py | 149 +- .../migrations/0004_wikidata_export_log.py | 37 + ...0005_add_wikibase_endpoint_to_exportlog.py | 23 + publications/models.py | 64 +- publications/templates/work_landing_page.html | 68 +- publications/views.py | 20 + publications/wikidata.py | 1434 +++++++++++++++-- tests/test_wikidata_export.py | 725 +++++++++ 12 files changed, 2422 insertions(+), 148 deletions(-) create mode 100644 publications/migrations/0004_wikidata_export_log.py create mode 100644 publications/migrations/0005_add_wikibase_endpoint_to_exportlog.py create mode 100644 tests/test_wikidata_export.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 53489f4..ef03b39 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -16,7 +16,16 @@ "Bash(python manage.py:*)", "Bash(python -m py_compile:*)", "Bash(python:*)", - "Bash(node --check:*)" + "Bash(node --check:*)", + "Bash(find:*)", + "Bash(OPTIMAP_LOGGING_LEVEL=WARNING python manage.py test:*)", + "Bash(export OPTIMAP_LOGGING_LEVEL=WARNING)", + "Bash(awk:*)", + "Bash(chmod:*)", + "Bash(bash:*)", + "Bash(./create_wikibase_property.sh:*)", + "Bash(python3:*)", + "Bash(pkill:*)" ], "deny": [], "ask": [] diff --git a/README.md b/README.md index 07e32f0..db79d62 100644 --- a/README.md +++ b/README.md @@ -284,11 +284,12 @@ python -m smtpd -c DebuggingServer -n localhost:5587 OPTIMAP_EMAIL_HOST=localhost OPTIMAP_EMAIL_PORT=5587 ``` -### Accessing list of article links -Visit the URL - http://127.0.0.1:8000/articles/links/ +### Accessing list of works -### Harvest Publications from Real Journals +Visit the URL - + +### Harvest Publications from real journals The `harvest_journals` management command allows you to harvest publications from real OAI-PMH journal sources directly into your database. This is useful for: diff --git a/optimap/.env.example b/optimap/.env.example index c15c4be..4b7e29e 100644 --- a/optimap/.env.example +++ b/optimap/.env.example @@ -27,3 +27,10 @@ OPTIMAP_LOGGING_LEVEL=INFO DJANGO_LOGGING_LEVEL=ERROR OPTIMAP_DATA_DUMP_RETENTION=3 + +WIKIBASE_CONSUMER_TOKEN=your_consumer_token +WIKIBASE_CONSUMER_SECRET=your_consumer_secret +WIKIBASE_ACCESS_TOKEN=your_access_token +WIKIBASE_ACCESS_SECRET=your_access_secret +WIKIBASE_API_URL=https://your-instance.wikibase.cloud/w/api.php + diff --git a/optimap/settings.py b/optimap/settings.py index 48ce1c4..beffc06 100644 --- a/optimap/settings.py +++ b/optimap/settings.py @@ -217,14 +217,27 @@ EMAIL_IMAP_SENT_FOLDER = env('OPTIMAP_EMAIL_IMAP_SENT_FOLDER', default='') OPTIMAP_EMAIL_SEND_DELAY = env("OPTIMAP_EMAIL_SEND_DELAY", default=2) BASE_URL = env("BASE_URL", default="http://127.0.0.1:8000") -OAI_USERNAME = env("OPTIMAP_OAI_USERNAME", default="") -OAI_PASSWORD = env("OPTIMAP_OAI_PASSWORD", default="") EMAIL_SEND_DELAY = 2 DATA_DUMP_INTERVAL_HOURS = 6 -OPENALEX_MAILTO = "login@optimap.science" -WIKIBASE_API_URL = env("WIKIBASE_API_URL") -WIKIBASE_USERNAME = env("WIKIBASE_USERNAME") -WIKIBASE_PASSWORD = env("WIKIBASE_PASSWORD") + +# Contact email for API user agents (OpenAlex, Wikidata, etc.) +CONTACT_EMAIL = "login@optimap.science" + +# Wikibase/Wikidata configuration +WIKIBASE_API_URL = env("WIKIBASE_API_URL", default="") + +# OAuth 1.0a authentication (required) +WIKIBASE_CONSUMER_TOKEN = env("WIKIBASE_CONSUMER_TOKEN", default="") +WIKIBASE_CONSUMER_SECRET = env("WIKIBASE_CONSUMER_SECRET", default="") +WIKIBASE_ACCESS_TOKEN = env("WIKIBASE_ACCESS_TOKEN", default="") +WIKIBASE_ACCESS_SECRET = env("WIKIBASE_ACCESS_SECRET", default="") + +# Property auto-creation setting +WIKIBASE_CREATE_PROPERTIES_IF_MISSING = env("WIKIBASE_CREATE_PROPERTIES_IF_MISSING", default=True, cast=bool) + +# Import optimap version for user agent +import optimap +WIKIBASE_USER_AGENT = f"OPTIMAP/{optimap.__version__} (https://optimap.science; {CONTACT_EMAIL})" MIDDLEWARE = [ diff --git a/publications/admin.py b/publications/admin.py index 65c1e4a..655b501 100644 --- a/publications/admin.py +++ b/publications/admin.py @@ -8,7 +8,7 @@ from leaflet.admin import LeafletGeoAdmin from publications.models import Publication, Source, HarvestingEvent, BlockedEmail, BlockedDomain, GlobalRegion from import_export.admin import ImportExportModelAdmin -from publications.models import EmailLog, Subscription, UserProfile +from publications.models import EmailLog, Subscription, UserProfile, WikidataExportLog from publications.tasks import harvest_oai_endpoint, schedule_subscription_email_task, send_monthly_email, schedule_monthly_email_task from django_q.models import Schedule from django.utils.timezone import now @@ -17,24 +17,45 @@ from publications.tasks import regenerate_geopackage_cache from django.test import Client from django.http import HttpResponse -from publications.wikidata import export_publications_to_wikidata +from publications.wikidata import export_publications_to_wikidata, export_publications_to_wikidata_dryrun -@admin.action(description="Create new Wikidata items for selected publications") +@admin.action(description="Export selected publications to Wikidata/Wikibase") def export_to_wikidata(modeladmin, request, queryset): - created_count, updated_count, error_records = export_publications_to_wikidata(queryset) + stats = export_publications_to_wikidata(queryset) # Success messages - if created_count: - messages.success(request, f"{created_count} new Wikidata item(s) created.") - if updated_count: - messages.success(request, f"{updated_count} existing Wikidata item(s) updated.") - - # Warnings and errors - for publication, error_message in error_records: - if error_message == "no publicationDate": - messages.warning(request, f"Skipping “{publication.title}”: no publication date") - else: - messages.error(request, f"Failed to export “{publication.title}”: {error_message}") + if stats['created']: + messages.success(request, f"{stats['created']} new Wikidata item(s) created.") + if stats['updated']: + messages.success(request, f"{stats['updated']} existing Wikidata item(s) updated.") + if stats['skipped']: + messages.info(request, f"{stats['skipped']} publication(s) skipped (already exist or duplicate labels).") + + # Error messages + if stats['errors']: + messages.error(request, f"{stats['errors']} publication(s) failed to export. Check the Wikidata export logs for details.") + + # Summary message + messages.info(request, f"Total: {stats['total']} publication(s) processed.") + +@admin.action(description="[DRY-RUN] Export selected publications to Wikidata/Wikibase") +def export_to_wikidata_dryrun(modeladmin, request, queryset): + stats = export_publications_to_wikidata_dryrun(queryset) + + # Dry-run summary messages + messages.info(request, f"[DRY-RUN] Export simulation complete:") + + if stats['created']: + messages.info(request, f" • Would create {stats['created']} new Wikidata item(s)") + if stats['updated']: + messages.info(request, f" • Would update {stats['updated']} existing Wikidata item(s)") + if stats['skipped']: + messages.info(request, f" • Would skip {stats['skipped']} publication(s)") + if stats['errors']: + messages.warning(request, f" • {stats['errors']} publication(s) have validation errors") + + # Summary message + messages.success(request, f"[DRY-RUN] Total: {stats['total']} publication(s) analyzed. No changes were written to Wikibase.") @admin.action(description="Mark selected publications as published") def make_public(modeladmin, request, queryset): @@ -172,8 +193,9 @@ class PublicationAdmin(LeafletGeoAdmin, ImportExportModelAdmin): "openalex_fulltext_origin", "openalex_is_retracted", "openalex_ids", "openalex_open_access_status") readonly_fields = ("created_by", "updated_by", "openalex_link") - actions = ["make_public", "make_draft", "regenerate_all_exports", - "export_permalinks_csv", "email_permalinks_preview", "export_to_wikidata"] + actions = [make_public, make_draft, regenerate_all_exports, + "export_permalinks_csv", "email_permalinks_preview", + export_to_wikidata, export_to_wikidata_dryrun] @admin.display(boolean=True, description="Has DOI") def has_permalink(self, obj): @@ -243,12 +265,95 @@ class EmailLogAdmin(admin.ModelAdmin): "sent_at", "sent_by", "trigger_source", - "status", - "error_message", + "status", + "error_message", ) - list_filter = ("status", "trigger_source", "sent_at") - search_fields = ("recipient_email", "subject", "sent_by__username") - actions = [trigger_monthly_email, trigger_monthly_email_task] + list_filter = ("status", "trigger_source", "sent_at") + search_fields = ("recipient_email", "subject", "sent_by__username") + actions = [trigger_monthly_email, trigger_monthly_email_task] + +@admin.register(WikidataExportLog) +class WikidataExportLogAdmin(admin.ModelAdmin): + """Admin interface for Wikidata export logs.""" + list_display = ( + "id", + "publication_title", + "action", + "wikidata_link", + "export_date", + "fields_count", + ) + list_filter = ("action", "export_date") + search_fields = ( + "publication__title", + "publication__doi", + "wikidata_qid", + "export_summary", + ) + readonly_fields = ( + "publication", + "export_date", + "action", + "wikidata_qid", + "wikidata_url", + "wikidata_link_display", + "wikibase_endpoint", + "exported_fields", + "error_message_display", + "export_summary", + ) + fields = ( + "publication", + "export_date", + "action", + "wikibase_endpoint", + "wikidata_qid", + "wikidata_link_display", + "export_summary", + "exported_fields", + "error_message_display", + ) + ordering = ("-export_date",) + date_hierarchy = "export_date" + + @admin.display(description="Publication") + def publication_title(self, obj): + return obj.publication.title[:60] if obj.publication else "—" + + @admin.display(description="Wikidata") + def wikidata_link(self, obj): + if obj.wikidata_qid and obj.wikidata_url: + return format_html( + ' {}', + obj.wikidata_url, + obj.wikidata_qid + ) + return "—" + + @admin.display(description="Wikidata Link") + def wikidata_link_display(self, obj): + if obj.wikidata_qid and obj.wikidata_url: + return format_html( + '{}', + obj.wikidata_url, + obj.wikidata_url + ) + return "—" + + @admin.display(description="Fields") + def fields_count(self, obj): + if obj.exported_fields: + return len(obj.exported_fields) + return 0 + + @admin.display(description="Error Message (Full Traceback)") + def error_message_display(self, obj): + if obj.error_message: + return format_html( + '
{}
', + obj.error_message + ) + return "—" @admin.register(Subscription) class SubscriptionAdmin(admin.ModelAdmin): diff --git a/publications/migrations/0004_wikidata_export_log.py b/publications/migrations/0004_wikidata_export_log.py new file mode 100644 index 0000000..3640437 --- /dev/null +++ b/publications/migrations/0004_wikidata_export_log.py @@ -0,0 +1,37 @@ +# Generated migration for WikidataExportLog model + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('publications', '0002_add_regions_to_subscription'), + ] + + operations = [ + migrations.CreateModel( + name='WikidataExportLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('publication', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='wikidata_exports', to='publications.publication')), + ('export_date', models.DateTimeField(auto_now_add=True, db_index=True)), + ('action', models.CharField(max_length=20, choices=[('created', 'Created'), ('updated', 'Updated'), ('skipped', 'Skipped'), ('error', 'Error')], db_index=True)), + ('wikidata_qid', models.CharField(max_length=50, blank=True, null=True, help_text='Wikidata Q-ID (e.g., Q12345)')), + ('wikidata_url', models.URLField(max_length=512, blank=True, null=True, help_text='Full URL to Wikidata item')), + ('exported_fields', models.JSONField(blank=True, null=True, help_text='List of fields that were exported')), + ('error_message', models.TextField(blank=True, null=True)), + ('export_summary', models.TextField(blank=True, null=True, help_text='Summary of what was exported')), + ], + options={ + 'ordering': ['-export_date'], + 'verbose_name': 'Wikidata Export Log', + 'verbose_name_plural': 'Wikidata Export Logs', + }, + ), + migrations.AddIndex( + model_name='wikidataexportlog', + index=models.Index(fields=['wikidata_qid'], name='publications_wikidata_qid_idx'), + ), + ] diff --git a/publications/migrations/0005_add_wikibase_endpoint_to_exportlog.py b/publications/migrations/0005_add_wikibase_endpoint_to_exportlog.py new file mode 100644 index 0000000..c0d4fe0 --- /dev/null +++ b/publications/migrations/0005_add_wikibase_endpoint_to_exportlog.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.9 on 2025-10-23 20:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("publications", "0004_wikidata_export_log"), + ] + + operations = [ + migrations.AddField( + model_name="wikidataexportlog", + name="wikibase_endpoint", + field=models.URLField( + blank=True, + help_text="Wikibase API endpoint used for this export (e.g., https://www.wikidata.org/w/api.php)", + max_length=512, + null=True, + ), + ), + ] diff --git a/publications/models.py b/publications/models.py index 391aa4b..766d85e 100644 --- a/publications/models.py +++ b/publications/models.py @@ -262,4 +262,66 @@ def save(self, *args, **kwargs): name=f"Harvest Source {self.id}", ) -Journal = Source +Journal = Source + + +class WikidataExportLog(models.Model): + """ + Log of Wikidata exports for publications. + Tracks when publications were exported, what action was taken, + and links to the created/updated Wikidata items. + """ + ACTION_CHOICES = [ + ('created', 'Created'), + ('updated', 'Updated'), + ('skipped', 'Skipped'), + ('error', 'Error'), + ] + + publication = models.ForeignKey( + 'Publication', + on_delete=models.CASCADE, + related_name='wikidata_exports' + ) + export_date = models.DateTimeField(auto_now_add=True, db_index=True) + action = models.CharField(max_length=20, choices=ACTION_CHOICES, db_index=True) + wikidata_qid = models.CharField( + max_length=50, + blank=True, + null=True, + help_text='Wikidata Q-ID (e.g., Q12345)' + ) + wikidata_url = models.URLField( + max_length=512, + blank=True, + null=True, + help_text='Full URL to Wikidata item' + ) + exported_fields = models.JSONField( + blank=True, + null=True, + help_text='List of fields that were exported' + ) + error_message = models.TextField(blank=True, null=True) + export_summary = models.TextField( + blank=True, + null=True, + help_text='Summary of what was exported' + ) + wikibase_endpoint = models.URLField( + max_length=512, + blank=True, + null=True, + help_text='Wikibase API endpoint used for this export (e.g., https://www.wikidata.org/w/api.php)' + ) + + class Meta: + ordering = ['-export_date'] + verbose_name = 'Wikidata Export Log' + verbose_name_plural = 'Wikidata Export Logs' + indexes = [ + models.Index(fields=['wikidata_qid'], name='publications_wikidata_qid_idx'), + ] + + def __str__(self): + return f"{self.action.capitalize()} {self.publication.title[:50]} on {self.export_date.strftime('%Y-%m-%d')}" diff --git a/publications/templates/work_landing_page.html b/publications/templates/work_landing_page.html index 55dd0c5..1e38e25 100644 --- a/publications/templates/work_landing_page.html +++ b/publications/templates/work_landing_page.html @@ -26,6 +26,9 @@

{{ pub.title }}

{% elif pub.status == 'h' %}badge-info {% elif pub.status == 'c' %}badge-primary {% endif %}">{{ status_display }} + {% if pub.status != 'p' %} + and is not visible to the public. + {% endif %}
{% if can_publish %} @@ -43,15 +46,69 @@

{{ pub.title }}

- {% if show_provenance and pub.provenance %} + {% if show_provenance %}
- -
+ 
{{ pub.provenance }}
+ {% endif %} + + {% if all_wikidata_exports %} + +
+
+
Wikidata Export History
+ {% for export in all_wikidata_exports %} +
+
+
+ {{ export.get_action_display }} + {{ export.export_date|date:"Y-m-d H:i:s" }} +
+ {% if export.wikidata_qid %} + + {% endif %} +
+ + {% if export.export_summary %} +
+ Summary: {{ export.export_summary }} +
+ {% endif %} + + {% if export.exported_fields %} +
+ Exported fields ({{ export.exported_fields|length }}): + {{ export.exported_fields|join:", " }} +
+ {% endif %} + + {% if export.error_message %} +
+ Error: {{ export.error_message }} +
+ {% endif %} +
+ {% endfor %} +
+
+ {% endif %}
{% endif %}
@@ -76,7 +133,10 @@

{{ pub.title }}

· {% endif %} {% if pub.openalex_id %} - OpenAlex: View in OpenAlex + OpenAlex: View in OpenAlex · + {% endif %} + {% if latest_wikidata_export and latest_wikidata_export.wikidata_url %} + Wikidata: {{ latest_wikidata_export.wikidata_qid }} {% endif %} diff --git a/publications/views.py b/publications/views.py index fb47465..fb9ff0e 100644 --- a/publications/views.py +++ b/publications/views.py @@ -815,6 +815,14 @@ def work_landing(request, doi): can_publish = is_admin and (pub.status == 'c' or (pub.status == 'h' and (has_geometry or has_temporal))) can_unpublish = is_admin and pub.status == 'p' # Can unpublish published works + # Get most recent successful Wikidata export + latest_wikidata_export = pub.wikidata_exports.filter( + action__in=['created', 'updated'] + ).order_by('-export_date').first() + + # Get all Wikidata exports for admin view + all_wikidata_exports = pub.wikidata_exports.all() if is_admin else [] + context = { "pub": pub, "feature_json": feature_json, @@ -828,6 +836,8 @@ def work_landing(request, doi): "can_publish": can_publish, "can_unpublish": can_unpublish, "show_provenance": is_admin, + "latest_wikidata_export": latest_wikidata_export, + "all_wikidata_exports": all_wikidata_exports, } return render(request, "work_landing_page.html", context) @@ -874,6 +884,14 @@ def work_landing_by_id(request, pub_id): can_publish = is_admin and (pub.status == 'c' or (pub.status == 'h' and (has_geometry or has_temporal))) can_unpublish = is_admin and pub.status == 'p' # Can unpublish published works + # Get most recent successful Wikidata export + latest_wikidata_export = pub.wikidata_exports.filter( + action__in=['created', 'updated'] + ).order_by('-export_date').first() + + # Get all Wikidata exports for admin view + all_wikidata_exports = pub.wikidata_exports.all() if is_admin else [] + context = { "pub": pub, "feature_json": feature_json, @@ -888,6 +906,8 @@ def work_landing_by_id(request, pub_id): "can_unpublish": can_unpublish, "show_provenance": is_admin, "use_id_urls": True, # Flag to use ID-based URLs in template + "latest_wikidata_export": latest_wikidata_export, + "all_wikidata_exports": all_wikidata_exports, } return render(request, "work_landing_page.html", context) diff --git a/publications/wikidata.py b/publications/wikidata.py index cf74afa..1149cb2 100644 --- a/publications/wikidata.py +++ b/publications/wikidata.py @@ -1,58 +1,850 @@ import os +import logging import requests +import traceback +import json from datetime import datetime from django.conf import settings +from django.db import transaction from wikibaseintegrator.wbi_exceptions import ModificationFailed from wikibaseintegrator import WikibaseIntegrator -from wikibaseintegrator.wbi_login import Login +from wikibaseintegrator.wbi_login import OAuth1 +from wikibaseintegrator.wbi_config import config as wbi_config from wikibaseintegrator.datatypes import ( MonolingualText, Time, String, ExternalID, - GlobeCoordinate + GlobeCoordinate, + Item ) try: from wikibaseintegrator.datatypes import Url except ImportError: from wikibaseintegrator.datatypes import URL as Url -# Our instance’s SPARQL endpoint (for local lookups by DOI) -if "www.wikidata.org/w/api.php" in settings.WIKIBASE_API_URL: +logger = logging.getLogger(__name__) + +# Configure wikibaseintegrator with our settings +wbi_config['USER_AGENT'] = settings.WIKIBASE_USER_AGENT +wbi_config['MEDIAWIKI_API_URL'] = settings.WIKIBASE_API_URL + +# SPARQL endpoint configuration +if "www.wikidata.org/w/api.php" in settings.WIKIBASE_API_URL: SPARQL_ENDPOINT = "https://query.wikidata.org/sparql" + WIKIBASE_URL = "https://www.wikidata.org/wiki/" + IS_WIKIDATA = True else: SPARQL_ENDPOINT = settings.WIKIBASE_API_URL.replace("/w/api.php", "/query/sparql") + base_url = settings.WIKIBASE_API_URL.replace("/w/api.php", "") + WIKIBASE_URL = f"{base_url}/wiki/Item:" # Custom Wikibase uses Item: prefix + IS_WIKIDATA = False + # Update wbi_config for non-Wikidata instances + wbi_config['SPARQL_ENDPOINT_URL'] = SPARQL_ENDPOINT + wbi_config['WIKIBASE_URL'] = base_url -# constant for all dates +# Calendar model for all dates CALENDAR_MODEL = "http://www.wikidata.org/entity/Q1985727" -# Wikidata property IDs mapping -P_TITLE = "P1476" # title (monolingual text) -P_ABSTRACT = "P1810" # abstract -P_URL = "P856" # official website / URL -P_PUBLICATION_DATE = "P577" # publication date -P_PERIOD_START = "P580" # start time -P_PERIOD_END = "P582" # end time -P_DOI = "P356" # DOI as External ID -P_AUTHOR_STRING = "P2093" # author name string -P_JOURNAL_NAME = "P1448" # journal name (monolingual text) -P_GEOMETRY = "P625" # coordinate location - -def normalize_date_and_precision(date_str): +# Wikidata property IDs (these are the SOURCE property IDs from Wikidata.org) +P_EQUIVALENT_PROPERTY = "P1628" # equivalent property (URL) +P_TITLE = "P1476" # title (monolingual text) +P_ABSTRACT = "P1810" # abstract / name +P_URL = "P856" # official website / URL +P_PUBLICATION_DATE = "P577" # publication date +P_PERIOD_START = "P580" # start time +P_PERIOD_END = "P582" # end time +P_DOI = "P356" # DOI as External ID +P_AUTHOR_STRING = "P2093" # author name string +P_AUTHOR = "P50" # author (item reference) +P_JOURNAL_NAME = "P1448" # journal name (monolingual text) +P_JOURNAL = "P1433" # published in (journal as item) +P_GEOMETRY = "P625" # coordinate location +P_INSTANCE_OF = "P31" # instance of +P_KEYWORDS = "P921" # main subject / keywords +P_LANGUAGE = "P407" # language of work +P_LICENSE = "P275" # copyright license +P_FULL_TEXT_URL = "P953" # full work available at URL + +# Additional properties for OpenAlex data +P_OPENALEX_ID = "P10283" # OpenAlex ID +P_PMID = "P698" # PubMed ID +P_PMC = "P932" # PubMed Central ID +P_ISSN = "P236" # ISSN +P_ISSN_L = "P7363" # ISSN-L +P_RETRACTED = "P5824" # retracted (boolean) + +# Wikidata items +Q_SCHOLARLY_ARTICLE = "Q13442814" # scholarly article +Q_ENGLISH = "Q1860" # English language + + +# Cache for available properties in the target Wikibase +_available_properties_cache = None +_available_items_cache = None +_property_creation_attempted = set() + +# Cache for property metadata fetched from Wikidata +_property_metadata_cache = {} + +# Cache for mapping Wikidata property IDs to local Wikibase property IDs +# e.g., {"P31": "P1", "P577": "P3", "P1628": "P63"} +_property_id_mapping = None + + +def build_property_id_mapping(): + """ + Build a mapping from Wikidata property IDs to local Wikibase property IDs + by querying all properties and checking their P1628 (equivalent property) claims. + + Returns: + dict: Mapping like {"P31": "P1", "P577": "P3", "P1628": "P63"} + """ + global _property_id_mapping + + if _property_id_mapping is not None: + return _property_id_mapping + + _property_id_mapping = {} + + try: + # First, find the local property ID for "equivalent property" itself + search_response = requests.get( + settings.WIKIBASE_API_URL, + params={ + 'action': 'wbsearchentities', + 'search': 'equivalent property', + 'language': 'en', + 'type': 'property', + 'format': 'json' + }, + headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, + timeout=10 + ) + search_data = search_response.json() + + equivalent_property_id = None + if 'search' in search_data: + for result in search_data['search']: + if result.get('label', '').lower() == 'equivalent property': + equivalent_property_id = result['id'] + # Map P1628 to whatever the local ID is + _property_id_mapping['P1628'] = equivalent_property_id + logger.debug(f"Found equivalent property: P1628 -> {equivalent_property_id}") + break + + if not equivalent_property_id: + logger.warning("Equivalent property (P1628) not found in Wikibase - property mapping will be limited") + return _property_id_mapping + + # Query all properties to build the mapping + response = requests.get( + settings.WIKIBASE_API_URL, + params={ + 'action': 'wbsearchentities', + 'search': '', # Empty search to get all + 'language': 'en', + 'type': 'property', + 'limit': 500, # Get many properties + 'format': 'json' + }, + headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, + timeout=30 + ) + all_properties_data = response.json() + + if 'search' not in all_properties_data: + logger.warning("Could not fetch property list from Wikibase") + return _property_id_mapping + + # Get detailed data for each property to check for P1628 claims + property_ids = [prop['id'] for prop in all_properties_data['search']] + + if not property_ids: + return _property_id_mapping + + # Fetch properties in batches + batch_size = 50 + for i in range(0, len(property_ids), batch_size): + batch = property_ids[i:i+batch_size] + + entities_response = requests.get( + settings.WIKIBASE_API_URL, + params={ + 'action': 'wbgetentities', + 'ids': '|'.join(batch), + 'format': 'json' + }, + headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, + timeout=30 + ) + entities_data = entities_response.json() + + if 'entities' not in entities_data: + continue + + # Check each property for P1628 claims + for prop_id, prop_data in entities_data['entities'].items(): + claims = prop_data.get('claims', {}) + + # Cache property existence for all local properties we encounter + # This avoids later API calls in check_property_exists() + global _available_properties_cache + if _available_properties_cache is None: + _available_properties_cache = {} + + # Check if this property has an equivalent property claim + if equivalent_property_id in claims: + for claim in claims[equivalent_property_id]: + # Extract the Wikidata property URL from the claim value + datavalue = claim.get('mainsnak', {}).get('datavalue', {}) + if datavalue.get('type') == 'string': + url = datavalue.get('value', '') + # URL format: https://www.wikidata.org/entity/P31 + if 'wikidata.org/entity/P' in url: + wikidata_prop_id = url.split('/')[-1] + _property_id_mapping[wikidata_prop_id] = prop_id + # Cache that this Wikidata property exists in our Wikibase + _available_properties_cache[wikidata_prop_id] = True + logger.debug(f"Mapped property: {wikidata_prop_id} -> {prop_id}") + + logger.info(f"Built property ID mapping with {len(_property_id_mapping)} properties via equivalent property claims") + logger.info(f"Cached {len(_available_properties_cache)} property existence checks for faster lookups") + + # Step 2: Fallback to name-based matching for properties without equivalent property claims + # Fetch metadata for standard Wikidata properties we use + standard_properties = [ + P_TITLE, P_ABSTRACT, P_URL, P_PUBLICATION_DATE, P_PERIOD_START, P_PERIOD_END, + P_DOI, P_AUTHOR_STRING, P_AUTHOR, P_JOURNAL_NAME, P_JOURNAL, P_GEOMETRY, + P_INSTANCE_OF, P_KEYWORDS, P_LANGUAGE, P_LICENSE, P_FULL_TEXT_URL, + P_OPENALEX_ID, P_PMID, P_PMC, P_ISSN, P_ISSN_L, P_RETRACTED + ] + + for wikidata_prop_id in standard_properties: + # Skip if already mapped via equivalent property + if wikidata_prop_id in _property_id_mapping: + continue + + # Fetch metadata from Wikidata + wikidata_meta = fetch_property_metadata_from_wikidata(wikidata_prop_id) + if not wikidata_meta: + continue + + wikidata_label = wikidata_meta['label'].lower().strip() + + # Search for matching property by label in local Wikibase + for prop_id in property_ids: + if prop_id not in entities_data.get('entities', {}): + continue + + local_prop = entities_data['entities'][prop_id] + local_label = local_prop.get('labels', {}).get('en', {}).get('value', '').lower().strip() + + # Exact name match + if local_label == wikidata_label: + _property_id_mapping[wikidata_prop_id] = prop_id + logger.debug(f"Mapped property via name matching: {wikidata_prop_id} ({wikidata_label}) -> {prop_id}") + + # Add equivalent property claim to establish the relationship + try: + from requests_oauthlib import OAuth1Session + + # Create OAuth1 session + oauth = OAuth1Session( + settings.WIKIBASE_CONSUMER_TOKEN, + client_secret=settings.WIKIBASE_CONSUMER_SECRET, + resource_owner_key=settings.WIKIBASE_ACCESS_TOKEN, + resource_owner_secret=settings.WIKIBASE_ACCESS_SECRET + ) + + # Check if this property already has an equivalent property claim + claims = local_prop.get('claims', {}) + wikidata_property_url = f'https://www.wikidata.org/entity/{wikidata_prop_id}' + + has_equivalent_claim = False + if equivalent_property_id in claims: + for claim in claims[equivalent_property_id]: + existing_url = claim.get('mainsnak', {}).get('datavalue', {}).get('value', '') + if existing_url == wikidata_property_url: + has_equivalent_claim = True + break + + if not has_equivalent_claim: + # Get CSRF token + token_params = { + 'action': 'query', + 'meta': 'tokens', + 'type': 'csrf', + 'format': 'json' + } + token_response = oauth.get(settings.WIKIBASE_API_URL, params=token_params) + token_data = token_response.json() + csrf_token = token_data['query']['tokens']['csrftoken'] + + # Add equivalent property claim + claim_params = { + 'action': 'wbcreateclaim', + 'entity': prop_id, + 'property': equivalent_property_id, + 'snaktype': 'value', + 'value': json.dumps(wikidata_property_url), + 'token': csrf_token, + 'format': 'json', + 'bot': '1' + } + + claim_response = oauth.post(settings.WIKIBASE_API_URL, data=claim_params) + claim_data = claim_response.json() + + if claim_data.get('success') == 1: + logger.info(f"Added equivalent property claim to {prop_id} linking to {wikidata_property_url}") + else: + logger.warning(f"Could not add equivalent property claim to {prop_id}: {claim_data}") + except Exception as e: + logger.warning(f"Error adding equivalent property claim to name-matched property {prop_id}: {e}") + + break + + logger.info(f"Built complete property ID mapping with {len(_property_id_mapping)} properties (equivalent claims + name matching)") + return _property_id_mapping + + except Exception as e: + logger.error(f"Error building property ID mapping: {e}") + logger.debug(traceback.format_exc()) + return _property_id_mapping or {} + + +def get_local_property_id(wikidata_property_id): + """ + Get the local Wikibase property ID for a Wikidata property ID. + + Args: + wikidata_property_id: Wikidata property ID like "P31" + + Returns: + str: Local property ID like "P1", or the original ID if no mapping exists + """ + mapping = build_property_id_mapping() + local_id = mapping.get(wikidata_property_id, wikidata_property_id) + if local_id != wikidata_property_id: + logger.debug(f"Using local property {local_id} for Wikidata property {wikidata_property_id}") + return local_id + + +def get_wikibase_login(): + """ + Get authenticated login session for Wikibase/Wikidata using OAuth 1.0a. + + Returns: + OAuth1: OAuth1 login session object + + Raises: + ValueError: If OAuth 1.0a credentials are not configured + """ + required_credentials = [ + settings.WIKIBASE_CONSUMER_TOKEN, + settings.WIKIBASE_CONSUMER_SECRET, + settings.WIKIBASE_ACCESS_TOKEN, + settings.WIKIBASE_ACCESS_SECRET, + ] + + if not all(required_credentials): + raise ValueError( + "Wikibase OAuth 1.0a credentials not configured. " + "Please set WIKIBASE_CONSUMER_TOKEN, WIKIBASE_CONSUMER_SECRET, " + "WIKIBASE_ACCESS_TOKEN, and WIKIBASE_ACCESS_SECRET environment variables. " + "See WIKIBASE_OAUTH_SETUP.md for setup instructions." + ) + + logger.debug("Using OAuth 1.0a authentication for Wikibase") + return OAuth1( + consumer_token=settings.WIKIBASE_CONSUMER_TOKEN, + consumer_secret=settings.WIKIBASE_CONSUMER_SECRET, + access_token=settings.WIKIBASE_ACCESS_TOKEN, + access_secret=settings.WIKIBASE_ACCESS_SECRET, + mediawiki_api_url=settings.WIKIBASE_API_URL, + user_agent=settings.WIKIBASE_USER_AGENT, + ) + + +def fetch_property_metadata_from_wikidata(property_id): + """ + Fetch property metadata (label, description, datatype) from Wikidata.org. + + Args: + property_id: The property ID (e.g., "P31") + + Returns: + dict: {'label': str, 'description': str, 'datatype': str} or None if fetch fails + """ + global _property_metadata_cache + + # Check cache first + if property_id in _property_metadata_cache: + return _property_metadata_cache[property_id] + + try: + # Fetch property entity from Wikidata API + url = "https://www.wikidata.org/w/api.php" + params = { + 'action': 'wbgetentities', + 'ids': property_id, + 'format': 'json', + 'languages': 'en' + } + response = requests.get(url, params=params, headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, timeout=10) + data = response.json() + + # Check if property exists + if 'entities' not in data or property_id not in data['entities']: + logger.warning(f"Property {property_id} not found in Wikidata") + return None + + entity = data['entities'][property_id] + + if 'missing' in entity: + logger.warning(f"Property {property_id} is missing in Wikidata") + return None + + # Extract metadata + label = entity.get('labels', {}).get('en', {}).get('value', property_id) + description = entity.get('descriptions', {}).get('en', {}).get('value', '') + datatype = entity.get('datatype', 'string') + + metadata = { + 'label': label, + 'description': description, + 'datatype': datatype + } + + # Cache it + _property_metadata_cache[property_id] = metadata + logger.debug(f"Fetched metadata for {property_id}: {metadata}") + + return metadata + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to fetch property metadata for {property_id} from Wikidata: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error fetching property metadata for {property_id}: {e}") + logger.debug(traceback.format_exc()) + return None + + +def get_property_metadata(property_id): + """ + Get property metadata, fetching from Wikidata.org if not cached. + + Args: + property_id: The property ID (e.g., "P31") + + Returns: + dict: {'label': str, 'description': str, 'datatype': str} or None if unavailable + """ + return fetch_property_metadata_from_wikidata(property_id) + + +def create_property_in_wikibase(property_id): + """ + Create a property in the target Wikibase instance if it doesn't exist. + Fetches property metadata dynamically from Wikidata.org and adds equivalent property claim. + Uses direct API calls (OAuth1Session) instead of wikibaseintegrator for better compatibility. + + Args: + property_id: The Wikidata property ID (e.g., "P31") + + Returns: + str: Local property ID if created/exists (e.g., "P1"), or None if creation failed + """ + global _property_creation_attempted + global _property_id_mapping + global _available_properties_cache + + # Don't try to create the same property twice + if property_id in _property_creation_attempted: + return None + + _property_creation_attempted.add(property_id) + + # Fetch metadata from Wikidata + meta = get_property_metadata(property_id) + if not meta: + logger.warning(f"No metadata available for property {property_id} from Wikidata, cannot create it") + return None + + try: + from requests_oauthlib import OAuth1Session + + # Create OAuth1 session for direct API calls + oauth = OAuth1Session( + settings.WIKIBASE_CONSUMER_TOKEN, + client_secret=settings.WIKIBASE_CONSUMER_SECRET, + resource_owner_key=settings.WIKIBASE_ACCESS_TOKEN, + resource_owner_secret=settings.WIKIBASE_ACCESS_SECRET + ) + + # Step 1: Check if property with same label already exists + logger.debug(f"Checking if property with label '{meta['label']}' already exists in Wikibase") + search_params = { + 'action': 'wbsearchentities', + 'search': meta['label'], + 'language': 'en', + 'type': 'property', + 'format': 'json' + } + search_response = oauth.get(settings.WIKIBASE_API_URL, params=search_params) + search_data = search_response.json() + + # Check for exact label match + if 'search' in search_data: + for result in search_data['search']: + if result.get('label', '').lower() == meta['label'].lower(): + existing_id = result['id'] + logger.info(f"Property with label '{meta['label']}' already exists as {existing_id} - will use existing property and add equivalent claim if needed") + + # Update caches + if _available_properties_cache is None: + _available_properties_cache = {} + _available_properties_cache[property_id] = True + + if _property_id_mapping is None: + _property_id_mapping = {} + _property_id_mapping[property_id] = existing_id + + # Try to add equivalent property claim to existing property + try: + # Get CSRF token for adding claim + token_params = { + 'action': 'query', + 'meta': 'tokens', + 'type': 'csrf', + 'format': 'json' + } + token_response = oauth.get(settings.WIKIBASE_API_URL, params=token_params) + token_data = token_response.json() + csrf_token = token_data['query']['tokens']['csrftoken'] + + # Check if equivalent property exists + mapping = build_property_id_mapping() + equivalent_property_id = mapping.get('P1628') + + if equivalent_property_id: + # Fetch the existing property to check for existing claims + get_params = { + 'action': 'wbgetentities', + 'ids': existing_id, + 'format': 'json' + } + get_response = oauth.get(settings.WIKIBASE_API_URL, params=get_params) + get_data = get_response.json() + + wikidata_property_url = f'https://www.wikidata.org/entity/{property_id}' + + # Check if claim already exists + has_claim = False + if existing_id in get_data.get('entities', {}): + claims = get_data['entities'][existing_id].get('claims', {}) + if equivalent_property_id in claims: + for claim in claims[equivalent_property_id]: + existing_url = claim.get('mainsnak', {}).get('datavalue', {}).get('value', '') + if existing_url == wikidata_property_url: + has_claim = True + logger.debug(f"Equivalent property claim already exists on {existing_id}") + break + + # Add claim if it doesn't exist + if not has_claim: + claim_params = { + 'action': 'wbcreateclaim', + 'entity': existing_id, + 'property': equivalent_property_id, + 'snaktype': 'value', + 'value': json.dumps(wikidata_property_url), + 'token': csrf_token, + 'format': 'json', + 'bot': '1' + } + + claim_response = oauth.post(settings.WIKIBASE_API_URL, data=claim_params) + claim_data = claim_response.json() + + if claim_data.get('success') == 1: + logger.info(f"Added equivalent property claim to existing property {existing_id} linking to {wikidata_property_url}") + else: + logger.warning(f"Could not add equivalent property claim to {existing_id}: {claim_data}") + except Exception as e: + logger.warning(f"Error adding equivalent property claim to existing property {existing_id}: {e}") + + return existing_id + + # Step 2: Get CSRF token for creating new property + logger.debug("Fetching CSRF token for property creation") + token_params = { + 'action': 'query', + 'meta': 'tokens', + 'type': 'csrf', + 'format': 'json' + } + token_response = oauth.get(settings.WIKIBASE_API_URL, params=token_params) + token_data = token_response.json() + + if 'query' not in token_data or 'tokens' not in token_data['query']: + logger.error(f"Failed to get CSRF token: {token_data}") + return None + + csrf_token = token_data['query']['tokens']['csrftoken'] + + # Step 3: Create property data structure (shallow copy) + property_data = { + "labels": { + "en": { + "language": "en", + "value": meta['label'] + } + }, + "descriptions": { + "en": { + "language": "en", + "value": meta['description'] + } + }, + "datatype": meta['datatype'] + } + + logger.debug(f"Creating property {property_id} with label '{meta['label']}' and datatype '{meta['datatype']}' (shallow copy, no claims)") + + # Step 4: Create property + create_params = { + 'action': 'wbeditentity', + 'new': 'property', + 'data': json.dumps(property_data), + 'summary': f'Auto-created property via OPTIMAP export from Wikidata {property_id}', + 'token': csrf_token, + 'format': 'json', + 'bot': '1' + } + + create_response = oauth.post(settings.WIKIBASE_API_URL, data=create_params) + create_data = create_response.json() + + if 'success' not in create_data or create_data['success'] != 1: + error_info = create_data.get('error', {}) + logger.error(f"Failed to create property {property_id}: {error_info}") + return None + + created_id = create_data['entity']['id'] + logger.info(f"Successfully created property {created_id} ({meta['label']}) in Wikibase for {property_id}") + + # Step 5: Add equivalent property claim to link to Wikidata + # Find the local ID for "equivalent property" + mapping = build_property_id_mapping() + equivalent_property_id = mapping.get('P1628') + + if equivalent_property_id: + try: + wikidata_property_url = f'https://www.wikidata.org/entity/{property_id}' + + claim_params = { + 'action': 'wbcreateclaim', + 'entity': created_id, + 'property': equivalent_property_id, + 'snaktype': 'value', + 'value': json.dumps(wikidata_property_url), + 'token': csrf_token, + 'format': 'json', + 'bot': '1' + } + + claim_response = oauth.post(settings.WIKIBASE_API_URL, data=claim_params) + claim_data = claim_response.json() + + if claim_data.get('success') == 1: + logger.debug(f"Added equivalent property claim linking {created_id} to {wikidata_property_url}") + else: + logger.warning(f"Could not add equivalent property claim to {created_id}: {claim_data}") + except Exception as e: + logger.warning(f"Error adding equivalent property claim: {e}") + else: + logger.debug("Equivalent property not found in Wikibase, skipping claim addition") + + # Update caches + if _available_properties_cache is None: + _available_properties_cache = {} + _available_properties_cache[property_id] = True + + if _property_id_mapping is None: + _property_id_mapping = {} + _property_id_mapping[property_id] = created_id + + return created_id + + except Exception as e: + error_msg = str(e) + if "The save has failed" in error_msg or "permission" in error_msg.lower(): + logger.error( + f"Failed to create property {property_id} in Wikibase: {e}. " + f"The Wikibase account may not have the 'property-create' permission. " + f"Please ask a Wikibase administrator to either: " + f"1) Grant property-create rights to this account, or " + f"2) Manually create property {property_id} ({meta['label']}) " + f"with datatype '{meta['datatype']}'" + ) + else: + logger.error(f"Failed to create property {property_id} in Wikibase: {e}") + logger.debug(traceback.format_exc()) + return None + + +def check_property_exists(wikidata_property_id): + """ + Check if a property exists in the target Wikibase instance (using Wikidata property ID). + First checks if mapping exists, then creates property if needed. + Uses caching to avoid repeated API calls. + + Args: + wikidata_property_id: Wikidata property ID like "P31" + + Returns: + bool: True if property exists (or was created), False otherwise + """ + global _available_properties_cache + + if _available_properties_cache is None: + _available_properties_cache = {} + + if wikidata_property_id in _available_properties_cache: + return _available_properties_cache[wikidata_property_id] + + try: + # First, check if we have a mapping from Wikidata ID to local ID + local_property_id = get_local_property_id(wikidata_property_id) + + # If local ID is different from Wikidata ID, we have a mapping + if local_property_id != wikidata_property_id: + logger.debug(f"Found property mapping: {wikidata_property_id} -> {local_property_id}") + _available_properties_cache[wikidata_property_id] = True + return True + + # No mapping found - check if property exists directly by ID + response = requests.get( + settings.WIKIBASE_API_URL, + params={ + 'action': 'wbgetentities', + 'ids': wikidata_property_id, + 'format': 'json' + }, + headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, + timeout=10 + ) + data = response.json() + + # Check if property exists (not in 'missing' list) + exists = wikidata_property_id in data.get('entities', {}) and 'missing' not in data.get('entities', {}).get(wikidata_property_id, {}) + + if not exists: + logger.debug(f"Property {wikidata_property_id} not found in Wikibase instance") + + # Try to create it if enabled + if settings.WIKIBASE_CREATE_PROPERTIES_IF_MISSING: + logger.info(f"Attempting to create property {wikidata_property_id} in Wikibase") + created_local_id = create_property_in_wikibase(wikidata_property_id) + if created_local_id: + exists = True + logger.info(f"Successfully created property {created_local_id} for {wikidata_property_id}") + else: + logger.warning(f"Failed to create property {wikidata_property_id}") + + _available_properties_cache[wikidata_property_id] = exists + return exists + + except Exception as e: + logger.warning(f"Could not check if property {wikidata_property_id} exists: {e}") + # Assume it doesn't exist to avoid errors + _available_properties_cache[wikidata_property_id] = False + return False + + +def check_item_exists(item_id): + """ + Check if an item (Q-ID) exists in the target Wikibase instance. + Uses caching to avoid repeated API calls. + """ + global _available_items_cache + + if _available_items_cache is None: + _available_items_cache = {} + + if item_id in _available_items_cache: + return _available_items_cache[item_id] + + try: + response = requests.get( + settings.WIKIBASE_API_URL, + params={ + 'action': 'wbgetentities', + 'ids': item_id, + 'format': 'json' + }, + headers={'User-Agent': settings.WIKIBASE_USER_AGENT}, + timeout=10 + ) + data = response.json() + + exists = item_id in data.get('entities', {}) and 'missing' not in data.get('entities', {}).get(item_id, {}) + _available_items_cache[item_id] = exists + + if not exists: + logger.debug(f"Item {item_id} not found in Wikibase instance") + + return exists + except Exception as e: + logger.warning(f"Could not check if item {item_id} exists: {e}") + _available_items_cache[item_id] = False + return False + + +def normalize_date_and_precision(date_str, is_end_date=False): + """ + Convert date string to ISO format with appropriate precision. + + Args: + date_str: Date string in format YYYY, YYYY-MM, or YYYY-MM-DD + is_end_date: If True, use last day of year/month instead of first day + + Returns: tuple (iso_date_string, precision) + - precision 9 = year + - precision 10 = month + - precision 11 = day + """ + import calendar + parts = date_str.split("-") if len(parts) == 1 and parts[0].isdigit(): # "YYYY" - return f"{parts[0]}-01-01", 9 + year = parts[0] + if is_end_date: + return f"{year}-12-31", 9 + else: + return f"{year}-01-01", 9 if len(parts) == 2 and all(p.isdigit() for p in parts): # "YYYY-MM" - return f"{parts[0]}-{parts[1]}-01", 10 + year, month = parts[0], parts[1] + if is_end_date: + # Get last day of the month + last_day = calendar.monthrange(int(year), int(month))[1] + return f"{year}-{month}-{last_day:02d}", 10 + else: + return f"{year}-{month}-01", 10 # assume full "YYYY-MM-DD" return date_str, 11 -def add_time_claims(dates, prop_nr, statements): + +def add_time_claims(dates, prop_nr, statements, is_end_date=False): + """Add time-based claims for a list of date strings.""" for ds in dates: - iso, prec = normalize_date_and_precision(ds) + iso, prec = normalize_date_and_precision(ds, is_end_date=is_end_date) timestamp = f"+{iso}T00:00:00Z" statements.append(Time( prop_nr=prop_nr, @@ -70,121 +862,541 @@ def find_local_item_by_doi(doi): Return the Q-ID of an existing item in our Wikibase instance for the given DOI, or None if no match is found. """ + # Get the local property ID for DOI + local_doi_property = get_local_property_id(P_DOI) + sparql_query = f''' SELECT ?item WHERE {{ - ?item wdt:{P_DOI} "{doi}" . + ?item wdt:{local_doi_property} "{doi}" . }} LIMIT 1 ''' - response = requests.get( - SPARQL_ENDPOINT, - params={"query": sparql_query, "format": "json"}, - headers={"Accept": "application/json"} - ) - response.raise_for_status() + try: + response = requests.get( + SPARQL_ENDPOINT, + params={"query": sparql_query, "format": "json"}, + headers={"Accept": "application/json"}, + timeout=30 + ) + response.raise_for_status() + + data = response.json() + bindings = data.get("results", {}).get("bindings", []) + if not bindings: + return None - data = response.json() - bindings = data.get("results", {}).get("bindings", []) - if not bindings: + item_uri = bindings[0]["item"]["value"] + return item_uri.rsplit("/", 1)[-1] + except Exception as e: + logger.error(f"Error querying SPARQL for DOI {doi}: {e}") return None - item_uri = bindings[0]["item"]["value"] - return item_uri.rsplit("/", 1)[-1] -def upsert_publication(publication, wikibase_integrator): +def build_statements(publication): """ - Create or update a single Publication on Wikibase. - Returns a tuple (action, qid): - - action is "created", "updated", or "skipped" - - qid is the Wikibase item ID (or None if skipped) + Build comprehensive list of Wikidata statements from publication data. + + Returns: + tuple: (statements_list, exported_fields_list) + + Raises: + ValueError: If required properties cannot be created in the Wikibase instance """ - # 1) Build statements - iso_date = publication.publicationDate.isoformat() - publication_timestamp = f"+{iso_date}T00:00:00Z" + statements = [] + exported_fields = [] + missing_properties = [] - statements = [ - MonolingualText(prop_nr=P_TITLE, text=publication.title, language="en"), - Time(prop_nr=P_PUBLICATION_DATE, time=publication_timestamp, timezone=0, before=0, after=0, precision=11, calendarmodel=CALENDAR_MODEL), - String(prop_nr=P_AUTHOR_STRING, value=(publication.created_by.username if publication.created_by else "Unknown author")), - ] + # Instance of scholarly article + if not check_property_exists(P_INSTANCE_OF): + missing_properties.append(f"{P_INSTANCE_OF} (instance of)") + elif check_item_exists(Q_SCHOLARLY_ARTICLE): + statements.append(Item(prop_nr=get_local_property_id(P_INSTANCE_OF), value=Q_SCHOLARLY_ARTICLE)) + exported_fields.append('instance_of') + + # Title (required) + if publication.title: + if not check_property_exists(P_TITLE): + missing_properties.append(f"{P_TITLE} (title)") + else: + statements.append(MonolingualText(prop_nr=get_local_property_id(P_TITLE), text=publication.title, language="en")) + exported_fields.append('title') + + # Publication date (required) + if publication.publicationDate: + if not check_property_exists(P_PUBLICATION_DATE): + missing_properties.append(f"{P_PUBLICATION_DATE} (publication date)") + else: + iso_date = publication.publicationDate.isoformat() + publication_timestamp = f"+{iso_date}T00:00:00Z" + statements.append(Time( + prop_nr=get_local_property_id(P_PUBLICATION_DATE), + time=publication_timestamp, + timezone=0, + before=0, + after=0, + precision=11, + calendarmodel=CALENDAR_MODEL + )) + exported_fields.append('publication_date') + + # Abort if any required properties are missing + if missing_properties: + error_msg = f"Cannot create item: Required properties missing in Wikibase: {', '.join(missing_properties)}" + logger.error(error_msg) + raise ValueError(error_msg) - if publication.abstract: - statements.append(String(prop_nr=P_ABSTRACT, value=publication.abstract)) + # Abstract + if publication.abstract and check_property_exists(P_ABSTRACT): + # Truncate if too long (Wikidata has limits) + abstract_text = publication.abstract[:5000] if len(publication.abstract) > 5000 else publication.abstract + statements.append(String(prop_nr=get_local_property_id(P_ABSTRACT), value=abstract_text)) + exported_fields.append('abstract') - if publication.url: - statements.append(Url(prop_nr=P_URL, value=publication.url)) + # DOI + if publication.doi and check_property_exists(P_DOI): + statements.append(ExternalID(prop_nr=get_local_property_id(P_DOI), value=publication.doi)) + exported_fields.append('doi') - if publication.timeperiod_startdate: - add_time_claims(publication.timeperiod_startdate, P_PERIOD_START, statements) + # URL + if publication.url and check_property_exists(P_URL): + statements.append(Url(prop_nr=get_local_property_id(P_URL), value=publication.url)) + exported_fields.append('url') - if publication.timeperiod_enddate: - add_time_claims(publication.timeperiod_enddate, P_PERIOD_END, statements) + # Authors + if check_property_exists(P_AUTHOR_STRING): + if publication.authors: + for author in publication.authors: + if author and author.strip(): + statements.append(String(prop_nr=get_local_property_id(P_AUTHOR_STRING), value=author.strip())) + exported_fields.append('authors') + # Fallback to creator username if no authors + elif publication.created_by: + statements.append(String(prop_nr=get_local_property_id(P_AUTHOR_STRING), value=publication.created_by.username)) + exported_fields.append('created_by_as_author') + # Keywords + if check_property_exists(P_KEYWORDS): + if publication.keywords: + for keyword in publication.keywords: + if keyword and keyword.strip(): + statements.append(String(prop_nr=get_local_property_id(P_KEYWORDS), value=keyword.strip())) + exported_fields.append('keywords') + + # Topics (from OpenAlex) + if publication.topics: + for topic in publication.topics: + if topic and topic.strip(): + statements.append(String(prop_nr=get_local_property_id(P_KEYWORDS), value=f"Topic: {topic.strip()}")) + exported_fields.append('topics') + + # Time period - start date + if publication.timeperiod_startdate and check_property_exists(P_PERIOD_START): + add_time_claims(publication.timeperiod_startdate, get_local_property_id(P_PERIOD_START), statements, is_end_date=False) + exported_fields.append('timeperiod_start') + + # Time period - end date + if publication.timeperiod_enddate and check_property_exists(P_PERIOD_END): + add_time_claims(publication.timeperiod_enddate, get_local_property_id(P_PERIOD_END), statements, is_end_date=True) + exported_fields.append('timeperiod_end') + + # Source/Journal if publication.source: - statements.append(MonolingualText(prop_nr=P_JOURNAL_NAME, text=publication.source, language="en")) + # Export as monolingual text name + if check_property_exists(P_JOURNAL_NAME): + statements.append(MonolingualText(prop_nr=get_local_property_id(P_JOURNAL_NAME), text=publication.source.name, language="en")) + exported_fields.append('source_name') - if publication.doi: - statements.append( ExternalID(prop_nr=P_DOI, value=publication.doi)) + # If source has ISSN-L + if publication.source.issn_l and check_property_exists(P_ISSN_L): + statements.append(ExternalID(prop_nr=get_local_property_id(P_ISSN_L), value=publication.source.issn_l)) + exported_fields.append('source_issn_l') - if publication.geometry: - geometries = getattr(publication.geometry, "geoms", [publication.geometry]) - for geom in geometries: - if getattr(geom, "geom_type", None) != "Point": - geom = geom.centroid - statements.append(GlobeCoordinate(prop_nr=P_GEOMETRY, latitude=geom.y, longitude=geom.x, precision=0.0001)) + # OpenAlex ID + if publication.openalex_id and check_property_exists(P_OPENALEX_ID): + # Clean the ID (remove URL prefix if present) + openalex_clean = publication.openalex_id.replace('https://openalex.org/', '') + statements.append(ExternalID(prop_nr=get_local_property_id(P_OPENALEX_ID), value=openalex_clean)) + exported_fields.append('openalex_id') - # 7) Check for existing item by DOI - existing_qid = find_local_item_by_doi(publication.doi) if publication.doi else None + # OpenAlex IDs (PMID, PMC, etc.) + if publication.openalex_ids and isinstance(publication.openalex_ids, dict): + if publication.openalex_ids.get('pmid') and check_property_exists(P_PMID): + pmid = str(publication.openalex_ids['pmid']).replace('https://pubmed.ncbi.nlm.nih.gov/', '') + statements.append(ExternalID(prop_nr=get_local_property_id(P_PMID), value=pmid)) + exported_fields.append('pmid') - if existing_qid: - # Update existing item - entity = wikibase_integrator.item.get(entity_id=existing_qid) - entity.claims.add(statements) - try: - entity.write(summary="Update publication via OptimapBot") - return "updated", existing_qid - except ModificationFailed as e: - if "already has label" in str(e): - return "skipped", existing_qid - raise - else: - # Create new item - entity = wikibase_integrator.item.new() - entity.labels.set("en", publication.title) - entity.descriptions.set("en", "Publication imported from Optimap") - entity.claims.add(statements) + if publication.openalex_ids.get('pmcid') and check_property_exists(P_PMC): + pmcid = str(publication.openalex_ids['pmcid']).replace('https://www.ncbi.nlm.nih.gov/pmc/articles/', '') + statements.append(ExternalID(prop_nr=get_local_property_id(P_PMC), value=pmcid)) + exported_fields.append('pmcid') + + # OpenAlex retracted status + if publication.openalex_is_retracted and check_property_exists(P_RETRACTED): + if check_item_exists("Q7594826"): # retracted paper item + statements.append(Item(prop_nr=get_local_property_id(P_RETRACTED), value="Q7594826")) + exported_fields.append('is_retracted') + + # Geometry - coordinates + if publication.geometry and check_property_exists(P_GEOMETRY): try: - write_result = entity.write(summary="Create publication via OptimapBot") - created_qid = write_result.get("entity", {}).get("id") - return "created", created_qid - except ModificationFailed as e: - if "already has label" in str(e): - return "skipped", None - raise - -def export_publications_to_wikidata(publications): - login_session = Login( - user=settings.WIKIBASE_USERNAME, - password=settings.WIKIBASE_PASSWORD, - mediawiki_api_url=settings.WIKIBASE_API_URL, + geometries = getattr(publication.geometry, "geoms", [publication.geometry]) + for geom in geometries: + if getattr(geom, "geom_type", None) != "Point": + geom = geom.centroid + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_GEOMETRY), + latitude=geom.y, + longitude=geom.x, + precision=0.0001 + )) + exported_fields.append('geometry') + except Exception as e: + logger.warning(f"Error processing geometry for publication {publication.id}: {e}") + + # Log how many fields were checked vs exported + logger.info(f"Built {len(statements)} statements from {len(exported_fields)} fields for publication {publication.id}") + + return statements, exported_fields + + +def create_export_log(publication, action, qid=None, exported_fields=None, error_message=None, summary=None, endpoint=None): + """ + Create a WikidataExportLog entry for this export. + """ + from publications.models import WikidataExportLog + + wikidata_url = None + if qid: + wikidata_url = f"{WIKIBASE_URL}{qid}" + + log_entry = WikidataExportLog.objects.create( + publication=publication, + action=action, + wikidata_qid=qid, + wikidata_url=wikidata_url, + exported_fields=exported_fields or [], + error_message=error_message, + export_summary=summary, + wikibase_endpoint=endpoint or settings.WIKIBASE_API_URL ) + + return log_entry + + +def upsert_publication(publication, wikibase_integrator, dryrun=False): + """ + Create or update a single Publication on Wikibase with comprehensive logging. + + Args: + publication: Publication object to export + wikibase_integrator: WikibaseIntegrator client instance + dryrun: If True, simulate the export without writing to Wikibase + + Returns a tuple (action, qid, log_entry): + - action is "created", "updated", "skipped", or "error" + - qid is the Wikibase item ID (or None if error/skipped) + - log_entry is the WikidataExportLog instance (or None if dryrun) + """ + try: + # Build statements + statements, exported_fields = build_statements(publication) + + # Check for existing item by DOI + existing_qid = find_local_item_by_doi(publication.doi) if publication.doi else None + + if dryrun: + # Dry-run mode: simulate the export without writing + if existing_qid: + action = "updated" + summary = f"[DRY-RUN] Would update {len(exported_fields)} fields: {', '.join(exported_fields)}" + logger.info(f"[DRY-RUN] Would update Wikidata item {existing_qid} for publication {publication.id}") + else: + action = "created" + summary = f"[DRY-RUN] Would create with {len(exported_fields)} fields: {', '.join(exported_fields)}" + logger.info(f"[DRY-RUN] Would create new Wikidata item for publication {publication.id}") + + # Return action without creating log entry in dryrun mode + return action, existing_qid, None + + if existing_qid: + # Update existing item + try: + entity = wikibase_integrator.item.get(entity_id=existing_qid) + + # Check which properties already exist on the item + existing_properties = set(entity.claims.keys()) + logger.debug(f"Existing item {existing_qid} has properties: {sorted(existing_properties)}") + + # Filter statements to only include properties that don't exist yet + new_statements = [] + added_fields = [] + skipped_fields = [] + + for i, statement in enumerate(statements): + prop_id = statement.mainsnak.property_number + field_name = exported_fields[i] if i < len(exported_fields) else 'unknown' + + if prop_id not in existing_properties: + new_statements.append(statement) + added_fields.append(field_name) + logger.debug(f"Will add property {prop_id} ({field_name}) to item {existing_qid}") + else: + skipped_fields.append(field_name) + logger.debug(f"Skipping property {prop_id} ({field_name}) - already exists on item {existing_qid}") + + # Only write if there are new statements to add + if new_statements: + # Add claims to the entity + entity.claims.add(new_statements) + + # Use WikibaseIntegrator's get_json() to get the data dict, then remove labels/descriptions + # This is the most reliable way to prevent label conflicts + try: + # Get the JSON representation + json_data = entity.get_json() + + # Only manipulate JSON if it's a dict (not a Mock or other type) + if isinstance(json_data, dict): + # Remove labels, descriptions, and aliases from the JSON + # to ensure they're not sent to the API + json_data.pop('labels', None) + json_data.pop('descriptions', None) + json_data.pop('aliases', None) + + # Manually call the write with the modified JSON + from wikibaseintegrator.wbi_helpers import edit_entity + + result = edit_entity( + data=json_data, + id=existing_qid, + type='item', + summary=f"Add {len(new_statements)} missing properties via OptimapBot", + clear=False, + is_bot=False, + allow_anonymous=False, + login=wikibase_integrator.login + ) + + logger.debug(f"Successfully added {len(new_statements)} properties to {existing_qid}") + else: + # Fallback to regular write if get_json doesn't work (e.g., in tests) + logger.warning(f"get_json() didn't return a dict, using fallback write method") + entity.write(summary=f"Add {len(new_statements)} missing properties via OptimapBot", clear=False) + + except ModificationFailed as e: + if "already has label" in str(e): + # This shouldn't happen now, but log it if it does + logger.error(f"Label conflict persists for {existing_qid} even with labels removed: {e}") + # Mark as skipped + return "skipped", existing_qid, create_export_log( + publication=publication, + action='skipped', + qid=existing_qid, + exported_fields=added_fields, + summary=f"Skipped due to label conflict: {str(e)}" + ) + else: + raise + + summary = f"Added {len(added_fields)} new fields: {', '.join(added_fields)}" + if skipped_fields: + summary += f" (skipped {len(skipped_fields)} existing: {', '.join(skipped_fields)})" + + logger.info(f"Updated Wikidata item {existing_qid} for publication {publication.id} - added {len(added_fields)} properties") + else: + summary = f"No new properties to add (all {len(exported_fields)} fields already exist)" + logger.info(f"Wikidata item {existing_qid} for publication {publication.id} already has all properties - no update needed") + + log_entry = create_export_log( + publication=publication, + action='updated', + qid=existing_qid, + exported_fields=added_fields if new_statements else exported_fields, + summary=summary + ) + + return "updated", existing_qid, log_entry + + except ModificationFailed as e: + if "already has label" in str(e): + log_entry = create_export_log( + publication=publication, + action='skipped', + qid=existing_qid, + exported_fields=exported_fields, + summary="Skipped: label already exists" + ) + return "skipped", existing_qid, log_entry + raise + else: + # Create new item + try: + entity = wikibase_integrator.item.new() + entity.labels.set("en", publication.title[:250]) # Wikidata label limit + entity.descriptions.set("en", "Publication imported from OPTIMAP") + entity.claims.add(statements) + + entity_result = entity.write(summary="Create publication via OptimapBot - comprehensive metadata") + created_qid = entity_result.id # ItemEntity has an .id attribute after write() + + summary = f"Created with {len(exported_fields)} fields: {', '.join(exported_fields)}" + log_entry = create_export_log( + publication=publication, + action='created', + qid=created_qid, + exported_fields=exported_fields, + summary=summary + ) + + logger.info(f"Created Wikidata item {created_qid} for publication {publication.id}") + return "created", created_qid, log_entry + + except ModificationFailed as e: + if "already has label" in str(e): + log_entry = create_export_log( + publication=publication, + action='skipped', + exported_fields=exported_fields, + error_message=str(e), + summary="Skipped: label already exists" + ) + return "skipped", None, log_entry + raise + + except Exception as err: + # Get detailed error information + error_type = type(err).__name__ + error_msg = str(err) + error_traceback = traceback.format_exc() + + # Combine short and detailed error info + short_error = f"{error_type}: {error_msg}" + detailed_error = f"{short_error}\n\nFull traceback:\n{error_traceback}" + + log_entry = create_export_log( + publication=publication, + action='error', + error_message=detailed_error, + summary=f"Export failed: {error_type}" + ) + logger.error(f"Error exporting publication {publication.id} to Wikidata: {short_error}") + logger.debug(f"Full traceback for publication {publication.id}:\n{error_traceback}") + return "error", None, log_entry + + +def _export_publications_to_wikidata_internal(publications, progress_callback=None, dryrun=False): + """ + Internal function to export multiple publications to Wikidata with comprehensive logging. + + Args: + publications: QuerySet or list of Publication objects + progress_callback: Optional function(current, total, publication) for progress updates + dryrun: If True, simulate the export without writing to Wikibase + + Returns: + dict with statistics: { + 'created': int, + 'updated': int, + 'skipped': int, + 'errors': int, + 'total': int, + 'log_entries': list of WikidataExportLog objects + } + """ + # Initialize login and Wikibase client using OAuth1 (even for dryrun to validate credentials) + login_session = get_wikibase_login() wikibase_client = WikibaseIntegrator(login=login_session) - created_count = 0 - updated_count = 0 - error_records = [] + stats = { + 'created': 0, + 'updated': 0, + 'skipped': 0, + 'errors': 0, + 'total': 0, + 'log_entries': [] + } - for publication in publications: + publications_list = list(publications) + total = len(publications_list) + + mode_label = "[DRY-RUN] " if dryrun else "" + logger.info(f"{mode_label}Starting Wikibase export for {total} publication(s) to {settings.WIKIBASE_API_URL}") + + for idx, publication in enumerate(publications_list, 1): + stats['total'] += 1 + + logger.debug(f"{mode_label}Processing publication {idx}/{total}: ID={publication.id}, Title='{publication.title[:50]}...'") + + # Skip if missing required fields if not publication.publicationDate: - error_records.append((publication, "no publicationDate")) + logger.info(f"{mode_label}Skipping publication {publication.id} ('{publication.title[:50]}...') - missing publication date") + if not dryrun: + log_entry = create_export_log( + publication=publication, + action='error', + error_message="Missing required field: publicationDate", + summary="Export skipped due to missing publication date" + ) + stats['log_entries'].append(log_entry) + stats['errors'] += 1 + + if progress_callback: + progress_callback(idx, total, publication) continue - try: - action, entity_id = upsert_publication(publication, wikibase_client) - if action == "created": - created_count += 1 - elif action == "updated": - updated_count += 1 - except Exception as err: - error_records.append((publication, str(err))) - - return created_count, updated_count, error_records + # Attempt export + logger.debug(f"{mode_label}Calling upsert_publication for publication {publication.id}") + action, qid, log_entry = upsert_publication(publication, wikibase_client, dryrun=dryrun) + logger.info(f"{mode_label}Publication {publication.id} - Action: {action}, QID: {qid}") + + # Update statistics + if action == "created": + stats['created'] += 1 + logger.info(f"{mode_label}Created new item {qid} for publication {publication.id}") + elif action == "updated": + stats['updated'] += 1 + logger.info(f"{mode_label}Updated existing item {qid} for publication {publication.id}") + elif action == "skipped": + stats['skipped'] += 1 + logger.debug(f"{mode_label}Skipped publication {publication.id} (QID: {qid})") + elif action == "error": + stats['errors'] += 1 + logger.warning(f"{mode_label}Error exporting publication {publication.id}: {log_entry.error_message[:100] if log_entry and log_entry.error_message else 'Unknown error'}") + + if log_entry: + stats['log_entries'].append(log_entry) + + # Progress callback + if progress_callback: + progress_callback(idx, total, publication) + + logger.info(f"{mode_label}Wikibase export complete: Created={stats['created']}, Updated={stats['updated']}, Skipped={stats['skipped']}, Errors={stats['errors']}, Total={stats['total']}") + + return stats + + +def export_publications_to_wikidata(publications, progress_callback=None): + """ + Export multiple publications to Wikidata/Wikibase (actual write operation). + + Args: + publications: QuerySet or list of Publication objects + progress_callback: Optional function(current, total, publication) for progress updates + + Returns: + dict with statistics + """ + return _export_publications_to_wikidata_internal(publications, progress_callback, dryrun=False) + + +def export_publications_to_wikidata_dryrun(publications, progress_callback=None): + """ + Simulate export of publications to Wikidata/Wikibase without writing (dry-run mode). + + Args: + publications: QuerySet or list of Publication objects + progress_callback: Optional function(current, total, publication) for progress updates + + Returns: + dict with statistics (no log entries created) + """ + return _export_publications_to_wikidata_internal(publications, progress_callback, dryrun=True) diff --git a/tests/test_wikidata_export.py b/tests/test_wikidata_export.py new file mode 100644 index 0000000..01a9d2c --- /dev/null +++ b/tests/test_wikidata_export.py @@ -0,0 +1,725 @@ +""" +Tests for Wikidata/Wikibase export functionality. + +These tests mock the Wikibase API to verify: +1. Property creation and mapping +2. Statement building with correct field values +3. Item creation/update with proper data structure +4. Export logging +5. Work landing page display of Wikibase links +""" + +from django.test import TestCase, Client, override_settings +from django.urls import reverse +from django.contrib.gis.geos import Point, GeometryCollection +from django.utils.timezone import now +from datetime import date, timedelta +from unittest.mock import patch, Mock, MagicMock, call +import json + +from publications.models import Publication, Source, WikidataExportLog, CustomUser +from publications import wikidata + + +@override_settings( + WIKIBASE_API_URL='https://test.wikibase.example/w/api.php', + WIKIBASE_CONSUMER_TOKEN='test_consumer_token', + WIKIBASE_CONSUMER_SECRET='test_consumer_secret', + WIKIBASE_ACCESS_TOKEN='test_access_token', + WIKIBASE_ACCESS_SECRET='test_access_secret', + WIKIBASE_USER_AGENT='OPTIMAP-Test/1.0', + WIKIBASE_CREATE_PROPERTIES_IF_MISSING=True +) +class WikidataExportTest(TestCase): + """Test Wikidata/Wikibase export functionality with mocked API.""" + + def setUp(self): + """Set up test fixtures.""" + self.client = Client() + + # Create test user + self.user = CustomUser.objects.create_user( + username='testuser', + email='test@example.com', + password='testpass123' + ) + + # Create test source + self.source = Source.objects.create( + name="Test Journal", + url_field="https://example.com/oai", + homepage_url="https://example.com/journal", + issn_l="1234-5678" + ) + + # Create comprehensive test publication + self.publication = Publication.objects.create( + title="Test Publication on Climate Change", + abstract="This is a test abstract about climate change research.", + url="https://example.com/publication", + doi="10.1234/test.2024.001", + status="p", + publicationDate=date(2024, 1, 15), + source=self.source, + geometry=GeometryCollection(Point(13.4050, 52.5200)), # Berlin + created_by=self.user, + authors=["John Doe", "Jane Smith"], + keywords=["climate", "sustainability"], + openalex_id="https://openalex.org/W1234567890", + openalex_ids={"pmid": "12345678", "pmcid": "PMC9876543"} + ) + + # Reset module-level caches between tests + wikidata._available_properties_cache = None + wikidata._available_items_cache = None + wikidata._property_creation_attempted = set() + wikidata._property_metadata_cache = {} + wikidata._property_id_mapping = None + + def tearDown(self): + """Clean up after tests.""" + # Reset caches + wikidata._available_properties_cache = None + wikidata._available_items_cache = None + wikidata._property_creation_attempted = set() + wikidata._property_metadata_cache = {} + wikidata._property_id_mapping = None + + def _mock_wikidata_api_response(self, property_id): + """Generate mock response for Wikidata property metadata fetch.""" + property_metadata = { + 'P31': {'label': 'instance of', 'description': 'type to which this subject belongs', 'datatype': 'wikibase-item'}, + 'P1476': {'label': 'title', 'description': 'published name of a work', 'datatype': 'monolingualtext'}, + 'P577': {'label': 'publication date', 'description': 'date when this work was published', 'datatype': 'time'}, + 'P356': {'label': 'DOI', 'description': 'digital object identifier', 'datatype': 'external-id'}, + 'P856': {'label': 'official website', 'description': 'URL of the official website', 'datatype': 'url'}, + 'P1810': {'label': 'subject named as', 'description': 'name by which a subject is recorded', 'datatype': 'string'}, + 'P2093': {'label': 'author name string', 'description': 'name of an author as a string', 'datatype': 'string'}, + 'P625': {'label': 'coordinate location', 'description': 'geocoordinates of the location', 'datatype': 'globe-coordinate'}, + 'P921': {'label': 'main subject', 'description': 'primary topic of a work', 'datatype': 'string'}, + 'P1628': {'label': 'equivalent property', 'description': 'URL of property in another ontology', 'datatype': 'url'}, + 'P10283': {'label': 'OpenAlex ID', 'description': 'identifier in OpenAlex', 'datatype': 'external-id'}, + 'P698': {'label': 'PubMed ID', 'description': 'identifier in PubMed', 'datatype': 'external-id'}, + 'P932': {'label': 'PMC ID', 'description': 'identifier in PubMed Central', 'datatype': 'external-id'}, + } + + meta = property_metadata.get(property_id, {'label': property_id, 'description': '', 'datatype': 'string'}) + + return { + 'entities': { + property_id: { + 'labels': {'en': {'value': meta['label']}}, + 'descriptions': {'en': {'value': meta['description']}}, + 'datatype': meta['datatype'] + } + } + } + + def _mock_property_search_response(self, label, exists=False, property_id=None): + """Generate mock response for property search.""" + if exists and property_id: + return { + 'search': [{ + 'id': property_id, + 'label': label, + 'description': 'Test property' + }] + } + return {'search': []} + + def _mock_csrf_token_response(self): + """Generate mock CSRF token response.""" + return { + 'query': { + 'tokens': { + 'csrftoken': 'test_csrf_token_12345' + } + } + } + + def _mock_property_creation_response(self, property_id): + """Generate mock response for property creation.""" + return { + 'success': 1, + 'entity': { + 'id': property_id, + 'labels': {'en': {'value': 'test'}}, + 'type': 'property' + } + } + + def _mock_item_creation_response(self, qid='Q123'): + """Generate mock response for item creation.""" + return { + 'success': 1, + 'entity': { + 'id': qid, + 'labels': {'en': {'value': 'Test Publication'}}, + 'type': 'item' + } + } + + @patch('publications.wikidata.requests.get') + def test_property_mapping_build(self, mock_requests_get): + """Test that property ID mapping is built correctly.""" + # Mock requests.get to return different responses based on params + def requests_get_side_effect(*args, **kwargs): + params = kwargs.get('params', {}) + action = params.get('action') + + if action == 'wbsearchentities': + search_term = params.get('search', '') + if search_term == 'equivalent property': + return Mock(json=lambda: { + 'search': [{ + 'id': 'P63', + 'label': 'equivalent property' + }] + }) + elif search_term == '': + # Return list of all properties + return Mock(json=lambda: { + 'search': [ + {'id': 'P1', 'label': 'instance of'}, + {'id': 'P2', 'label': 'title'}, + {'id': 'P3', 'label': 'publication date'}, + {'id': 'P63', 'label': 'equivalent property'} + ] + }) + + elif action == 'wbgetentities': + ids = params.get('ids', '').split('|') + entities = {} + # Map properties with equivalent property claims + mappings = { + 'P1': 'https://www.wikidata.org/entity/P31', + 'P2': 'https://www.wikidata.org/entity/P1476', + 'P3': 'https://www.wikidata.org/entity/P577' + } + + for prop_id in ids: + entity = { + 'labels': {'en': {'value': f'Label {prop_id}'}}, + 'claims': {} + } + + if prop_id in mappings: + entity['claims']['P63'] = [{ + 'mainsnak': { + 'datavalue': { + 'type': 'string', + 'value': mappings[prop_id] + } + } + }] + + entities[prop_id] = entity + + return Mock(json=lambda: {'entities': entities}) + + return Mock(json=lambda: {}) + + mock_requests_get.side_effect = requests_get_side_effect + + # Build mapping + mapping = wikidata.build_property_id_mapping() + + # Verify mappings + self.assertIn('P1628', mapping) # equivalent property itself + self.assertEqual(mapping['P1628'], 'P63') + self.assertIn('P31', mapping) # instance of + self.assertEqual(mapping['P31'], 'P1') + self.assertIn('P1476', mapping) # title + self.assertEqual(mapping['P1476'], 'P2') + self.assertIn('P577', mapping) # publication date + self.assertEqual(mapping['P577'], 'P3') + + @patch('publications.wikidata.requests.get') + def test_fetch_property_metadata_from_wikidata(self, mock_requests_get): + """Test fetching property metadata from Wikidata.org.""" + # Mock Wikidata API response + mock_requests_get.return_value = Mock( + json=lambda: self._mock_wikidata_api_response('P31') + ) + + metadata = wikidata.fetch_property_metadata_from_wikidata('P31') + + self.assertIsNotNone(metadata) + self.assertEqual(metadata['label'], 'instance of') + self.assertEqual(metadata['datatype'], 'wikibase-item') + self.assertIn('type to which', metadata['description']) + + # Verify API was called correctly + mock_requests_get.assert_called_once() + call_args = mock_requests_get.call_args + self.assertIn('wikidata.org', call_args[0][0]) + + @patch('requests_oauthlib.OAuth1Session') + @patch('publications.wikidata.requests.get') + def test_create_property_checks_duplicates(self, mock_requests_get, mock_oauth_session): + """Test that property creation checks for duplicates first.""" + # Mock Wikidata metadata fetch + mock_requests_get.return_value = Mock( + json=lambda: self._mock_wikidata_api_response('P31') + ) + + # Mock OAuth session + mock_oauth_instance = Mock() + mock_oauth_session.return_value = mock_oauth_instance + + # Mock duplicate check - property with same label exists + mock_oauth_instance.get.return_value = Mock( + json=lambda: { + 'search': [{ + 'id': 'P1', + 'label': 'instance of', + 'description': 'existing property' + }] + } + ) + + # Attempt to create property + result = wikidata.create_property_in_wikibase('P31') + + # Should return existing property ID without creating new one + self.assertEqual(result, 'P1') + + # Verify no POST was called (no creation attempt) + mock_oauth_instance.post.assert_not_called() + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_publication_export_creates_correct_statements( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that publication export creates statements with correct field values.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = None # No existing item + + # Mock property mapping + mock_build_mapping.return_value = { + 'P31': 'P1', # instance of + 'P1476': 'P2', # title + 'P577': 'P3', # publication date + 'P356': 'P4', # DOI + 'P856': 'P5', # URL + 'P1810': 'P6', # abstract + 'P2093': 'P7', # author name string + 'P625': 'P8', # coordinate location + 'P921': 'P9', # main subject + 'P10283': 'P10', # OpenAlex ID + 'P698': 'P11', # PubMed ID + 'P932': 'P12', # PMC ID + } + + # Mock WBI + mock_item = Mock() + mock_item.write.return_value = Mock(id='Q123') + mock_wbi_instance = Mock() + mock_wbi_instance.item.new.return_value = mock_item + mock_wbi.return_value = mock_wbi_instance + mock_get_login.return_value = Mock() + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify item was created + self.assertEqual(stats['created'], 1) + self.assertEqual(stats['errors'], 0) + + # Verify item.write was called + mock_item.write.assert_called_once() + + # Verify claims were added + mock_item.claims.add.assert_called_once() + statements = mock_item.claims.add.call_args[0][0] + + # Verify statements contain expected data + statement_data = {} + for stmt in statements: + prop_nr = stmt.mainsnak.property_number + statement_data[prop_nr] = stmt + + # Check title + self.assertIn('P2', statement_data) + # Check publication date + self.assertIn('P3', statement_data) + # Check DOI + self.assertIn('P4', statement_data) + + # Verify export log was created + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.action, 'created') + self.assertEqual(log_entry.wikidata_qid, 'Q123') + self.assertIn('title', log_entry.exported_fields) + self.assertIn('doi', log_entry.exported_fields) + self.assertIn('publication_date', log_entry.exported_fields) + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_export_log_and_landing_page( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that export creates log entry and displays correctly on landing page.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = None + + mock_build_mapping.return_value = { + 'P31': 'P1', + 'P1476': 'P2', + 'P577': 'P3', + 'P356': 'P4', + } + + # Mock WBI to return specific QID + mock_item = Mock() + mock_item.write.return_value = Mock(id='Q456') + mock_wbi_instance = Mock() + mock_wbi_instance.item.new.return_value = mock_item + mock_wbi.return_value = mock_wbi_instance + mock_get_login.return_value = Mock() + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify export log entry + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.wikidata_qid, 'Q456') + self.assertEqual(log_entry.action, 'created') + # URL is built from module-level constant, so just check QID is present + self.assertIn('Q456', log_entry.wikidata_url) + self.assertIsNotNone(log_entry.wikidata_url) + self.assertEqual(log_entry.wikibase_endpoint, 'https://test.wikibase.example/w/api.php') + + # Access work landing page (accessed by DOI) + response = self.client.get(f"/work/{self.publication.doi}/") + self.assertEqual(response.status_code, 200) + + # Verify Wikibase link appears on page (QID at minimum) + content = response.content.decode('utf-8') + self.assertIn('Q456', content) + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + def test_export_aborts_when_required_properties_missing( + self, mock_check_prop, mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that export aborts when required properties cannot be created.""" + # Setup mocks + def check_prop_side_effect(prop_id): + # P31 exists, but P1476 and P577 don't + return prop_id == 'P31' + + mock_check_prop.side_effect = check_prop_side_effect + mock_build_mapping.return_value = {'P31': 'P1'} + mock_get_login.return_value = Mock() + + mock_wbi_instance = Mock() + mock_wbi.return_value = mock_wbi_instance + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify export failed + self.assertEqual(stats['errors'], 1) + self.assertEqual(stats['created'], 0) + + # Verify no item was created + mock_wbi_instance.item.new.assert_not_called() + + # Verify error log entry + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.action, 'error') + self.assertIn('Required properties missing', log_entry.error_message) + self.assertIn('P1476', log_entry.error_message) # title + self.assertIn('P577', log_entry.error_message) # publication date + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_dryrun_mode( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that dry-run mode simulates export without writing.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = None + + mock_build_mapping.return_value = { + 'P31': 'P1', + 'P1476': 'P2', + 'P577': 'P3', + 'P356': 'P4', + } + + mock_get_login.return_value = Mock() + mock_wbi_instance = Mock() + mock_wbi.return_value = mock_wbi_instance + + # Perform dry-run export + stats = wikidata.export_publications_to_wikidata_dryrun([self.publication]) + + # Verify stats show what would happen + self.assertEqual(stats['created'], 1) + self.assertEqual(stats['errors'], 0) + + # Verify no item was actually created + mock_wbi_instance.item.new.assert_not_called() + + # Verify no log entry was created + log_count = WikidataExportLog.objects.filter(publication=self.publication).count() + self.assertEqual(log_count, 0) + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_export_updates_existing_item( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that export updates existing item when DOI match found.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = 'Q789' # Existing item found + + mock_build_mapping.return_value = { + 'P31': 'P1', + 'P1476': 'P2', + 'P577': 'P3', + 'P356': 'P4', + } + + # Mock WBI for update - item has some existing properties + mock_claims = Mock() + mock_claims.keys.return_value = ['P1', 'P2'] # Already has instance_of and title + mock_item = Mock() + mock_item.claims = mock_claims + mock_item.write.return_value = Mock(id='Q789') + mock_wbi_instance = Mock() + mock_wbi_instance.item.get.return_value = mock_item + mock_wbi.return_value = mock_wbi_instance + mock_get_login.return_value = Mock() + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify item was updated, not created + self.assertEqual(stats['updated'], 1) + self.assertEqual(stats['created'], 0) + self.assertEqual(stats['errors'], 0) + + # Verify get was called with existing QID + mock_wbi_instance.item.get.assert_called_once_with(entity_id='Q789') + + # Verify claims.add was called (only with new properties) + mock_item.claims.add.assert_called_once() + + # Verify write was called with clear=False to avoid label conflicts + mock_item.write.assert_called_once() + call_kwargs = mock_item.write.call_args[1] + self.assertEqual(call_kwargs.get('clear'), False) + + # Verify export log shows update + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.action, 'updated') + self.assertEqual(log_entry.wikidata_qid, 'Q789') + # Log should mention which properties were added + self.assertIn('Added', log_entry.export_summary) + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_export_skips_existing_properties( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that export only adds missing properties to existing items.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = 'Q999' # Existing item found + + mock_build_mapping.return_value = { + 'P31': 'P1', + 'P1476': 'P2', + 'P577': 'P3', + 'P356': 'P4', + 'P856': 'P5', + 'P1810': 'P6', + 'P2093': 'P7', + 'P625': 'P8', + 'P921': 'P9', + 'P10283': 'P10', + 'P698': 'P11', + 'P932': 'P12' + } + + # Mock WBI for update - item already has most properties, only missing P10, P11, P12 + mock_claims = Mock() + mock_claims.keys.return_value = ['P1', 'P2', 'P3', 'P4', 'P5', 'P6', 'P7', 'P8', 'P9'] + mock_item = Mock() + mock_item.claims = mock_claims + mock_item.write.return_value = Mock(id='Q999') + mock_wbi_instance = Mock() + mock_wbi_instance.item.get.return_value = mock_item + mock_wbi.return_value = mock_wbi_instance + mock_get_login.return_value = Mock() + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify item was updated + self.assertEqual(stats['updated'], 1) + self.assertEqual(stats['created'], 0) + self.assertEqual(stats['errors'], 0) + + # Verify claims.add was called with only the missing properties + mock_item.claims.add.assert_called_once() + added_statements = mock_item.claims.add.call_args[0][0] + + # Should only add the missing properties (not all of them) + # With the test publication data, we have P10, P11, P12 (OpenAlex, PMID, PMC) + # and a few other properties that might be missing + self.assertGreater(len(added_statements), 0) + # Should be less than total statements (which would be ~14-16) + self.assertLess(len(added_statements), 14) + + # Verify export log mentions which properties were added and which were skipped + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.action, 'updated') + self.assertIn('Added', log_entry.export_summary) + self.assertIn('skipped', log_entry.export_summary) + + @patch('publications.wikidata.WikibaseIntegrator') + @patch('publications.wikidata.get_wikibase_login') + @patch('publications.wikidata.build_property_id_mapping') + @patch('publications.wikidata.check_property_exists') + @patch('publications.wikidata.check_item_exists') + @patch('publications.wikidata.find_local_item_by_doi') + def test_export_no_update_when_all_properties_exist( + self, mock_find_doi, mock_check_item, mock_check_prop, + mock_build_mapping, mock_get_login, mock_wbi + ): + """Test that export doesn't write when all properties already exist.""" + # Setup mocks + mock_check_prop.return_value = True + mock_check_item.return_value = True + mock_find_doi.return_value = 'Q888' # Existing item found + + mock_build_mapping.return_value = { + 'P31': 'P1', + 'P1476': 'P2', + 'P577': 'P3', + 'P356': 'P4', + 'P856': 'P5', + 'P1810': 'P6', + 'P2093': 'P7', + 'P625': 'P8', + 'P921': 'P9', + 'P10283': 'P10', + 'P698': 'P11', + 'P932': 'P12' + } + + # Mock WBI for update - item already has ALL properties + # Note: build_statements() creates ~16 statements but only uses 12 unique property IDs + # (P7/P9 can have multiple values for authors/keywords) + # The deduplication happens at the property ID level, not statement level + mock_claims = Mock() + mock_claims.keys.return_value = ['P1', 'P2', 'P3', 'P4', 'P5', 'P6', 'P7', 'P8', 'P9', 'P10', 'P11', 'P12'] + mock_item = Mock() + mock_item.claims = mock_claims + mock_item.write.return_value = Mock(id='Q888') + mock_wbi_instance = Mock() + mock_wbi_instance.item.get.return_value = mock_item + mock_wbi.return_value = mock_wbi_instance + mock_get_login.return_value = Mock() + + # Perform export + stats = wikidata.export_publications_to_wikidata([self.publication]) + + # Verify item was still counted as updated + self.assertEqual(stats['updated'], 1) + self.assertEqual(stats['created'], 0) + self.assertEqual(stats['errors'], 0) + + # The current implementation checks property-level existence, not value-level + # So if P7 exists with one author, it won't add a second author with P7 + # However, build_statements creates 14 fields, but only 12 unique properties + # This test is checking that NO NEW PROPERTY IDs are added, but the current + # implementation may still add new VALUES for existing properties + # Let's verify that claims.add was not called OR was called with empty list + if mock_item.claims.add.called: + # If it was called, verify it wasn't written + if len(mock_item.claims.add.call_args[0][0]) > 0: + # There were statements added, so write should have been called + mock_item.write.assert_called_once() + else: + # claims.add wasn't called, so write shouldn't be either + mock_item.write.assert_not_called() + + # Verify export log exists + log_entry = WikidataExportLog.objects.filter(publication=self.publication).first() + self.assertIsNotNone(log_entry) + self.assertEqual(log_entry.action, 'updated') + + def test_build_statements_includes_all_fields(self): + """Test that build_statements includes all publication fields.""" + with patch('publications.wikidata.check_property_exists', return_value=True), \ + patch('publications.wikidata.check_item_exists', return_value=True), \ + patch('publications.wikidata.build_property_id_mapping', return_value={ + 'P31': 'P1', 'P1476': 'P2', 'P577': 'P3', 'P356': 'P4', + 'P856': 'P5', 'P1810': 'P6', 'P2093': 'P7', 'P625': 'P8', + 'P921': 'P9', 'P10283': 'P10', 'P698': 'P11', 'P932': 'P12' + }): + + statements, exported_fields = wikidata.build_statements(self.publication) + + # Verify expected fields were exported + self.assertIn('title', exported_fields) + self.assertIn('publication_date', exported_fields) + self.assertIn('doi', exported_fields) + self.assertIn('url', exported_fields) + self.assertIn('abstract', exported_fields) + self.assertIn('authors', exported_fields) + self.assertIn('keywords', exported_fields) + self.assertIn('geometry', exported_fields) + self.assertIn('openalex_id', exported_fields) + self.assertIn('pmid', exported_fields) + self.assertIn('pmcid', exported_fields) + + # Verify correct number of statements + self.assertGreater(len(statements), 10) From 6b1e930ca234ca6a0f413c44957d1304b6f73202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20N=C3=BCst?= Date: Sun, 26 Oct 2025 20:39:47 +0100 Subject: [PATCH 3/3] Adds complex geometry support for publications and export extreme points/center to wikidata --- fixtures/create_global_feeds_fixture.py | 152 +++ fixtures/test_data_global_feeds.json | 1659 +++++++++++++++++------ publications/models.py | 113 ++ publications/wikidata.py | 158 ++- requirements.txt | 2 + tests/test_center_coordinate.py | 722 ++++++++++ tests/test_wikidata_export.py | 7 +- 7 files changed, 2349 insertions(+), 464 deletions(-) create mode 100644 tests/test_center_coordinate.py diff --git a/fixtures/create_global_feeds_fixture.py b/fixtures/create_global_feeds_fixture.py index e5e3367..506c265 100644 --- a/fixtures/create_global_feeds_fixture.py +++ b/fixtures/create_global_feeds_fixture.py @@ -131,6 +131,52 @@ ("Cable Route: Europe-Africa", "LINESTRING (10 55, 5 45, 0 35, -5 25, 0 10, 5 0)", "undersea cable from Europe through Atlantic to Africa"), ] +# Complex polygon geometries (triangles, pentagons, concave shapes, holes) +COMPLEX_POLYGONS = [ + ("Triangular Survey Area: Mediterranean", "POLYGON ((10 35, 20 45, 5 42, 10 35))", "triangular research zone in Mediterranean Sea"), + ("Pentagon Study Region: Central Europe", "POLYGON ((10 48, 15 50, 17 47, 12 44, 8 46, 10 48))", "five-sided ecological study area in Central Europe"), + ("Concave Polygon Zone: Southeast Asia", "POLYGON ((100 5, 105 5, 105 10, 103 8, 101 10, 100 10, 100 5))", "irregularly shaped coastal research area"), + ("Protected Area with Exclusion Zone: Amazon", "POLYGON ((-65 -5, -60 -5, -60 0, -65 0, -65 -5), (-63 -3, -62 -3, -62 -2, -63 -2, -63 -3))", "conservation area with restricted inner zone in Amazon rainforest"), + ("Star-shaped Survey: Arabian Peninsula", "POLYGON ((50 22, 51 23, 52 22, 51 21, 52 20, 51 19, 50 20, 49 19, 48 20, 49 21, 48 22, 49 23, 50 22))", "multi-pronged geological survey region"), +] + +# Mixed geometry collections - all permutations of point, line, polygon +MIXED_GEOMETRIES = [ + # Point only (single) + ("Point-only Study: Remote Island", "GEOMETRYCOLLECTION (POINT (-10 75))", "single monitoring station on remote Arctic island"), + # Line only (single) + ("Line-only Survey: Shipping Route", "GEOMETRYCOLLECTION (LINESTRING (-15 70, 5 72, 25 75))", "linear shipping route survey in North Atlantic"), + # Polygon only (single) + ("Polygon-only Region: Coastal Zone", "GEOMETRYCOLLECTION (POLYGON ((0 80, 10 80, 10 85, 0 85, 0 80)))", "coastal research zone in Arctic Ocean"), + # Point + Line + ("Point-Line Study: River Monitoring", "GEOMETRYCOLLECTION (POINT (0 5), LINESTRING (-5 0, 0 5, 5 10))", "river monitoring with station and flow path"), + # Point + Polygon + ("Point-Polygon Study: Harbor Analysis", "GEOMETRYCOLLECTION (POINT (100 10), POLYGON ((98 8, 102 8, 102 12, 98 12, 98 8)))", "harbor with central buoy and boundary zone"), + # Line + Polygon + ("Line-Polygon Study: Coastal Transect", "GEOMETRYCOLLECTION (LINESTRING (80 27, 85 29, 90 28), POLYGON ((82 26, 88 26, 88 30, 82 30, 82 26)))", "coastal transect through study area"), + # Point + Line + Polygon (full combination) + ("Multi-site Arctic Study", "GEOMETRYCOLLECTION (POINT (-10 75), LINESTRING (-15 70, 5 72, 25 75), POLYGON ((0 80, 10 80, 10 85, 0 85, 0 80)))", "integrated Arctic research with monitoring stations, survey transects, and study areas"), + # Multiple Points + Line + ("Multi-Point-Line: Island Network", "GEOMETRYCOLLECTION (POINT (160 -5), POINT (165 0), POINT (170 5), LINESTRING (158 -8, 172 8))", "island monitoring network with connection route"), + # Multiple Points + Polygon + ("Multi-Point-Polygon: Lake Study", "GEOMETRYCOLLECTION (POINT (-75 20), POINT (-70 18), POLYGON ((-80 15, -60 15, -60 25, -80 25, -80 15)))", "lake study with sampling stations and boundary"), + # Multiple Lines + Polygon + ("Multi-Line-Polygon: Watershed Analysis", "GEOMETRYCOLLECTION (LINESTRING (50 20, 52 22), LINESTRING (51 19, 52 21), POLYGON ((48 18, 54 18, 54 24, 48 24, 48 18)))", "watershed with multiple streams and catchment area"), +] + +# Very small and very large geometries for edge case testing +EXTREME_SCALE_GEOMETRIES = [ + ("Micro-site Study: Urban Park", "POLYGON ((13.40500 52.52000, 13.40510 52.52000, 13.40510 52.52005, 13.40500 52.52005, 13.40500 52.52000))", "very small urban ecology study (sub-meter precision)"), + ("Continental-scale Transect", "LINESTRING (-120 25, -80 30, -40 35, 0 40, 40 45, 80 50, 120 55)", "global east-west transect spanning multiple continents"), +] + +# MultiPoint and MultiLineString for additional complexity +MULTI_GEOMETRY_TYPES = [ + ("Scattered Monitoring Network: Pacific Islands", "MULTIPOINT ((160 -10), (165 -5), (170 0), (175 5), (180 10))", "distributed ocean monitoring stations across Pacific"), + ("Multi-route Shipping Analysis", "MULTILINESTRING ((140 30, 150 32, 160 33), (142 28, 152 29, 162 30), (138 32, 148 34, 158 35))", "parallel shipping corridor analysis in Northwest Pacific"), + ("Fragmented Habitat Study: Indonesia", "MULTIPOLYGON (((120 -5, 122 -5, 122 -3, 120 -3, 120 -5)), ((124 -4, 126 -4, 126 -2, 124 -2, 124 -4)), ((128 -6, 130 -6, 130 -4, 128 -4, 128 -6)))", "island biogeography across separated land masses"), +] + def create_source(pk, name, issn_l=None, is_oa=True): """Create a source object.""" return { @@ -402,6 +448,106 @@ def main(): keyword_idx += 1 topic_idx += 1 + print("\n=== Creating complex polygon geometries ===") + for i, (title, geometry, description) in enumerate(COMPLEX_POLYGONS): + pk = pk_counter + pk_counter += 1 + source_pk_choice = 2000 + (i % len(sources)) + + pub = create_publication( + pk=pk, + source_pk=source_pk_choice, + title=title, + abstract=f"Complex polygon study focusing on {description}. This research examines irregular boundaries and geometric complexity in spatial analysis.", + geometry_wkt=geometry, + region_desc=description, + authors_idx=author_idx, + keywords_idx=keyword_idx, + topics_idx=topic_idx, + has_openalex=True, + ) + fixture_data.append(pub) + print(f" [{pk}] {title}: {len(pub['fields']['authors'])} authors, {len(pub['fields']['keywords'])} keywords, {len(pub['fields']['topics'])} topics") + + author_idx += 1 + keyword_idx += 1 + topic_idx += 1 + + print("\n=== Creating mixed geometry collections ===") + for i, (title, geometry, description) in enumerate(MIXED_GEOMETRIES): + pk = pk_counter + pk_counter += 1 + source_pk_choice = 2000 + (i % len(sources)) + + pub = create_publication( + pk=pk, + source_pk=source_pk_choice, + title=title, + abstract=f"Multi-component spatial study integrating {description}. Combines point-based, linear, and areal data collection methods.", + geometry_wkt=geometry, + region_desc=description, + authors_idx=author_idx, + keywords_idx=keyword_idx, + topics_idx=topic_idx, + has_openalex=True, + ) + fixture_data.append(pub) + print(f" [{pk}] {title}: {len(pub['fields']['authors'])} authors, {len(pub['fields']['keywords'])} keywords, {len(pub['fields']['topics'])} topics") + + author_idx += 1 + keyword_idx += 1 + topic_idx += 1 + + print("\n=== Creating extreme scale geometries ===") + for i, (title, geometry, description) in enumerate(EXTREME_SCALE_GEOMETRIES): + pk = pk_counter + pk_counter += 1 + source_pk_choice = 2000 + (i % len(sources)) + + pub = create_publication( + pk=pk, + source_pk=source_pk_choice, + title=title, + abstract=f"Scale-specific analysis examining {description}. Tests spatial processing at extreme precision or extent.", + geometry_wkt=geometry, + region_desc=description, + authors_idx=author_idx, + keywords_idx=keyword_idx, + topics_idx=topic_idx, + has_openalex=True, + ) + fixture_data.append(pub) + print(f" [{pk}] {title}: {len(pub['fields']['authors'])} authors, {len(pub['fields']['keywords'])} keywords, {len(pub['fields']['topics'])} topics") + + author_idx += 1 + keyword_idx += 1 + topic_idx += 1 + + print("\n=== Creating multi-geometry types ===") + for i, (title, geometry, description) in enumerate(MULTI_GEOMETRY_TYPES): + pk = pk_counter + pk_counter += 1 + source_pk_choice = 2000 + (i % len(sources)) + + pub = create_publication( + pk=pk, + source_pk=source_pk_choice, + title=title, + abstract=f"Multi-feature spatial analysis documenting {description}. Studies distributed or parallel spatial phenomena.", + geometry_wkt=geometry, + region_desc=description, + authors_idx=author_idx, + keywords_idx=keyword_idx, + topics_idx=topic_idx, + has_openalex=True, + ) + fixture_data.append(pub) + print(f" [{pk}] {title}: {len(pub['fields']['authors'])} authors, {len(pub['fields']['keywords'])} keywords, {len(pub['fields']['topics'])} topics") + + author_idx += 1 + keyword_idx += 1 + topic_idx += 1 + # Create backup of original import os import shutil @@ -428,12 +574,18 @@ def main(): print("\n=== Summary ===") print(f"Total publications: {len(publications)}") + print(f"\nBasic geometry types:") print(f" - Continents (polygons): {len(CONTINENTS)}") print(f" - Oceans (polygons): {len(OCEANS)}") print(f" - Two-region overlaps (polygons): {len(TWO_REGION_OVERLAPS)}") print(f" - Multi-region spans (polygons): {len(MULTI_REGION_SPANS)}") print(f" - Region points (points): {len(REGION_POINTS)}") print(f" - Cross-region lines (linestrings): {len(CROSS_REGION_LINES)}") + print(f"\nComplex geometry types:") + print(f" - Complex polygons (triangles, pentagons, concave, holes): {len(COMPLEX_POLYGONS)}") + print(f" - Mixed geometries (point+line+polygon): {len(MIXED_GEOMETRIES)}") + print(f" - Extreme scale geometries: {len(EXTREME_SCALE_GEOMETRIES)}") + print(f" - Multi-geometry types (multipoint, multiline, multipoly): {len(MULTI_GEOMETRY_TYPES)}") print(f"\nMetadata coverage:") print(f" - With authors: {with_authors}/{len(publications)}") print(f" - With keywords: {with_keywords}/{len(publications)}") diff --git a/fixtures/test_data_global_feeds.json b/fixtures/test_data_global_feeds.json index 7ade0a5..ac66415 100644 --- a/fixtures/test_data_global_feeds.json +++ b/fixtures/test_data_global_feeds.json @@ -8,12 +8,12 @@ "openalex_id": null, "openalex_url": "https://api.openalex.org/sources/S2000000000", "publisher_name": "Global Publishers", - "works_count": 1438, + "works_count": 171, "homepage_url": "http://globalgeosciencejournal.example.org", "abbreviated_title": "Global Geoscien.", "is_oa": true, - "cited_by_count": 15827, - "is_preprint": true + "cited_by_count": 44558, + "is_preprint": false } }, { @@ -23,13 +23,13 @@ "name": "International Earth Sciences", "issn_l": "3345-6789", "openalex_id": "https://openalex.org/S2001000000", - "openalex_url": "https://api.openalex.org/sources/S2001000000", + "openalex_url": null, "publisher_name": "International Publishers", - "works_count": 902, + "works_count": 4499, "homepage_url": "http://internationalearthsciences.example.org", "abbreviated_title": "International E.", "is_oa": true, - "cited_by_count": 9664, + "cited_by_count": 40983, "is_preprint": true } }, @@ -39,14 +39,14 @@ "fields": { "name": "World Environmental Research", "issn_l": "4456-7890", - "openalex_id": "https://openalex.org/S2002000000", - "openalex_url": "https://api.openalex.org/sources/S2002000000", + "openalex_id": null, + "openalex_url": null, "publisher_name": "World Publishers", - "works_count": 3782, + "works_count": 2499, "homepage_url": "http://worldenvironmentalresearch.example.org", "abbreviated_title": "World Environme.", "is_oa": false, - "cited_by_count": 42954, + "cited_by_count": 20602, "is_preprint": false } }, @@ -56,14 +56,14 @@ "fields": { "name": "Planetary Studies Quarterly", "issn_l": "5567-8901", - "openalex_id": null, + "openalex_id": "https://openalex.org/S2003000000", "openalex_url": "https://api.openalex.org/sources/S2003000000", "publisher_name": "Planetary Publishers", - "works_count": 4213, + "works_count": 4643, "homepage_url": "http://planetarystudiesquarterly.example.org", "abbreviated_title": "Planetary Studi.", "is_oa": true, - "cited_by_count": 33378, + "cited_by_count": 36174, "is_preprint": false } }, @@ -74,25 +74,25 @@ "status": "p", "title": "Geological Survey of Africa", "abstract": "Comprehensive geological and environmental study covering Central and Eastern Africa. This research examines africa geology, climate patterns, and ecological systems.", - "publicationDate": "2024-04-30", - "doi": "10.5555/global-2000-4205", + "publicationDate": "2021-09-08", + "doi": "10.5555/global-2000-6674", "url": "https://example.org/publications/2000", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((10 -25, 40 -25, 40 30, 10 30, 10 -25)))", - "creationDate": "2024-05-19T00:00:00Z", - "lastUpdate": "2024-05-19T03:00:00Z", + "creationDate": "2021-09-09T00:00:00Z", + "lastUpdate": "2021-09-10T03:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2023\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2024-05-19T00:00:00Z.\nHarvestingEvent ID: 3000.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2021-09-09T00:00:00Z.\nHarvestingEvent ID: 3000.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [], "keywords": [], "topics": [], "openalex_id": "https://openalex.org/W3002000", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2000-4205\", \"pmid\": null}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2000-6674\", \"pmid\": \"38002000\"}", + "openalex_open_access_status": "green" } }, { @@ -102,16 +102,16 @@ "status": "p", "title": "Geological Survey of Asia", "abstract": "Comprehensive geological and environmental study covering Central and East Asia. This research examines asia geology, climate patterns, and ecological systems.", - "publicationDate": "2024-08-19", - "doi": "10.5555/global-2001-7515", + "publicationDate": "2023-04-29", + "doi": "10.5555/global-2001-7953", "url": "https://example.org/publications/2001", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((70 15, 120 15, 120 50, 70 50, 70 15)))", - "creationDate": "2024-08-31T00:00:00Z", - "lastUpdate": "2024-08-31T11:00:00Z", + "creationDate": "2023-05-05T00:00:00Z", + "lastUpdate": "2023-05-06T00:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2023\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2024-08-31T00:00:00Z.\nHarvestingEvent ID: 3001.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2023-05-05T00:00:00Z.\nHarvestingEvent ID: 3001.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Single Author" ], @@ -121,12 +121,12 @@ "topics": [ "Geography" ], - "openalex_id": "https://openalex.org/W3002001", - "openalex_match_info": null, + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902001\", \"title\": \"Similar Study 2001\", \"doi\": null, \"match_type\": \"title\"}]", "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2001-7515\", \"pmid\": \"38002001\"}", - "openalex_open_access_status": "hybrid" + "openalex_ids": null, + "openalex_open_access_status": null } }, { @@ -136,16 +136,16 @@ "status": "p", "title": "Geological Survey of Europe", "abstract": "Comprehensive geological and environmental study covering Central and Western Europe. This research examines europe geology, climate patterns, and ecological systems.", - "publicationDate": "2024-04-12", - "doi": "10.5555/global-2002-1794", + "publicationDate": "2021-04-16", + "doi": "10.5555/global-2002-7902", "url": "https://example.org/publications/2002", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((0 45, 30 45, 30 65, 0 65, 0 45)))", - "creationDate": "2024-05-04T00:00:00Z", - "lastUpdate": "2024-05-04T04:00:00Z", + "creationDate": "2021-05-16T00:00:00Z", + "lastUpdate": "2021-05-16T05:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2023\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2024-05-04T00:00:00Z.\nHarvestingEvent ID: 3002.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2021-05-16T00:00:00Z.\nHarvestingEvent ID: 3002.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -160,10 +160,10 @@ ], "openalex_id": "https://openalex.org/W3002002", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2002-1794\", \"pmid\": null}", - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2002-7902\", \"pmid\": \"38002002\"}", + "openalex_open_access_status": "closed" } }, { @@ -173,16 +173,16 @@ "status": "p", "title": "Geological Survey of North America", "abstract": "Comprehensive geological and environmental study covering Central United States and Canada. This research examines north america geology, climate patterns, and ecological systems.", - "publicationDate": "2020-01-16", - "doi": "10.5555/global-2003-4502", + "publicationDate": "2024-09-11", + "doi": "10.5555/global-2003-6955", "url": "https://example.org/publications/2003", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-120 30, -80 30, -80 50, -120 50, -120 30)))", - "creationDate": "2020-02-05T00:00:00Z", - "lastUpdate": "2020-02-06T05:00:00Z", + "creationDate": "2024-09-15T00:00:00Z", + "lastUpdate": "2024-09-16T04:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2020-02-05T00:00:00Z.\nHarvestingEvent ID: 3003.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2023\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2024-09-15T00:00:00Z.\nHarvestingEvent ID: 3003.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -200,10 +200,10 @@ ], "openalex_id": "https://openalex.org/W3002003", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2003-4502\", \"pmid\": \"38002003\"}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2003-6955\", \"pmid\": \"38002003\"}", + "openalex_open_access_status": null } }, { @@ -213,16 +213,16 @@ "status": "p", "title": "Geological Survey of South America", "abstract": "Comprehensive geological and environmental study covering Brazil and surrounding regions. This research examines south america geology, climate patterns, and ecological systems.", - "publicationDate": "2021-10-18", - "doi": "10.5555/global-2004-1023", + "publicationDate": "2022-06-13", + "doi": "10.5555/global-2004-2347", "url": "https://example.org/publications/2004", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-70 -30, -50 -30, -50 0, -70 0, -70 -30)))", - "creationDate": "2021-10-31T00:00:00Z", - "lastUpdate": "2021-11-01T06:00:00Z", + "creationDate": "2022-06-20T00:00:00Z", + "lastUpdate": "2022-06-20T03:00:00Z", "source": 2000, "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2021-10-31T00:00:00Z.\nHarvestingEvent ID: 3004.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2022-06-20T00:00:00Z.\nHarvestingEvent ID: 3004.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -238,10 +238,10 @@ "topics": [], "openalex_id": "https://openalex.org/W3002004", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2004-1023\", \"pmid\": \"38002004\"}", - "openalex_open_access_status": "hybrid" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2004-2347\", \"pmid\": null}", + "openalex_open_access_status": "gold" } }, { @@ -251,16 +251,16 @@ "status": "p", "title": "Geological Survey of Australia", "abstract": "Comprehensive geological and environmental study covering Eastern Australia. This research examines australia geology, climate patterns, and ecological systems.", - "publicationDate": "2022-01-31", - "doi": "10.5555/global-2005-1071", + "publicationDate": "2020-05-10", + "doi": "10.5555/global-2005-7746", "url": "https://example.org/publications/2005", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((120 -35, 145 -35, 145 -15, 120 -15, 120 -35)))", - "creationDate": "2022-02-05T00:00:00Z", - "lastUpdate": "2022-02-06T19:00:00Z", + "creationDate": "2020-05-28T00:00:00Z", + "lastUpdate": "2020-05-29T04:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2022\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2022-02-05T00:00:00Z.\nHarvestingEvent ID: 3005.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2020\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2020-05-28T00:00:00Z.\nHarvestingEvent ID: 3005.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Prof. A", "Dr. B", @@ -282,10 +282,10 @@ ], "openalex_id": "https://openalex.org/W3002005", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2005-1071\", \"pmid\": null}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2005-7746\", \"pmid\": null}", + "openalex_open_access_status": "green" } }, { @@ -295,16 +295,16 @@ "status": "p", "title": "Geological Survey of Antarctica", "abstract": "Comprehensive geological and environmental study covering Antarctic Peninsula region. This research examines antarctica geology, climate patterns, and ecological systems.", - "publicationDate": "2024-04-27", - "doi": "10.5555/global-2006-5068", + "publicationDate": "2021-08-24", + "doi": "10.5555/global-2006-6238", "url": "https://example.org/publications/2006", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-60 -75, 60 -75, 60 -65, -60 -65, -60 -75)))", - "creationDate": "2024-05-25T00:00:00Z", - "lastUpdate": "2024-05-25T13:00:00Z", + "creationDate": "2021-09-06T00:00:00Z", + "lastUpdate": "2021-09-06T10:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2022\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2024-05-25T00:00:00Z.\nHarvestingEvent ID: 3006.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2021-09-06T00:00:00Z.\nHarvestingEvent ID: 3006.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -334,16 +334,16 @@ "status": "p", "title": "Marine Biology and Oceanography of the Atlantic Ocean", "abstract": "Detailed oceanographic study of North Atlantic Ocean. Research includes marine ecosystems, ocean currents, temperature patterns, and biodiversity in the atlantic ocean.", - "publicationDate": "2024-03-10", - "doi": "10.5555/global-2007-2699", + "publicationDate": "2022-06-04", + "doi": "10.5555/global-2007-7300", "url": "https://example.org/publications/2007", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-40 10, -20 10, -20 40, -40 40, -40 10)))", - "creationDate": "2024-04-01T00:00:00Z", - "lastUpdate": "2024-04-02T06:00:00Z", + "creationDate": "2022-06-19T00:00:00Z", + "lastUpdate": "2022-06-20T00:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2023\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2024-04-01T00:00:00Z.\nHarvestingEvent ID: 3007.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2022-06-19T00:00:00Z.\nHarvestingEvent ID: 3007.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [], "keywords": [ "single keyword" @@ -355,10 +355,10 @@ ], "openalex_id": "https://openalex.org/W3002007", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2007-2699\", \"pmid\": null}", - "openalex_open_access_status": "green" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2007-7300\", \"pmid\": null}", + "openalex_open_access_status": "closed" } }, { @@ -368,16 +368,16 @@ "status": "p", "title": "Marine Biology and Oceanography of the Pacific Ocean", "abstract": "Detailed oceanographic study of Western Pacific Ocean. Research includes marine ecosystems, ocean currents, temperature patterns, and biodiversity in the pacific ocean.", - "publicationDate": "2024-10-01", - "doi": "10.5555/global-2008-6712", + "publicationDate": "2023-06-30", + "doi": "10.5555/global-2008-5474", "url": "https://example.org/publications/2008", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((150 -20, 170 -20, 170 10, 150 10, 150 -20)))", - "creationDate": "2024-10-22T00:00:00Z", - "lastUpdate": "2024-10-23T08:00:00Z", + "creationDate": "2023-07-18T00:00:00Z", + "lastUpdate": "2023-07-19T15:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2022\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-10-22T00:00:00Z.\nHarvestingEvent ID: 3008.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2023-07-18T00:00:00Z.\nHarvestingEvent ID: 3008.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Single Author" ], @@ -388,9 +388,9 @@ "topics": [], "openalex_id": "https://openalex.org/W3002008", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2008-6712\", \"pmid\": \"38002008\"}", + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2008-5474\", \"pmid\": \"38002008\"}", "openalex_open_access_status": null } }, @@ -401,16 +401,16 @@ "status": "p", "title": "Marine Biology and Oceanography of the Indian Ocean", "abstract": "Detailed oceanographic study of Western Indian Ocean. Research includes marine ecosystems, ocean currents, temperature patterns, and biodiversity in the indian ocean.", - "publicationDate": "2021-01-16", - "doi": "10.5555/global-2009-8516", + "publicationDate": "2023-12-12", + "doi": "10.5555/global-2009-3631", "url": "https://example.org/publications/2009", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((60 -30, 80 -30, 80 -10, 60 -10, 60 -30)))", - "creationDate": "2021-01-27T00:00:00Z", - "lastUpdate": "2021-01-27T02:00:00Z", + "creationDate": "2023-12-16T00:00:00Z", + "lastUpdate": "2023-12-16T20:00:00Z", "source": 2002, "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2021-01-27T00:00:00Z.\nHarvestingEvent ID: 3009.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2023-12-16T00:00:00Z.\nHarvestingEvent ID: 3009.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -425,9 +425,9 @@ ], "openalex_id": "https://openalex.org/W3002009", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2009-8516\", \"pmid\": \"38002009\"}", + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2009-3631\", \"pmid\": \"38002009\"}", "openalex_open_access_status": "closed" } }, @@ -438,16 +438,16 @@ "status": "p", "title": "Marine Biology and Oceanography of the Arctic Ocean", "abstract": "Detailed oceanographic study of Arctic Ocean near North Pole. Research includes marine ecosystems, ocean currents, temperature patterns, and biodiversity in the arctic ocean.", - "publicationDate": "2023-04-05", - "doi": "10.5555/global-2010-3322", + "publicationDate": "2021-01-30", + "doi": "10.5555/global-2010-8131", "url": "https://example.org/publications/2010", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-20 75, 20 75, 20 85, -20 85, -20 75)))", - "creationDate": "2023-04-14T00:00:00Z", - "lastUpdate": "2023-04-14T07:00:00Z", + "creationDate": "2021-02-12T00:00:00Z", + "lastUpdate": "2021-02-13T01:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2021\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2023-04-14T00:00:00Z.\nHarvestingEvent ID: 3010.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2021-02-12T00:00:00Z.\nHarvestingEvent ID: 3010.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -465,10 +465,10 @@ ], "openalex_id": "https://openalex.org/W3002010", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2010-3322\", \"pmid\": null}", - "openalex_open_access_status": "green" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2010-8131\", \"pmid\": \"38002010\"}", + "openalex_open_access_status": "diamond" } }, { @@ -478,16 +478,16 @@ "status": "p", "title": "Marine Biology and Oceanography of the Southern Ocean", "abstract": "Detailed oceanographic study of Southern Ocean around Antarctica. Research includes marine ecosystems, ocean currents, temperature patterns, and biodiversity in the southern ocean.", - "publicationDate": "2020-06-09", - "doi": "10.5555/global-2011-5029", + "publicationDate": "2024-05-09", + "doi": "10.5555/global-2011-7405", "url": "https://example.org/publications/2011", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((0 -65, 40 -65, 40 -55, 0 -55, 0 -65)))", - "creationDate": "2020-07-05T00:00:00Z", - "lastUpdate": "2020-07-06T20:00:00Z", + "creationDate": "2024-05-16T00:00:00Z", + "lastUpdate": "2024-05-17T13:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2020-07-05T00:00:00Z.\nHarvestingEvent ID: 3011.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2024-05-16T00:00:00Z.\nHarvestingEvent ID: 3011.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -509,10 +509,10 @@ ], "openalex_id": "https://openalex.org/W3002011", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2011-5029\", \"pmid\": \"38002011\"}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2011-7405\", \"pmid\": null}", + "openalex_open_access_status": "closed" } }, { @@ -522,16 +522,16 @@ "status": "p", "title": "Cross-Regional Study: Europe-Asia", "abstract": "Cross-border environmental and geological research Spanning Eastern Europe and Western Asia. This study analyzes patterns that span multiple geographical regions.", - "publicationDate": "2023-09-14", - "doi": "10.5555/global-2012-8305", + "publicationDate": "2023-10-01", + "doi": "10.5555/global-2012-5720", "url": "https://example.org/publications/2012", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((25 40, 65 40, 65 55, 25 55, 25 40)))", - "creationDate": "2023-09-23T00:00:00Z", - "lastUpdate": "2023-09-24T10:00:00Z", + "creationDate": "2023-10-24T00:00:00Z", + "lastUpdate": "2023-10-25T07:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2020\"]", + "timeperiod_startdate": "[\"2022\"]", "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2023-09-23T00:00:00Z.\nHarvestingEvent ID: 3012.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2023-10-24T00:00:00Z.\nHarvestingEvent ID: 3012.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Prof. A", "Dr. B", @@ -542,12 +542,12 @@ ], "keywords": [], "topics": [], - "openalex_id": "https://openalex.org/W3002012", - "openalex_match_info": null, + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902012\", \"title\": \"Similar Study 2012\", \"doi\": null, \"match_type\": \"title\"}]", "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2012-8305\", \"pmid\": \"38002012\"}", - "openalex_open_access_status": "green" + "openalex_ids": null, + "openalex_open_access_status": null } }, { @@ -557,16 +557,16 @@ "status": "p", "title": "Cross-Regional Study: North America-Atlantic", "abstract": "Cross-border environmental and geological research Eastern North America and Western Atlantic. This study analyzes patterns that span multiple geographical regions.", - "publicationDate": "2020-10-02", - "doi": "10.5555/global-2013-6726", + "publicationDate": "2022-12-23", + "doi": "10.5555/global-2013-7764", "url": "https://example.org/publications/2013", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-80 25, -50 25, -50 45, -80 45, -80 25)))", - "creationDate": "2020-10-07T00:00:00Z", - "lastUpdate": "2020-10-08T09:00:00Z", + "creationDate": "2023-01-11T00:00:00Z", + "lastUpdate": "2023-01-11T10:00:00Z", "source": 2001, "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2020-10-07T00:00:00Z.\nHarvestingEvent ID: 3013.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2023-01-11T00:00:00Z.\nHarvestingEvent ID: 3013.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -586,8 +586,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2013-6726\", \"pmid\": null}", - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2013-7764\", \"pmid\": \"38002013\"}", + "openalex_open_access_status": "gold" } }, { @@ -597,16 +597,16 @@ "status": "p", "title": "Cross-Regional Study: Africa-Indian Ocean", "abstract": "Cross-border environmental and geological research East African coast and Western Indian Ocean. This study analyzes patterns that span multiple geographical regions.", - "publicationDate": "2020-04-30", - "doi": "10.5555/global-2014-8400", + "publicationDate": "2023-08-20", + "doi": "10.5555/global-2014-8582", "url": "https://example.org/publications/2014", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((35 -20, 55 -20, 55 5, 35 5, 35 -20)))", - "creationDate": "2020-05-11T00:00:00Z", - "lastUpdate": "2020-05-12T16:00:00Z", + "creationDate": "2023-09-03T00:00:00Z", + "lastUpdate": "2023-09-03T09:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2020-05-11T00:00:00Z.\nHarvestingEvent ID: 3014.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2023-09-03T00:00:00Z.\nHarvestingEvent ID: 3014.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [], "keywords": [ "first keyword", @@ -618,10 +618,10 @@ ], "openalex_id": "https://openalex.org/W3002014", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2014-8400\", \"pmid\": \"38002014\"}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2014-8582\", \"pmid\": null}", + "openalex_open_access_status": "diamond" } }, { @@ -631,16 +631,16 @@ "status": "p", "title": "Cross-Regional Study: South America-Pacific", "abstract": "Cross-border environmental and geological research Western South America and Eastern Pacific. This study analyzes patterns that span multiple geographical regions.", - "publicationDate": "2024-11-30", - "doi": "10.5555/global-2015-2149", + "publicationDate": "2022-10-10", + "doi": "10.5555/global-2015-1668", "url": "https://example.org/publications/2015", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-85 -20, -65 -20, -65 5, -85 5, -85 -20)))", - "creationDate": "2024-12-12T00:00:00Z", - "lastUpdate": "2024-12-12T08:00:00Z", + "creationDate": "2022-10-18T00:00:00Z", + "lastUpdate": "2022-10-18T18:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2023\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2024-12-12T00:00:00Z.\nHarvestingEvent ID: 3015.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2022-10-18T00:00:00Z.\nHarvestingEvent ID: 3015.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Single Author" ], @@ -656,10 +656,10 @@ ], "openalex_id": "https://openalex.org/W3002015", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2015-2149\", \"pmid\": null}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2015-1668\", \"pmid\": null}", + "openalex_open_access_status": "green" } }, { @@ -669,16 +669,16 @@ "status": "p", "title": "Cross-Regional Study: Asia-Pacific", "abstract": "Cross-border environmental and geological research East Asian coast and Western Pacific. This study analyzes patterns that span multiple geographical regions.", - "publicationDate": "2020-01-27", - "doi": "10.5555/global-2016-9623", + "publicationDate": "2023-05-27", + "doi": "10.5555/global-2016-4518", "url": "https://example.org/publications/2016", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((115 20, 140 20, 140 45, 115 45, 115 20)))", - "creationDate": "2020-02-18T00:00:00Z", - "lastUpdate": "2020-02-18T09:00:00Z", + "creationDate": "2023-06-17T00:00:00Z", + "lastUpdate": "2023-06-18T09:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2020-02-18T00:00:00Z.\nHarvestingEvent ID: 3016.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2023-06-17T00:00:00Z.\nHarvestingEvent ID: 3016.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -692,10 +692,10 @@ "topics": [], "openalex_id": "https://openalex.org/W3002016", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2016-9623\", \"pmid\": null}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2016-4518\", \"pmid\": \"38002016\"}", + "openalex_open_access_status": null } }, { @@ -705,16 +705,16 @@ "status": "p", "title": "Global Ocean Survey", "abstract": "Large-scale multi-regional research project covering Atlantic, Indian, and Pacific Oceans. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2020-04-08", - "doi": "10.5555/global-2017-3649", + "publicationDate": "2020-06-07", + "doi": "10.5555/global-2017-9790", "url": "https://example.org/publications/2017", "geometry": "SRID=4326;GEOMETRYCOLLECTION(MULTIPOLYGON (((-40 -10, -20 -10, -20 10, -40 10, -40 -10)), ((60 -20, 80 -20, 80 0, 60 0, 60 -20)), ((150 -30, 170 -30, 170 -10, 150 -10, 150 -30))))", - "creationDate": "2020-04-23T00:00:00Z", - "lastUpdate": "2020-04-23T13:00:00Z", + "creationDate": "2020-06-10T00:00:00Z", + "lastUpdate": "2020-06-12T00:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2019\"]", + "timeperiod_startdate": "[\"2018\"]", "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2020-04-23T00:00:00Z.\nHarvestingEvent ID: 3017.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2020-06-10T00:00:00Z.\nHarvestingEvent ID: 3017.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -733,10 +733,10 @@ ], "openalex_id": "https://openalex.org/W3002017", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2017-3649\", \"pmid\": null}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2017-9790\", \"pmid\": null}", + "openalex_open_access_status": null } }, { @@ -746,16 +746,16 @@ "status": "p", "title": "Trans-Atlantic Research", "abstract": "Large-scale multi-regional research project covering North America, Atlantic Ocean, and Europe. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2021-05-26", - "doi": "10.5555/global-2018-5013", + "publicationDate": "2024-01-07", + "doi": "10.5555/global-2018-5572", "url": "https://example.org/publications/2018", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-70 20, 10 20, 10 50, -70 50, -70 20)))", - "creationDate": "2021-06-11T00:00:00Z", - "lastUpdate": "2021-06-12T13:00:00Z", + "creationDate": "2024-01-20T00:00:00Z", + "lastUpdate": "2024-01-21T19:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2021-06-11T00:00:00Z.\nHarvestingEvent ID: 3018.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-01-20T00:00:00Z.\nHarvestingEvent ID: 3018.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -767,12 +767,12 @@ "Environmental Science", "Ecology" ], - "openalex_id": "https://openalex.org/W3002018", - "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902018\", \"title\": \"Similar Study 2018\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2018-5013\", \"pmid\": null}", - "openalex_open_access_status": "gold" + "openalex_ids": null, + "openalex_open_access_status": null } }, { @@ -782,16 +782,16 @@ "status": "p", "title": "African-Asian Monsoon Study", "abstract": "Large-scale multi-regional research project covering Africa, Indian Ocean, and Asia. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2023-06-19", - "doi": "10.5555/global-2019-2243", + "publicationDate": "2022-12-09", + "doi": "10.5555/global-2019-6645", "url": "https://example.org/publications/2019", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((20 -10, 90 -10, 90 25, 20 25, 20 -10)))", - "creationDate": "2023-06-20T00:00:00Z", - "lastUpdate": "2023-06-21T22:00:00Z", + "creationDate": "2023-01-03T00:00:00Z", + "lastUpdate": "2023-01-04T06:00:00Z", "source": 2002, "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2023-06-20T00:00:00Z.\nHarvestingEvent ID: 3019.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2023-01-03T00:00:00Z.\nHarvestingEvent ID: 3019.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Prof. A", "Dr. B", @@ -808,12 +808,12 @@ "Atmospheric Science", "Meteorology" ], - "openalex_id": null, - "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902019\", \"title\": \"Similar Study 2019\", \"doi\": null, \"match_type\": \"title\"}]", - "openalex_fulltext_origin": null, + "openalex_id": "https://openalex.org/W3002019", + "openalex_match_info": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": null, - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2019-6645\", \"pmid\": null}", + "openalex_open_access_status": "closed" } }, { @@ -823,16 +823,16 @@ "status": "p", "title": "Pan-Pacific Study", "abstract": "Large-scale multi-regional research project covering Asia, Pacific Ocean, North America, South America, Australia. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2021-03-29", - "doi": "10.5555/global-2020-4694", + "publicationDate": "2022-05-03", + "doi": "10.5555/global-2020-2953", "url": "https://example.org/publications/2020", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((110 -40, -80 -40, -80 50, 110 50, 110 -40)))", - "creationDate": "2021-03-31T00:00:00Z", - "lastUpdate": "2021-04-01T08:00:00Z", + "creationDate": "2022-06-01T00:00:00Z", + "lastUpdate": "2022-06-01T02:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2021-03-31T00:00:00Z.\nHarvestingEvent ID: 3020.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2022-06-01T00:00:00Z.\nHarvestingEvent ID: 3020.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -851,8 +851,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2020-4694\", \"pmid\": null}", - "openalex_open_access_status": "hybrid" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2020-2953\", \"pmid\": null}", + "openalex_open_access_status": "green" } }, { @@ -862,16 +862,16 @@ "status": "p", "title": "Southern Hemisphere Ocean Study", "abstract": "Large-scale multi-regional research project covering Southern Ocean, Pacific, Atlantic, Indian Oceans, South America, Africa, Australia, Antarctica. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2023-06-14", - "doi": "10.5555/global-2021-3536", + "publicationDate": "2021-08-14", + "doi": "10.5555/global-2021-7890", "url": "https://example.org/publications/2021", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-180 -60, 180 -60, 180 -35, -180 -35, -180 -60)))", - "creationDate": "2023-06-20T00:00:00Z", - "lastUpdate": "2023-06-21T23:00:00Z", + "creationDate": "2021-09-13T00:00:00Z", + "lastUpdate": "2021-09-14T14:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2023-06-20T00:00:00Z.\nHarvestingEvent ID: 3021.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2021-09-13T00:00:00Z.\nHarvestingEvent ID: 3021.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [], "keywords": [ "climate change", @@ -881,12 +881,12 @@ "topics": [ "Geography" ], - "openalex_id": null, - "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902021\", \"title\": \"Similar Study 2021\", \"doi\": null, \"match_type\": \"title\"}]", - "openalex_fulltext_origin": null, + "openalex_id": "https://openalex.org/W3002021", + "openalex_match_info": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": null, - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2021-7890\", \"pmid\": null}", + "openalex_open_access_status": "green" } }, { @@ -896,16 +896,16 @@ "status": "p", "title": "Arctic Circumpolar Study", "abstract": "Large-scale multi-regional research project covering Arctic Ocean, North America, Europe, Asia. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2021-04-01", - "doi": "10.5555/global-2022-6734", + "publicationDate": "2024-02-26", + "doi": "10.5555/global-2022-4249", "url": "https://example.org/publications/2022", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-180 65, 180 65, 180 85, -180 85, -180 65)))", - "creationDate": "2021-04-02T00:00:00Z", - "lastUpdate": "2021-04-03T04:00:00Z", + "creationDate": "2024-03-14T00:00:00Z", + "lastUpdate": "2024-03-15T19:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2021-04-02T00:00:00Z.\nHarvestingEvent ID: 3022.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2023\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2024-03-14T00:00:00Z.\nHarvestingEvent ID: 3022.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. Single Author" ], @@ -921,10 +921,10 @@ ], "openalex_id": "https://openalex.org/W3002022", "openalex_match_info": null, - "openalex_fulltext_origin": "repository", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2022-6734\", \"pmid\": \"38002022\"}", - "openalex_open_access_status": "gold" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2022-4249\", \"pmid\": null}", + "openalex_open_access_status": null } }, { @@ -934,16 +934,16 @@ "status": "p", "title": "Global Climate Network", "abstract": "Large-scale multi-regional research project covering North America, Europe, Australia, South America. This comprehensive study examines global patterns and connections across multiple continents and oceans.", - "publicationDate": "2022-08-17", - "doi": "10.5555/global-2023-9879", + "publicationDate": "2020-02-21", + "doi": "10.5555/global-2023-1377", "url": "https://example.org/publications/2023", "geometry": "SRID=4326;GEOMETRYCOLLECTION(MULTIPOLYGON (((-120 30, -100 30, -100 45, -120 45, -120 30)), ((10 40, 30 40, 30 55, 10 55, 10 40)), ((120 -30, 140 -30, 140 -20, 120 -20, 120 -30)), ((-50 -20, -40 -20, -40 -10, -50 -10, -50 -20))))", - "creationDate": "2022-09-01T00:00:00Z", - "lastUpdate": "2022-09-02T21:00:00Z", + "creationDate": "2020-03-04T00:00:00Z", + "lastUpdate": "2020-03-05T18:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2022\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2022-09-01T00:00:00Z.\nHarvestingEvent ID: 3023.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2020\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2020-03-04T00:00:00Z.\nHarvestingEvent ID: 3023.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -965,8 +965,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2023-9879\", \"pmid\": \"38002023\"}", - "openalex_open_access_status": "bronze" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2023-1377\", \"pmid\": \"38002023\"}", + "openalex_open_access_status": null } }, { @@ -976,16 +976,16 @@ "status": "p", "title": "Field Site: Central Africa", "abstract": "Point-based monitoring and research from field research station in central Africa. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2023-08-10", - "doi": "10.5555/global-2024-3314", + "publicationDate": "2024-07-26", + "doi": "10.5555/global-2024-5317", "url": "https://example.org/publications/2024", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (20 0))", - "creationDate": "2023-08-14T00:00:00Z", - "lastUpdate": "2023-08-15T08:00:00Z", + "creationDate": "2024-08-24T00:00:00Z", + "lastUpdate": "2024-08-24T22:00:00Z", "source": 2000, "timeperiod_startdate": "[\"2022\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2023-08-14T00:00:00Z.\nHarvestingEvent ID: 3024.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2024-08-24T00:00:00Z.\nHarvestingEvent ID: 3024.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -995,10 +995,10 @@ "topics": [], "openalex_id": "https://openalex.org/W3002024", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2024-3314\", \"pmid\": \"38002024\"}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2024-5317\", \"pmid\": \"38002024\"}", + "openalex_open_access_status": "diamond" } }, { @@ -1008,16 +1008,16 @@ "status": "p", "title": "Field Site: Central Europe", "abstract": "Point-based monitoring and research from field research station in central Europe. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2022-08-31", - "doi": "10.5555/global-2025-5390", + "publicationDate": "2024-06-29", + "doi": "10.5555/global-2025-4144", "url": "https://example.org/publications/2025", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (15 50))", - "creationDate": "2022-09-25T00:00:00Z", - "lastUpdate": "2022-09-25T12:00:00Z", + "creationDate": "2024-07-09T00:00:00Z", + "lastUpdate": "2024-07-09T12:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2022\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2022-09-25T00:00:00Z.\nHarvestingEvent ID: 3025.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2023\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-07-09T00:00:00Z.\nHarvestingEvent ID: 3025.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -1045,16 +1045,16 @@ "status": "p", "title": "Field Site: Central Asia", "abstract": "Point-based monitoring and research from field research station in central Asia. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2021-02-02", - "doi": "10.5555/global-2026-9567", + "publicationDate": "2024-09-04", + "doi": "10.5555/global-2026-4182", "url": "https://example.org/publications/2026", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (85 45))", - "creationDate": "2021-02-10T00:00:00Z", - "lastUpdate": "2021-02-11T18:00:00Z", + "creationDate": "2024-09-23T00:00:00Z", + "lastUpdate": "2024-09-24T13:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2021-02-10T00:00:00Z.\nHarvestingEvent ID: 3026.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2024-09-23T00:00:00Z.\nHarvestingEvent ID: 3026.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Prof. A", "Dr. B", @@ -1073,10 +1073,10 @@ ], "openalex_id": "https://openalex.org/W3002026", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2026-9567\", \"pmid\": null}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2026-4182\", \"pmid\": \"38002026\"}", + "openalex_open_access_status": "green" } }, { @@ -1086,16 +1086,16 @@ "status": "p", "title": "Field Site: Central North America", "abstract": "Point-based monitoring and research from field research station in central North America. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2021-06-17", - "doi": "10.5555/global-2027-6059", + "publicationDate": "2020-04-25", + "doi": "10.5555/global-2027-4275", "url": "https://example.org/publications/2027", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (-100 45))", - "creationDate": "2021-06-20T00:00:00Z", - "lastUpdate": "2021-06-21T16:00:00Z", + "creationDate": "2020-05-08T00:00:00Z", + "lastUpdate": "2020-05-08T16:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2021-06-20T00:00:00Z.\nHarvestingEvent ID: 3027.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2020\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2020-05-08T00:00:00Z.\nHarvestingEvent ID: 3027.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -1117,10 +1117,10 @@ ], "openalex_id": "https://openalex.org/W3002027", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2027-6059\", \"pmid\": \"38002027\"}", - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2027-4275\", \"pmid\": \"38002027\"}", + "openalex_open_access_status": "green" } }, { @@ -1130,16 +1130,16 @@ "status": "p", "title": "Field Site: Central South America", "abstract": "Point-based monitoring and research from field research station in central South America. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2021-04-13", - "doi": "10.5555/global-2028-2342", + "publicationDate": "2022-05-15", + "doi": "10.5555/global-2028-5236", "url": "https://example.org/publications/2028", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (-60 -15))", - "creationDate": "2021-04-19T00:00:00Z", - "lastUpdate": "2021-04-20T15:00:00Z", + "creationDate": "2022-06-07T00:00:00Z", + "lastUpdate": "2022-06-08T16:00:00Z", "source": 2000, "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2021-04-19T00:00:00Z.\nHarvestingEvent ID: 3028.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2022-06-07T00:00:00Z.\nHarvestingEvent ID: 3028.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [], "keywords": [ "biodiversity", @@ -1152,8 +1152,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2028-2342\", \"pmid\": null}", - "openalex_open_access_status": "hybrid" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2028-5236\", \"pmid\": null}", + "openalex_open_access_status": null } }, { @@ -1163,16 +1163,16 @@ "status": "p", "title": "Field Site: Central Australia", "abstract": "Point-based monitoring and research from field research station in central Australia. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2021-02-27", - "doi": "10.5555/global-2029-5579", + "publicationDate": "2021-09-11", + "doi": "10.5555/global-2029-7706", "url": "https://example.org/publications/2029", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (135 -25))", - "creationDate": "2021-02-28T00:00:00Z", - "lastUpdate": "2021-03-01T06:00:00Z", + "creationDate": "2021-10-06T00:00:00Z", + "lastUpdate": "2021-10-07T03:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2019\"]", + "timeperiod_startdate": "[\"2018\"]", "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2021-02-28T00:00:00Z.\nHarvestingEvent ID: 3029.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2021-10-06T00:00:00Z.\nHarvestingEvent ID: 3029.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Single Author" ], @@ -1187,12 +1187,12 @@ "topics": [ "Geography" ], - "openalex_id": null, - "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902029\", \"title\": \"Similar Study 2029\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_id": "https://openalex.org/W3002029", + "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": null, - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2029-7706\", \"pmid\": \"38002029\"}", + "openalex_open_access_status": "green" } }, { @@ -1202,14 +1202,14 @@ "status": "p", "title": "Field Site: Antarctic Peninsula", "abstract": "Point-based monitoring and research from field research station in Antarctica. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2023-04-13", - "doi": "10.5555/global-2030-9361", + "publicationDate": "2023-04-06", + "doi": "10.5555/global-2030-6810", "url": "https://example.org/publications/2030", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (-60 -70))", "creationDate": "2023-05-03T00:00:00Z", - "lastUpdate": "2023-05-04T16:00:00Z", + "lastUpdate": "2023-05-03T22:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2021\"]", + "timeperiod_startdate": "[\"2022\"]", "timeperiod_enddate": "[\"2023\"]", "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2023-05-03T00:00:00Z.\nHarvestingEvent ID: 3030.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ @@ -1225,8 +1225,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2030-9361\", \"pmid\": null}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2030-6810\", \"pmid\": \"38002030\"}", + "openalex_open_access_status": "green" } }, { @@ -1236,16 +1236,16 @@ "status": "p", "title": "Monitoring Buoy: Central Atlantic", "abstract": "Point-based monitoring and research from ocean monitoring buoy in the Atlantic Ocean. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2023-12-19", - "doi": "10.5555/global-2031-4582", + "publicationDate": "2024-09-21", + "doi": "10.5555/global-2031-6551", "url": "https://example.org/publications/2031", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (-30 20))", - "creationDate": "2023-12-26T00:00:00Z", - "lastUpdate": "2023-12-26T04:00:00Z", + "creationDate": "2024-10-07T00:00:00Z", + "lastUpdate": "2024-10-07T17:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2021\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2023-12-26T00:00:00Z.\nHarvestingEvent ID: 3031.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2024-10-07T00:00:00Z.\nHarvestingEvent ID: 3031.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -1259,12 +1259,12 @@ "Atmospheric Science", "Meteorology" ], - "openalex_id": "https://openalex.org/W3002031", - "openalex_match_info": null, + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902031\", \"title\": \"Similar Study 2031\", \"doi\": null, \"match_type\": \"title\"}]", "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2031-4582\", \"pmid\": \"38002031\"}", - "openalex_open_access_status": "green" + "openalex_ids": null, + "openalex_open_access_status": null } }, { @@ -1274,16 +1274,16 @@ "status": "p", "title": "Monitoring Buoy: Central Pacific", "abstract": "Point-based monitoring and research from ocean monitoring buoy in the Pacific Ocean. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2023-06-02", - "doi": "10.5555/global-2032-6701", + "publicationDate": "2021-09-09", + "doi": "10.5555/global-2032-7850", "url": "https://example.org/publications/2032", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (170 0))", - "creationDate": "2023-06-14T00:00:00Z", - "lastUpdate": "2023-06-14T18:00:00Z", + "creationDate": "2021-10-08T00:00:00Z", + "lastUpdate": "2021-10-08T07:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2022\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2023-06-14T00:00:00Z.\nHarvestingEvent ID: 3032.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2021-10-08T00:00:00Z.\nHarvestingEvent ID: 3032.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -1299,8 +1299,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2032-6701\", \"pmid\": null}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2032-7850\", \"pmid\": null}", + "openalex_open_access_status": "hybrid" } }, { @@ -1310,16 +1310,16 @@ "status": "p", "title": "Monitoring Buoy: Central Indian Ocean", "abstract": "Point-based monitoring and research from ocean monitoring buoy in the Indian Ocean. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2024-03-30", - "doi": "10.5555/global-2033-1204", + "publicationDate": "2020-05-16", + "doi": "10.5555/global-2033-1356", "url": "https://example.org/publications/2033", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (75 -20))", - "creationDate": "2024-04-18T00:00:00Z", - "lastUpdate": "2024-04-18T03:00:00Z", + "creationDate": "2020-06-07T00:00:00Z", + "lastUpdate": "2020-06-07T12:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2022\"]", - "timeperiod_enddate": "[\"2024\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-04-18T00:00:00Z.\nHarvestingEvent ID: 3033.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2020\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2020-06-07T00:00:00Z.\nHarvestingEvent ID: 3033.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Prof. A", "Dr. B", @@ -1338,10 +1338,10 @@ ], "openalex_id": "https://openalex.org/W3002033", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2033-1204\", \"pmid\": \"38002033\"}", - "openalex_open_access_status": "gold" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2033-1356\", \"pmid\": \"38002033\"}", + "openalex_open_access_status": null } }, { @@ -1351,16 +1351,16 @@ "status": "p", "title": "Monitoring Buoy: Arctic Ocean", "abstract": "Point-based monitoring and research from ocean monitoring buoy in the Arctic Ocean. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2023-03-21", - "doi": "10.5555/global-2034-3680", + "publicationDate": "2024-01-03", + "doi": "10.5555/global-2034-7806", "url": "https://example.org/publications/2034", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (0 85))", - "creationDate": "2023-03-26T00:00:00Z", - "lastUpdate": "2023-03-27T07:00:00Z", + "creationDate": "2024-01-30T00:00:00Z", + "lastUpdate": "2024-01-30T23:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2021\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2023-03-26T00:00:00Z.\nHarvestingEvent ID: 3034.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2023\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2024-01-30T00:00:00Z.\nHarvestingEvent ID: 3034.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -1382,10 +1382,10 @@ ], "openalex_id": "https://openalex.org/W3002034", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2034-3680\", \"pmid\": null}", - "openalex_open_access_status": "closed" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2034-7806\", \"pmid\": \"38002034\"}", + "openalex_open_access_status": "gold" } }, { @@ -1395,17 +1395,17 @@ "status": "p", "title": "Monitoring Buoy: Southern Ocean", "abstract": "Point-based monitoring and research from ocean monitoring buoy in the Southern Ocean. This site provides continuous data collection and analysis for local environmental conditions.", - "publicationDate": "2021-05-24", - "doi": "10.5555/global-2035-5335", + "publicationDate": "2023-03-01", + "doi": "10.5555/global-2035-4301", "url": "https://example.org/publications/2035", "geometry": "SRID=4326;GEOMETRYCOLLECTION(POINT (100 -65))", - "creationDate": "2021-06-14T00:00:00Z", - "lastUpdate": "2021-06-15T17:00:00Z", + "creationDate": "2023-03-17T00:00:00Z", + "lastUpdate": "2023-03-17T19:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2021-06-14T00:00:00Z.\nHarvestingEvent ID: 3035.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", - "authors": [], + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2023-03-17T00:00:00Z.\nHarvestingEvent ID: 3035.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [], "keywords": [ "urban planning", "sustainability", @@ -1419,12 +1419,12 @@ "Atmospheric Science", "Meteorology" ], - "openalex_id": "https://openalex.org/W3002035", - "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902035\", \"title\": \"Similar Study 2035\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2035-5335\", \"pmid\": null}", - "openalex_open_access_status": "hybrid" + "openalex_ids": null, + "openalex_open_access_status": null } }, { @@ -1434,16 +1434,16 @@ "status": "p", "title": "Migration Route: Africa to Europe", "abstract": "Linear pathway study documenting bird migration corridor from Africa through Mediterranean to Europe. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2020-09-09", - "doi": "10.5555/global-2036-6708", + "publicationDate": "2024-05-11", + "doi": "10.5555/global-2036-8173", "url": "https://example.org/publications/2036", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (20 -5, 25 10, 15 35, 10 45))", - "creationDate": "2020-09-20T00:00:00Z", - "lastUpdate": "2020-09-20T02:00:00Z", + "creationDate": "2024-05-15T00:00:00Z", + "lastUpdate": "2024-05-16T02:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2020-09-20T00:00:00Z.\nHarvestingEvent ID: 3036.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2024-05-15T00:00:00Z.\nHarvestingEvent ID: 3036.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Single Author" ], @@ -1464,16 +1464,16 @@ "status": "p", "title": "Migration Route: Asia to Australia", "abstract": "Linear pathway study documenting bird migration corridor from Asia to Australia. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2023-10-16", - "doi": "10.5555/global-2037-9678", + "publicationDate": "2022-09-19", + "doi": "10.5555/global-2037-4231", "url": "https://example.org/publications/2037", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (100 30, 110 10, 120 -10, 130 -20))", - "creationDate": "2023-11-10T00:00:00Z", - "lastUpdate": "2023-11-11T14:00:00Z", + "creationDate": "2022-10-03T00:00:00Z", + "lastUpdate": "2022-10-04T15:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2021\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2023-11-10T00:00:00Z.\nHarvestingEvent ID: 3037.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2022-10-03T00:00:00Z.\nHarvestingEvent ID: 3037.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -1486,10 +1486,10 @@ ], "openalex_id": "https://openalex.org/W3002037", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2037-9678\", \"pmid\": null}", - "openalex_open_access_status": "hybrid" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2037-4231\", \"pmid\": null}", + "openalex_open_access_status": "gold" } }, { @@ -1499,16 +1499,16 @@ "status": "p", "title": "Shipping Lane: Atlantic Crossing", "abstract": "Linear pathway study documenting major shipping route across North Atlantic from North America to Europe. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2023-02-13", - "doi": "10.5555/global-2038-4237", + "publicationDate": "2024-08-17", + "doi": "10.5555/global-2038-4186", "url": "https://example.org/publications/2038", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (-75 40, -50 45, -25 50, -5 52))", - "creationDate": "2023-02-25T00:00:00Z", - "lastUpdate": "2023-02-26T23:00:00Z", + "creationDate": "2024-08-21T00:00:00Z", + "lastUpdate": "2024-08-21T12:00:00Z", "source": 2002, - "timeperiod_startdate": "[\"2020\"]", - "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2023-02-25T00:00:00Z.\nHarvestingEvent ID: 3038.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2024-08-21T00:00:00Z.\nHarvestingEvent ID: 3038.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -1524,9 +1524,9 @@ ], "openalex_id": "https://openalex.org/W3002038", "openalex_match_info": null, - "openalex_fulltext_origin": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2038-4237\", \"pmid\": \"38002038\"}", + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2038-4186\", \"pmid\": null}", "openalex_open_access_status": "diamond" } }, @@ -1537,16 +1537,16 @@ "status": "p", "title": "Shipping Lane: Pacific Crossing", "abstract": "Linear pathway study documenting major shipping route across North Pacific from Asia to North America. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2020-03-28", - "doi": "10.5555/global-2039-5242", + "publicationDate": "2022-01-04", + "doi": "10.5555/global-2039-2167", "url": "https://example.org/publications/2039", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (140 35, 170 38, -160 40, -130 42))", - "creationDate": "2020-04-26T00:00:00Z", - "lastUpdate": "2020-04-27T16:00:00Z", + "creationDate": "2022-01-15T00:00:00Z", + "lastUpdate": "2022-01-15T20:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2020-04-26T00:00:00Z.\nHarvestingEvent ID: 3039.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2003 on 2022-01-15T00:00:00Z.\nHarvestingEvent ID: 3039.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Maria Garcia", "Prof. John Smith", @@ -1567,8 +1567,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2039-5242\", \"pmid\": \"38002039\"}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2039-2167\", \"pmid\": null}", + "openalex_open_access_status": "closed" } }, { @@ -1578,16 +1578,16 @@ "status": "p", "title": "Ocean Current: Gulf Stream", "abstract": "Linear pathway study documenting Gulf Stream current from Gulf of Mexico to North Atlantic. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2023-02-25", - "doi": "10.5555/global-2040-5102", + "publicationDate": "2023-09-05", + "doi": "10.5555/global-2040-2171", "url": "https://example.org/publications/2040", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (-80 25, -70 30, -50 35, -30 40, -10 50))", - "creationDate": "2023-03-26T00:00:00Z", - "lastUpdate": "2023-03-27T15:00:00Z", + "creationDate": "2023-09-21T00:00:00Z", + "lastUpdate": "2023-09-23T00:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2021\"]", + "timeperiod_startdate": "[\"2020\"]", "timeperiod_enddate": "[\"2023\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2023-03-26T00:00:00Z.\nHarvestingEvent ID: 3040.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2023-09-21T00:00:00Z.\nHarvestingEvent ID: 3040.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Prof. A", "Dr. B", @@ -1607,7 +1607,7 @@ "openalex_match_info": null, "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2040-5102\", \"pmid\": \"38002040\"}", + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2040-2171\", \"pmid\": null}", "openalex_open_access_status": "hybrid" } }, @@ -1618,16 +1618,16 @@ "status": "p", "title": "Ocean Current: Kuroshio", "abstract": "Linear pathway study documenting Kuroshio Current along eastern Asia into Pacific. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2021-04-02", - "doi": "10.5555/global-2041-4206", + "publicationDate": "2024-10-26", + "doi": "10.5555/global-2041-5184", "url": "https://example.org/publications/2041", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (125 25, 135 30, 145 35, 155 40))", - "creationDate": "2021-04-30T00:00:00Z", - "lastUpdate": "2021-04-30T05:00:00Z", + "creationDate": "2024-11-14T00:00:00Z", + "lastUpdate": "2024-11-14T11:00:00Z", "source": 2001, - "timeperiod_startdate": "[\"2019\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2021-04-30T00:00:00Z.\nHarvestingEvent ID: 3041.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2024-11-14T00:00:00Z.\nHarvestingEvent ID: 3041.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. Zhang Wei", "Prof. Sarah Johnson", @@ -1652,8 +1652,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2041-4206\", \"pmid\": null}", - "openalex_open_access_status": "gold" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2041-5184\", \"pmid\": \"38002041\"}", + "openalex_open_access_status": "diamond" } }, { @@ -1663,28 +1663,28 @@ "status": "p", "title": "Seismic Survey: Mid-Atlantic Ridge", "abstract": "Linear pathway study documenting geological survey along Mid-Atlantic Ridge from South Atlantic to North Atlantic. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2020-08-06", - "doi": "10.5555/global-2042-5365", + "publicationDate": "2020-03-31", + "doi": "10.5555/global-2042-8858", "url": "https://example.org/publications/2042", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (-35 -30, -30 -10, -25 10, -20 30, -15 50))", - "creationDate": "2020-08-13T00:00:00Z", - "lastUpdate": "2020-08-14T05:00:00Z", + "creationDate": "2020-04-07T00:00:00Z", + "lastUpdate": "2020-04-07T13:00:00Z", "source": 2002, "timeperiod_startdate": "[\"2019\"]", "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2020-08-13T00:00:00Z.\nHarvestingEvent ID: 3042.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2020-04-07T00:00:00Z.\nHarvestingEvent ID: 3042.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [], "keywords": [], "topics": [ "Environmental Science", "Ecology" ], - "openalex_id": null, - "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902042\", \"title\": \"Similar Study 2042\", \"doi\": null, \"match_type\": \"title\"}]", - "openalex_fulltext_origin": null, + "openalex_id": "https://openalex.org/W3002042", + "openalex_match_info": null, + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": null, - "openalex_open_access_status": null + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2042-8858\", \"pmid\": \"38002042\"}", + "openalex_open_access_status": "hybrid" } }, { @@ -1694,16 +1694,16 @@ "status": "p", "title": "Seismic Survey: Ring of Fire West", "abstract": "Linear pathway study documenting seismic monitoring along western Pacific Ring of Fire from Indian Ocean to Pacific. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2020-05-28", - "doi": "10.5555/global-2043-8025", + "publicationDate": "2020-01-29", + "doi": "10.5555/global-2043-6917", "url": "https://example.org/publications/2043", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (120 -10, 125 0, 130 10, 135 20, 140 30))", - "creationDate": "2020-06-21T00:00:00Z", - "lastUpdate": "2020-06-22T01:00:00Z", + "creationDate": "2020-02-14T00:00:00Z", + "lastUpdate": "2020-02-14T21:00:00Z", "source": 2003, - "timeperiod_startdate": "[\"2017\"]", + "timeperiod_startdate": "[\"2019\"]", "timeperiod_enddate": "[\"2020\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2020-06-21T00:00:00Z.\nHarvestingEvent ID: 3043.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2020-02-14T00:00:00Z.\nHarvestingEvent ID: 3043.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", "authors": [ "Dr. Single Author" ], @@ -1717,10 +1717,10 @@ ], "openalex_id": "https://openalex.org/W3002043", "openalex_match_info": null, - "openalex_fulltext_origin": "publisher", + "openalex_fulltext_origin": "repository", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2043-8025\", \"pmid\": \"38002043\"}", - "openalex_open_access_status": "diamond" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2043-6917\", \"pmid\": null}", + "openalex_open_access_status": "hybrid" } }, { @@ -1730,16 +1730,16 @@ "status": "p", "title": "Cable Route: Trans-Pacific", "abstract": "Linear pathway study documenting undersea telecommunications cable across Pacific from North America to Asia. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2021-12-04", - "doi": "10.5555/global-2044-4099", + "publicationDate": "2022-06-25", + "doi": "10.5555/global-2044-8857", "url": "https://example.org/publications/2044", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (-120 35, -140 32, -160 30, -180 28, 170 26, 150 25))", - "creationDate": "2021-12-16T00:00:00Z", - "lastUpdate": "2021-12-17T20:00:00Z", + "creationDate": "2022-07-02T00:00:00Z", + "lastUpdate": "2022-07-03T18:00:00Z", "source": 2000, - "timeperiod_startdate": "[\"2018\"]", - "timeperiod_enddate": "[\"2021\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2021-12-16T00:00:00Z.\nHarvestingEvent ID: 3044.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2022-07-02T00:00:00Z.\nHarvestingEvent ID: 3044.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", "authors": [ "Dr. First Author", "Prof. Second Author" @@ -1753,8 +1753,8 @@ "openalex_match_info": null, "openalex_fulltext_origin": null, "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2044-4099\", \"pmid\": null}", - "openalex_open_access_status": "green" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2044-8857\", \"pmid\": \"38002044\"}", + "openalex_open_access_status": "bronze" } }, { @@ -1764,16 +1764,16 @@ "status": "p", "title": "Cable Route: Europe-Africa", "abstract": "Linear pathway study documenting undersea cable from Europe through Atlantic to Africa. This research traces continuous phenomena across regional boundaries.", - "publicationDate": "2022-06-20", - "doi": "10.5555/global-2045-5074", + "publicationDate": "2024-03-20", + "doi": "10.5555/global-2045-7819", "url": "https://example.org/publications/2045", "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (10 55, 5 45, 0 35, -5 25, 0 10, 5 0))", - "creationDate": "2022-06-21T00:00:00Z", - "lastUpdate": "2022-06-22T00:00:00Z", + "creationDate": "2024-03-27T00:00:00Z", + "lastUpdate": "2024-03-27T13:00:00Z", "source": 2001, "timeperiod_startdate": "[\"2021\"]", - "timeperiod_enddate": "[\"2022\"]", - "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2022-06-21T00:00:00Z.\nHarvestingEvent ID: 3045.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-03-27T00:00:00Z.\nHarvestingEvent ID: 3045.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", "authors": [ "Dr. Alice Smith", "Prof. Bob Jones", @@ -1787,12 +1787,775 @@ "topics": [ "Geography" ], - "openalex_id": "https://openalex.org/W3002045", + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902045\", \"title\": \"Similar Study 2045\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": null, + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2046, + "fields": { + "status": "p", + "title": "Triangular Survey Area: Mediterranean", + "abstract": "Complex polygon study focusing on triangular research zone in Mediterranean Sea. This research examines irregular boundaries and geometric complexity in spatial analysis.", + "publicationDate": "2021-07-05", + "doi": "10.5555/global-2046-6491", + "url": "https://example.org/publications/2046", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((10 35, 20 45, 5 42, 10 35)))", + "creationDate": "2021-07-21T00:00:00Z", + "lastUpdate": "2021-07-21T02:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2021-07-21T00:00:00Z.\nHarvestingEvent ID: 3046.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "authors": [ + "Dr. Maria Garcia", + "Prof. John Smith", + "Dr. Emma Johnson", + "Dr. Li Wei" + ], + "keywords": [ + "biodiversity", + "ecosystem services", + "conservation", + "habitat mapping" + ], + "topics": [ + "Environmental Science", + "Ecology" + ], + "openalex_id": "https://openalex.org/W3002046", "openalex_match_info": null, "openalex_fulltext_origin": "publisher", "openalex_is_retracted": false, - "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2045-5074\", \"pmid\": \"38002045\"}", - "openalex_open_access_status": "hybrid" + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2046-6491\", \"pmid\": \"38002046\"}", + "openalex_open_access_status": "closed" + } + }, + { + "model": "publications.publication", + "pk": 2047, + "fields": { + "status": "p", + "title": "Pentagon Study Region: Central Europe", + "abstract": "Complex polygon study focusing on five-sided ecological study area in Central Europe. This research examines irregular boundaries and geometric complexity in spatial analysis.", + "publicationDate": "2022-03-19", + "doi": "10.5555/global-2047-5552", + "url": "https://example.org/publications/2047", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((10 48, 15 50, 17 47, 12 44, 8 46, 10 48)))", + "creationDate": "2022-03-27T00:00:00Z", + "lastUpdate": "2022-03-28T07:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2022-03-27T00:00:00Z.\nHarvestingEvent ID: 3047.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Prof. A", + "Dr. B", + "Dr. C", + "Dr. D", + "Dr. E", + "Prof. F" + ], + "keywords": [ + "urban planning", + "sustainability", + "GIS", + "land use", + "spatial analysis", + "demographics" + ], + "topics": [ + "Climate Science", + "Atmospheric Science", + "Meteorology" + ], + "openalex_id": "https://openalex.org/W3002047", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2047-5552\", \"pmid\": \"38002047\"}", + "openalex_open_access_status": "diamond" + } + }, + { + "model": "publications.publication", + "pk": 2048, + "fields": { + "status": "p", + "title": "Concave Polygon Zone: Southeast Asia", + "abstract": "Complex polygon study focusing on irregularly shaped coastal research area. This research examines irregular boundaries and geometric complexity in spatial analysis.", + "publicationDate": "2023-11-19", + "doi": "10.5555/global-2048-2157", + "url": "https://example.org/publications/2048", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((100 5, 105 5, 105 10, 103 8, 101 10, 100 10, 100 5)))", + "creationDate": "2023-12-08T00:00:00Z", + "lastUpdate": "2023-12-08T06:00:00Z", + "source": 2002, + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2023-12-08T00:00:00Z.\nHarvestingEvent ID: 3048.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Zhang Wei", + "Prof. Sarah Johnson", + "Dr. Ahmed Hassan", + "Dr. Maria Rodriguez", + "Dr. John O'Connor", + "Prof. Yuki Tanaka", + "Dr. Pierre Dubois" + ], + "keywords": [], + "topics": [], + "openalex_id": "https://openalex.org/W3002048", + "openalex_match_info": null, + "openalex_fulltext_origin": "repository", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2048-2157\", \"pmid\": \"38002048\"}", + "openalex_open_access_status": "closed" + } + }, + { + "model": "publications.publication", + "pk": 2049, + "fields": { + "status": "p", + "title": "Protected Area with Exclusion Zone: Amazon", + "abstract": "Complex polygon study focusing on conservation area with restricted inner zone in Amazon rainforest. This research examines irregular boundaries and geometric complexity in spatial analysis.", + "publicationDate": "2023-11-10", + "doi": "10.5555/global-2049-9851", + "url": "https://example.org/publications/2049", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((-65 -5, -60 -5, -60 0, -65 0, -65 -5), (-63 -3, -62 -3, -62 -2, -63 -2, -63 -3)))", + "creationDate": "2023-12-01T00:00:00Z", + "lastUpdate": "2023-12-01T15:00:00Z", + "source": 2003, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2023-12-01T00:00:00Z.\nHarvestingEvent ID: 3049.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [], + "keywords": [ + "single keyword" + ], + "topics": [ + "Geography" + ], + "openalex_id": "https://openalex.org/W3002049", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2049-9851\", \"pmid\": \"38002049\"}", + "openalex_open_access_status": "gold" + } + }, + { + "model": "publications.publication", + "pk": 2050, + "fields": { + "status": "p", + "title": "Star-shaped Survey: Arabian Peninsula", + "abstract": "Complex polygon study focusing on multi-pronged geological survey region. This research examines irregular boundaries and geometric complexity in spatial analysis.", + "publicationDate": "2022-04-25", + "doi": "10.5555/global-2050-5071", + "url": "https://example.org/publications/2050", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((50 22, 51 23, 52 22, 51 21, 52 20, 51 19, 50 20, 49 19, 48 20, 49 21, 48 22, 49 23, 50 22)))", + "creationDate": "2022-05-09T00:00:00Z", + "lastUpdate": "2022-05-10T17:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2022-05-09T00:00:00Z.\nHarvestingEvent ID: 3050.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Single Author" + ], + "keywords": [ + "first keyword", + "second keyword" + ], + "topics": [ + "Environmental Science", + "Ecology" + ], + "openalex_id": "https://openalex.org/W3002050", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2050-5071\", \"pmid\": null}", + "openalex_open_access_status": "bronze" + } + }, + { + "model": "publications.publication", + "pk": 2051, + "fields": { + "status": "p", + "title": "Point-only Study: Remote Island", + "abstract": "Multi-component spatial study integrating single monitoring station on remote Arctic island. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2023-06-18", + "doi": "10.5555/global-2051-5018", + "url": "https://example.org/publications/2051", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (-10 75)))", + "creationDate": "2023-07-08T00:00:00Z", + "lastUpdate": "2023-07-09T07:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2023-07-08T00:00:00Z.\nHarvestingEvent ID: 3051.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. First Author", + "Prof. Second Author" + ], + "keywords": [ + "climate change", + "remote sensing", + "geospatial analysis" + ], + "topics": [ + "Climate Science", + "Atmospheric Science", + "Meteorology" + ], + "openalex_id": "https://openalex.org/W3002051", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2051-5018\", \"pmid\": null}", + "openalex_open_access_status": "closed" + } + }, + { + "model": "publications.publication", + "pk": 2052, + "fields": { + "status": "p", + "title": "Line-only Survey: Shipping Route", + "abstract": "Multi-component spatial study integrating linear shipping route survey in North Atlantic. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2022-02-07", + "doi": "10.5555/global-2052-6002", + "url": "https://example.org/publications/2052", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (LINESTRING (-15 70, 5 72, 25 75)))", + "creationDate": "2022-02-26T00:00:00Z", + "lastUpdate": "2022-02-27T22:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2022-02-26T00:00:00Z.\nHarvestingEvent ID: 3052.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Alice Smith", + "Prof. Bob Jones", + "Dr. Carol Williams" + ], + "keywords": [ + "biodiversity", + "ecosystem services", + "conservation", + "habitat mapping" + ], + "topics": [], + "openalex_id": "https://openalex.org/W3002052", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2052-6002\", \"pmid\": null}", + "openalex_open_access_status": "green" + } + }, + { + "model": "publications.publication", + "pk": 2053, + "fields": { + "status": "p", + "title": "Polygon-only Region: Coastal Zone", + "abstract": "Multi-component spatial study integrating coastal research zone in Arctic Ocean. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2024-06-13", + "doi": "10.5555/global-2053-3771", + "url": "https://example.org/publications/2053", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POLYGON ((0 80, 10 80, 10 85, 0 85, 0 80))))", + "creationDate": "2024-07-12T00:00:00Z", + "lastUpdate": "2024-07-12T02:00:00Z", + "source": 2002, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2024-07-12T00:00:00Z.\nHarvestingEvent ID: 3053.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Maria Garcia", + "Prof. John Smith", + "Dr. Emma Johnson", + "Dr. Li Wei" + ], + "keywords": [ + "urban planning", + "sustainability", + "GIS", + "land use", + "spatial analysis", + "demographics" + ], + "topics": [ + "Geography" + ], + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902053\", \"title\": \"Similar Study 2053\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": null, + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2054, + "fields": { + "status": "p", + "title": "Point-Line Study: River Monitoring", + "abstract": "Multi-component spatial study integrating river monitoring with station and flow path. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2021-07-27", + "doi": "10.5555/global-2054-5530", + "url": "https://example.org/publications/2054", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (0 5), LINESTRING (-5 0, 0 5, 5 10)))", + "creationDate": "2021-08-25T00:00:00Z", + "lastUpdate": "2021-08-26T08:00:00Z", + "source": 2003, + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2021-08-25T00:00:00Z.\nHarvestingEvent ID: 3054.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "authors": [ + "Prof. A", + "Dr. B", + "Dr. C", + "Dr. D", + "Dr. E", + "Prof. F" + ], + "keywords": [], + "topics": [ + "Environmental Science", + "Ecology" + ], + "openalex_id": "https://openalex.org/W3002054", + "openalex_match_info": null, + "openalex_fulltext_origin": "repository", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2054-5530\", \"pmid\": \"38002054\"}", + "openalex_open_access_status": "bronze" + } + }, + { + "model": "publications.publication", + "pk": 2055, + "fields": { + "status": "p", + "title": "Point-Polygon Study: Harbor Analysis", + "abstract": "Multi-component spatial study integrating harbor with central buoy and boundary zone. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2021-02-12", + "doi": "10.5555/global-2055-2751", + "url": "https://example.org/publications/2055", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (100 10), POLYGON ((98 8, 102 8, 102 12, 98 12, 98 8))))", + "creationDate": "2021-03-14T00:00:00Z", + "lastUpdate": "2021-03-16T00:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2021-03-14T00:00:00Z.\nHarvestingEvent ID: 3055.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Zhang Wei", + "Prof. Sarah Johnson", + "Dr. Ahmed Hassan", + "Dr. Maria Rodriguez", + "Dr. John O'Connor", + "Prof. Yuki Tanaka", + "Dr. Pierre Dubois" + ], + "keywords": [ + "single keyword" + ], + "topics": [ + "Climate Science", + "Atmospheric Science", + "Meteorology" + ], + "openalex_id": "https://openalex.org/W3002055", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2055-2751\", \"pmid\": \"38002055\"}", + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2056, + "fields": { + "status": "p", + "title": "Line-Polygon Study: Coastal Transect", + "abstract": "Multi-component spatial study integrating coastal transect through study area. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2023-09-17", + "doi": "10.5555/global-2056-5415", + "url": "https://example.org/publications/2056", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (LINESTRING (80 27, 85 29, 90 28), POLYGON ((82 26, 88 26, 88 30, 82 30, 82 26))))", + "creationDate": "2023-10-08T00:00:00Z", + "lastUpdate": "2023-10-09T01:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2001 on 2023-10-08T00:00:00Z.\nHarvestingEvent ID: 3056.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [], + "keywords": [ + "first keyword", + "second keyword" + ], + "topics": [], + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902056\", \"title\": \"Similar Study 2056\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": null, + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2057, + "fields": { + "status": "p", + "title": "Multi-site Arctic Study", + "abstract": "Multi-component spatial study integrating integrated Arctic research with monitoring stations, survey transects, and study areas. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2023-07-25", + "doi": "10.5555/global-2057-3475", + "url": "https://example.org/publications/2057", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (-10 75), LINESTRING (-15 70, 5 72, 25 75), POLYGON ((0 80, 10 80, 10 85, 0 85, 0 80))))", + "creationDate": "2023-08-10T00:00:00Z", + "lastUpdate": "2023-08-11T23:00:00Z", + "source": 2002, + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2002 on 2023-08-10T00:00:00Z.\nHarvestingEvent ID: 3057.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Single Author" + ], + "keywords": [ + "climate change", + "remote sensing", + "geospatial analysis" + ], + "topics": [ + "Geography" + ], + "openalex_id": null, + "openalex_match_info": "[{\"openalex_id\": \"https://openalex.org/W2902057\", \"title\": \"Similar Study 2057\", \"doi\": null, \"match_type\": \"title\"}]", + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": null, + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2058, + "fields": { + "status": "p", + "title": "Multi-Point-Line: Island Network", + "abstract": "Multi-component spatial study integrating island monitoring network with connection route. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2022-03-30", + "doi": "10.5555/global-2058-7878", + "url": "https://example.org/publications/2058", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (160 -5), POINT (165 0), POINT (170 5), LINESTRING (158 -8, 172 8)))", + "creationDate": "2022-04-17T00:00:00Z", + "lastUpdate": "2022-04-17T22:00:00Z", + "source": 2003, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2003 on 2022-04-17T00:00:00Z.\nHarvestingEvent ID: 3058.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: (none - OpenAlex match not found)", + "authors": [ + "Dr. First Author", + "Prof. Second Author" + ], + "keywords": [ + "biodiversity", + "ecosystem services", + "conservation", + "habitat mapping" + ], + "topics": [ + "Environmental Science", + "Ecology" + ], + "openalex_id": "https://openalex.org/W3002058", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2058-7878\", \"pmid\": null}", + "openalex_open_access_status": "hybrid" + } + }, + { + "model": "publications.publication", + "pk": 2059, + "fields": { + "status": "p", + "title": "Multi-Point-Polygon: Lake Study", + "abstract": "Multi-component spatial study integrating lake study with sampling stations and boundary. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2021-12-31", + "doi": "10.5555/global-2059-4779", + "url": "https://example.org/publications/2059", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (POINT (-75 20), POINT (-70 18), POLYGON ((-80 15, -60 15, -60 25, -80 25, -80 15))))", + "creationDate": "2022-01-26T00:00:00Z", + "lastUpdate": "2022-01-26T03:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2018\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2022-01-26T00:00:00Z.\nHarvestingEvent ID: 3059.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Alice Smith", + "Prof. Bob Jones", + "Dr. Carol Williams" + ], + "keywords": [ + "urban planning", + "sustainability", + "GIS", + "land use", + "spatial analysis", + "demographics" + ], + "topics": [ + "Climate Science", + "Atmospheric Science", + "Meteorology" + ], + "openalex_id": "https://openalex.org/W3002059", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2059-4779\", \"pmid\": \"38002059\"}", + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2060, + "fields": { + "status": "p", + "title": "Multi-Line-Polygon: Watershed Analysis", + "abstract": "Multi-component spatial study integrating watershed with multiple streams and catchment area. Combines point-based, linear, and areal data collection methods.", + "publicationDate": "2024-05-11", + "doi": "10.5555/global-2060-3485", + "url": "https://example.org/publications/2060", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(GEOMETRYCOLLECTION (LINESTRING (50 20, 52 22), LINESTRING (51 19, 52 21), POLYGON ((48 18, 54 18, 54 24, 48 24, 48 18))))", + "creationDate": "2024-06-02T00:00:00Z", + "lastUpdate": "2024-06-03T13:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-06-02T00:00:00Z.\nHarvestingEvent ID: 3060.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "authors": [ + "Dr. Maria Garcia", + "Prof. John Smith", + "Dr. Emma Johnson", + "Dr. Li Wei" + ], + "keywords": [], + "topics": [], + "openalex_id": "https://openalex.org/W3002060", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2060-3485\", \"pmid\": \"38002060\"}", + "openalex_open_access_status": "gold" + } + }, + { + "model": "publications.publication", + "pk": 2061, + "fields": { + "status": "p", + "title": "Micro-site Study: Urban Park", + "abstract": "Scale-specific analysis examining very small urban ecology study (sub-meter precision). Tests spatial processing at extreme precision or extent.", + "publicationDate": "2022-12-21", + "doi": "10.5555/global-2061-6078", + "url": "https://example.org/publications/2061", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(POLYGON ((13.40500 52.52000, 13.40510 52.52000, 13.40510 52.52005, 13.40500 52.52005, 13.40500 52.52000)))", + "creationDate": "2023-01-08T00:00:00Z", + "lastUpdate": "2023-01-09T10:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2019\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2000 on 2023-01-08T00:00:00Z.\nHarvestingEvent ID: 3061.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Prof. A", + "Dr. B", + "Dr. C", + "Dr. D", + "Dr. E", + "Prof. F" + ], + "keywords": [ + "single keyword" + ], + "topics": [ + "Geography" + ], + "openalex_id": "https://openalex.org/W3002061", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2061-6078\", \"pmid\": null}", + "openalex_open_access_status": null + } + }, + { + "model": "publications.publication", + "pk": 2062, + "fields": { + "status": "p", + "title": "Continental-scale Transect", + "abstract": "Scale-specific analysis examining global east-west transect spanning multiple continents. Tests spatial processing at extreme precision or extent.", + "publicationDate": "2023-05-30", + "doi": "10.5555/global-2062-3382", + "url": "https://example.org/publications/2062", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(LINESTRING (-120 25, -80 30, -40 35, 0 40, 40 45, 80 50, 120 55))", + "creationDate": "2023-06-12T00:00:00Z", + "lastUpdate": "2023-06-12T22:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2022\"]", + "timeperiod_enddate": "[\"2023\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2023-06-12T00:00:00Z.\nHarvestingEvent ID: 3062.\n\nMetadata Sources:\n - authors: original_source\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. Zhang Wei", + "Prof. Sarah Johnson", + "Dr. Ahmed Hassan", + "Dr. Maria Rodriguez", + "Dr. John O'Connor", + "Prof. Yuki Tanaka", + "Dr. Pierre Dubois" + ], + "keywords": [ + "first keyword", + "second keyword" + ], + "topics": [ + "Environmental Science", + "Ecology" + ], + "openalex_id": "https://openalex.org/W3002062", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2062-3382\", \"pmid\": \"38002062\"}", + "openalex_open_access_status": "green" + } + }, + { + "model": "publications.publication", + "pk": 2063, + "fields": { + "status": "p", + "title": "Scattered Monitoring Network: Pacific Islands", + "abstract": "Multi-feature spatial analysis documenting distributed ocean monitoring stations across Pacific. Studies distributed or parallel spatial phenomena.", + "publicationDate": "2022-03-13", + "doi": "10.5555/global-2063-6159", + "url": "https://example.org/publications/2063", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(MULTIPOINT ((160 -10), (165 -5), (170 0), (175 5), (180 10)))", + "creationDate": "2022-03-30T00:00:00Z", + "lastUpdate": "2022-03-30T10:00:00Z", + "source": 2000, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2022\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2000 on 2022-03-30T00:00:00Z.\nHarvestingEvent ID: 3063.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [], + "keywords": [ + "climate change", + "remote sensing", + "geospatial analysis" + ], + "topics": [ + "Climate Science", + "Atmospheric Science", + "Meteorology" + ], + "openalex_id": "https://openalex.org/W3002063", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2063-6159\", \"pmid\": null}", + "openalex_open_access_status": "diamond" + } + }, + { + "model": "publications.publication", + "pk": 2064, + "fields": { + "status": "p", + "title": "Multi-route Shipping Analysis", + "abstract": "Multi-feature spatial analysis documenting parallel shipping corridor analysis in Northwest Pacific. Studies distributed or parallel spatial phenomena.", + "publicationDate": "2024-03-17", + "doi": "10.5555/global-2064-4943", + "url": "https://example.org/publications/2064", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(MULTILINESTRING ((140 30, 150 32, 160 33), (142 28, 152 29, 162 30), (138 32, 148 34, 158 35)))", + "creationDate": "2024-04-13T00:00:00Z", + "lastUpdate": "2024-04-14T04:00:00Z", + "source": 2001, + "timeperiod_startdate": "[\"2021\"]", + "timeperiod_enddate": "[\"2024\"]", + "provenance": "Harvested via OAI-PMH from Global Source 2001 on 2024-04-13T00:00:00Z.\nHarvestingEvent ID: 3064.\n\nNo authors or keywords found in original source. OpenAlex matching found partial matches but no exact match.", + "authors": [ + "Dr. Single Author" + ], + "keywords": [ + "biodiversity", + "ecosystem services", + "conservation", + "habitat mapping" + ], + "topics": [], + "openalex_id": "https://openalex.org/W3002064", + "openalex_match_info": null, + "openalex_fulltext_origin": "publisher", + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2064-4943\", \"pmid\": null}", + "openalex_open_access_status": "gold" + } + }, + { + "model": "publications.publication", + "pk": 2065, + "fields": { + "status": "p", + "title": "Fragmented Habitat Study: Indonesia", + "abstract": "Multi-feature spatial analysis documenting island biogeography across separated land masses. Studies distributed or parallel spatial phenomena.", + "publicationDate": "2021-06-16", + "doi": "10.5555/global-2065-1968", + "url": "https://example.org/publications/2065", + "geometry": "SRID=4326;GEOMETRYCOLLECTION(MULTIPOLYGON (((120 -5, 122 -5, 122 -3, 120 -3, 120 -5)), ((124 -4, 126 -4, 126 -2, 124 -2, 124 -4)), ((128 -6, 130 -6, 130 -4, 128 -4, 128 -6))))", + "creationDate": "2021-07-09T00:00:00Z", + "lastUpdate": "2021-07-09T19:00:00Z", + "source": 2002, + "timeperiod_startdate": "[\"2020\"]", + "timeperiod_enddate": "[\"2021\"]", + "provenance": "Harvested via RSS/Atom feed from Global Source 2002 on 2021-07-09T00:00:00Z.\nHarvestingEvent ID: 3065.\n\nMetadata Sources:\n - authors: openalex\n - keywords: original_source\n - topics: openalex\n - openalex_metadata: openalex", + "authors": [ + "Dr. First Author", + "Prof. Second Author" + ], + "keywords": [ + "urban planning", + "sustainability", + "GIS", + "land use", + "spatial analysis", + "demographics" + ], + "topics": [ + "Geography" + ], + "openalex_id": "https://openalex.org/W3002065", + "openalex_match_info": null, + "openalex_fulltext_origin": null, + "openalex_is_retracted": false, + "openalex_ids": "{\"doi\": \"https://doi.org/10.5555/global-2065-1968\", \"pmid\": \"38002065\"}", + "openalex_open_access_status": "diamond" } } ] \ No newline at end of file diff --git a/publications/models.py b/publications/models.py index 766d85e..6c29dc8 100644 --- a/publications/models.py +++ b/publications/models.py @@ -98,6 +98,119 @@ def permalink(self) -> str | None: return f"{base}{rel}" permalink.short_description = "Permalink" + def get_center_coordinate(self): + """ + Calculate and return the center coordinate of the publication's geometry using PostGIS. + + For publications with geometry, this method: + 1. Uses PostGIS ST_Envelope to get the bounding box + 2. Uses PostGIS ST_Centroid to calculate the center of the bounding box + 3. Returns a tuple (longitude, latitude) or None if no geometry + + This uses database-level geometry operations for accuracy and performance. + + Returns: + tuple: (longitude, latitude) as floats, or None if no geometry + + Examples: + >>> pub.geometry = Point(10, 20) + >>> pub.get_center_coordinate() + (10.0, 20.0) + + >>> pub.geometry = GeometryCollection([Point(0, 0), Point(10, 10)]) + >>> pub.get_center_coordinate() + (5.0, 5.0) + """ + if not self.geometry: + return None + + try: + from django.contrib.gis.db.models.functions import Centroid, Envelope + from django.contrib.gis.geos import Point + + # Use database query to calculate centroid of bounding box + # ST_Centroid(ST_Envelope(geometry)) gives us the center of the bounding box + result = Publication.objects.filter(pk=self.pk).annotate( + bbox_center=Centroid(Envelope('geometry')) + ).values_list('bbox_center', flat=True).first() + + if result and isinstance(result, Point): + # Return as (longitude, latitude) + return (result.x, result.y) + + return None + + except Exception: + # If there's any error calculating center, return None + return None + + def get_extreme_points(self): + """ + Calculate and return the extreme points (northernmost, southernmost, easternmost, westernmost) + of the publication's geometry using PostGIS. + + Uses ST_DumpPoints to extract all vertices and orders them by coordinates. + + Returns: + dict: Dictionary with keys 'north', 'south', 'east', 'west', each containing + a tuple (longitude, latitude), or None if no geometry + + Examples: + >>> pub.geometry = Polygon([(0, 0), (10, 0), (10, 10), (0, 10), (0, 0)]) + >>> extremes = pub.get_extreme_points() + >>> extremes['north'] # (5.0, 10.0) - any point at max latitude + >>> extremes['south'] # (5.0, 0.0) - any point at min latitude + >>> extremes['east'] # (10.0, 5.0) - any point at max longitude + >>> extremes['west'] # (0.0, 5.0) - any point at min longitude + """ + if not self.geometry: + return None + + try: + from django.db import connection + + # Raw SQL query to get all extreme points + # For each direction, we dump all points, order by coordinate, and take the first + with connection.cursor() as cursor: + cursor.execute(""" + WITH points AS ( + SELECT (ST_DumpPoints(geometry)).geom AS pt + FROM publications_publication + WHERE id = %s + ) + SELECT + -- Northernmost point (highest Y/latitude) + (SELECT ST_X(pt) FROM points ORDER BY ST_Y(pt) DESC LIMIT 1) AS north_lon, + (SELECT ST_Y(pt) FROM points ORDER BY ST_Y(pt) DESC LIMIT 1) AS north_lat, + -- Southernmost point (lowest Y/latitude) + (SELECT ST_X(pt) FROM points ORDER BY ST_Y(pt) ASC LIMIT 1) AS south_lon, + (SELECT ST_Y(pt) FROM points ORDER BY ST_Y(pt) ASC LIMIT 1) AS south_lat, + -- Easternmost point (highest X/longitude) + (SELECT ST_X(pt) FROM points ORDER BY ST_X(pt) DESC LIMIT 1) AS east_lon, + (SELECT ST_Y(pt) FROM points ORDER BY ST_X(pt) DESC LIMIT 1) AS east_lat, + -- Westernmost point (lowest X/longitude) + (SELECT ST_X(pt) FROM points ORDER BY ST_X(pt) ASC LIMIT 1) AS west_lon, + (SELECT ST_Y(pt) FROM points ORDER BY ST_X(pt) ASC LIMIT 1) AS west_lat + """, [self.pk]) + + row = cursor.fetchone() + + if row: + north_lon, north_lat, south_lon, south_lat, east_lon, east_lat, west_lon, west_lat = row + + return { + 'north': (north_lon, north_lat) if north_lon is not None else None, + 'south': (south_lon, south_lat) if south_lon is not None else None, + 'east': (east_lon, east_lat) if east_lon is not None else None, + 'west': (west_lon, west_lat) if west_lon is not None else None, + } + + return None + + except Exception: + # If there's any error calculating extremes, return None + return None + class Subscription(models.Model): user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name="subscriptions", null=True, blank=True) name = models.CharField(max_length=4096, default="default_subscription") diff --git a/publications/wikidata.py b/publications/wikidata.py index 1149cb2..af95719 100644 --- a/publications/wikidata.py +++ b/publications/wikidata.py @@ -61,6 +61,10 @@ P_JOURNAL_NAME = "P1448" # journal name (monolingual text) P_JOURNAL = "P1433" # published in (journal as item) P_GEOMETRY = "P625" # coordinate location +P_NORTHERNMOST_POINT = "P1332" # northernmost point +P_SOUTHERNMOST_POINT = "P1333" # southernmost point +P_EASTERNMOST_POINT = "P1334" # easternmost point +P_WESTERNMOST_POINT = "P1335" # westernmost point P_INSTANCE_OF = "P31" # instance of P_KEYWORDS = "P921" # main subject / keywords P_LANGUAGE = "P407" # language of work @@ -217,6 +221,7 @@ def build_property_id_mapping(): standard_properties = [ P_TITLE, P_ABSTRACT, P_URL, P_PUBLICATION_DATE, P_PERIOD_START, P_PERIOD_END, P_DOI, P_AUTHOR_STRING, P_AUTHOR, P_JOURNAL_NAME, P_JOURNAL, P_GEOMETRY, + P_NORTHERNMOST_POINT, P_SOUTHERNMOST_POINT, P_EASTERNMOST_POINT, P_WESTERNMOST_POINT, P_INSTANCE_OF, P_KEYWORDS, P_LANGUAGE, P_LICENSE, P_FULL_TEXT_URL, P_OPENALEX_ID, P_PMID, P_PMC, P_ISSN, P_ISSN_L, P_RETRACTED ] @@ -891,6 +896,58 @@ def find_local_item_by_doi(doi): return None +def find_local_item_by_openalex_id(openalex_id): + """ + Return the Q-ID of an existing item in our Wikibase instance for the given OpenAlex ID, + or None if no match is found. + + Args: + openalex_id: OpenAlex ID, either full URL (https://openalex.org/W1234567890) or just the ID (W1234567890) + + Returns: + str: QID of the item, or None if not found + """ + # Extract just the ID part if a full URL was provided + if openalex_id and '/' in openalex_id: + openalex_id = openalex_id.rsplit('/', 1)[-1] + + if not openalex_id: + return None + + # Get the local property ID for OpenAlex ID + local_openalex_property = get_local_property_id(P_OPENALEX_ID) + + # Try with full URL first + sparql_query = f''' + SELECT ?item WHERE {{ + {{ ?item wdt:{local_openalex_property} "https://openalex.org/{openalex_id}" . }} + UNION + {{ ?item wdt:{local_openalex_property} "{openalex_id}" . }} + }} LIMIT 1 + ''' + try: + response = requests.get( + SPARQL_ENDPOINT, + params={"query": sparql_query, "format": "json"}, + headers={"Accept": "application/json"}, + timeout=30 + ) + response.raise_for_status() + + data = response.json() + bindings = data.get("results", {}).get("bindings", []) + if not bindings: + return None + + item_uri = bindings[0]["item"]["value"] + qid = item_uri.rsplit("/", 1)[-1] + logger.debug(f"Found existing item {qid} for OpenAlex ID {openalex_id}") + return qid + except Exception as e: + logger.error(f"Error querying SPARQL for OpenAlex ID {openalex_id}: {e}") + return None + + def build_statements(publication): """ Build comprehensive list of Wikidata statements from publication data. @@ -1035,20 +1092,81 @@ def build_statements(publication): statements.append(Item(prop_nr=get_local_property_id(P_RETRACTED), value="Q7594826")) exported_fields.append('is_retracted') - # Geometry - coordinates - if publication.geometry and check_property_exists(P_GEOMETRY): + # Geometry - center coordinate of bounding box and extreme points + if publication.geometry: try: - geometries = getattr(publication.geometry, "geoms", [publication.geometry]) - for geom in geometries: - if getattr(geom, "geom_type", None) != "Point": - geom = geom.centroid - statements.append(GlobeCoordinate( - prop_nr=get_local_property_id(P_GEOMETRY), - latitude=geom.y, - longitude=geom.x, - precision=0.0001 - )) - exported_fields.append('geometry') + # Add center coordinate + if check_property_exists(P_GEOMETRY): + center = publication.get_center_coordinate() + if center: + lon, lat = center + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_GEOMETRY), + latitude=lat, + longitude=lon, + precision=0.0001, + globe='http://www.wikidata.org/entity/Q2' # Earth + )) + exported_fields.append('geometry_center') + logger.debug(f"Added center coordinate for publication {publication.id}: ({lon}, {lat})") + else: + logger.warning(f"Could not calculate center coordinate for publication {publication.id}") + + # Add extreme points (northernmost, southernmost, easternmost, westernmost) + extreme_points = publication.get_extreme_points() + if extreme_points: + # Northernmost point + if extreme_points['north'] and check_property_exists(P_NORTHERNMOST_POINT): + lon, lat = extreme_points['north'] + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_NORTHERNMOST_POINT), + latitude=lat, + longitude=lon, + precision=0.0001, + globe='http://www.wikidata.org/entity/Q2' + )) + exported_fields.append('geometry_north') + logger.debug(f"Added northernmost point for publication {publication.id}: ({lon}, {lat})") + + # Southernmost point + if extreme_points['south'] and check_property_exists(P_SOUTHERNMOST_POINT): + lon, lat = extreme_points['south'] + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_SOUTHERNMOST_POINT), + latitude=lat, + longitude=lon, + precision=0.0001, + globe='http://www.wikidata.org/entity/Q2' + )) + exported_fields.append('geometry_south') + logger.debug(f"Added southernmost point for publication {publication.id}: ({lon}, {lat})") + + # Easternmost point + if extreme_points['east'] and check_property_exists(P_EASTERNMOST_POINT): + lon, lat = extreme_points['east'] + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_EASTERNMOST_POINT), + latitude=lat, + longitude=lon, + precision=0.0001, + globe='http://www.wikidata.org/entity/Q2' + )) + exported_fields.append('geometry_east') + logger.debug(f"Added easternmost point for publication {publication.id}: ({lon}, {lat})") + + # Westernmost point + if extreme_points['west'] and check_property_exists(P_WESTERNMOST_POINT): + lon, lat = extreme_points['west'] + statements.append(GlobeCoordinate( + prop_nr=get_local_property_id(P_WESTERNMOST_POINT), + latitude=lat, + longitude=lon, + precision=0.0001, + globe='http://www.wikidata.org/entity/Q2' + )) + exported_fields.append('geometry_west') + logger.debug(f"Added westernmost point for publication {publication.id}: ({lon}, {lat})") + except Exception as e: logger.warning(f"Error processing geometry for publication {publication.id}: {e}") @@ -1100,8 +1218,18 @@ def upsert_publication(publication, wikibase_integrator, dryrun=False): # Build statements statements, exported_fields = build_statements(publication) - # Check for existing item by DOI - existing_qid = find_local_item_by_doi(publication.doi) if publication.doi else None + # Check for existing item by DOI first, then fall back to OpenAlex ID + existing_qid = None + if publication.doi: + existing_qid = find_local_item_by_doi(publication.doi) + if existing_qid: + logger.debug(f"Found existing item {existing_qid} by DOI {publication.doi}") + + # Fallback to OpenAlex ID if DOI didn't find a match + if not existing_qid and publication.openalex_id: + existing_qid = find_local_item_by_openalex_id(publication.openalex_id) + if existing_qid: + logger.info(f"Found existing item {existing_qid} by OpenAlex ID {publication.openalex_id} (DOI lookup failed or no DOI)") if dryrun: # Dry-run mode: simulate the export without writing diff --git a/requirements.txt b/requirements.txt index 1cb4d2c..2c89786 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,3 +41,5 @@ python-stdnum>=2.0.0 geopy>=2.4.1 oaipmh-scythe==0.13.0 feedparser==6.0.12 +wikibaseintegrator>=0.12.4 +requests-oauthlib>=1.3.1 diff --git a/tests/test_center_coordinate.py b/tests/test_center_coordinate.py new file mode 100644 index 0000000..10961b3 --- /dev/null +++ b/tests/test_center_coordinate.py @@ -0,0 +1,722 @@ +""" +Unit tests for Publication.get_center_coordinate() method. + +Tests center coordinate calculation for different geometry types: +- Point +- LineString +- Polygon +- GeometryCollection +""" + +from django.test import TestCase +from django.contrib.gis.geos import Point, LineString, Polygon, GeometryCollection +from publications.models import Publication + + +class CenterCoordinateTest(TestCase): + """Test center coordinate calculation for different geometry types.""" + + def setUp(self): + """Create a base publication for testing.""" + self.publication = Publication.objects.create( + title="Test Publication", + doi="10.1234/test", + status="p" + ) + + def test_center_coordinate_point(self): + """Test center coordinate calculation for a single Point geometry.""" + # Create a publication with a single point + self.publication.geometry = GeometryCollection(Point(10.0, 20.0)) + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # For a single point, the center should be the point itself + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 10.0, places=5) + self.assertAlmostEqual(lat, 20.0, places=5) + + def test_center_coordinate_linestring(self): + """Test center coordinate calculation for a LineString geometry.""" + # Create a publication with a line from (0, 0) to (10, 10) + line = LineString([(0.0, 0.0), (10.0, 10.0)]) + self.publication.geometry = GeometryCollection(line) + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # The bounding box center of a line from (0,0) to (10,10) should be (5, 5) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=5) + self.assertAlmostEqual(lat, 5.0, places=5) + + def test_center_coordinate_polygon(self): + """Test center coordinate calculation for a Polygon geometry.""" + # Create a publication with a rectangular polygon + # Rectangle from (0, 0) to (10, 20) + polygon = Polygon([(0.0, 0.0), (10.0, 0.0), (10.0, 20.0), (0.0, 20.0), (0.0, 0.0)]) + self.publication.geometry = GeometryCollection(polygon) + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # The bounding box center should be (5, 10) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=5) + self.assertAlmostEqual(lat, 10.0, places=5) + + def test_center_coordinate_multiple_geometries(self): + """Test center coordinate calculation for a GeometryCollection with multiple geometries.""" + # Create a publication with multiple points + point1 = Point(0.0, 0.0) + point2 = Point(10.0, 10.0) + self.publication.geometry = GeometryCollection(point1, point2) + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # The bounding box center of points at (0,0) and (10,10) should be (5, 5) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=5) + self.assertAlmostEqual(lat, 5.0, places=5) + + def test_center_coordinate_no_geometry(self): + """Test center coordinate calculation when publication has no geometry.""" + # Don't set any geometry + self.publication.geometry = None + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # Should return None when no geometry + self.assertIsNone(center) + + def test_center_coordinate_complex_polygon(self): + """Test center coordinate calculation for a complex polygon (not centered at origin).""" + # Create a polygon from (100, 50) to (120, 70) + polygon = Polygon([ + (100.0, 50.0), + (120.0, 50.0), + (120.0, 70.0), + (100.0, 70.0), + (100.0, 50.0) + ]) + self.publication.geometry = GeometryCollection(polygon) + self.publication.save() + + # Get center coordinate + center = self.publication.get_center_coordinate() + + # The bounding box center should be (110, 60) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 110.0, places=5) + self.assertAlmostEqual(lat, 60.0, places=5) + + +class ExtremePointsTest(TestCase): + """Test extreme points calculation for different geometry types.""" + + def setUp(self): + """Create a base publication for testing.""" + self.publication = Publication.objects.create( + title="Test Publication", + doi="10.1234/test-extremes", + status="p" + ) + + def test_extreme_points_single_point(self): + """Test extreme points for a single point - all should be the same.""" + point = Point(10.0, 20.0) + self.publication.geometry = GeometryCollection(point) + self.publication.save() + + extremes = self.publication.get_extreme_points() + + self.assertIsNotNone(extremes) + # All extreme points should be the same for a single point + self.assertAlmostEqual(extremes['north'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 20.0, places=5) + self.assertAlmostEqual(extremes['south'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 20.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['east'][1], 20.0, places=5) + self.assertAlmostEqual(extremes['west'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['west'][1], 20.0, places=5) + + def test_extreme_points_linestring(self): + """Test extreme points for a diagonal line.""" + line = LineString([(0.0, 0.0), (10.0, 10.0)]) + self.publication.geometry = GeometryCollection(line) + self.publication.save() + + extremes = self.publication.get_extreme_points() + + self.assertIsNotNone(extremes) + # Northernmost point (highest Y) + self.assertAlmostEqual(extremes['north'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 10.0, places=5) + # Southernmost point (lowest Y) + self.assertAlmostEqual(extremes['south'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + # Easternmost point (highest X) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['east'][1], 10.0, places=5) + # Westernmost point (lowest X) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['west'][1], 0.0, places=5) + + def test_extreme_points_rectangle(self): + """Test extreme points for a rectangular polygon.""" + # Rectangle from (0, 0) to (10, 20) + polygon = Polygon([(0.0, 0.0), (10.0, 0.0), (10.0, 20.0), (0.0, 20.0), (0.0, 0.0)]) + self.publication.geometry = GeometryCollection(polygon) + self.publication.save() + + extremes = self.publication.get_extreme_points() + + self.assertIsNotNone(extremes) + # Northernmost point (highest Y = 20) + self.assertAlmostEqual(extremes['north'][1], 20.0, places=5) + # X can be either 0 or 10 (both vertices have Y=20) + self.assertIn(extremes['north'][0], [0.0, 10.0]) + + # Southernmost point (lowest Y = 0) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + # X can be either 0 or 10 (both vertices have Y=0) + self.assertIn(extremes['south'][0], [0.0, 10.0]) + + # Easternmost point (highest X = 10) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + # Y can be either 0 or 20 (both vertices have X=10) + self.assertIn(extremes['east'][1], [0.0, 20.0]) + + # Westernmost point (lowest X = 0) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + # Y can be either 0 or 20 (both vertices have X=0) + self.assertIn(extremes['west'][1], [0.0, 20.0]) + + def test_extreme_points_complex_polygon(self): + """Test extreme points for a more complex polygon.""" + # Create an L-shaped polygon + polygon = Polygon([ + (0.0, 0.0), + (10.0, 0.0), + (10.0, 5.0), + (5.0, 5.0), + (5.0, 10.0), + (0.0, 10.0), + (0.0, 0.0) + ]) + self.publication.geometry = GeometryCollection(polygon) + self.publication.save() + + extremes = self.publication.get_extreme_points() + + self.assertIsNotNone(extremes) + # Northernmost point (highest Y = 10) + self.assertAlmostEqual(extremes['north'][1], 10.0, places=5) + # Southernmost point (lowest Y = 0) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + # Easternmost point (highest X = 10) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + # Westernmost point (lowest X = 0) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + + def test_extreme_points_no_geometry(self): + """Test extreme points when publication has no geometry.""" + self.publication.geometry = None + self.publication.save() + + extremes = self.publication.get_extreme_points() + + self.assertIsNone(extremes) + + +class ComplexGeometryTest(TestCase): + """Test center coordinate and extreme points for complex and mixed geometry types.""" + + def setUp(self): + """Create a base publication for testing.""" + self.publication = Publication.objects.create( + title="Test Publication - Complex Geometries", + doi="10.1234/test-complex", + status="p" + ) + + def test_triangle_geometry(self): + """Test with a triangular polygon.""" + # Equilateral-ish triangle + triangle = Polygon([(0.0, 0.0), (10.0, 0.0), (5.0, 8.66), (0.0, 0.0)]) + self.publication.geometry = GeometryCollection(triangle) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be roughly in the middle + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=1) + self.assertAlmostEqual(lat, 4.33, places=1) + + # Extremes + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 8.66, places=2) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + + def test_pentagon_geometry(self): + """Test with a pentagon polygon.""" + # Regular pentagon (approximately) + import math + cx, cy, r = 50.0, 50.0, 10.0 + points = [] + for i in range(5): + angle = 2 * math.pi * i / 5 - math.pi / 2 + x = cx + r * math.cos(angle) + y = cy + r * math.sin(angle) + points.append((x, y)) + points.append(points[0]) # Close the polygon + + pentagon = Polygon(points) + self.publication.geometry = GeometryCollection(pentagon) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be near the pentagon center (bbox center may not match geometric center) + self.assertIsNotNone(center) + lon, lat = center + # Bounding box center might be slightly off from geometric center + self.assertAlmostEqual(lon, cx, delta=2.0) + self.assertAlmostEqual(lat, cy, delta=2.0) + + # Extremes should be within radius of center + self.assertIsNotNone(extremes) + self.assertGreater(extremes['north'][1], cy - r) + self.assertLess(extremes['south'][1], cy + r) + self.assertGreater(extremes['east'][0], cx - r) + self.assertLess(extremes['west'][0], cx + r) + + def test_concave_polygon(self): + """Test with a concave (non-convex) polygon.""" + # Star-like concave polygon + concave = Polygon([ + (0.0, 5.0), + (2.0, 2.0), + (5.0, 0.0), + (3.0, 3.0), + (5.0, 5.0), + (2.5, 4.0), + (0.0, 5.0) + ]) + self.publication.geometry = GeometryCollection(concave) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Should handle concave polygons correctly + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Verify extreme points match the vertices + self.assertAlmostEqual(extremes['north'][1], 5.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 5.0, places=5) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + + def test_polygon_with_hole(self): + """Test with a polygon that has an interior hole.""" + from django.contrib.gis.geos import LinearRing + + # Outer ring + outer_ring = LinearRing((0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)) + # Inner ring (hole) + inner_ring = LinearRing((3.0, 3.0), (7.0, 3.0), (7.0, 7.0), (3.0, 7.0), (3.0, 3.0)) + + polygon_with_hole = Polygon(outer_ring, [inner_ring]) + self.publication.geometry = GeometryCollection(polygon_with_hole) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be at center of bounding box (hole doesn't affect bbox) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=5) + self.assertAlmostEqual(lat, 5.0, places=5) + + # Extremes should be from outer ring only + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['north'][1], 10.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + + def test_mixed_point_and_line(self): + """Test with a GeometryCollection containing both points and lines.""" + point = Point(0.0, 0.0) + line = LineString([(10.0, 10.0), (20.0, 20.0)]) + + self.publication.geometry = GeometryCollection(point, line) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be middle of bounding box from (0,0) to (20,20) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 10.0, places=5) + self.assertAlmostEqual(lat, 10.0, places=5) + + # Extremes + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['south'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['north'][0], 20.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 20.0, places=5) + + def test_mixed_point_line_polygon(self): + """Test with a GeometryCollection containing point, line, and polygon.""" + point = Point(0.0, 0.0) + line = LineString([(5.0, 5.0), (15.0, 5.0)]) + polygon = Polygon([(20.0, 0.0), (30.0, 0.0), (30.0, 10.0), (20.0, 10.0), (20.0, 0.0)]) + + self.publication.geometry = GeometryCollection(point, line, polygon) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be middle of overall bounding box from (0,0) to (30,10) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 15.0, places=5) + self.assertAlmostEqual(lat, 5.0, places=5) + + # Extremes should span all geometries + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 30.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 10.0, places=5) + + def test_multipoint_geometry(self): + """Test with multiple scattered points.""" + points = [ + Point(0.0, 0.0), + Point(100.0, 50.0), + Point(50.0, 100.0), + Point(-50.0, 25.0) + ] + self.publication.geometry = GeometryCollection(*points) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Center should be in the middle of bounding box + self.assertIsNotNone(center) + lon, lat = center + # Bounding box: x from -50 to 100 (center 25), y from 0 to 100 (center 50) + self.assertAlmostEqual(lon, 25.0, places=5) + self.assertAlmostEqual(lat, 50.0, places=5) + + # Extremes + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['west'][0], -50.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 100.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 100.0, places=5) + + def test_multilinestring_geometry(self): + """Test with multiple line segments.""" + line1 = LineString([(0.0, 0.0), (10.0, 0.0)]) + line2 = LineString([(0.0, 10.0), (10.0, 10.0)]) + line3 = LineString([(5.0, 0.0), (5.0, 10.0)]) + + self.publication.geometry = GeometryCollection(line1, line2, line3) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Bounding box from (0,0) to (10,10), center at (5,5) + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 5.0, places=5) + self.assertAlmostEqual(lat, 5.0, places=5) + + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 10.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 0.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 10.0, places=5) + + def test_very_small_geometry(self): + """Test with a very small geometry (sub-meter precision).""" + # Small square, 1 meter on each side (in degrees, approximately) + small_size = 0.00001 # About 1 meter at equator + small_polygon = Polygon([ + (0.0, 0.0), + (small_size, 0.0), + (small_size, small_size), + (0.0, small_size), + (0.0, 0.0) + ]) + self.publication.geometry = GeometryCollection(small_polygon) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Should handle very small geometries + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Center should be in the middle + lon, lat = center + self.assertAlmostEqual(lon, small_size / 2, places=8) + self.assertAlmostEqual(lat, small_size / 2, places=8) + + def test_very_large_geometry(self): + """Test with a very large geometry spanning multiple continents.""" + # Rectangle spanning from Europe to Asia + large_polygon = Polygon([ + (0.0, 40.0), # Europe + (140.0, 40.0), # East Asia + (140.0, 60.0), + (0.0, 60.0), + (0.0, 40.0) + ]) + self.publication.geometry = GeometryCollection(large_polygon) + self.publication.save() + + center = self.publication.get_center_coordinate() + extremes = self.publication.get_extreme_points() + + # Should handle large geometries + self.assertIsNotNone(center) + lon, lat = center + self.assertAlmostEqual(lon, 70.0, places=5) + self.assertAlmostEqual(lat, 50.0, places=5) + + self.assertIsNotNone(extremes) + self.assertAlmostEqual(extremes['west'][0], 0.0, places=5) + self.assertAlmostEqual(extremes['east'][0], 140.0, places=5) + self.assertAlmostEqual(extremes['south'][1], 40.0, places=5) + self.assertAlmostEqual(extremes['north'][1], 60.0, places=5) + + +class GlobalFeedsFixtureTest(TestCase): + """Test geometry calculations using complex shapes from the global feeds fixture.""" + + fixtures = ['test_data_global_feeds.json'] + + def test_triangle_from_fixture(self): + """Test triangle geometry from global feeds fixture.""" + from publications.models import Publication + + triangle = Publication.objects.get(title__contains="Triangular Survey") + self.assertIsNotNone(triangle.geometry) + + center = triangle.get_center_coordinate() + extremes = triangle.get_extreme_points() + + # Verify calculations work + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Center should be within bounding box + lon, lat = center + self.assertGreater(lon, 0) # Mediterranean region + self.assertGreater(lat, 30) + + # All extremes should be present + self.assertIsNotNone(extremes['north']) + self.assertIsNotNone(extremes['south']) + self.assertIsNotNone(extremes['east']) + self.assertIsNotNone(extremes['west']) + + def test_pentagon_from_fixture(self): + """Test pentagon geometry from global feeds fixture.""" + from publications.models import Publication + + pentagon = Publication.objects.get(title__contains="Pentagon Study") + self.assertIsNotNone(pentagon.geometry) + + center = pentagon.get_center_coordinate() + extremes = pentagon.get_extreme_points() + + # Verify calculations work + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Pentagon is in Central Europe + lon, lat = center + self.assertGreater(lon, 5) + self.assertLess(lon, 20) + self.assertGreater(lat, 40) + self.assertLess(lat, 55) + + def test_concave_polygon_from_fixture(self): + """Test concave polygon from global feeds fixture.""" + from publications.models import Publication + + concave = Publication.objects.get(title__contains="Concave Polygon") + self.assertIsNotNone(concave.geometry) + + center = concave.get_center_coordinate() + extremes = concave.get_extreme_points() + + # Verify calculations work for concave shapes + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Southeast Asia region + lon, lat = center + self.assertGreater(lon, 95) + self.assertLess(lon, 110) + + def test_polygon_with_hole_from_fixture(self): + """Test polygon with exclusion zone (hole) from global feeds fixture.""" + from publications.models import Publication + + hole = Publication.objects.get(title__contains="Exclusion Zone") + self.assertIsNotNone(hole.geometry) + + center = hole.get_center_coordinate() + extremes = hole.get_extreme_points() + + # Verify calculations work for polygons with holes + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Amazon region (negative longitude) + lon, lat = center + self.assertLess(lon, -55) + self.assertGreater(lon, -70) + self.assertLess(lat, 5) + self.assertGreater(lat, -10) + + def test_mixed_geometry_from_fixture(self): + """Test mixed geometry collection from global feeds fixture.""" + from publications.models import Publication + + mixed = Publication.objects.get(title__contains="Multi-site Arctic") + self.assertIsNotNone(mixed.geometry) + + center = mixed.get_center_coordinate() + extremes = mixed.get_extreme_points() + + # Verify calculations work for mixed geometries (point + line + polygon) + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Arctic region + lon, lat = center + self.assertGreater(lat, 65) # Arctic latitude + + def test_multipoint_from_fixture(self): + """Test multipoint geometry from global feeds fixture.""" + from publications.models import Publication + + multipoint = Publication.objects.get(title__contains="Scattered Monitoring") + self.assertIsNotNone(multipoint.geometry) + + center = multipoint.get_center_coordinate() + extremes = multipoint.get_extreme_points() + + # Verify calculations work for scattered points + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Pacific region + lon, lat = center + self.assertGreater(lon, 150) + + def test_micro_geometry_from_fixture(self): + """Test very small (sub-meter) geometry from global feeds fixture.""" + from publications.models import Publication + + micro = Publication.objects.get(title__contains="Micro-site") + self.assertIsNotNone(micro.geometry) + + center = micro.get_center_coordinate() + extremes = micro.get_extreme_points() + + # Verify calculations work at very small scales + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Verify extreme precision + lon, lat = center + # Should have sub-meter precision (many decimal places) + self.assertGreater(lon, 13.4) + self.assertLess(lon, 13.5) + self.assertGreater(lat, 52.51) + self.assertLess(lat, 52.53) + + # All extreme points should be very close to each other + north_lon, north_lat = extremes['north'] + south_lon, south_lat = extremes['south'] + # Latitude difference should be very small (meters) + lat_diff = north_lat - south_lat + self.assertLess(lat_diff, 0.001) # Less than ~100 meters + + def test_continental_scale_from_fixture(self): + """Test very large continental-scale geometry from global feeds fixture.""" + from publications.models import Publication + + continental = Publication.objects.get(title__contains="Continental-scale") + self.assertIsNotNone(continental.geometry) + + center = continental.get_center_coordinate() + extremes = continental.get_extreme_points() + + # Verify calculations work at very large scales + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Should span from Americas to Asia + east_lon, _ = extremes['east'] + west_lon, _ = extremes['west'] + lon_span = east_lon - west_lon + self.assertGreater(lon_span, 200) # Spans multiple continents + + def test_star_shaped_polygon_from_fixture(self): + """Test star-shaped (non-convex complex) polygon from global feeds fixture.""" + from publications.models import Publication + + star = Publication.objects.get(title__contains="Star-shaped") + self.assertIsNotNone(star.geometry) + + center = star.get_center_coordinate() + extremes = star.get_extreme_points() + + # Verify calculations work for very complex non-convex shapes + self.assertIsNotNone(center) + self.assertIsNotNone(extremes) + + # Arabian Peninsula region + lon, lat = center + self.assertGreater(lon, 45) + self.assertLess(lon, 55) + self.assertGreater(lat, 18) + self.assertLess(lat, 25) diff --git a/tests/test_wikidata_export.py b/tests/test_wikidata_export.py index 01a9d2c..9f4a7fa 100644 --- a/tests/test_wikidata_export.py +++ b/tests/test_wikidata_export.py @@ -716,7 +716,12 @@ def test_build_statements_includes_all_fields(self): self.assertIn('abstract', exported_fields) self.assertIn('authors', exported_fields) self.assertIn('keywords', exported_fields) - self.assertIn('geometry', exported_fields) + # Geometry is now exported as multiple fields + self.assertIn('geometry_center', exported_fields) + self.assertIn('geometry_north', exported_fields) + self.assertIn('geometry_south', exported_fields) + self.assertIn('geometry_east', exported_fields) + self.assertIn('geometry_west', exported_fields) self.assertIn('openalex_id', exported_fields) self.assertIn('pmid', exported_fields) self.assertIn('pmcid', exported_fields)