Update scrapers new data model

.gitignore

@@ -3,6 +3,7 @@
 *.parquet
 *.dbs
 data/*
+!data/models/*png
 !data/.gitkeep
 logs/*
 *.log

README.md

@@ -27,13 +27,6 @@ Create a virtual environment:
 uv sync
 ```
 
-> **Note:** This project requires PyArrow >= 23.0.1. If you encounter errors
-> related to parquet files, upgrade PyArrow by running:
->
-> ```sh
-> uv add --upgrade pyarrow
-> ```
-
 ## Scrape MD data repositories
 
 Currently, we are scraping the following data repositories:
@@ -50,6 +43,14 @@ Soon:
 - [OSF](docs/osf.md)
 - [NMRlipids](docs/nmrlipids.md)
 
+### Data model for scraped data
+
+Generate a visual diagram of the Pydantic model schema to see how scraped metadata are represented:
+
+```sh
+uv run draw-pydantic-schema --out-path data/models/pydantic_schema.png
+```
+
 ### Scraping Zenodo, Figshare and OSF
 
 To scrape Zenodo, Figshare and OSF, you need a token. Because these data repositories are generic data repositories,
@@ -83,14 +84,6 @@ Aggregated results are stored in the `data/` folder in `stats_*.tsv` files.
 
 The notebook `notebooks/scraper_stats.ipynb` provides more in-depth analysis and figures.
 
-### Visualise Pydantic schemas
-
-Generate a visual diagram of the Pydantic models schema to see how MDverse entities (datasets, simulations, files, and publications) link together:
-
-```sh
-uv run draw-pydantic-schema --out_path data/models/pydantic_schema.png
-```
-
 ## Analyze Gromacs mdp and gro files
 
 ### Download files

docs/atlas.md

@@ -40,7 +40,7 @@ Example with dataset id `1k5n_A`:
 Remarks:
 
 - The title of the dataset is the protein name.
-- No comment or description is provided. We used the organism name as description.
+- No description is provided. We generate one based on other available metadata.
 - Parameters of molecular dynamics simulations are provided through the API endpoint <https://www.dsimb.inserm.fr/ATLAS/api/MD_parameters>.
 
 ### Metadata for files

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     "certifi>=2026.1.4",
     "html5lib>=1.1",
     "pandas>=2.3.3",
-    "pyarrow>=23.0.0",
+    "pyarrow>=23.0.1",
     "tqdm>=4.67.3",
     "sqlmodel>=0.0.37",
     "fastparquet>=2025.12.0",
@@ -62,7 +62,7 @@ dev = [
     "jupyterlab>=4.4.9",
     "matplotlib>=3.10.1",
     "plotly>=6.0.0",
-    "prek>=0.3.2",
+    "prek>=0.4.10",
     "pysankeybeta>=1.4.2",
     "pytest>=9.0.2",
     "ruff>=0.15.5",

src/mdverse/models/draw_pydantic_schema.py

@@ -14,14 +14,14 @@
 
 @click.command()
 @click.option(
-    "--out_path",
+    "--out-path",
     type=click.Path(writable=True, path_type=Path),
-    help="Path where the generated ERD image will be saved.",
+    help="Path where the generated data model image will be saved.",
 )
 def main(out_path: Path) -> None:
     """Generate and save an Entity-Relationship Diagram for Pydantic metadata models."""
     logger = create_logger()
-    logger.info("Initializing Entity-Relationship Diagram.")
+    logger.info("Building schema for data model...")
     diagram = erd.EntityRelationshipDiagram()
     # Add Pydantic models to the diagram
     diagram.add_model(DatasetMetadata)

src/mdverse/models/simulation.py

@@ -77,6 +77,9 @@ class Molecule(BaseModel):
         None, description="Sequence of the molecule for protein and nucleic acid."
     )
     inchikey: str | None = Field(None, description="InChIKey of the molecule.")
+    organism: str | None = Field(
+        None, description="Organism the molecule is coming from."
+    )
     external_identifiers: list[ExternalIdentifier] = Field(
         default_factory=list,
         description=("List of external database identifiers for this molecule."),

src/mdverse/scrapers/atlas.py

@@ -14,11 +14,13 @@
 from mdverse.models.dataset import DatasetMetadata
 from mdverse.models.enums import DatasetSourceName, ExternalDatabaseName, MoleculeType
 from mdverse.models.file import FileMetadata
+from mdverse.models.person import Person
 from mdverse.models.scraper import ScraperContext
 from mdverse.models.simulation import (
     ExternalIdentifier,
     ForceFieldModel,
     Molecule,
+    SimulationMetadata,
     Software,
 )
 from mdverse.models.utils import (
@@ -41,14 +43,48 @@
 BASE_URL = "https://www.dsimb.inserm.fr/ATLAS/"
 BASE_API_URL = "https://www.dsimb.inserm.fr/ATLAS/api"
 PDB_LIST_URL = "https://www.dsimb.inserm.fr/ATLAS/data/download/distributions/2024_11_18_ATLAS_pdb.txt"
+AFFILIATION = (
+    "Université Paris Cité and Université des Antilles and Université de la Réunion, "
+    "INSERM, BIGR, F-75014 Paris, France"
+)
 ATLAS_METADATA = {
     "license": "CC-BY-NC",  # https://www.dsimb.inserm.fr/ATLAS/download.html
-    "author_name": [  # https://academic.oup.com/nar/article/52/D1/D384/7438909
-        "Yann Vander Meersche",
-        "Gabriel Cretin",
-        "Aria Gheeraert",
-        "Jean-Christophe Gelly",
-        "Tatiana Galochkina",
+    "authors": [  # https://academic.oup.com/nar/article/52/D1/D384/7438909
+        Person(
+            full_name="Yann Vander Meersche",
+            first_name="Yann",
+            last_name="Vander Meersche",
+            orcid="0000-0002-6680-8104",
+            affiliation=AFFILIATION,
+        ),
+        Person(
+            full_name="Gabriel Cretin",
+            first_name="Gabriel",
+            last_name="Cretin",
+            orcid="0000-0002-3522-7083",
+            affiliation=AFFILIATION,
+        ),
+        Person(
+            full_name="Aria Gheeraert",
+            first_name="Aria",
+            last_name="Gheeraert",
+            orcid="0000-0001-7493-8702",
+            affiliation=AFFILIATION,
+        ),
+        Person(
+            full_name="Jean-Christophe Gelly",
+            first_name="Jean-Christophe",
+            last_name="Gelly",
+            orcid="0000-0001-5138-361X",
+            affiliation=AFFILIATION,
+        ),
+        Person(
+            full_name="Tatiana Galochkina",
+            first_name="Tatiana",
+            last_name="Galochkina",
+            orcid="0000-0002-3608-5208",
+            affiliation=AFFILIATION,
+        ),
     ],
     "doi": "10.1093/nar/gkad1084",  # https://academic.oup.com/nar/article/52/D1/D384/7438909
     "external_link": ["https://www.dsimb.inserm.fr/ATLAS/"],
@@ -106,6 +142,47 @@ def extract_files_from_html(
     return files_metadata
 
 
+def generate_description(
+    metadata: SimulationMetadata, logger: "loguru.Logger" = loguru.logger
+) -> str:
+    """Generate description for ATLAS dataset based on metadata.
+
+    Parameters
+    ----------
+    metadata : SimulationMetadata
+        Metadata object for the dataset.
+    logger : loguru.Logger
+        Logger for logging messages.
+
+    Returns
+    -------
+    str
+        Generated description string.
+    """
+    description = "[automatically generated] Molecular dynamics simulation of "
+    for molecule in metadata.molecules:
+        description += f"{molecule.name} "
+        description += f"from organism {molecule.organism} "
+        external_identifiers = [
+            f"{ext_id.database_name.value} ID: {ext_id.identifier}"
+            for ext_id in molecule.external_identifiers
+        ]
+        if external_identifiers:
+            description += "(" + ", ".join(external_identifiers) + ") "
+    description += f"with {metadata.software[0].name} {metadata.software[0].version} "
+    forcefields_models = [
+        f"{ffm.name} {ffm.version}" if ffm.version else f"{ffm.name}"
+        for ffm in metadata.forcefields_models
+    ]
+    if metadata.forcefields_models:
+        description += "using " + " and ".join(forcefields_models) + " "
+    description += f"at {metadata.simulation_temperatures_in_kelvin[0]} K "
+    description += f"for {metadata.simulation_times[0]}."
+    logger.debug("Dataset generated description:")
+    logger.debug(description)
+    return description
+
+
 def scrape_metadata_for_one_dataset(
     client: httpx.Client,
     chain_id: str,
@@ -150,13 +227,12 @@ def scrape_metadata_for_one_dataset(
         "dataset_id_in_repository": chain_id,
         "dataset_url_in_repository": dataset_url,
         "title": meta_json.get("protein_name"),
-        "description": meta_json.get("organism"),
         "license": ATLAS_METADATA["license"],
-        "author_names": ATLAS_METADATA["author_name"],
+        "authors": ATLAS_METADATA["authors"],
         "doi": ATLAS_METADATA["doi"],
         "external_links": ATLAS_METADATA["external_link"],
     }
-    # Add molecules.
+    # Gather metadata for molecules.
     external_identifiers = []
     if meta_json.get("PDB"):
         external_identifiers.append(
@@ -172,37 +248,39 @@ def scrape_metadata_for_one_dataset(
                 identifier=meta_json["UniProt"],
             )
         )
-    metadata["molecules"] = [
-        Molecule(
-            name=meta_json.get("protein_name"),
-            sequence=meta_json.get("sequence"),
-            external_identifiers=external_identifiers,
-            type=MoleculeType.PROTEIN,
-        )
-    ]
-    # Add software.
-    metadata["software"] = [
+    molecule = Molecule(
+        name=meta_json.get("protein_name"),
+        sequence=meta_json.get("sequence"),
+        organism=meta_json.get("organism"),
+        external_identifiers=external_identifiers,
+        type=MoleculeType.PROTEIN,
+    )
+    # Gather metadata for software.
+    software = [
         Software(
             name=ATLAS_METADATA["software_name"],
             version=ATLAS_METADATA["software_version"],
         )
     ]
-    # Add forcefields and models.
-    metadata["forcefields_models"] = [
+    # Gather metadata for forcefields and models.
+    forcefields_models = [
         ForceFieldModel(
             name=ATLAS_METADATA["forcefield_name"],
             version=ATLAS_METADATA["forcefield_version"],
         ),
         ForceFieldModel(name=ATLAS_METADATA["water_model"]),
     ]
-    # Add simulation temperature.
-    metadata["simulation_temperatures_in_kelvin"] = [
-        ATLAS_METADATA["simulation_temperature"]
-    ]
-    # Add simulation time.
-    metadata["simulation_times"] = [ATLAS_METADATA["simulation_time"]]
-    # Add simulation time step.
-    metadata["simulation_timesteps_in_fs"] = [ATLAS_METADATA["simulation_timestep"]]
+    # Create simulation metadata.
+    simulation_metadata = SimulationMetadata(
+        molecules=[molecule],
+        software=software,
+        forcefields_models=forcefields_models,
+        simulation_temperatures_in_kelvin=[ATLAS_METADATA["simulation_temperature"]],
+        simulation_times=[ATLAS_METADATA["simulation_time"]],
+        simulation_timesteps_in_fs=[ATLAS_METADATA["simulation_timestep"]],
+    )
+    metadata["simulation"] = simulation_metadata
+    metadata["description"] = generate_description(simulation_metadata)
     logger.info("Done.")
     return metadata
 

src/mdverse/scrapers/figshare.py

@@ -12,6 +12,7 @@
 from mdverse.core.logger import create_logger
 from mdverse.models.enums import DatasetSourceName
 from mdverse.models.file import FileMetadata
+from mdverse.models.person import Person
 from mdverse.models.scraper import ScraperContext
 from mdverse.models.utils import (
     export_list_of_models_to_parquet,
@@ -200,7 +201,7 @@ def scrap_zip_files_content(
             f"({zip_files_counter:,}/{len(zip_files):,}"
             f":{zip_files_counter / len(zip_files):.0%})"
         )
-    logger.success("Done extracting files from zip archives.")
+    logger.success("Done extracting files from zip archives")
     return files_in_zip_metadata
 
 
@@ -244,8 +245,13 @@ def extract_metadata_from_single_dataset_record(
         "date_created": record_json.get("created_date"),
         "date_last_updated": record_json.get("modified_date"),
         "title": clean_text(record_json.get("title", "")),
-        "author_names": [
-            clean_text(author.get("full_name"))
+        "authors": [
+            Person(
+                full_name=clean_text(author.get("full_name")),
+                first_name=clean_text(author.get("first_name")),
+                last_name=clean_text(author.get("last_name")),
+                orcid=clean_text(author.get("orcid_id")),
+            )
             for author in record_json.get("authors", [])
         ],
         # Remove far too many line breaks in description.
@@ -365,7 +371,7 @@ def search_all_datasets(
                 found_datasets_per_keyword += found_datasets_per_keyword_per_page
                 logger.info(
                     f"Page {page} fetched "
-                    f"({len(found_datasets_per_keyword_per_page)} datasets)."
+                    f"({len(found_datasets_per_keyword_per_page)} datasets)"
                 )
                 page += 1
             found_datasets_per_filetype.update(found_datasets_per_keyword)
@@ -430,7 +436,7 @@ def get_metadata_for_datasets_and_files(
         dataset_metadata, files_metadata = extract_metadata_from_single_dataset_record(
             resp_json_article, scraper
         )
-        logger.info("Done.")
+        logger.info("Done")
         # Append non-empty metadata to datasets and files lists.
         if dataset_metadata:
             datasets_lst.append(dataset_metadata)