From 10a59faaf12213b8ea44aeff95b8551343904ea4 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 23 Jul 2025 11:05:59 -0600 Subject: [PATCH 01/24] upgrade stac-fastapi.pgstac to root_path fix version --- stac_api/runtime/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_api/runtime/setup.py b/stac_api/runtime/setup.py index cd6e810f..feb03507 100644 --- a/stac_api/runtime/setup.py +++ b/stac_api/runtime/setup.py @@ -12,7 +12,7 @@ "stac-fastapi.api~=5.0", "stac-fastapi.types~=5.0", "stac-fastapi.extensions~=5.0", - "stac-fastapi.pgstac~=5.0", + "stac-fastapi.pgstac>=5.0.3,<6.0", "jinja2>=2.11.2,<4.0.0", "starlette-cramjam>=0.3.2,<0.4", "importlib_resources>=1.1.0;python_version<='3.11'", # https://github.com/cogeotiff/rio-tiler/pull/379 From 0fb438c1b5d1c4c3fb53497482d5469f09dc7070 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 23 Jul 2025 14:48:45 -0600 Subject: [PATCH 02/24] remove json dumps from ConfigDict --- stac_api/runtime/src/render.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/stac_api/runtime/src/render.py b/stac_api/runtime/src/render.py index 1d09aead..4499e343 100644 --- a/stac_api/runtime/src/render.py +++ b/stac_api/runtime/src/render.py @@ -6,12 +6,6 @@ import orjson from pydantic import BaseModel, ConfigDict - -def orjson_dumps(v: Dict[str, Any], *args: Any, default: Any) -> str: - """orjson.dumps returns bytes, to match standard json.dumps we need to decode.""" - return orjson.dumps(v, default=default).decode() - - def get_param_str(params: Dict[str, Any]) -> str: """Get parameter string from a dictionary of parameters.""" for k, v in params.items(): @@ -53,7 +47,7 @@ def get_render_params(self) -> str: params = self.render_params.copy() return f"{get_param_str(params)}" - model_config = ConfigDict(json_loads=orjson.loads, json_dumps=orjson_dumps) + model_config = ConfigDict(json_loads=orjson.loads) def get_render_config(render_params) -> RenderConfig: From 14533af304030a81f7d19408cbb8096550677761 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 23 Jul 2025 15:07:38 -0600 Subject: [PATCH 03/24] fix lint error --- stac_api/runtime/src/render.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stac_api/runtime/src/render.py b/stac_api/runtime/src/render.py index 4499e343..efee8b20 100644 --- a/stac_api/runtime/src/render.py +++ b/stac_api/runtime/src/render.py @@ -6,6 +6,7 @@ import orjson from pydantic import BaseModel, ConfigDict + def get_param_str(params: Dict[str, Any]) -> str: """Get parameter string from a dictionary of parameters.""" for k, v in params.items(): From 330f580c7c4d15268094a7ef626053726baf95fd Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 23 Jul 2025 15:38:43 -0600 Subject: [PATCH 04/24] include tileMatrixSetId in STAC Item map links --- stac_api/runtime/src/links.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stac_api/runtime/src/links.py b/stac_api/runtime/src/links.py index c763d1e8..2b093c83 100644 --- a/stac_api/runtime/src/links.py +++ b/stac_api/runtime/src/links.py @@ -54,7 +54,8 @@ def inject_item(self, item: Item) -> None: def _get_item_map_link(self, item_id: str, collection_id: str) -> Dict[str, Any]: qs = self.render_config.get_full_render_qs() href = urljoin( - self.tiler_href, f"collections/{collection_id}/items/{item_id}/map?{qs}" + self.tiler_href, + f"collections/{collection_id}/items/{item_id}/WebMercatorQuad/map?{qs}", ) return { From bc87871e784bd5bfd7b82ba5fea4f7eefd678fb9 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Fri, 25 Jul 2025 14:42:57 -0600 Subject: [PATCH 05/24] include collection search extension in stac extensions config --- stac_api/runtime/src/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index da8fad8a..1f2555e1 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -26,6 +26,7 @@ SortExtension, TokenPaginationExtension, TransactionExtension, + FreeTextExtension, ) from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.pgstac.config import PostgresSettings, Settings @@ -142,6 +143,7 @@ def TilesApiSettings() -> _TilesApiSettings: FilterExtension(), QueryExtension(), SortExtension(), + FreeTextExtension(), pagination_extension, ] From 06c18b6d2a3b420ae44f1b1bb7fd9bcdd04580e8 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Fri, 25 Jul 2025 14:44:01 -0600 Subject: [PATCH 06/24] fix lint error --- stac_api/runtime/src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index 1f2555e1..e8f014b0 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -22,11 +22,11 @@ from stac_fastapi.extensions.core import ( FieldsExtension, FilterExtension, + FreeTextExtension, QueryExtension, SortExtension, TokenPaginationExtension, TransactionExtension, - FreeTextExtension, ) from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.pgstac.config import PostgresSettings, Settings From 8c196be2d467024d117fcc873bfee0f80a3ad4f1 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 6 Aug 2025 15:32:11 -0600 Subject: [PATCH 07/24] Update stac_api/runtime/src/render.py --- stac_api/runtime/src/render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stac_api/runtime/src/render.py b/stac_api/runtime/src/render.py index efee8b20..0b0ffd8a 100644 --- a/stac_api/runtime/src/render.py +++ b/stac_api/runtime/src/render.py @@ -48,7 +48,6 @@ def get_render_params(self) -> str: params = self.render_params.copy() return f"{get_param_str(params)}" - model_config = ConfigDict(json_loads=orjson.loads) def get_render_config(render_params) -> RenderConfig: From 6b573897a54cc26e238878fc293b714df98653d5 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 6 Aug 2025 15:47:46 -0600 Subject: [PATCH 08/24] lint --- stac_api/runtime/src/render.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stac_api/runtime/src/render.py b/stac_api/runtime/src/render.py index 0b0ffd8a..1e82b816 100644 --- a/stac_api/runtime/src/render.py +++ b/stac_api/runtime/src/render.py @@ -3,8 +3,7 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlencode -import orjson -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel def get_param_str(params: Dict[str, Any]) -> str: @@ -49,7 +48,6 @@ def get_render_params(self) -> str: return f"{get_param_str(params)}" - def get_render_config(render_params) -> RenderConfig: """This is a placeholder for what may be a more complex function in the future. As of now, it isn't clear how we should get this rendering information as it should From 449ed670e8a1b2e607c7f7a18d24b13f75dc7b25 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 7 Aug 2025 10:57:46 -0600 Subject: [PATCH 09/24] fix(db): remove custom collection id search UDF --- database/runtime/handler.py | 97 +------------------------------------ 1 file changed, 1 insertion(+), 96 deletions(-) diff --git a/database/runtime/handler.py b/database/runtime/handler.py index 079f1b60..6a389db4 100644 --- a/database/runtime/handler.py +++ b/database/runtime/handler.py @@ -145,96 +145,6 @@ def create_dashboard_schema(cursor, username: str) -> None: ) -def create_collection_search_functions(cursor) -> None: - """Create custom functions for collection-level search.""" - - search_collection_ids_sql = """ - CREATE OR REPLACE FUNCTION pgstac.collection_id_search(_search jsonb = '{}'::jsonb) RETURNS SETOF text AS $$ - DECLARE - searches searches%ROWTYPE; - _where text; - token_where text; - full_where text; - orderby text; - query text; - token_type text := substr(_search->>'token',1,4); - curs refcursor; - iter_record items%ROWTYPE; - prev_query text; - next text; - prev_id text; - has_next boolean := false; - has_prev boolean := false; - prev text; - total_count bigint; - context jsonb; - includes text[]; - excludes text[]; - exit_flag boolean := FALSE; - batches int := 0; - timer timestamptz := clock_timestamp(); - pstart timestamptz; - pend timestamptz; - pcurs refcursor; - search_where search_wheres%ROWTYPE; - id text; - collections text[]; - BEGIN - CREATE TEMP TABLE results (collection text, content jsonb) ON COMMIT DROP; - -- if ids is set, short circuit and just use direct ids query for each id - -- skip any paging or caching - -- hard codes ordering in the same order as the array of ids - searches := search_query(_search); - _where := searches._where; - orderby := searches.orderby; - search_where := where_stats(_where); - total_count := coalesce(search_where.total_count, search_where.estimated_count); - - IF token_type='prev' THEN - token_where := get_token_filter(_search, null::jsonb); - orderby := sort_sqlorderby(_search, TRUE); - END IF; - IF token_type='next' THEN - token_where := get_token_filter(_search, null::jsonb); - END IF; - - full_where := concat_ws(' AND ', _where, token_where); - RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; - timer := clock_timestamp(); - - FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) - LOOP - timer := clock_timestamp(); - RAISE NOTICE 'Partition Query: %', query; - batches := batches + 1; - -- curs = create_cursor(query); - RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); - OPEN curs FOR EXECUTE query; - LOOP - FETCH curs into iter_record; - EXIT WHEN NOT FOUND; - - INSERT INTO results (collection, content) VALUES (iter_record.collection, iter_record.content); - - END LOOP; - CLOSE curs; - RAISE NOTICE 'Query took %.', clock_timestamp()-timer; - timer := clock_timestamp(); - EXIT WHEN exit_flag; - END LOOP; - RAISE NOTICE 'Scanned through % partitions.', batches; - - RETURN QUERY SELECT DISTINCT collection FROM results WHERE content is not NULL; - - DROP TABLE results; - - RETURN; - END; - $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; - """ - cursor.execute(sql.SQL(search_collection_ids_sql)) - - def create_collection_extents_functions(cursor) -> None: """ Functions to update spatial and temporal extents off all items in a collection @@ -451,12 +361,7 @@ def handler(event, context): ) ) - # As admin, create custom search functions - with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: - with conn.cursor() as cur: - print("Creating custom STAC search functions...") - create_collection_search_functions(cursor=cur) - + # As admin, create custom dashboard functions with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: with conn.cursor() as cur: print("Creating dashboard schema...") From c75109842b5a7363707c7d5f0e1191792de98572 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 7 Aug 2025 10:58:49 -0600 Subject: [PATCH 10/24] fix(stac-api): remove deprecated collection id search endpoints --- stac_api/runtime/src/api.py | 58 ------------- stac_api/runtime/src/app.py | 4 +- stac_api/runtime/src/core.py | 100 +--------------------- stac_api/runtime/src/search.py | 152 --------------------------------- 4 files changed, 3 insertions(+), 311 deletions(-) delete mode 100644 stac_api/runtime/src/api.py delete mode 100644 stac_api/runtime/src/search.py diff --git a/stac_api/runtime/src/api.py b/stac_api/runtime/src/api.py deleted file mode 100644 index ada32b25..00000000 --- a/stac_api/runtime/src/api.py +++ /dev/null @@ -1,58 +0,0 @@ -"""FastAPI extensions for the VEDA STAC API.""" -from typing import List - -import attr - -from stac_fastapi.api.app import StacApi -from stac_fastapi.api.routes import create_async_endpoint - -from .core import VedaCrudClient -from .search import CollectionSearchGet, CollectionSearchPost - - -class VedaStacApi(StacApi): - """Veda STAC API.""" - - client: VedaCrudClient = attr.ib() - - def register_post_search(self): - """Register search endpoint (POST /search). - Returns: - None - """ - super().register_post_search() - self.router.add_api_route( - name="Search", - path="/collection-id-search", - response_model=List[str] if self.settings.enable_response_models else None, - response_class=self.response_class, - response_model_exclude_unset=True, - response_model_exclude_none=True, - methods=["POST"], - endpoint=create_async_endpoint( - self.client.collection_id_post_search, - CollectionSearchPost, - ), - include_in_schema=False, - ) - - def register_get_search(self): - """Register search endpoint (GET /search). - Returns: - None - """ - super().register_get_search() - self.router.add_api_route( - name="Search", - path="/collection-id-search", - response_model=List[str] if self.settings.enable_response_models else None, - response_class=self.response_class, - response_model_exclude_unset=True, - response_model_exclude_none=True, - methods=["GET"], - endpoint=create_async_endpoint( - self.client.collection_id_get_search, - CollectionSearchGet, - ), - include_in_schema=False, - ) diff --git a/stac_api/runtime/src/app.py b/stac_api/runtime/src/app.py index acb06260..663a50ee 100644 --- a/stac_api/runtime/src/app.py +++ b/stac_api/runtime/src/app.py @@ -14,6 +14,7 @@ from fastapi import APIRouter, FastAPI from fastapi.responses import ORJSONResponse +from stac_fastapi.api.app import StacApi from stac_fastapi.pgstac.db import close_db_connection, connect_to_db from starlette.middleware import Middleware from starlette.middleware.cors import CORSMiddleware @@ -22,7 +23,6 @@ from starlette.templating import Jinja2Templates from starlette_cramjam.middleware import CompressionMiddleware -from .api import VedaStacApi from .core import VedaCrudClient from .monitoring import LoggerRouteHandler, logger, metrics, tracer from .validation import ValidationMiddleware @@ -50,7 +50,7 @@ async def lifespan(app: FastAPI): await close_db_connection(app) -api = VedaStacApi( +api = StacApi( app=FastAPI( title=f"{api_settings.project_name} STAC API", openapi_url="/openapi.json", diff --git a/stac_api/runtime/src/core.py b/stac_api/runtime/src/core.py index 54cd3a0f..64b317f1 100644 --- a/stac_api/runtime/src/core.py +++ b/stac_api/runtime/src/core.py @@ -1,23 +1,12 @@ """CoreCrudClient extensions for the VEDA STAC API.""" -from datetime import datetime -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Union -import orjson -from asyncpg.exceptions import InvalidDatetimeFormatError -from buildpg import render -from pydantic import ValidationError -from pygeofilter.backends.cql2_json import to_cql2 -from pygeofilter.parsers.cql2_text import parse as parse_cql2_text - -from fastapi import HTTPException from stac_fastapi.pgstac.core import CoreCrudClient from stac_fastapi.pgstac.types.search import PgstacSearch -from stac_fastapi.types.errors import InvalidQueryParameter from stac_fastapi.types.stac import Item, ItemCollection from starlette.requests import Request from .links import LinkInjector -from .search import CollectionSearchPost NumType = Union[float, int] @@ -25,93 +14,6 @@ class VedaCrudClient(CoreCrudClient): """Veda STAC API Client.""" - async def _collection_id_search_base( - self, - search_request: CollectionSearchPost, - **kwargs: Any, - ) -> List[str]: - """Cross catalog search (POST). - Called with `POST /search`. - Args: - search_request: search request parameters. - Returns: - ItemCollection containing items which match the search criteria. - """ - request: Request = kwargs["request"] - pool = request.app.state.readpool - - search_request.conf = search_request.conf or {} - req = search_request.json(exclude_none=True, by_alias=True) - - try: - async with pool.acquire() as conn: - q, p = render( - """ - SELECT * FROM collection_id_search(:req::text::jsonb); - """, - req=req, - ) - collections = await conn.fetch(q, *p) - except InvalidDatetimeFormatError: - raise InvalidQueryParameter( - f"Datetime parameter {search_request.datetime} is invalid." - ) - - return [collection["collection_id_search"] for collection in collections] - - async def collection_id_post_search( - self, search_request: CollectionSearchPost, **kwargs - ) -> List[str]: - """Cross catalog search (POST). - Called with `POST /collection-id-search`. - Args: - search_request: search request parameters. - Returns: - A list of collection IDs which match the search criteria. - """ - collection_ids = await self._collection_id_search_base(search_request, **kwargs) - return collection_ids - - async def collection_id_get_search( - self, - bbox: Optional[List[NumType]] = None, - datetime: Optional[Union[str, datetime]] = None, - filter: Optional[str] = None, - **kwargs, - ) -> List[str]: - """Cross catalog search (GET). - Called with `GET /collection-id-search`. - Returns: - A list of collection IDs which match the search criteria. - """ - # Parse request parameters - base_args = {"bbox": bbox} - - if filter: - ast = parse_cql2_text(filter) - base_args["filter"] = orjson.loads(to_cql2(ast)) - base_args["filter-lang"] = "cql2-json" # type: ignore - - if datetime: - base_args["datetime"] = datetime # type: ignore - - # Remove None values from dict - clean = {} - for k, v in base_args.items(): - if v is not None and v != []: - clean[k] = v - - # Do the request - try: - search_request = CollectionSearchPost(**clean) - except ValidationError as e: - raise HTTPException( - status_code=400, detail=f"Invalid parameters provided {e}" - ) - return await self.collection_id_post_search( - search_request, request=kwargs["request"] - ) - def inject_item_links( self, item: Item, render_params: Dict[str, Any], request: Request ) -> Item: diff --git a/stac_api/runtime/src/search.py b/stac_api/runtime/src/search.py deleted file mode 100644 index 6f1b972e..00000000 --- a/stac_api/runtime/src/search.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Custom search models""" -from datetime import datetime as dt -from typing import Dict, Optional, Union - -import attr -from geojson_pydantic.geometries import ( # type: ignore - GeometryCollection, - LineString, - MultiLineString, - MultiPoint, - MultiPolygon, - Point, - Polygon, - _GeometryBase, -) -from pydantic import BaseModel, field_validator -from stac_pydantic.shared import BBox - -from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime, str_to_interval -from stac_fastapi.types.search import APIRequest, str2list - -Intersection = Union[ - Point, - MultiPoint, - LineString, - MultiLineString, - Polygon, - MultiPolygon, - GeometryCollection, -] - - -class CollectionSearchPost(BaseModel): - """ - The class for STAC API collection searches. - """ - - bbox: Optional[BBox] = None - intersects: Optional[Intersection] = None - datetime: Optional[str] = None - conf: Optional[Dict] = None - - @property - def start_date(self) -> Optional[dt]: - """Extract the start date from the datetime string.""" - interval = str_to_interval(self.datetime) - return interval[0] if interval else None - - @property - def end_date(self) -> Optional[dt]: - """Extract the end date from the datetime string.""" - interval = str_to_interval(self.datetime) - return interval[1] if interval else None - - @field_validator("intersects") - def validate_spatial(cls, v, values): - """Check bbox and intersects are not both supplied.""" - if v and values["bbox"]: - raise ValueError("intersects and bbox parameters are mutually exclusive") - return v - - @field_validator("bbox") - @classmethod - def validate_bbox(cls, v: BBox): - """Check order of supplied bbox coordinates.""" - if v: - # Validate order - if len(v) == 4: - xmin, ymin, xmax, ymax = v - else: - xmin, ymin, min_elev, xmax, ymax, max_elev = v - if max_elev < min_elev: - raise ValueError( - "Maximum elevation must greater than minimum elevation" - ) - - if xmax < xmin: - raise ValueError( - "Maximum longitude must be greater than minimum longitude" - ) - - if ymax < ymin: - raise ValueError( - "Maximum longitude must be greater than minimum longitude" - ) - - # Validate against WGS84 - if xmin < -180 or ymin < -90 or xmax > 180 or ymax > 90: - raise ValueError("Bounding box must be within (-180, -90, 180, 90)") - - return v - - @field_validator("datetime") - @classmethod - def validate_datetime(cls, v): - """Validate datetime.""" - if "/" in v: - values = v.split("/") - else: - # Single date is interpreted as end date - values = ["..", v] - - dates = [] - for value in values: - if value == ".." or value == "": - dates.append("..") - continue - - # throws ValueError if invalid RFC 3339 string - dates.append(rfc3339_str_to_datetime(value)) - - if dates[0] == ".." and dates[1] == "..": - raise ValueError( - "Invalid datetime range, both ends of range may not be open" - ) - - if ".." not in dates and dates[0] > dates[1]: - raise ValueError( - "Invalid datetime range, must match format (begin_date, end_date)" - ) - - return v - - @property - def spatial_filter(self) -> Optional[_GeometryBase]: - """Return a geojson-pydantic object representing the spatial filter for the search request. - Check for both because the ``bbox`` and ``intersects`` parameters are mutually exclusive. - """ - if self.bbox: - return Polygon( - coordinates=[ - [ - [self.bbox[0], self.bbox[3]], - [self.bbox[2], self.bbox[3]], - [self.bbox[2], self.bbox[1]], - [self.bbox[0], self.bbox[1]], - [self.bbox[0], self.bbox[3]], - ] - ] - ) - if self.intersects: - return self.intersects - return None - - -@attr.s -class CollectionSearchGet(APIRequest): - """Base arguments for GET Request.""" - - bbox: Optional[str] = attr.ib(default=None, converter=str2list) # type: ignore - intersects: Optional[str] = attr.ib(default=None, converter=str2list) # type: ignore - datetime: Optional[str] = attr.ib(default=None) From cfc3a7137c45485ac9bfd9ce89a0e6c0a24dc84a Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 8 Aug 2025 16:22:32 -0600 Subject: [PATCH 11/24] feat(stac-api): add collection search --- docker-compose.yml | 2 +- stac_api/runtime/src/app.py | 23 ++++++---- stac_api/runtime/src/config.py | 83 ++++++++++++++++++++++++++-------- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 75901713..864b58f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,7 +97,7 @@ services: database: container_name: veda.db platform: linux/amd64 - image: ghcr.io/stac-utils/pgstac:v0.7.10 + image: ghcr.io/stac-utils/pgstac:v0.9.6 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password diff --git a/stac_api/runtime/src/app.py b/stac_api/runtime/src/app.py index 663a50ee..6a65cf74 100644 --- a/stac_api/runtime/src/app.py +++ b/stac_api/runtime/src/app.py @@ -5,11 +5,15 @@ from contextlib import asynccontextmanager from aws_lambda_powertools.metrics import MetricUnit -from src.config import TilesApiSettings, api_settings -from src.config import extensions as PgStacExtensions -from src.config import get_request_model as GETModel -from src.config import items_get_request_model -from src.config import post_request_model as POSTModel +from src.config import ( + TilesApiSettings, + api_settings, + application_extensions, + collections_get_request_model, + get_request_model, + items_get_request_model, + post_request_model, +) from src.extension import TiTilerExtension from fastapi import APIRouter, FastAPI @@ -71,10 +75,11 @@ async def lifespan(app: FastAPI): title=f"{api_settings.project_name} STAC API", description=api_settings.project_description, settings=api_settings, - extensions=PgStacExtensions, - client=VedaCrudClient(pgstac_search_model=POSTModel), - search_get_request_model=GETModel, - search_post_request_model=POSTModel, + extensions=application_extensions, + client=VedaCrudClient(pgstac_search_model=post_request_model), + search_get_request_model=get_request_model, + search_post_request_model=post_request_model, + collections_get_request_model=collections_get_request_model, items_get_request_model=items_get_request_model, response_class=ORJSONResponse, middlewares=[Middleware(CompressionMiddleware), Middleware(ValidationMiddleware)], diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index e8f014b0..fa627db7 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -20,16 +20,25 @@ # from stac_fastapi.pgstac.extensions import QueryExtension from stac_fastapi.extensions.core import ( + CollectionSearchExtension, + CollectionSearchFilterExtension, FieldsExtension, FilterExtension, FreeTextExtension, - QueryExtension, + ItemCollectionFilterExtension, + OffsetPaginationExtension, SortExtension, TokenPaginationExtension, TransactionExtension, ) +from stac_fastapi.extensions.core.fields import FieldsConformanceClasses +from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses +from stac_fastapi.extensions.core.query import QueryConformanceClasses +from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.pgstac.config import PostgresSettings, Settings +from stac_fastapi.pgstac.extensions import QueryExtension +from stac_fastapi.pgstac.extensions.filter import FiltersClient from stac_fastapi.pgstac.transactions import BulkTransactionsClient, TransactionsClient from stac_fastapi.pgstac.types.search import PgstacSearch @@ -136,33 +145,67 @@ def TilesApiSettings() -> _TilesApiSettings: return _TilesApiSettings() -pagination_extension = TokenPaginationExtension() +# stac-fastapi-pgstac app.py example for configuring extensions https://github.com/stac-utils/stac-fastapi-pgstac/blob/5.0.3/stac_fastapi/pgstac/app.py +application_extensions = [] -extensions = [ +# TODO this was extensions = [ +# /search models +search_extensions = [ FieldsExtension(), FilterExtension(), QueryExtension(), SortExtension(), - FreeTextExtension(), - pagination_extension, + TokenPaginationExtension(), +] +post_request_model = create_post_request_model( + search_extensions, base_model=PgstacSearch +) +get_request_model = create_get_request_model(search_extensions) +application_extensions.extend(search_extensions) + +# /collections model +cs_extensions = [ + QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), + SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + CollectionSearchFilterExtension(client=FiltersClient()), + FreeTextExtension( + conformance_classes=[FreeTextConformanceClasses.COLLECTIONS], + ), + OffsetPaginationExtension(), +] +collection_search_extension = CollectionSearchExtension.from_extensions(cs_extensions) +collections_get_request_model = collection_search_extension.GET +application_extensions.append(collection_search_extension) + +# /collections/{collectionId}/items model +items_get_request_model = ItemCollectionUri +itm_col_extensions = [ + QueryExtension( + conformance_classes=[QueryConformanceClasses.ITEMS], + ), + SortExtension( + conformance_classes=[SortConformanceClasses.ITEMS], + ), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.ITEMS]), + ItemCollectionFilterExtension(client=FiltersClient()), + TokenPaginationExtension(), ] - items_get_request_model = create_request_model( - "ItemCollectionURI", + model_name="ItemCollectionUri", base_model=ItemCollectionUri, - mixins=[pagination_extension.GET], + extensions=itm_col_extensions, + request_type="GET", ) +application_extensions.extend(itm_col_extensions) if api_settings.enable_transactions: - extensions.extend( - [ - BulkTransactionExtension(client=BulkTransactionsClient()), - TransactionExtension( - client=TransactionsClient(), - settings=api_settings, - response_class=ORJSONResponse, - ), - ] - ) -post_request_model = create_post_request_model(extensions, base_model=PgstacSearch) -get_request_model = create_get_request_model(extensions) + transactions_model = [ + BulkTransactionExtension(client=BulkTransactionsClient()), + TransactionExtension( + client=TransactionsClient(), + settings=api_settings, + response_class=ORJSONResponse, + ), + ] + application_extensions.extend(transactions_model) From b7e4ae63df8de33e3e2b7a665c105f28eb33e966 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Aug 2025 14:58:57 -0600 Subject: [PATCH 12/24] feat(tests)!: modifications to docker compose and tests for pgstac upgrade --- .../data/noaa-emergency-response.json | 2 +- docker-compose.yml | 23 +++++++++++++++---- scripts/bin/load-data.sh | 6 ++--- scripts/run-local-tests.sh | 8 ++++--- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/data/noaa-emergency-response.json b/.github/workflows/data/noaa-emergency-response.json index 3d426cc7..dd9aabd4 100644 --- a/.github/workflows/data/noaa-emergency-response.json +++ b/.github/workflows/data/noaa-emergency-response.json @@ -20,7 +20,7 @@ "interval": [ [ "2005-01-01T00:00:00Z", - "null" + null ] ] } diff --git a/docker-compose.yml b/docker-compose.yml index 864b58f1..f609943b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: stac: container_name: veda.stac @@ -106,7 +104,7 @@ services: - PGPASSWORD=password - PGDATABASE=postgis ports: - - "5432:5432" + - "5439:5432" command: postgres -N 500 volumes: - ./scripts:/tmp/scripts @@ -114,6 +112,23 @@ services: # broken in github actions (definitely when run in act, possibly in tests involving ingest-api)? re-enable to persist local database # - ./.pgdata:/var/lib/postgresql/data + pypgstac: + container_name: veda.loadtestdata + platform: linux/amd64 + image: ghcr.io/stac-utils/pgstac-pypgstac:v0.9.6 + environment: + - PGUSER=username + - PGPASSWORD=password + - PGDATABASE=postgis + - PGHOST=database + - PGPORT=5432 + depends_on: + - database + command: bash -c "sleep 10 && /tmp/scripts/bin/load-data.sh" + volumes: + - ./scripts:/tmp/scripts + - ./.github/workflows/data:/tmp/data + ingestor: container_name: veda.ingestor platform: linux/amd64 @@ -128,7 +143,7 @@ services: - PGPASSWORD=password - PGDATABASE=postgis - DYNAMODB_ENDPOINT=http://localhost:8085 - - VEDA_DB_PGSTAC_VERSION=0.7.10 + - VEDA_DB_PGSTAC_VERSION=0.9.6 - AWS_REGION=us-west-2 - AWS_DEFAULT_REGION=us-west-2 - DYNAMODB_TABLE=veda diff --git a/scripts/bin/load-data.sh b/scripts/bin/load-data.sh index 8caa4aab..28842e95 100755 --- a/scripts/bin/load-data.sh +++ b/scripts/bin/load-data.sh @@ -3,17 +3,17 @@ set -e # Wait for the database to start -until pypgstac pgready --dsn postgresql://username:password@0.0.0.0:5432/postgis; do +until pypgstac pgready; do echo "Waiting for database to start..." sleep 1 done # Load collections echo "Loading collections..." -pypgstac load collections /tmp/data/noaa-emergency-response.json --dsn postgresql://username:password@0.0.0.0:5432/postgis --method upsert +pypgstac load collections /tmp/data/noaa-emergency-response.json --method upsert # Load items echo "Loading items..." -pypgstac load items /tmp/data/noaa-eri-nashville2020.json --dsn postgresql://username:password@0.0.0.0:5432/postgis --method upsert +pypgstac load items /tmp/data/noaa-eri-nashville2020.json --method upsert echo "Data loaded successfully!" diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index eccd1841..f6880778 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -5,7 +5,8 @@ set -e pre-commit run --all-files # Bring up stack for testing; ingestor not required -docker compose up -d stac raster database dynamodb +docker compose up -d stac raster database dynamodb pypgstac +# docker compose up -d --wait stac raster database dynamodb pypgstac # cleanup, logging in case of failure cleanup() { @@ -26,7 +27,7 @@ cleanup() { trap cleanup EXIT # Load data for tests -docker exec veda.db /tmp/scripts/bin/load-data.sh +docker exec veda.loadtestdata /tmp/scripts/bin/load-data.sh # Run tests python -m pytest .github/workflows/tests/ -vv -s @@ -35,4 +36,5 @@ python -m pytest .github/workflows/tests/ -vv -s NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s # Transactions tests -python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s +# Temp disable transactions tests +# python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s From 12cffd1cb8d5e59b1e80250661e74e07e30edd7a Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Aug 2025 15:05:32 -0600 Subject: [PATCH 13/24] use pypgstac container to load data --- scripts/load-data-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/load-data-container.sh b/scripts/load-data-container.sh index 917b19ef..3fb2836e 100755 --- a/scripts/load-data-container.sh +++ b/scripts/load-data-container.sh @@ -3,4 +3,4 @@ DOCKER_CONTAINER_NAME="veda.db" SCRIPT_PATH="/tmp/scripts/bin/load-data.sh" -docker exec "$DOCKER_CONTAINER_NAME" "$SCRIPT_PATH" +docker exec veda.loadtestdata "$DOCKER_CONTAINER_NAME" "$SCRIPT_PATH" From 485e0f2f3ced9f9358765424b10b05f6d6862ccb Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Aug 2025 15:09:54 -0600 Subject: [PATCH 14/24] no need to load items in actions because the database is loaded in docker compose pypgstac container --- .github/actions/cdk-deploy/action.yml | 12 ++++++------ .github/workflows/pr.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/cdk-deploy/action.yml b/.github/actions/cdk-deploy/action.yml index b1c03dc7..649d041e 100644 --- a/.github/actions/cdk-deploy/action.yml +++ b/.github/actions/cdk-deploy/action.yml @@ -66,12 +66,12 @@ runs: working-directory: ${{ inputs.dir }} run: docker compose up --build -d - - name: Ingest Stac Items/Collection - if: ${{ inputs.skip_tests == false }} - shell: bash - working-directory: ${{ inputs.dir }} - run: | - ./scripts/load-data-container.sh + # - name: Ingest Stac Items/Collection + # if: ${{ inputs.skip_tests == false }} + # shell: bash + # working-directory: ${{ inputs.dir }} + # run: | + # ./scripts/load-data-container.sh - name: Sleep for 10 seconds if: ${{ inputs.skip_tests == false }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b1ae1af1..c5dbab3b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -63,9 +63,9 @@ jobs: - name: Launch services run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d - - name: Ingest Stac Items/Collection - run: | - ./scripts/load-data-container.sh + # - name: Ingest Stac Items/Collection + # run: | + # ./scripts/load-data-container.sh - name: Sleep for 10 seconds run: sleep 10s From 4271091f49b70bd73bc1dfac8c456adbae88662c Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Aug 2025 16:09:27 -0600 Subject: [PATCH 15/24] use pg prefixed env vars in transactions conftest --- scripts/run-local-tests.sh | 3 +-- stac_api/runtime/tests/conftest.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index f6880778..decf4f9f 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -36,5 +36,4 @@ python -m pytest .github/workflows/tests/ -vv -s NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s # Transactions tests -# Temp disable transactions tests -# python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s +python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index e52ba97c..4d9ca287 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -239,6 +239,12 @@ def test_environ(): os.environ["POSTGRES_HOST_WRITER"] = "0.0.0.0" os.environ["POSTGRES_PORT"] = "5432" + os.environ["PGUSER"] = "username" + os.environ["PGPASSWORD"] = "password" + os.environ["PGDATABASE"] = "postgis" + os.environ["PGHOST"] = "0.0.0.0" + os.environ["PGPORT"] = "5432" + def override_validated_token(): """ From 7ac16fac774abf11c07c9e413902bd850a3e6b03 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 13 Aug 2025 13:35:18 -0600 Subject: [PATCH 16/24] some test debugging --- docker-compose.yml | 1 + stac_api/runtime/tests/conftest.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f609943b..8a3a3371 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: # https://github.com/developmentseed/eoAPI/issues/16 # - TITILER_ENDPOINT=raster - TITILER_ENDPOINT=http://0.0.0.0:8082 + - VEDA_STAC_ENABLE_TRANSACTIONS=True depends_on: - database - raster diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index 4d9ca287..5974a31b 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -6,6 +6,7 @@ setup for testing with mock AWS and PostgreSQL configurations. """ +import copy import os from unittest.mock import MagicMock, patch @@ -15,7 +16,7 @@ from stac_fastapi.pgstac.db import close_db_connection, connect_to_db VALID_COLLECTION = { - "id": "CMIP245-winter-median-pr", + "id": "valid-collection-test", "type": "Collection", "title": "Projected changes to winter (January, February, and March) cumulative daily precipitation", "links": [], @@ -75,7 +76,7 @@ } VALID_ITEM = { - "id": "OMI_trno2_0.10x0.10_2023_Col3_V4", + "id": "valid-item-test", "bbox": [-180.0, -90.0, 180.0, 90.0], "type": "Feature", "links": [ @@ -237,13 +238,13 @@ def test_environ(): os.environ["POSTGRES_DBNAME"] = "postgis" os.environ["POSTGRES_HOST_READER"] = "0.0.0.0" os.environ["POSTGRES_HOST_WRITER"] = "0.0.0.0" - os.environ["POSTGRES_PORT"] = "5432" + os.environ["POSTGRES_PORT"] = "5439" - os.environ["PGUSER"] = "username" - os.environ["PGPASSWORD"] = "password" - os.environ["PGDATABASE"] = "postgis" - os.environ["PGHOST"] = "0.0.0.0" - os.environ["PGPORT"] = "5432" + # os.environ["PGUSER"] = "username" + # os.environ["PGPASSWORD"] = "password" + # os.environ["PGDATABASE"] = "postgis" + # os.environ["PGHOST"] = "database" + # os.environ["PGPORT"] = "5439" def override_validated_token(): @@ -347,7 +348,7 @@ def invalid_stac_collection(): Returns: dict: An invalid STAC collection with the 'extent' field removed. """ - invalid = VALID_COLLECTION.copy() + invalid = copy.deepcopy(VALID_COLLECTION) invalid.pop("extent") return invalid @@ -371,6 +372,6 @@ def invalid_stac_item(): Returns: dict: An invalid STAC item with the 'properties' field removed. """ - invalid_item = VALID_ITEM.copy() + invalid_item = copy.deepcopy(VALID_ITEM) invalid_item.pop("properties") return invalid_item From 80da7a41c09eda1a1adc838bed95db633336c5c8 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 13 Aug 2025 13:51:52 -0600 Subject: [PATCH 17/24] revert transaction test data ids --- stac_api/runtime/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index 5974a31b..336d3def 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -16,7 +16,7 @@ from stac_fastapi.pgstac.db import close_db_connection, connect_to_db VALID_COLLECTION = { - "id": "valid-collection-test", + "id": "CMIP245-winter-median-pr", "type": "Collection", "title": "Projected changes to winter (January, February, and March) cumulative daily precipitation", "links": [], @@ -76,7 +76,7 @@ } VALID_ITEM = { - "id": "valid-item-test", + "id": "OMI_trno2_0.10x0.10_2023_Col3_V4", "bbox": [-180.0, -90.0, 180.0, 90.0], "type": "Feature", "links": [ From b49d629a2de8f2f4814865d735022a7ac7908245 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 13 Aug 2025 14:27:54 -0600 Subject: [PATCH 18/24] attempt to create collection_in_db before each test --- stac_api/runtime/tests/conftest.py | 17 +++++ stac_api/runtime/tests/test_transactions.py | 83 +++++++-------------- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index 336d3def..bd646027 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -375,3 +375,20 @@ def invalid_stac_item(): invalid_item = copy.deepcopy(VALID_ITEM) invalid_item.pop("properties") return invalid_item + + +@pytest.fixture +async def collection_in_db(api_client, valid_stac_collection): + """ + Fixture to ensure a valid STAC collection exists in the database. + + This fixture posts a valid collection before a test runs and yields + the collection ID. + """ + # Create the collection + response = await api_client.post("/collections", json=valid_stac_collection) + + # Ensure the setup was successful before the test proceeds + assert response.status_code == 201 + + yield valid_stac_collection["id"] diff --git a/stac_api/runtime/tests/test_transactions.py b/stac_api/runtime/tests/test_transactions.py index 57788bd7..8a6d54d0 100644 --- a/stac_api/runtime/tests/test_transactions.py +++ b/stac_api/runtime/tests/test_transactions.py @@ -11,7 +11,6 @@ - /collections/{}/bulk_items """ -import pytest collections_endpoint = "/collections" items_endpoint = "/collections/{}/items" @@ -28,110 +27,84 @@ class TestList: necessary data. """ - @pytest.fixture(autouse=True) - def setup( - self, - api_client, - valid_stac_collection, - valid_stac_item, - invalid_stac_collection, - invalid_stac_item, - ): - """ - Set up the test environment with the required fixtures. - - Args: - api_client: The API client for making requests. - valid_stac_collection: A valid STAC collection for testing. - valid_stac_item: A valid STAC item for testing. - invalid_stac_collection: An invalid STAC collection for testing. - invalid_stac_item: An invalid STAC item for testing. - """ - self.api_client = api_client - self.valid_stac_collection = valid_stac_collection - self.valid_stac_item = valid_stac_item - self.invalid_stac_collection = invalid_stac_collection - self.invalid_stac_item = invalid_stac_item - - async def test_post_invalid_collection(self): + async def test_post_invalid_collection(self, api_client, invalid_stac_collection): """ Test the API's response to posting an invalid STAC collection. Asserts that the response status code is 422 and the detail is "Validation Error". """ - response = await self.api_client.post( - collections_endpoint, json=self.invalid_stac_collection + response = await api_client.post( + collections_endpoint, json=invalid_stac_collection ) assert response.json()["detail"] == "Validation Error" assert response.status_code == 422 - async def test_post_valid_collection(self): + async def test_post_valid_collection(self, api_client, valid_stac_collection): """ Test the API's response to posting a valid STAC collection. Asserts that the response status code is 200. """ - response = await self.api_client.post( - collections_endpoint, json=self.valid_stac_collection + response = await api_client.post( + collections_endpoint, json=valid_stac_collection ) - # assert response.json() == {} assert response.status_code == 201 - async def test_post_invalid_item(self): + async def test_post_invalid_item(self, api_client, invalid_stac_item): """ Test the API's response to posting an invalid STAC item. Asserts that the response status code is 422 and the detail is "Validation Error". """ - response = await self.api_client.post( - items_endpoint.format(self.invalid_stac_item["collection"]), - json=self.invalid_stac_item, + collection_id = invalid_stac_item["collection"] + response = await api_client.post( + items_endpoint.format(collection_id), json=invalid_stac_item ) assert response.json()["detail"] == "Validation Error" assert response.status_code == 422 - async def test_post_valid_item(self): + async def test_post_valid_item(self, api_client, valid_stac_item, collection_in_db): """ Test the API's response to posting a valid STAC item. Asserts that the response status code is 200. """ - response = await self.api_client.post( - items_endpoint.format(self.valid_stac_item["collection"]), - json=self.valid_stac_item, + collection_id = valid_stac_item["collection"] + response = await api_client.post( + items_endpoint.format(collection_id), json=valid_stac_item ) - # assert response.json() == {} assert response.status_code == 201 - async def test_post_invalid_bulk_items(self): + async def test_post_invalid_bulk_items(self, api_client, invalid_stac_item): """ Test the API's response to posting invalid bulk STAC items. Asserts that the response status code is 422. """ - item_id = self.invalid_stac_item["id"] - collection_id = self.invalid_stac_item["collection"] - invalid_request = { - "items": {item_id: self.invalid_stac_item}, - "method": "upsert", - } - response = await self.api_client.post( + item_id = invalid_stac_item["id"] + collection_id = invalid_stac_item["collection"] + invalid_request = {"items": {item_id: invalid_stac_item}, "method": "upsert"} + + response = await api_client.post( bulk_endpoint.format(collection_id), json=invalid_request ) assert response.status_code == 422 - async def test_post_valid_bulk_items(self): + async def test_post_valid_bulk_items( + self, api_client, valid_stac_item, collection_in_db + ): """ Test the API's response to posting valid bulk STAC items. Asserts that the response status code is 200. """ - item_id = self.valid_stac_item["id"] - collection_id = self.valid_stac_item["collection"] - valid_request = {"items": {item_id: self.valid_stac_item}, "method": "upsert"} - response = await self.api_client.post( + item_id = valid_stac_item["id"] + collection_id = valid_stac_item["collection"] + valid_request = {"items": {item_id: valid_stac_item}, "method": "upsert"} + + response = await api_client.post( bulk_endpoint.format(collection_id), json=valid_request ) assert response.status_code == 200 From c9603fe243ceb814552d639c2a490be63fded089 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 13 Aug 2025 14:40:07 -0600 Subject: [PATCH 19/24] do not fail fixture if collection was already created --- stac_api/runtime/tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index bd646027..5eea6d28 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -389,6 +389,8 @@ async def collection_in_db(api_client, valid_stac_collection): response = await api_client.post("/collections", json=valid_stac_collection) # Ensure the setup was successful before the test proceeds - assert response.status_code == 201 + # The setup is successful if the collection was created (201) or if it + # already existed (409). Any other status code is a failure. + assert response.status_code in [201, 409] yield valid_stac_collection["id"] From 093f09676aa7ada6efc468894cba92328931984c Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 13 Aug 2025 14:45:50 -0600 Subject: [PATCH 20/24] fix indention failure on linting --- stac_api/runtime/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index 5eea6d28..762e8aa8 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -389,7 +389,7 @@ async def collection_in_db(api_client, valid_stac_collection): response = await api_client.post("/collections", json=valid_stac_collection) # Ensure the setup was successful before the test proceeds - # The setup is successful if the collection was created (201) or if it + # The setup is successful if the collection was created (201) or if it # already existed (409). Any other status code is a failure. assert response.status_code in [201, 409] From 3e0643c29166d788ae1061e2169f4041aeb23337 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 13 Aug 2025 15:02:45 -0600 Subject: [PATCH 21/24] attempt to test searching by free text word precipitation --- stac_api/runtime/tests/test_transactions.py | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/stac_api/runtime/tests/test_transactions.py b/stac_api/runtime/tests/test_transactions.py index 8a6d54d0..ed8bcacf 100644 --- a/stac_api/runtime/tests/test_transactions.py +++ b/stac_api/runtime/tests/test_transactions.py @@ -108,3 +108,45 @@ async def test_post_valid_bulk_items( bulk_endpoint.format(collection_id), json=valid_request ) assert response.status_code == 200 + + async def test_get_collection_by_id(self, api_client, collection_in_db): + """ + Test searching for a specific collection by its ID. + """ + # The `collection_in_db` fixture ensures the collection exists and provides its ID. + collection_id = collection_in_db + + # Perform a GET request to the /collections endpoint with an "ids" query + response = await api_client.get( + collections_endpoint, params={"ids": collection_id} + ) + + assert response.status_code == 200 + + response_data = response.json() + + assert response_data["collections"][0]["id"] == collection_id + + async def test_collection_freetext_search_by_title( + self, api_client, valid_stac_collection, collection_in_db + ): + """ + Test free-text search for a collection using a word from its title. + """ + + # The `collection_in_db` fixture ensures the collection exists. + collection_id = collection_in_db + + # Use a unique word from the collection's title for the query. + search_term = "precipitation" + + # Perform a GET request with the `q` free-text search parameter. + response = await api_client.get(collections_endpoint, params={"q": search_term}) + + assert response.status_code == 200 + response_data = response.json() + + assert len(response_data["collections"]) > 0 + + returned_ids = [col["id"] for col in response_data["collections"]] + assert collection_id in returned_ids From 0c1642c1303a79e4b84f07007746d72d4d66f4af Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 13 Aug 2025 17:00:19 -0600 Subject: [PATCH 22/24] remove commented code and rename stac api extensions test file --- .github/actions/cdk-deploy/action.yml | 7 - .github/workflows/pr.yml | 6 +- scripts/load-data-container.sh | 6 - scripts/run-local-tests.sh | 5 +- stac_api/runtime/tests/conftest.py | 6 - stac_api/runtime/tests/test_extensions.py | 152 ++++++++++++++++++++ stac_api/runtime/tests/test_transactions.py | 10 +- 7 files changed, 163 insertions(+), 29 deletions(-) delete mode 100755 scripts/load-data-container.sh create mode 100644 stac_api/runtime/tests/test_extensions.py diff --git a/.github/actions/cdk-deploy/action.yml b/.github/actions/cdk-deploy/action.yml index 649d041e..1766581e 100644 --- a/.github/actions/cdk-deploy/action.yml +++ b/.github/actions/cdk-deploy/action.yml @@ -66,13 +66,6 @@ runs: working-directory: ${{ inputs.dir }} run: docker compose up --build -d - # - name: Ingest Stac Items/Collection - # if: ${{ inputs.skip_tests == false }} - # shell: bash - # working-directory: ${{ inputs.dir }} - # run: | - # ./scripts/load-data-container.sh - - name: Sleep for 10 seconds if: ${{ inputs.skip_tests == false }} shell: bash diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c5dbab3b..2a0be301 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -63,10 +63,6 @@ jobs: - name: Launch services run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d - # - name: Ingest Stac Items/Collection - # run: | - # ./scripts/load-data-container.sh - - name: Sleep for 10 seconds run: sleep 10s shell: bash @@ -83,7 +79,7 @@ jobs: - name: Ingest unit tests run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s - - name: Stac-api transactions unit tests + - name: Stac-api extensions unit tests run: python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s - name: Stop services diff --git a/scripts/load-data-container.sh b/scripts/load-data-container.sh deleted file mode 100755 index 3fb2836e..00000000 --- a/scripts/load-data-container.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -DOCKER_CONTAINER_NAME="veda.db" -SCRIPT_PATH="/tmp/scripts/bin/load-data.sh" - -docker exec veda.loadtestdata "$DOCKER_CONTAINER_NAME" "$SCRIPT_PATH" diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index decf4f9f..62c6ce1e 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -5,8 +5,7 @@ set -e pre-commit run --all-files # Bring up stack for testing; ingestor not required -docker compose up -d stac raster database dynamodb pypgstac -# docker compose up -d --wait stac raster database dynamodb pypgstac +docker compose up -d --wait stac raster database dynamodb pypgstac # cleanup, logging in case of failure cleanup() { @@ -36,4 +35,4 @@ python -m pytest .github/workflows/tests/ -vv -s NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s # Transactions tests -python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s +python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s \ No newline at end of file diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index 762e8aa8..ede8bad1 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -240,12 +240,6 @@ def test_environ(): os.environ["POSTGRES_HOST_WRITER"] = "0.0.0.0" os.environ["POSTGRES_PORT"] = "5439" - # os.environ["PGUSER"] = "username" - # os.environ["PGPASSWORD"] = "password" - # os.environ["PGDATABASE"] = "postgis" - # os.environ["PGHOST"] = "database" - # os.environ["PGPORT"] = "5439" - def override_validated_token(): """ diff --git a/stac_api/runtime/tests/test_extensions.py b/stac_api/runtime/tests/test_extensions.py new file mode 100644 index 00000000..ed8bcacf --- /dev/null +++ b/stac_api/runtime/tests/test_extensions.py @@ -0,0 +1,152 @@ +""" +Test suite for STAC (SpatioTemporal Asset Catalog) Transactions API endpoints. + +This module contains tests for the collection and item endpoints of the STAC API. +It verifies the behavior of the API when posting valid and invalid STAC collections and items, +as well as bulk items. + +Endpoints tested: +- /collections +- /collections/{}/items +- /collections/{}/bulk_items +""" + + +collections_endpoint = "/collections" +items_endpoint = "/collections/{}/items" +bulk_endpoint = "/collections/{}/bulk_items" + + +class TestList: + """ + Test cases for STAC API's collection and item endpoints. + + This class contains tests to ensure that the STAC API correctly handles + posting valid and invalid STAC collections and items, both individually + and in bulk. It uses pytest fixtures to set up the test environment with + necessary data. + """ + + async def test_post_invalid_collection(self, api_client, invalid_stac_collection): + """ + Test the API's response to posting an invalid STAC collection. + + Asserts that the response status code is 422 and the detail + is "Validation Error". + """ + response = await api_client.post( + collections_endpoint, json=invalid_stac_collection + ) + assert response.json()["detail"] == "Validation Error" + assert response.status_code == 422 + + async def test_post_valid_collection(self, api_client, valid_stac_collection): + """ + Test the API's response to posting a valid STAC collection. + + Asserts that the response status code is 200. + """ + response = await api_client.post( + collections_endpoint, json=valid_stac_collection + ) + assert response.status_code == 201 + + async def test_post_invalid_item(self, api_client, invalid_stac_item): + """ + Test the API's response to posting an invalid STAC item. + + Asserts that the response status code is 422 and the detail + is "Validation Error". + """ + collection_id = invalid_stac_item["collection"] + response = await api_client.post( + items_endpoint.format(collection_id), json=invalid_stac_item + ) + assert response.json()["detail"] == "Validation Error" + assert response.status_code == 422 + + async def test_post_valid_item(self, api_client, valid_stac_item, collection_in_db): + """ + Test the API's response to posting a valid STAC item. + + Asserts that the response status code is 200. + """ + collection_id = valid_stac_item["collection"] + response = await api_client.post( + items_endpoint.format(collection_id), json=valid_stac_item + ) + assert response.status_code == 201 + + async def test_post_invalid_bulk_items(self, api_client, invalid_stac_item): + """ + Test the API's response to posting invalid bulk STAC items. + + Asserts that the response status code is 422. + """ + item_id = invalid_stac_item["id"] + collection_id = invalid_stac_item["collection"] + invalid_request = {"items": {item_id: invalid_stac_item}, "method": "upsert"} + + response = await api_client.post( + bulk_endpoint.format(collection_id), json=invalid_request + ) + assert response.status_code == 422 + + async def test_post_valid_bulk_items( + self, api_client, valid_stac_item, collection_in_db + ): + """ + Test the API's response to posting valid bulk STAC items. + + Asserts that the response status code is 200. + """ + item_id = valid_stac_item["id"] + collection_id = valid_stac_item["collection"] + valid_request = {"items": {item_id: valid_stac_item}, "method": "upsert"} + + response = await api_client.post( + bulk_endpoint.format(collection_id), json=valid_request + ) + assert response.status_code == 200 + + async def test_get_collection_by_id(self, api_client, collection_in_db): + """ + Test searching for a specific collection by its ID. + """ + # The `collection_in_db` fixture ensures the collection exists and provides its ID. + collection_id = collection_in_db + + # Perform a GET request to the /collections endpoint with an "ids" query + response = await api_client.get( + collections_endpoint, params={"ids": collection_id} + ) + + assert response.status_code == 200 + + response_data = response.json() + + assert response_data["collections"][0]["id"] == collection_id + + async def test_collection_freetext_search_by_title( + self, api_client, valid_stac_collection, collection_in_db + ): + """ + Test free-text search for a collection using a word from its title. + """ + + # The `collection_in_db` fixture ensures the collection exists. + collection_id = collection_in_db + + # Use a unique word from the collection's title for the query. + search_term = "precipitation" + + # Perform a GET request with the `q` free-text search parameter. + response = await api_client.get(collections_endpoint, params={"q": search_term}) + + assert response.status_code == 200 + response_data = response.json() + + assert len(response_data["collections"]) > 0 + + returned_ids = [col["id"] for col in response_data["collections"]] + assert collection_id in returned_ids diff --git a/stac_api/runtime/tests/test_transactions.py b/stac_api/runtime/tests/test_transactions.py index ed8bcacf..8ff18e81 100644 --- a/stac_api/runtime/tests/test_transactions.py +++ b/stac_api/runtime/tests/test_transactions.py @@ -1,14 +1,20 @@ """ -Test suite for STAC (SpatioTemporal Asset Catalog) Transactions API endpoints. +Test suite for STAC (SpatioTemporal Asset Catalog) Extensions including Transactions and Collections Search API endpoints. This module contains tests for the collection and item endpoints of the STAC API. It verifies the behavior of the API when posting valid and invalid STAC collections and items, as well as bulk items. Endpoints tested: +Transactions - /collections - /collections/{}/items - /collections/{}/bulk_items + +Collection Search +This module adds search options to collections GET method +- /Collections search by id and free text search + """ @@ -128,7 +134,7 @@ async def test_get_collection_by_id(self, api_client, collection_in_db): assert response_data["collections"][0]["id"] == collection_id async def test_collection_freetext_search_by_title( - self, api_client, valid_stac_collection, collection_in_db + self, api_client, collection_in_db ): """ Test free-text search for a collection using a word from its title. From 23d3b936e8d8fc705ee06754ac74b654262030a1 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 13 Aug 2025 17:08:29 -0600 Subject: [PATCH 23/24] remove deleted test file --- stac_api/runtime/tests/test_extensions.py | 10 +- stac_api/runtime/tests/test_transactions.py | 158 -------------------- 2 files changed, 8 insertions(+), 160 deletions(-) delete mode 100644 stac_api/runtime/tests/test_transactions.py diff --git a/stac_api/runtime/tests/test_extensions.py b/stac_api/runtime/tests/test_extensions.py index ed8bcacf..8ff18e81 100644 --- a/stac_api/runtime/tests/test_extensions.py +++ b/stac_api/runtime/tests/test_extensions.py @@ -1,14 +1,20 @@ """ -Test suite for STAC (SpatioTemporal Asset Catalog) Transactions API endpoints. +Test suite for STAC (SpatioTemporal Asset Catalog) Extensions including Transactions and Collections Search API endpoints. This module contains tests for the collection and item endpoints of the STAC API. It verifies the behavior of the API when posting valid and invalid STAC collections and items, as well as bulk items. Endpoints tested: +Transactions - /collections - /collections/{}/items - /collections/{}/bulk_items + +Collection Search +This module adds search options to collections GET method +- /Collections search by id and free text search + """ @@ -128,7 +134,7 @@ async def test_get_collection_by_id(self, api_client, collection_in_db): assert response_data["collections"][0]["id"] == collection_id async def test_collection_freetext_search_by_title( - self, api_client, valid_stac_collection, collection_in_db + self, api_client, collection_in_db ): """ Test free-text search for a collection using a word from its title. diff --git a/stac_api/runtime/tests/test_transactions.py b/stac_api/runtime/tests/test_transactions.py deleted file mode 100644 index 8ff18e81..00000000 --- a/stac_api/runtime/tests/test_transactions.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Test suite for STAC (SpatioTemporal Asset Catalog) Extensions including Transactions and Collections Search API endpoints. - -This module contains tests for the collection and item endpoints of the STAC API. -It verifies the behavior of the API when posting valid and invalid STAC collections and items, -as well as bulk items. - -Endpoints tested: -Transactions -- /collections -- /collections/{}/items -- /collections/{}/bulk_items - -Collection Search -This module adds search options to collections GET method -- /Collections search by id and free text search - -""" - - -collections_endpoint = "/collections" -items_endpoint = "/collections/{}/items" -bulk_endpoint = "/collections/{}/bulk_items" - - -class TestList: - """ - Test cases for STAC API's collection and item endpoints. - - This class contains tests to ensure that the STAC API correctly handles - posting valid and invalid STAC collections and items, both individually - and in bulk. It uses pytest fixtures to set up the test environment with - necessary data. - """ - - async def test_post_invalid_collection(self, api_client, invalid_stac_collection): - """ - Test the API's response to posting an invalid STAC collection. - - Asserts that the response status code is 422 and the detail - is "Validation Error". - """ - response = await api_client.post( - collections_endpoint, json=invalid_stac_collection - ) - assert response.json()["detail"] == "Validation Error" - assert response.status_code == 422 - - async def test_post_valid_collection(self, api_client, valid_stac_collection): - """ - Test the API's response to posting a valid STAC collection. - - Asserts that the response status code is 200. - """ - response = await api_client.post( - collections_endpoint, json=valid_stac_collection - ) - assert response.status_code == 201 - - async def test_post_invalid_item(self, api_client, invalid_stac_item): - """ - Test the API's response to posting an invalid STAC item. - - Asserts that the response status code is 422 and the detail - is "Validation Error". - """ - collection_id = invalid_stac_item["collection"] - response = await api_client.post( - items_endpoint.format(collection_id), json=invalid_stac_item - ) - assert response.json()["detail"] == "Validation Error" - assert response.status_code == 422 - - async def test_post_valid_item(self, api_client, valid_stac_item, collection_in_db): - """ - Test the API's response to posting a valid STAC item. - - Asserts that the response status code is 200. - """ - collection_id = valid_stac_item["collection"] - response = await api_client.post( - items_endpoint.format(collection_id), json=valid_stac_item - ) - assert response.status_code == 201 - - async def test_post_invalid_bulk_items(self, api_client, invalid_stac_item): - """ - Test the API's response to posting invalid bulk STAC items. - - Asserts that the response status code is 422. - """ - item_id = invalid_stac_item["id"] - collection_id = invalid_stac_item["collection"] - invalid_request = {"items": {item_id: invalid_stac_item}, "method": "upsert"} - - response = await api_client.post( - bulk_endpoint.format(collection_id), json=invalid_request - ) - assert response.status_code == 422 - - async def test_post_valid_bulk_items( - self, api_client, valid_stac_item, collection_in_db - ): - """ - Test the API's response to posting valid bulk STAC items. - - Asserts that the response status code is 200. - """ - item_id = valid_stac_item["id"] - collection_id = valid_stac_item["collection"] - valid_request = {"items": {item_id: valid_stac_item}, "method": "upsert"} - - response = await api_client.post( - bulk_endpoint.format(collection_id), json=valid_request - ) - assert response.status_code == 200 - - async def test_get_collection_by_id(self, api_client, collection_in_db): - """ - Test searching for a specific collection by its ID. - """ - # The `collection_in_db` fixture ensures the collection exists and provides its ID. - collection_id = collection_in_db - - # Perform a GET request to the /collections endpoint with an "ids" query - response = await api_client.get( - collections_endpoint, params={"ids": collection_id} - ) - - assert response.status_code == 200 - - response_data = response.json() - - assert response_data["collections"][0]["id"] == collection_id - - async def test_collection_freetext_search_by_title( - self, api_client, collection_in_db - ): - """ - Test free-text search for a collection using a word from its title. - """ - - # The `collection_in_db` fixture ensures the collection exists. - collection_id = collection_in_db - - # Use a unique word from the collection's title for the query. - search_term = "precipitation" - - # Perform a GET request with the `q` free-text search parameter. - response = await api_client.get(collections_endpoint, params={"q": search_term}) - - assert response.status_code == 200 - response_data = response.json() - - assert len(response_data["collections"]) > 0 - - returned_ids = [col["id"] for col in response_data["collections"]] - assert collection_id in returned_ids From 11d1d688bde6212e108642f945b4436f0d5ed9e0 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 24 Mar 2026 13:09:30 -0700 Subject: [PATCH 24/24] fix: adjust lifespan handling --- raster_api/runtime/handler.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/raster_api/runtime/handler.py b/raster_api/runtime/handler.py index a357f53d..302c9607 100644 --- a/raster_api/runtime/handler.py +++ b/raster_api/runtime/handler.py @@ -17,17 +17,13 @@ logging.getLogger("mangum.http").setLevel(logging.ERROR) -@app.on_event("startup") -async def startup_event() -> None: - """Connect to database on startup.""" - await connect_to_db(app, settings=settings.load_postgres_settings(), pool_kwargs={}) - - handler = Mangum(app, lifespan="off", api_gateway_base_path=app.root_path) if "AWS_EXECUTION_ENV" in os.environ: loop = asyncio.get_event_loop() - loop.run_until_complete(app.router.startup()) + loop.run_until_complete( + connect_to_db(app, settings=settings.load_postgres_settings(), pool_kwargs={}) + ) # Add tracing handler.__name__ = "handler" # tracer requires __name__ to be set