From 917671fcf4e51d78f8cf8e1bee7e8c64e4801cfa Mon Sep 17 00:00:00 2001 From: "Justin Singh-M." Date: Tue, 30 Sep 2025 14:38:51 -0700 Subject: [PATCH 1/2] add duckdb functions; better caching of auth creds --- DESCRIPTION | 2 ++ NAMESPACE | 3 +++ R/auth.R | 34 ++++++++++++++++++++++++------ R/duckdb.R | 39 +++++++++++++++++++++++++++++++++++ man/duckdb_connection.Rd | 19 +++++++++++++++++ man/lynker_spatial_auth.Rd | 5 ++++- man/lynker_spatial_refresh.Rd | 6 +++++- man/lynker_spatial_url.Rd | 17 +++++++++++++++ man/tbl_http.Rd | 26 +++++++++++++++++++++++ 9 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 R/duckdb.R create mode 100644 man/duckdb_connection.Rd create mode 100644 man/lynker_spatial_url.Rd create mode 100644 man/tbl_http.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 32ecb47..e3decc2 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -20,6 +20,7 @@ Imports: DBI, dbplyr, dplyr, + duckdb, glue, httr2, igraph, @@ -46,6 +47,7 @@ Collate: 'OGRSQLDriver.R' 'OGRSQLResult.R' 'auth.R' + 'duckdb.R' 'geom.R' 'hfio.R' 'hfutils-package.R' diff --git a/NAMESPACE b/NAMESPACE index 126db2f..bf0dac4 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,11 +12,13 @@ export(add_measures) export(as_ogr) export(clean_geometry) export(collect) +export(duckdb_connection) export(flowpaths_to_linestrings) export(get_hydroseq) export(get_node) export(layer_exists) export(lynker_spatial_auth) +export(lynker_spatial_url) export(node_geometry) export(read_hydrofabric) export(read_sf_dataset) @@ -24,6 +26,7 @@ export(rename_geometry) export(st_as_sf) export(st_read_parquet) export(st_write_parquet) +export(tbl_http) export(union_linestrings) export(union_polygons) export(write_hydrofabric) diff --git a/R/auth.R b/R/auth.R index 2e33767..f8b2d0d 100644 --- a/R/auth.R +++ b/R/auth.R @@ -5,9 +5,13 @@ #' @param libs Supported libraries to configure auth for. #' @param duckdb_con A DuckDB DBI connection to add a bearer token secret to. #' @returns The `token` argument, or a newly provisioned token +#' +#' @details +#' lynker_spatial.auth.token +#' #' @export lynker_spatial_auth <- function( - token = NULL, + token = getOption("lynker_spatial.token"), ..., libs = c("gdal", "duckdb"), duckdb_con = NULL @@ -79,21 +83,39 @@ lynker_spatial_client <- function() { #' @keywords internal lynker_spatial_token <- function(..., client = lynker_spatial_client()) { # Get the token using the OIDC client - httr2::oauth_flow_auth_code( + token <- httr2::oauth_flow_auth_code( client, auth_url = client$provider$authorization_endpoint, scope = "openid profile email phone", redirect_uri = client$redirect_uri, pkce = TRUE ) + + options("lynker_spatial.token" = token) + token } #' Refresh an existing Lynker Spatial token #' @keywords internal -lynker_spatial_refresh <- function(token, ..., client = lynker_spatial_client()) { - if (!inherits(token, "httr2_token")) { +lynker_spatial_refresh <- function(token = getOption("lynker_spatial.token"), ..., client = lynker_spatial_client()) { + if (inherits(token, "httr2_token")) { + refresh_token <- token$refresh_token + } else if (is.character(token)) { + refresh_token <- token + } else { stop("token is malformed", call. = FALSE) } - httr2::oauth_flow_refresh(client, token$refresh_token, scope = "openid profile email phone") -} \ No newline at end of file + new_token <- httr2::oauth_flow_refresh(client, token$refresh_token, scope = "openid profile email phone") + options("lynker_spatial.token" = new_token) + new_token +} + +#' Create an authenticated URL connection for Lynker Spatial +#' @param url URL passed to the connection +#' @returns a URL connection object +#' @export +lynker_spatial_url <- function(url) { + token <- lynker_spatial_auth(libs = "gdal") + url(url, headers = list(Authorization = paste("Bearer", token$id_token))) +} diff --git a/R/duckdb.R b/R/duckdb.R new file mode 100644 index 0000000..2b73bd0 --- /dev/null +++ b/R/duckdb.R @@ -0,0 +1,39 @@ +#' Create a new DuckDB connection. +#' @param ... Arguments passed to [DBI::dbConnect()]. +#' @param extensions Character vector of extensions to install and load on connect. +#' @returns A DBI connection to a DuckDB instance. +#' @export +duckdb_connection <- function(..., extensions = character(0)) { + conn <- DBI::dbConnect(duckdb::duckdb(), ...) + + if (length(extensions) > 0) { + for (ext in extensions) { + DBI::dbExecute(conn, paste("INSTALL", ext)) + DBI::dbExecute(conn, paste("LOAD", ext)) + } + } + + conn +} + +#' Read DuckDB File(s) over HTTP +#' @param urls 1 or more URLs +#' @param ... Unused +#' @param conn A DuckDB connection +#' @param read_func The DuckDB SQL function to call against the list of urls. +#' Defaults to `read_parquet`. +#' @export +tbl_http <- function( + urls, + ..., + conn = duckdb_connection(extensions = "httpfs"), + read_func = c("read_parquet", "read_csv", ) +) { + + # TODO(justin): allow read_func arguments i.e. union_by_name + query <- paste0("SELECT * FROM ", read_func, "([", + paste0("'", urls, "'", collapse = ","), + "])") + + dplyr::tbl(conn, dbplyr::sql(query)) +} diff --git a/man/duckdb_connection.Rd b/man/duckdb_connection.Rd new file mode 100644 index 0000000..f03bab7 --- /dev/null +++ b/man/duckdb_connection.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/duckdb.R +\name{duckdb_connection} +\alias{duckdb_connection} +\title{Create a new DuckDB connection.} +\usage{ +duckdb_connection(..., extensions = character(0)) +} +\arguments{ +\item{...}{Arguments passed to [DBI::dbConnect()].} + +\item{extensions}{Character vector of extensions to install and load on connect.} +} +\value{ +A DBI connection to a DuckDB instance. +} +\description{ +Create a new DuckDB connection. +} diff --git a/man/lynker_spatial_auth.Rd b/man/lynker_spatial_auth.Rd index bc3ae39..f8e9feb 100644 --- a/man/lynker_spatial_auth.Rd +++ b/man/lynker_spatial_auth.Rd @@ -5,7 +5,7 @@ \title{Authenticate with Lynker Spatial} \usage{ lynker_spatial_auth( - token = NULL, + token = getOption("lynker_spatial.token"), ..., libs = c("gdal", "duckdb"), duckdb_con = NULL @@ -27,3 +27,6 @@ The `token` argument, or a newly provisioned token \description{ Authenticate with Lynker Spatial } +\details{ +lynker_spatial.auth.token +} diff --git a/man/lynker_spatial_refresh.Rd b/man/lynker_spatial_refresh.Rd index 850acdb..2bf2517 100644 --- a/man/lynker_spatial_refresh.Rd +++ b/man/lynker_spatial_refresh.Rd @@ -4,7 +4,11 @@ \alias{lynker_spatial_refresh} \title{Refresh an existing Lynker Spatial token} \usage{ -lynker_spatial_refresh(token, ..., client = lynker_spatial_client()) +lynker_spatial_refresh( + token = getOption("lynker_spatial.token"), + ..., + client = lynker_spatial_client() +) } \description{ Refresh an existing Lynker Spatial token diff --git a/man/lynker_spatial_url.Rd b/man/lynker_spatial_url.Rd new file mode 100644 index 0000000..9d4d90a --- /dev/null +++ b/man/lynker_spatial_url.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/auth.R +\name{lynker_spatial_url} +\alias{lynker_spatial_url} +\title{Create an authenticated URL connection for Lynker Spatial} +\usage{ +lynker_spatial_url(url) +} +\arguments{ +\item{url}{URL passed to the connection} +} +\value{ +a URL connection object +} +\description{ +Create an authenticated URL connection for Lynker Spatial +} diff --git a/man/tbl_http.Rd b/man/tbl_http.Rd new file mode 100644 index 0000000..59da579 --- /dev/null +++ b/man/tbl_http.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/duckdb.R +\name{tbl_http} +\alias{tbl_http} +\title{Read DuckDB File(s) over HTTP} +\usage{ +tbl_http( + urls, + ..., + conn = duckdb_connection(extensions = "httpfs"), + read_func = c("read_parquet", "read_csv", ) +) +} +\arguments{ +\item{urls}{1 or more URLs} + +\item{...}{Unused} + +\item{conn}{A DuckDB connection} + +\item{read_func}{The DuckDB SQL function to call against the list of urls. +Defaults to `read_parquet`.} +} +\description{ +Read DuckDB File(s) over HTTP +} From 1838cc93f7a5f9b6c76e4d9e23a6b3a39f00e7cd Mon Sep 17 00:00:00 2001 From: "Justin Singh-M." Date: Tue, 30 Sep 2025 14:47:59 -0700 Subject: [PATCH 2/2] auto authenticate duckdb connections --- R/duckdb.R | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/R/duckdb.R b/R/duckdb.R index 2b73bd0..b2df3cb 100644 --- a/R/duckdb.R +++ b/R/duckdb.R @@ -1,9 +1,10 @@ #' Create a new DuckDB connection. #' @param ... Arguments passed to [DBI::dbConnect()]. #' @param extensions Character vector of extensions to install and load on connect. +#' @param add_auth Include Lynker Spatial authentication. #' @returns A DBI connection to a DuckDB instance. #' @export -duckdb_connection <- function(..., extensions = character(0)) { +duckdb_connection <- function(..., extensions = character(0), add_auth = TRUE) { conn <- DBI::dbConnect(duckdb::duckdb(), ...) if (length(extensions) > 0) { @@ -13,6 +14,10 @@ duckdb_connection <- function(..., extensions = character(0)) { } } + if (add_auth) { + lynker_spatial_auth(libs = c("gdal", "duckdb"), duckdb_con = conn) + } + conn }