Fix tests

.github/workflows/R_CMD_check_Hades.yaml

@@ -20,6 +20,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
+          - {os: windows-latest, r: 'release'}
+          - {os: macOS-latest, r: 'release'}
           - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
 
     env:
@@ -63,7 +65,7 @@ jobs:
       CDM_BIG_QUERY_OHDSI_SCHEMA: ${{ secrets.CDM_BIG_QUERY_OHDSI_SCHEMA }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Java
         if: runner.os != 'Linux'
@@ -103,22 +105,37 @@ jobs:
 
       - name: Upload source package
         if: success() && runner.os == 'macOS' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: package_tarball
           path: check/*.tar.gz
 
       - name: Install covr
-        if: runner.os == 'macOS'
+        if: runner.os == 'Linux'
         run: |
-          install.packages("covr")
+          remotes::install_cran("covr")
+          remotes::install_cran("xml2")
         shell: Rscript {0}
-
+        
       - name: Test coverage
-        if: runner.os == 'macOS'
-        run: covr::codecov()
+        if: runner.os == 'Linux'
+        run: |
+          cov <- covr::package_coverage(
+            quiet = FALSE,
+            clean = FALSE,
+            install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
+          )
+          covr::to_cobertura(cov)
         shell: Rscript {0}
 
+      - uses: codecov/codecov-action@v4
+        if: runner.os == 'Linux'
+        with:
+          file: ./cobertura.xml
+          plugin: noop
+          disable_search: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   Release:
     needs: R-CMD-Check
 
@@ -131,7 +148,7 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -173,7 +190,7 @@ jobs:
 
       - name: Download package tarball
         if: ${{ env.new_version != '' }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4.1.7
         with:
           name: package_tarball
 

.github/workflows/R_CMD_check_main_weekly.yaml

@@ -52,7 +52,7 @@ jobs:
       CDM5_SPARK_OHDSI_SCHEMA: ${{ secrets.CDM5_SPARK_OHDSI_SCHEMA }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Java
         uses: actions/setup-java@v4

.gitignore

@@ -5,4 +5,6 @@ R/LocalEnvironment.R
 /bin/
 /DatabaseConnector.iml
 .idea/
-errorReportSql.txt
+errorReportSql.txt
+/work/
+.settings/org.eclipse.core.resources.prefs

CRAN-SUBMISSION

@@ -1,3 +1,3 @@
-Version: 6.3.2
-Date: 2023-12-11 13:54:30 UTC
-SHA: dd511ec8b23927ffb61a17bedd9ee8bb81cbe476
+Version: 6.4.0
+Date: 2025-01-30 11:09:57 UTC
+SHA: 168b5dbd79128199da65551f84b5d128c1f08d67

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: DatabaseConnector
 Type: Package
 Title: Connecting to Various Database Platforms
-Version: 6.3.3.9000
-Date: 2024-06-13
+Version: 6.4.0
+Date: 2025-01-30
 Authors@R: c(
     person("Martijn", "Schuemie", email = "schuemie@ohdsi.org", role = c("aut", "cre")),
     person("Marc", "Suchard", role = c("aut")),
@@ -13,14 +13,15 @@ Authors@R: c(
     person("Amazon Inc.", role = c("cph"), comment = "RedShift  JDBC driver")
   )
 Description: An R 'DataBase Interface' ('DBI') compatible interface to various database platforms ('PostgreSQL', 'Oracle', 'Microsoft SQL Server', 
-    'Amazon Redshift', 'Microsoft Parallel Database Warehouse', 'IBM Netezza', 'Apache Impala', 'Google BigQuery', 'Snowflake', 'Spark', and 'SQLite'). Also includes support for
-    fetching data as 'Andromeda' objects. Uses either 'Java Database Connectivity' ('JDBC') or other 'DBI' drivers to connect to databases.
+    'Amazon Redshift', 'Microsoft Parallel Database Warehouse', 'IBM Netezza', 'Apache Impala', 'Google BigQuery', 'Snowflake', 'Spark', 'SQLite', 
+    and 'InterSystems IRIS'). Also includes support for fetching data as 'Andromeda' objects. Uses either 'Java Database Connectivity' ('JDBC') or 
+    other 'DBI' drivers to connect to databases.
 SystemRequirements: Java (>= 8)
 Depends:
     R (>= 4.0.0)
 Imports:
     rJava,
-    SqlRender (>= 1.16.0),
+    SqlRender (>= 1.19.1),
     methods,
     stringr,
     readr,
@@ -47,8 +48,10 @@ Suggests:
     RPostgres,
     odbc,
     duckdb,
+    bigrquery,
     pool,
-    ParallelLogger
+    ParallelLogger,
+    AzureStor
 License: Apache License
 VignetteBuilder: knitr
 URL: https://ohdsi.github.io/DatabaseConnector/, https://github.com/OHDSI/DatabaseConnector

NAMESPACE

@@ -1,10 +1,20 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(compileReconnectCode,DatabaseConnectorDbiConnection)
+S3method(compileReconnectCode,default)
 S3method(dbplyr_edition,DatabaseConnectorConnection)
 S3method(disconnect,DatabaseConnectorDbiConnection)
 S3method(disconnect,default)
+S3method(getCatalogs,DatabaseConnectorDbiConnection)
+S3method(getCatalogs,default)
+S3method(getSchemaNames,DatabaseConnectorDbiConnection)
+S3method(getSchemaNames,default)
+S3method(getServer,DatabaseConnectorDbiConnection)
+S3method(getServer,default)
 S3method(insertTable,DatabaseConnectorDbiConnection)
 S3method(insertTable,default)
+S3method(listDatabaseConnectorColumns,DatabaseConnectorDbiConnection)
+S3method(listDatabaseConnectorColumns,default)
 S3method(lowLevelExecuteSql,DatabaseConnectorDbiConnection)
 S3method(lowLevelExecuteSql,default)
 S3method(lowLevelQuerySql,DatabaseConnectorDbiConnection)
@@ -13,6 +23,7 @@ S3method(lowLevelQuerySqlToAndromeda,DatabaseConnectorDbiConnection)
 S3method(lowLevelQuerySqlToAndromeda,default)
 S3method(renderTranslateQueryApplyBatched,DatabaseConnectorDbiConnection)
 S3method(renderTranslateQueryApplyBatched,default)
+S3method(sql_query_select,DatabaseConnectorJdbcConnection)
 S3method(sql_translation,DatabaseConnectorJdbcConnection)
 export(DatabaseConnectorDriver)
 export(assertTempEmulationSchemaSet)
@@ -77,6 +88,7 @@ import(methods)
 import(rJava)
 importFrom(bit64,integer64)
 importFrom(dbplyr,dbplyr_edition)
+importFrom(dbplyr,sql_query_select)
 importFrom(dbplyr,sql_translation)
 importFrom(rlang,abort)
 importFrom(rlang,inform)

NEWS.md

@@ -1,3 +1,22 @@
+DatabaseConnector 6.4.1
+=======================
+
+Bugfixes:
+
+- Remove direct call to `bit64` S3 method to avoid issues in the future.
+
+- Fixed error when calling `getTableNames()` on a `DuckDB` connection.
+
+
+DatabaseConnector 6.4.0
+=======================
+
+Changes:
+
+- Adding support for InterSystems IRIS.
+
+
+
 DatabaseConnector 6.3.3
 =======================
 
@@ -7,6 +26,10 @@ Changes:
 
 - Updated Databricks driver to 2.6.36.
 
+- Updated BigQuery driver to 1.6.2.
+
+- Using `INSERT` with multiple values in `insertTable()` for DataBricks for faster inserts.
+
 
 DatabaseConnector 6.3.2
 =======================

R/Andromeda.R

@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

R/BulkLoad.R

@@ -1,6 +1,6 @@
 # @file BulkLoad.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -62,6 +62,25 @@
       return(FALSE)
     }
     return(TRUE)
+  } else if (dbms(connection) == "spark") {
+    envSet <- FALSE
+    container <- FALSE
+
+    if (Sys.getenv("AZR_STORAGE_ACCOUNT") != "" && Sys.getenv("AZR_ACCOUNT_KEY") != "" && Sys.setenv("AZR_CONTAINER_NAME") != "") {
+      envSet <- TRUE
+    }
+
+    # List storage containers to confirm the container
+    # specified in the configuration exists
+    ensure_installed("AzureStor")
+    azureEndpoint <- getAzureEndpoint()
+    containerList <- getAzureContainerNames(azureEndpoint)
+
+    if (Sys.getenv("AZR_CONTAINER_NAME") %in% containerList) {
+      container <- TRUE
+    }
+
+    return(envSet & container)
   } else {
     return(FALSE)
   }
@@ -72,6 +91,18 @@
   return(if (sshUser == "") "root" else sshUser)
 }
 
+getAzureEndpoint <- function() {
+  azureEndpoint <- AzureStor::storage_endpoint(
+    paste0("https://", Sys.getenv("AZR_STORAGE_ACCOUNT"), ".dfs.core.windows.net"),
+    key = Sys.getenv("AZR_ACCOUNT_KEY")
+  )
+  return(azureEndpoint)  
+}
+
+getAzureContainerNames <- function(azureEndpoint) {
+  return(names(AzureStor::list_storage_containers(azureEndpoint)))
+}
+
 countRows <- function(connection, sqlTableName) {
   sql <- "SELECT COUNT(*) FROM @table"
   count <- renderTranslateQuerySql(
@@ -299,7 +330,19 @@
   readr::write_excel_csv(data, csvFileName, na = "")
   on.exit(unlink(csvFileName))
 
-  hostServerDb <- strsplit(attr(connection, "server")(), "/")[[1]]
+  server <- attr(connection, "server")()
+  if (is.null(server)) {
+    # taken directly from DatabaseConnector R/RStudio.R - getServer.default, could an attr too?
+    databaseMetaData <- rJava::.jcall(
+      connection@jConnection,
+      "Ljava/sql/DatabaseMetaData;",
+      "getMetaData"
+    )
+    server <- rJava::.jcall(databaseMetaData, "Ljava/lang/String;", "getURL")
+    server <- strsplit(server, "//")[[1]][2]
+  }
+
+  hostServerDb <- strsplit(server, "/")[[1]]
   port <- attr(connection, "port")()
   user <- attr(connection, "user")()
   password <- attr(connection, "password")()
@@ -342,3 +385,53 @@
   delta <- Sys.time() - startTime
   inform(paste("Bulk load to PostgreSQL took", signif(delta, 3), attr(delta, "units")))
 }
+
+bulkLoadSpark <- function(connection, sqlTableName, data) {
+  ensure_installed("AzureStor")
+  logTrace(sprintf("Inserting %d rows into table '%s' using DataBricks bulk load", nrow(data), sqlTableName))
+  start <- Sys.time()
+
+  csvFileName <- tempfile("spark_insert_", fileext = ".csv")
+  write.csv(x = data, na = "", file = csvFileName, row.names = FALSE, quote = TRUE)
+  on.exit(unlink(csvFileName))
+
+  azureEndpoint <- getAzureEndpoint()
+  containers <- AzureStor::list_storage_containers(azureEndpoint)
+  targetContainer <- containers[[Sys.getenv("AZR_CONTAINER_NAME")]]
+  AzureStor::storage_upload(
+    targetContainer, 
+    src=csvFileName, 
+    dest=csvFileName
+  )  
+
+  on.exit(
+    AzureStor::delete_storage_file(
+      targetContainer, 
+      file = csvFileName,
+      confirm = FALSE
+    ),
+    add = TRUE
+  )
+
+  sql <- SqlRender::loadRenderTranslateSql(
+    sqlFilename = "sparkCopy.sql",
+    packageName = "DatabaseConnector",
+    dbms = "spark",
+    sqlTableName = sqlTableName,
+    fileName = basename(csvFileName),
+    azureAccountKey = Sys.getenv("AZR_ACCOUNT_KEY"),
+    azureStorageAccount = Sys.getenv("AZR_STORAGE_ACCOUNT")
+  )
+
+  tryCatch(
+    {
+      DatabaseConnector::executeSql(connection = connection, sql = sql, reportOverallTime = FALSE)
+    },
+    error = function(e) {
+      abort("Error in DataBricks bulk upload. Please check DataBricks/Azure Storage access.")
+    }
+  )
+  delta <- Sys.time() - start
+  inform(paste("Bulk load to DataBricks took", signif(delta, 3), attr(delta, "units")))
+}
+

R/Compression.R

@@ -1,6 +1,6 @@
 # @file InsertTable.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #