diff --git a/.github/workflows/dsBase_test_suite.yaml b/.github/workflows/dsBase_test_suite.yaml new file mode 100644 index 00000000..e951ca3a --- /dev/null +++ b/.github/workflows/dsBase_test_suite.yaml @@ -0,0 +1,211 @@ +################################################################################ +# DataSHIELD GHA test suite - dsBase +# Adapted from `azure-pipelines.yml` by Roberto Villegas-Diaz +# +# Inside the root directory $(Pipeline.Workspace) will be a file tree like: +# /dsBase <- Checked out version of datashield/dsBase +# /dsBase/logs <- Where results of tests and logs are collated +# /testStatus <- Checked out version of datashield/testStatus +# +# As of Jul 2025 this takes ~ 9 mins to run. +################################################################################ +name: dsBase tests' suite + +on: + push: + branches: + - master + - v6.3.3-dev + + schedule: + - cron: '0 0 * * 0' # Weekly on master + - cron: '0 1 * * *' # Nightly on v6.3.3-dev + +jobs: + dsBase_test_suite: + runs-on: ubuntu-latest + timeout-minutes: 120 + + # These should all be constant, except TEST_FILTER. This can be used to test + # subsets of test files in the testthat directory. Options are like: + # '*' <- Run all tests. + # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests. + # '*_smk_*' <- Run all the smoke tests for all functions. + env: + TEST_FILTER: '*' + _r_check_system_clock_: 0 + WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }} + PROJECT_NAME: dsBase + BRANCH_NAME: ${{ github.ref_name }} + REPO_OWNER: ${{ github.repository_owner }} + R_KEEP_PKG_SOURCE: yes + + steps: + - name: Checkout dsBase + uses: actions/checkout@v4 + with: + path: dsBase + + - name: Checkout testStatus + uses: actions/checkout@v4 + with: + repository: ${{ env.REPO_OWNER }}/testStatus + token: ${{ secrets.GH_TOKEN }} + ref: master + path: testStatus + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: | + any::rcmdcheck + cran::devtools + cran::git2r + cran::RCurl + cran::readr + cran::magrittr + cran::xml2 + cran::purrr + cran::dplyr + cran::stringr + cran::tidyr + cran::quarto + cran::knitr + cran::kableExtra + cran::rmarkdown + cran::downlit + needs: check + + - name: Check man files up-to-date + run: | + orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) + R -e "devtools::document()" + new_sum=$(find man -type f | sort -u | xargs cat | md5sum) + if [ "$orig_sum" != "$new_sum" ]; then + echo "Your committed manual files (man/*.Rd) are out of sync with the R files. Run devtools::document() locally then commit." + exit 1 + else + echo "Documentation up-to-date." + fi + working-directory: dsBase + continue-on-error: true + + - name: Run devtools::check + run: | + R -q -e "library('devtools'); devtools::check(args = c('--no-tests', '--no-examples'))" | tee ../check.Rout + grep -q "^0 errors" ../check.Rout && grep -q " 0 warnings" ../check.Rout && grep -q " 0 notes" ../check.Rout + working-directory: dsBase + continue-on-error: true + + - name: Run tests with coverage & JUnit report + run: | + mkdir -p logs + R -q -e "devtools::reload();" + R -q -e ' + write.csv( + covr::coverage_to_list( + covr::package_coverage( + type = c("none"), + code = c('"'"' + output_file <- file("test_console_output.txt"); + sink(output_file); + sink(output_file, type = "message"); + junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml")); + progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); + multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' + ) + ) + ), + "coveragelist.csv" + )' + + mv coveragelist.csv logs/ + mv test_* logs/ + grep -q " FAIL 0 " logs/test_console_output.txt + working-directory: dsBase + + - name: Check for JUnit errors + run: | + issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true) + echo "Number of testsuites with issues: $issue_count" + sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true + cat issues.log || true + exit $issue_count + working-directory: dsBase/logs + + - name: Write versions to file + run: | + echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt + echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt + echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt + working-directory: dsBase/logs + + - name: Parse results from testthat and covr + run: | + Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ + working-directory: dsBase + + - name: Commit results to testStatus + # if: github.repository == 'villegar/dsBase' && github.event_name != 'pull_request' + run: | + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + cd testStatus + git checkout master + git pull + + mkdir -p logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ + # clear the latest directory + rm -rf docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/* + + # Copy logs to new logs directory location + cp -rv ../dsBase/logs/* logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + cp -rv ../dsBase/logs/${{ env.WORKFLOW_ID }}.txt logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + + # Create symbolic links + ln -sf ${{ env.WORKFLOW_ID }}/ logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/.LATEST + # ln -sf docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/latest + + R -e 'input_dir <- file.path("../logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' + mv source/test_report.html docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html + cp -r docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest + + git add . + git commit -m "Auto test for ${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }} @ ${{ env.WORKFLOW_ID }}" + git push + + env: + PROJECT_NAME: ${{ env.PROJECT_NAME }} + BRANCH_NAME: ${{ env.BRANCH_NAME }} + WORKFLOW_ID: ${{ env.WORKFLOW_ID }} + + - name: Dump environment info + run: | + echo -e "\n#############################" + echo -e "ls /: ######################" + ls -al . + echo -e "\n#############################" + echo -e "lscpu: ######################" + lscpu + echo -e "\n#############################" + echo -e "memory: #####################" + free -m + echo -e "\n#############################" + echo -e "env: ########################" + env + echo -e "\n#############################" + echo -e "R sessionInfo(): ############" + R -e 'sessionInfo()' + sudo apt install tree -y + tree . + diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml new file mode 100644 index 00000000..bfc9f4db --- /dev/null +++ b/.github/workflows/pkgdown.yaml @@ -0,0 +1,49 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + release: + types: [published] + workflow_dispatch: + +name: pkgdown.yaml + +permissions: read-all + +jobs: + pkgdown: + runs-on: ubuntu-latest + # Only restrict concurrency for non-PR jobs + concurrency: + group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgdown, local::. + needs: website + + - name: Build site + run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) + shell: Rscript {0} + + - name: Deploy to GitHub pages 🚀 + if: github.event_name != 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4.5.0 + with: + clean: false + branch: gh-pages + folder: docs diff --git a/README.md b/README.md index aac29667..de46eade 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,35 @@ -dsBase -====== +## dsBase: 'DataSHIELD' Server Side Base Functions -DataSHIELD server side base R library. +[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) +[![](https://www.r-pkg.org/badges/version/dsBase?color=black)](https://cran.r-project.org/package=dsBase) +[![R build +status](https://github.com/datashield/dsBase/workflows/R-CMD-check/badge.svg)](https://github.com/datashield/dsBase/actions) -[![License](https://img.shields.io/badge/license-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) +## Installation -About -===== +You can install the released version of dsBase from +[CRAN](https://cran.r-project.org/package=dsBase) with: + +``` r +install.packages("dsBase") +``` + +And the development version from +[GitHub](https://github.com/datashield/dsBase/) with: + + +``` r +install.packages("remotes") +remotes::install_github("datashield/dsBase", "") + +# Install v6.4.0 with the following +remotes::install_github("datashield/dsBase", "v6.4.0-dev") +``` + +For a full list of development branches, checkout https://github.com/datashield/dsBase/branches + + +## About DataSHIELD is a software package which allows you to do non-disclosive federated analysis on sensitive data. Our website (https://www.datashield.org) has in depth descriptions of what it is, how it works and how to install it. A key point to highlight is that DataSHIELD has a client-server infrastructure, so the dsBase package (https://github.com/datashield/dsBase) needs to be used in conjuction with the dsBaseClient package (https://github.com/datashield/dsBaseClient) - trying to use one without the other makes no sense. @@ -18,3 +41,30 @@ Detailed instructions on how to install DataSHIELD are at https://wiki.datashiel | obiba CRAN | Where you probably should install DataSHIELD from. | | releases | Stable releases. | | master branch | Mostly in sync with the latest release, changes rarely. | + +## References + +[1] Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, + Avraam D, Marcon Y, Bishop T, Gaye A, Escribà Montagut X, Wheater S (2025). _dsBase: 'DataSHIELD' Server Side Base Functions_. R + package version 6.3.3. . + +[2] Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio + M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, + Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, + Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, + Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, + Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). + “DataSHIELD: taking the analysis to the data, not the data to the analysis.” _International + Journal of Epidemiology_, *43*(6), 1929-1944. . + +[3] Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). + “DataSHIELD – New Directions and Dimensions.” _Data Science Journal_, *16*(21), 1-21. + . + +[4] Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, + Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, + Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, + Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, + Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). “DataSHIELD: + mitigating disclosure risk in a multi-site federated analysis platform.” _Bioinformatics + Advances_, *5*(1), 1-21. . diff --git a/docs/authors.html b/docs/authors.html index 345f4351..70570f2a 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -88,16 +88,62 @@

Citation

-

Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Wheater S (2025). +

Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Wheater S (????). dsBase: 'DataSHIELD' Server Site Base Functions. R package version 6.3.3. 

@Manual{,
   title = {dsBase: 'DataSHIELD' Server Site Base Functions},
   author = {Paul Burton and Rebecca Wilson and Olly Butters and Patricia Ryser-Welch and Alex Westerberg and Leire Abarrategui and Roberto Villegas-Diaz and Demetris Avraam and Yannick Marcon and Stuart Wheater},
-  year = {2025},
   note = {R package version 6.3.3},
 }
+

Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). +“DataSHIELD: taking the analysis to the data, not the data to the analysis.” +International Journal of Epidemiology, 43(6), 1929–1944. +doi:10.1093/ije/dyu188. +

+ 
@Article{,
+  title = {{DataSHIELD: taking the analysis to the data, not the data to the analysis}},
+  author = {Amadou Gaye and Yannick Marcon and Julia Isaeva and Philippe {LaFlamme} and Andrew Turner and Elinor M Jones and Joel Minion and Andrew W Boyd and Christopher J Newby and Marja-Liisa Nuotio and Rebecca Wilson and Oliver Butters and Barnaby Murtagh and Ipek Demir and Dany Doiron and Lisette Giepmans and Susan E Wallace and Isabelle Budin-Lj{\o}sne and Carsten {{Oliver Schmidt}} and Paolo Boffetta and Mathieu Boniol and Maria Bota and Kim W Carter and Nick {deKlerk} and Chris Dibben and Richard W Francis and Tero Hiekkalinna and Kristian Hveem and Kirsti Kval{\o}y and Sean Millar and Ivan J Perry and Annette Peters and Catherine M Phillips and Frank Popham and Gillian Raab and Eva Reischl and Nuala Sheehan and Melanie Waldenberger and Markus Perola and Edwin {{van den Heuvel}} and John Macleod and Bartha M Knoppers and Ronald P Stolk and Isabel Fortier and Jennifer R Harris and Bruce H R Woffenbuttel and Madeleine J Murtagh and Vincent Ferretti and Paul R Burton},
+  journal = {International Journal of Epidemiology},
+  year = {2014},
+  volume = {43},
+  number = {6},
+  pages = {1929--1944},
+  doi = {10.1093/ije/dyu188},
+}
+

Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). +“DataSHIELD – New Directions and Dimensions.” +Data Science Journal, 16(21), 1–21. +doi:10.5334/dsj-2017-021. +

+ 
@Article{,
+  title = {{DataSHIELD – New Directions and Dimensions}},
+  author = {Rebecca C. Wilson and Oliver {W. Butters} and Demetris Avraam and James Baker and Jonathan A. Tedds and Andrew Turner and Madeleine Murtagh and Paul {R. Burton}},
+  journal = {Data Science Journal},
+  year = {2017},
+  volume = {16},
+  number = {21},
+  pages = {1--21},
+  doi = {10.5334/dsj-2017-021},
+}
+

Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). +“DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform.” +Bioinformatics Advances, 5(1), 1–21. +doi:10.1093/bioadv/vbaf046. +

+ 
@Article{,
+  title = {{DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform}},
+  author = {Demetris Avraam and Rebecca C Wilson and Noemi {{Aguirre Chan}} and Soumya Banerjee and Tom R P Bishop and Olly Butters and Tim Cadman and Luise Cederkvist and Liesbeth Duijts and Xavier {{Escrib{\a`a} Montagut}} and Hugh Garner and Gon{\c c}alo {Gon{\c c}alves} and Juan R Gonz{\a'a}lez and Sido Haakma and Mette Hartlev and Jan Hasenauer and Manuel Huth and Eleanor Hyde and Vincent W V Jaddoe and Yannick Marcon and Michaela Th Mayrhofer and Fruzsina Molnar-Gabor and Andrei Scott Morgan and Madeleine Murtagh and Marc Nestor and Anne-Marie {{Nybo Andersen}} and Simon Parker and Angela {{Pinot de Moira}} and Florian Schwarz and Katrine Strandberg-Larsen and {Morris AvSwertz} and Marieke Welten and Stuart Wheater and Paul Burton},
+  journal = {Bioinformatics Advances},
+  year = {2024},
+  volume = {5},
+  number = {1},
+  pages = {1--21},
+  doi = {10.1093/bioadv/vbaf046},
+  editor = {Thomas Lengauer},
+  publisher = {Oxford University Press (OUP)},
+}
diff --git a/docs/index.html b/docs/index.html index 54bbe6e5..ca9ac5a7 100644 --- a/docs/index.html +++ b/docs/index.html @@ -12,7 +12,7 @@ - +

+
+install.packages("remotes")
+remotes::install_github("datashield/dsBase", "<BRANCH>")
+
+# Install v6.4.0 with the following
+remotes::install_github("datashield/dsBase", "v6.4.0-dev")
+

For a full list of development branches, checkout https://github.com/datashield/dsBase/branches

+ +
+

About +

DataSHIELD is a software package which allows you to do non-disclosive federated analysis on sensitive data. Our website (https://www.datashield.org) has in depth descriptions of what it is, how it works and how to install it. A key point to highlight is that DataSHIELD has a client-server infrastructure, so the dsBase package (https://github.com/datashield/dsBase) needs to be used in conjuction with the dsBaseClient package (https://github.com/datashield/dsBaseClient) - trying to use one without the other makes no sense.

Detailed instructions on how to install DataSHIELD are at https://wiki.datashield.org/. The code here is organised as:

@@ -72,31 +86,46 @@

About

- + - + - + - +
Location What is it?
obiba CRAN Where you probably should install DataSHIELD from.
releases Stable releases.
master branch Mostly in sync with the latest release, changes rarely.
+
+
+

References +

+

[1] Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Bishop T, Gaye A, Escribà Montagut X, Wheater S (2025). dsBase: ‘DataSHIELD’ Server Side Base Functions. R package version 6.3.3. https://doi.org/10.32614/CRAN.package.dsBase.

+

[2] Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). “DataSHIELD: taking the analysis to the data, not the data to the analysis.” International Journal of Epidemiology, 43(6), 1929-1944. https://doi.org/10.1093/ije/dyu188.

+

[3] Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). “DataSHIELD – New Directions and Dimensions.” Data Science Journal, 16(21), 1-21. https://doi.org/10.5334/dsj-2017-021.

+

[4] Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). “DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform.” Bioinformatics Advances, 5(1), 1-21. https://doi.org/10.1093/bioadv/vbaf046.