diff --git a/.DS_Store b/.DS_Store index a0330d96..6fee0cbf 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/workflows/run-integration-test.yaml b/.github/workflows/run-integration-test.yaml index 8656af63..209aa64b 100644 --- a/.github/workflows/run-integration-test.yaml +++ b/.github/workflows/run-integration-test.yaml @@ -31,12 +31,14 @@ jobs: uses: actions/checkout@v2 with: repository: 2DegreesInvesting/pacta-data + ssh-key: false token: ${{ secrets.MAURO_PAT_FOR_2DII }} path: pacta-data - name: Checkout create_interactive_report uses: actions/checkout@v2 with: repository: 2DegreesInvesting/create_interactive_report + ssh-key: false token: ${{ secrets.MAURO_PAT_FOR_2DII }} path: create_interactive_report @@ -65,4 +67,4 @@ jobs: - name: Run integration test run: | cd PACTA_analysis - Rscript -e "rmarkdown::render('integration-test.Rmd')" + Rscript -e "source('integration-test.R')" diff --git a/.gitignore b/.gitignore index 771a5e61..1f5981a1 100644 --- a/.gitignore +++ b/.gitignore @@ -33,10 +33,16 @@ vignettes/*.pdf *.knit.md .Ruserdata -**/working_dir/30_Processed_Inputs -**/working_dir/40_Results -**/working_dir/50_Outputs +**/working_dir/30_Processed_Inputs/* +!**/working_dir/30_Processed_Inputs/.gitkeep.txt +**/working_dir/40_Results/* +!**/working_dir/40_Results/.gitkeep.txt +**/working_dir/50_Outputs/* +!**/working_dir/50_Outputs/.gitkeep.txt setup.R data/isins_missing_bbg_data.csv data/fund_isins_without_bbg_data.csv + +# Mac files +working_dir/DS_STORE diff --git a/0_fund_processing.R b/0_fund_processing.R deleted file mode 100644 index bacc0aad..00000000 --- a/0_fund_processing.R +++ /dev/null @@ -1,79 +0,0 @@ -library(tidyverse) -library(xml2) - -path <- "/Users/vincentjerosch-herold/Dropbox (2° Investing)/PortCheck/00_Data/03_FundData/Morningstar_RawData/XML_files/FundPool_2019Q4/UniverseFiles/" -index <- read_csv("/Users/vincentjerosch-herold/Dropbox (2° Investing)/PortCheck/00_Data/03_FundData/Morningstar_RawData/Administration/GlobalUniverse/2dii_fundlist.csv") -master <- read_csv("/Users/vincentjerosch-herold/Dropbox (2° Investing)/PortCheck/00_Data/03_FundData/Morningstar_RawData/Administration/GlobalUniverse/Global_FundPool_2019Q4.csv") - - -master_fund_list -portfolio - - - - - -#first cross-reference what isins are associated with funds covered by morningstar. -project_fund_list <- portfolio %>% - distinct(isin) -inner_join(master_fund_list, by = "isin") - -#find out which funds we do not already cover and subset by id -missing_fund_list <- project_fund_list %>% - filter(covered != TRUE) %>% - distinct(id) - -#then create a index of ids we need to load fund data for. -project_fund_list <- project_fund_list %>% - distinct(id) - -#binding to the master list of missing funds and take only distinct ids -#from here we would run the python script that pulls the missing funds and does the funds of funds looping -global_missing_fund_list <- bind_rows(missing_fund_list, global_missing_fund_list) %>% - distinct(id) - - -#every fund holdings are stored as a seperated .csv so we only load what is project relevant! -#each fund will be stored with the columns: Fund.id, Holding.isin, Holding.Weight -for(i in 1:nrow(project_fund_list)) { - - id <- project_fund_list[i, "id"] - - fund <- read_csv(paste0(ANALYSIS.INPUTS.PATH, id, ".csv")) - - extracted_fund_list[[i]] <- fund - -} - -fund_data <- bind_rows(extracted_data_list)#binding all of the files together - -#since we each fund is stored based on the id we have to match these ids to isins in the master_fund_list -#we then use isin as the primary index, which corresponds with the existing process. -fund_data <- master_fund_list %>% - select(id, isin) %>% - right_join(fund_data, by = "id") %>% - select(-id) - - -for(i in 2:nrow(index)) { - Fund_id <- index[i, "Fundid"] - id <- index[i, "id"] - - xml <- read_xml(paste0(path, Fund_id, ".xml")) - - write_xml(xml, paste0(path, id, ".xml")) - - cat(paste0("#", i, "; ")) - - -} - - -master <- read_csv("/Users/vincentjerosch-herold/Dropbox (2° Investing)/PortCheck/00_Data/03_FundData/Morningstar_RawData/Administration/GlobalUniverse/Global_FundPool_2019Q4.csv") - -test <- master %>% - distinct(id) - -temp <- master %>% - left_join(index, by = "id") %>% - filter(!is.na(Fundid)) \ No newline at end of file diff --git a/0_web_functions.R b/0_web_functions.R index f87a953f..5b49ae9d 100644 --- a/0_web_functions.R +++ b/0_web_functions.R @@ -29,12 +29,11 @@ identify_portfolios <- function(portfolio_total){ } -create_portfolio_subfolders <- function(file_names, portfolio_name_ref_all){ +create_portfolio_subfolders <- function(portfolio_name_ref_all){ folders <- c("30_Processed_Inputs", "40_Results", "50_Outputs") locs_to_create <- folders %>% - # purrr::map(~ paste0(project_location, "/", .x, "/", file_names$portfolio_name)) %>% purrr::map(~ paste0(project_location, "/", .x, "/", portfolio_name_ref_all)) %>% flatten_chr() @@ -160,12 +159,17 @@ get_input_files <- function(portfolio_name_ref_all){ input_file_path <- paste0(input_path, input_files[i]) portfolio_name_ref = portfolio_name_ref_all[i] - portfolio_ <- read_web_input_file(input_file_path) portfolio_ <- portfolio_ %>% select(-contains("X")) set_portfolio_parameters(file_path = paste0(par_file_path,"/",portfolio_name_ref,"_PortfolioParameters.yml")) + + # this writes the portfolio and ivestor names that are provided from the parameter file to the pf + # as agreed with Constructiva. They ensure grouped portfolios will get one name only. + portfolio_ <- portfolio_ %>% + mutate(Portfolio.Name = portfolio_name_in, + Investor.Name = investor_name_in) # clean and check column names portfolio_ <- check_input_file_contents(portfolio_, portfolio_name_in, investor_name_in) diff --git a/integration-test.R b/integration-test.R new file mode 100644 index 00000000..f34d11f4 --- /dev/null +++ b/integration-test.R @@ -0,0 +1,235 @@ +## ----setup, include=FALSE-------------------------------------- +knitr::opts_chunk$set( + echo = TRUE, + comment = "#>", + collapse = TRUE, + cache = FALSE +) + + +## -------------------------------------------------------------- +suppressPackageStartupMessages(library(tidyverse)) +suppressPackageStartupMessages(library(devtools)) +suppressPackageStartupMessages(library(testthat)) +suppressPackageStartupMessages(library(config)) +suppressPackageStartupMessages(library(rlang)) +suppressPackageStartupMessages(library(renv)) +suppressPackageStartupMessages(library(glue)) +suppressPackageStartupMessages(library(fs)) +library(here) + + +## -------------------------------------------------------------- +detect_packages <- function() { + packages <- renv::dependencies()$Package + sort(unique(packages)) +} + +detect_packages() + + +## -------------------------------------------------------------- +devtools::session_info() + + +## -------------------------------------------------------------- +file_name <- "TestPortfolio_Input.csv" +example_dataset <- here("sample_files", "20_input_files", file_name) + +expect_true(file_exists(example_dataset)) + + +## -------------------------------------------------------------- +expected_dataset <- here("working_dir", "20_Raw_Inputs", file_name) + +if (file_exists(expected_dataset)) { + warn(glue("Removing existing file: {expected_dataset}")) + file_delete(expected_dataset) +} + +file_copy(example_dataset, expected_dataset) + +expect_true(file_exists(expected_dataset)) + + +## -------------------------------------------------------------- +ensure_empty_directory <- function(directory) { + if (dir_exists(directory)) { + dir_delete(directory) + } + + dir_create(directory) + + invisible(directory) +} + +children <- c("30_Processed_Inputs", "40_Results", "50_Outputs") +(paths <- here("working_dir", children)) + +walk(paths, ensure_empty_directory) + + +## -------------------------------------------------------------- +is_sibling <- function(x) { + parent <- path_dir(here()) + dir_exists(path(parent, x)) +} + +repos <- c("pacta-data", "create_interactive_report", "PACTA_analysis") +all_siblings <- all(map_lgl(repos, is_sibling)) + +expect_true(all_siblings) + + +## -------------------------------------------------------------- +expect_equal(path_file(here()), "PACTA_analysis") + + +## -------------------------------------------------------------- +# What value is currently assigned to the variable `portfolio_name_ref_all`? +show_pattern_in_file <- function(file, pattern) { + grep(pattern, readLines(file), value = TRUE) +} + +(files <- dir_ls(regexp = "web_tool_script")) + +this_pattern <- "portfolio_name_ref_all.*<-" +matched <- map(files, show_pattern_in_file, pattern = this_pattern) +walk(matched, writeLines) + +script_has_this_pattern <- grepl(this_pattern, matched) +expect_true(all(script_has_this_pattern)) + + +## -------------------------------------------------------------- +config_1 <- here( + "working_dir", + "10_Parameter_File", + "TestPortfolio_Input_PortfolioParameters.yml" +) + +expect_true(file_exists(config_1)) + + +## -------------------------------------------------------------- +look_into <- function(path, n = -1L) { + lines <- readLines(path, n, encoding = "UTF-8") + writeLines(lines) +} + +look_into(config_1) + + +## -------------------------------------------------------------- +config_2 <- here("parameter_files", "WebParameters_2dii.yml") + +expect_true(file_exists(config_2)) + + +## -------------------------------------------------------------- +make_config_portable <- function(config) { + lines <- readLines(config, encoding = "UTF-8") + lines <- make_paths_portable(lines) + writeLines(lines, config) + + invisible(config) +} + +make_paths_portable <- function(x) { + x %>% + root_field_path("project_location_ext", pattern = "PACTA_analysis") %>% + root_field_path("data_location_ext", pattern = "pacta-data") %>% + root_field_path("template_location", pattern = "create_interactive_report") +} + +root_field_path <- function(x, field, pattern) { + parent <- path_dir(here()) + value <- path(parent, extract_from(x, pattern)) + sub(glue("({field}:[ ]?).*"), glue("\\1{value}/"), x) +} + +extract_from <- function(x, pattern) { + line <- grep(pattern, x, value = TRUE) + sub(glue(".*({pattern}.*)"), "\\1", line) +} + +make_config_portable(config_2) + + +## -------------------------------------------------------------- +config_paths <- config::get(file = config_2)$paths +all_paths_exist <- all(map_lgl(config_paths, dir_exists)) + +expect_true(all_paths_exist) + + +## -------------------------------------------------------------- +look_into(config_2) + + +## ----message=FALSE--------------------------------------------- +dir_has_files <- function(path) { + stopifnot(is_dir(path)) + + contents <- dir_ls(path, recurse = TRUE) + has_files <- any(map_lgl(contents, is_file)) + has_files +} + +# source another script with arguments +source_with_args <- function(file, ...){ + passArgs <<- function(trailingOnly){ + list(...) + } + source(file) +} + +out_1 <- path("working_dir", "30_Processed_Inputs") + +expect_false(dir_has_files(out_1)) +source("web_tool_script_1.R") +# system(paste("Rscript web_tool_script_1.R", "TestPortfolio_Input")) +expect_true(dir_has_files(out_1)) + + +## ----message=FALSE--------------------------------------------- +out_2 <- path("working_dir", "40_Results") + +expect_false(dir_has_files(out_2)) +source("web_tool_script_2.R") +expect_true(dir_has_files(out_2)) + + +## ----message=FALSE--------------------------------------------- +out_3 <- path("working_dir", "50_Outputs") + +expect_false(dir_has_files(out_3)) +source("web_tool_script_3.R") +expect_true(dir_has_files(out_3)) + + +## -------------------------------------------------------------- +outputs <- path("working_dir", "50_Outputs") + +css <- dir_ls(outputs, recurse = TRUE, regexp = "[.]css") +expect_true(length(css) > 0L) + +js <- dir_ls(outputs, recurse = TRUE, regexp = "[.]js") +expect_true(length(js) > 0L) + +index <- dir_ls(outputs, recurse = TRUE, regexp = "index[.]html") +expect_true(length(index) > 0L) + +zip <- dir_ls(outputs, recurse = TRUE, regexp = "[.]zip") +expect_true(length(zip) > 0L) + + +## -------------------------------------------------------------- +look_into(index, n = 20L) + +dir_tree(path(outputs, "TestPortfolio_Input"), recurse = FALSE) + + +## -------------------------------------------------------------- +dir_ls(path("..", "pacta-data", "2019Q4", "cleaned_files")) + diff --git a/integration-test.Rmd b/integration-test.Rmd index 94ad29e4..610a295f 100644 --- a/integration-test.Rmd +++ b/integration-test.Rmd @@ -244,6 +244,7 @@ out_1 <- path("working_dir", "30_Processed_Inputs") expect_false(dir_has_files(out_1)) source("web_tool_script_1.R") +# system(paste("Rscript web_tool_script_1.R", "TestPortfolio_Input")) expect_true(dir_has_files(out_1)) ``` diff --git a/integration-test.md b/integration-test.md index 42d61475..e9f970d2 100644 --- a/integration-test.md +++ b/integration-test.md @@ -1,6 +1,6 @@ Integration test: Run the web tool ================ -2020-09-26 +2020-09-30 ## Introduction @@ -29,7 +29,7 @@ suppressPackageStartupMessages(library(renv)) suppressPackageStartupMessages(library(glue)) suppressPackageStartupMessages(library(fs)) library(here) -#> here() starts at /home/mauro/git/PACTA_analysis +#> here() starts at /Users/jacobkastl/Documents/git/PACTA_analysis ``` All packages detected in the directory PACTA\_analysis: @@ -41,16 +41,19 @@ detect_packages <- function() { } detect_packages() -#> Finding R package dependencies ... Done! -#> [1] "assertthat" "base" "config" "countrycode" "cowplot" -#> [6] "devtools" "dplyr" "extrafont" "fs" "fst" -#> [11] "ggforce" "ggmap" "ggplot2" "ggrepel" "ggthemes" -#> [16] "glue" "grid" "gridExtra" "here" "janitor" -#> [21] "jsonlite" "knitr" "lme4" "matrixStats" "plyr" -#> [26] "purrr" "r2dii.utils" "RColorBrewer" "readr" "readxl" -#> [31] "renv" "reshape2" "rlang" "rmarkdown" "rstudioapi" -#> [36] "rworldmap" "scales" "sitools" "stringr" "testthat" -#> [41] "tidyr" "tidyselect" "tidyverse" "tools" "xml2" +#> Finding R package dependencies ... [18/22] [19/22] [20/22] [21/22] [22/22] Done! +#> [1] "assertthat" "base" "config" "countrycode" +#> [5] "cowplot" "devtools" "dplyr" "extrafont" +#> [9] "fs" "fst" "ggforce" "ggmap" +#> [13] "ggplot2" "ggrepel" "ggthemes" "glue" +#> [17] "grid" "gridExtra" "here" "janitor" +#> [21] "jsonlite" "knitr" "lme4" "matrixStats" +#> [25] "plyr" "purrr" "r2dii.utils" "RColorBrewer" +#> [29] "readr" "readxl" "renv" "reshape2" +#> [33] "rlang" "rmarkdown" "rstudioapi" "rworldmap" +#> [37] "scales" "sitools" "stringr" "testthat" +#> [41] "tidyr" "tidyselect" "tidyverse" "tools" +#> [45] "xml2" ```
@@ -59,98 +62,98 @@ detect_packages() ``` r devtools::session_info() -#> ─ Session info ─────────────────────────────────────────────────────────────── +#> ─ Session info ──────────────────────────────────────────────── #> setting value #> version R version 4.0.2 (2020-06-22) -#> os Ubuntu 18.04.5 LTS -#> system x86_64, linux-gnu -#> ui X11 -#> language en_US:en +#> os macOS Catalina 10.15.4 +#> system x86_64, darwin17.0 +#> ui RStudio +#> language (EN) #> collate en_US.UTF-8 #> ctype en_US.UTF-8 -#> tz America/Chicago -#> date 2020-09-26 +#> tz Europe/Berlin +#> date 2020-09-30 #> -#> ─ Packages ─────────────────────────────────────────────────────────────────── -#> package * version date lib source -#> assertthat 0.2.1 2019-03-21 [1] CRAN (R 4.0.0) -#> backports 1.1.10 2020-09-15 [1] CRAN (R 4.0.2) -#> blob 1.2.1 2020-01-20 [1] CRAN (R 4.0.0) -#> broom 0.7.0 2020-07-09 [1] CRAN (R 4.0.0) -#> callr 3.4.4 2020-09-07 [1] RSPM (R 4.0.2) -#> cellranger 1.1.0 2016-07-27 [1] CRAN (R 4.0.0) -#> cli 2.0.2 2020-02-28 [1] CRAN (R 4.0.0) -#> colorspace 1.4-1 2019-03-18 [1] CRAN (R 4.0.0) -#> config * 0.3 2018-03-27 [1] CRAN (R 4.0.0) -#> crayon 1.3.4.9000 2020-09-03 [1] Github (r-lib/crayon@6b3f0c6) -#> DBI 1.1.0 2019-12-15 [1] CRAN (R 4.0.0) -#> dbplyr 1.4.4 2020-05-27 [1] CRAN (R 4.0.0) -#> desc 1.2.0 2018-05-01 [1] CRAN (R 4.0.0) -#> devtools * 2.3.2 2020-09-18 [1] RSPM (R 4.0.2) -#> digest 0.6.25 2020-02-23 [1] CRAN (R 4.0.0) -#> dplyr * 1.0.2 2020-08-18 [1] RSPM (R 4.0.2) -#> ellipsis 0.3.1 2020-05-15 [1] CRAN (R 4.0.0) -#> evaluate 0.14 2019-05-28 [1] CRAN (R 4.0.0) -#> fansi 0.4.1 2020-01-08 [1] CRAN (R 4.0.0) -#> forcats * 0.5.0 2020-03-01 [1] CRAN (R 4.0.0) -#> fs * 1.5.0 2020-07-31 [1] RSPM (R 4.0.2) -#> generics 0.0.2 2018-11-29 [1] CRAN (R 4.0.0) -#> ggplot2 * 3.3.2 2020-06-19 [1] CRAN (R 4.0.0) -#> glue * 1.4.2 2020-08-27 [1] RSPM (R 4.0.2) -#> gtable 0.3.0 2019-03-25 [1] CRAN (R 4.0.0) -#> haven 2.3.1 2020-06-01 [1] CRAN (R 4.0.0) -#> here * 0.1 2017-05-28 [1] CRAN (R 4.0.0) -#> hms 0.5.3 2020-01-08 [1] CRAN (R 4.0.0) -#> htmltools 0.5.0 2020-06-16 [1] CRAN (R 4.0.0) -#> httr 1.4.2 2020-07-20 [1] RSPM (R 4.0.2) -#> jsonlite 1.7.1 2020-09-07 [1] RSPM (R 4.0.2) -#> knitr 1.30 2020-09-22 [1] RSPM (R 4.0.2) -#> lifecycle 0.2.0 2020-03-06 [1] CRAN (R 4.0.0) -#> lubridate 1.7.9 2020-06-08 [1] CRAN (R 4.0.0) -#> magrittr 1.5.0.9000 2020-08-31 [1] Github (tidyverse/magrittr@15f6f07) -#> memoise 1.1.0 2017-04-21 [1] CRAN (R 4.0.0) -#> modelr 0.1.8 2020-05-19 [1] CRAN (R 4.0.0) -#> munsell 0.5.0 2018-06-12 [1] CRAN (R 4.0.0) -#> pillar 1.4.6 2020-07-10 [1] CRAN (R 4.0.0) -#> pkgbuild 1.1.0 2020-07-13 [1] RSPM (R 4.0.2) -#> pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.0.0) -#> pkgload 1.1.0 2020-05-29 [1] CRAN (R 4.0.0) -#> prettyunits 1.1.1 2020-01-24 [1] CRAN (R 4.0.0) -#> processx 3.4.4 2020-09-03 [1] CRAN (R 4.0.2) -#> ps 1.3.4 2020-08-11 [1] CRAN (R 4.0.0) -#> purrr * 0.3.4 2020-04-17 [1] CRAN (R 4.0.0) -#> R6 2.4.1 2019-11-12 [1] CRAN (R 4.0.0) -#> Rcpp 1.0.5 2020-07-06 [1] CRAN (R 4.0.0) -#> readr * 1.3.1 2018-12-21 [1] CRAN (R 4.0.0) -#> readxl 1.3.1 2019-03-13 [1] CRAN (R 4.0.0) -#> remotes 2.2.0 2020-07-21 [1] RSPM (R 4.0.2) -#> renv * 0.12.0 2020-08-28 [1] RSPM (R 4.0.2) -#> reprex 0.3.0 2019-05-16 [1] CRAN (R 4.0.0) -#> rlang * 0.4.7 2020-07-09 [1] CRAN (R 4.0.0) -#> rmarkdown 2.3 2020-06-18 [1] CRAN (R 4.0.0) -#> rprojroot 1.3-2 2018-01-03 [1] CRAN (R 4.0.0) -#> rstudioapi 0.11 2020-02-07 [1] CRAN (R 4.0.0) -#> rvest 0.3.6 2020-07-25 [1] RSPM (R 4.0.2) -#> scales 1.1.1 2020-05-11 [1] CRAN (R 4.0.0) -#> sessioninfo 1.1.1 2018-11-05 [1] CRAN (R 4.0.0) -#> stringi 1.5.3 2020-09-09 [1] RSPM (R 4.0.2) -#> stringr * 1.4.0 2019-02-10 [1] CRAN (R 4.0.0) -#> testthat * 2.99.0.9000 2020-08-18 [1] Github (r-lib/testthat@9e643d8) -#> tibble * 3.0.3 2020-07-10 [1] CRAN (R 4.0.0) -#> tidyr * 1.1.2 2020-08-27 [1] RSPM (R 4.0.2) -#> tidyselect 1.1.0 2020-05-11 [1] CRAN (R 4.0.0) -#> tidyverse * 1.3.0 2019-11-21 [1] RSPM (R 4.0.2) -#> usethis * 1.6.3 2020-09-17 [1] RSPM (R 4.0.2) -#> vctrs 0.3.4 2020-08-29 [1] RSPM (R 4.0.2) -#> withr 2.3.0 2020-09-22 [1] RSPM (R 4.0.2) -#> xfun 0.17 2020-09-09 [1] RSPM (R 4.0.2) -#> xml2 1.3.2 2020-04-23 [1] CRAN (R 4.0.0) -#> yaml 2.2.1 2020-02-01 [1] RSPM (R 4.0.0) +#> ─ Packages ──────────────────────────────────────────────────── +#> package * version date lib source +#> assertthat 0.2.1 2019-03-21 [1] CRAN (R 4.0.0) +#> backports 1.1.8 2020-06-17 [1] CRAN (R 4.0.2) +#> blob 1.2.1 2020-01-20 [1] CRAN (R 4.0.0) +#> broom 0.5.6 2020-04-20 [1] CRAN (R 4.0.0) +#> callr 3.4.3 2020-03-28 [1] CRAN (R 4.0.0) +#> cellranger 1.1.0 2016-07-27 [1] CRAN (R 4.0.0) +#> cli 2.0.2 2020-02-28 [1] CRAN (R 4.0.0) +#> colorspace 1.4-1 2019-03-18 [1] CRAN (R 4.0.0) +#> config * 0.3 2018-03-27 [1] CRAN (R 4.0.0) +#> crayon 1.3.4 2017-09-16 [1] CRAN (R 4.0.0) +#> DBI 1.1.0 2019-12-15 [1] CRAN (R 4.0.0) +#> dbplyr 1.4.4 2020-05-27 [1] CRAN (R 4.0.0) +#> desc 1.2.0 2018-05-01 [1] CRAN (R 4.0.0) +#> devtools * 2.3.1 2020-07-21 [1] CRAN (R 4.0.2) +#> digest 0.6.25 2020-02-23 [1] CRAN (R 4.0.0) +#> dplyr * 1.0.0 2020-05-29 [1] CRAN (R 4.0.0) +#> ellipsis 0.3.1 2020-05-15 [1] CRAN (R 4.0.0) +#> evaluate 0.14 2019-05-28 [1] CRAN (R 4.0.0) +#> fansi 0.4.1 2020-01-08 [1] CRAN (R 4.0.0) +#> forcats * 0.5.0 2020-03-01 [1] CRAN (R 4.0.0) +#> fs * 1.5.0 2020-07-31 [1] CRAN (R 4.0.2) +#> generics 0.0.2 2018-11-29 [1] CRAN (R 4.0.0) +#> ggplot2 * 3.3.1 2020-05-28 [1] CRAN (R 4.0.0) +#> glue * 1.4.1 2020-05-13 [1] CRAN (R 4.0.0) +#> gtable 0.3.0 2019-03-25 [1] CRAN (R 4.0.0) +#> haven 2.3.1 2020-06-01 [1] CRAN (R 4.0.0) +#> here * 0.1 2017-05-28 [1] CRAN (R 4.0.2) +#> hms 0.5.3 2020-01-08 [1] CRAN (R 4.0.0) +#> htmltools 0.5.0 2020-06-16 [1] CRAN (R 4.0.1) +#> httr 1.4.2 2020-07-20 [1] CRAN (R 4.0.2) +#> jsonlite 1.7.0 2020-06-25 [1] CRAN (R 4.0.2) +#> knitr 1.28 2020-02-06 [1] CRAN (R 4.0.0) +#> lattice 0.20-41 2020-04-02 [1] CRAN (R 4.0.2) +#> lifecycle 0.2.0 2020-03-06 [1] CRAN (R 4.0.0) +#> lubridate 1.7.9 2020-06-08 [1] CRAN (R 4.0.0) +#> magrittr 1.5 2014-11-22 [1] CRAN (R 4.0.0) +#> memoise 1.1.0 2017-04-21 [1] CRAN (R 4.0.0) +#> modelr 0.1.8 2020-05-19 [1] CRAN (R 4.0.0) +#> munsell 0.5.0 2018-06-12 [1] CRAN (R 4.0.0) +#> nlme 3.1-148 2020-05-24 [1] CRAN (R 4.0.2) +#> packrat 0.5.0 2018-11-14 [1] CRAN (R 4.0.2) +#> pillar 1.4.6 2020-07-10 [1] CRAN (R 4.0.2) +#> pkgbuild 1.0.8 2020-05-07 [1] CRAN (R 4.0.0) +#> pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.0.0) +#> pkgload 1.1.0 2020-05-29 [1] CRAN (R 4.0.0) +#> prettyunits 1.1.1 2020-01-24 [1] CRAN (R 4.0.0) +#> processx 3.4.2 2020-02-09 [1] CRAN (R 4.0.0) +#> ps 1.3.3 2020-05-08 [1] CRAN (R 4.0.0) +#> purrr * 0.3.4 2020-04-17 [1] CRAN (R 4.0.0) +#> R6 2.4.1 2019-11-12 [1] CRAN (R 4.0.0) +#> Rcpp 1.0.4.6 2020-04-09 [1] CRAN (R 4.0.0) +#> readr * 1.3.1 2018-12-21 [1] CRAN (R 4.0.0) +#> readxl 1.3.1 2019-03-13 [1] CRAN (R 4.0.0) +#> remotes 2.2.0 2020-07-21 [1] CRAN (R 4.0.2) +#> renv * 0.12.0 2020-08-28 [1] CRAN (R 4.0.2) +#> reprex 0.3.0 2019-05-16 [1] CRAN (R 4.0.0) +#> rlang * 0.4.7 2020-07-09 [1] CRAN (R 4.0.2) +#> rmarkdown 2.3 2020-06-18 [1] CRAN (R 4.0.0) +#> rprojroot 1.3-2 2018-01-03 [1] CRAN (R 4.0.0) +#> rstudioapi 0.11 2020-02-07 [1] CRAN (R 4.0.0) +#> rvest 0.3.5 2019-11-08 [1] CRAN (R 4.0.0) +#> scales 1.1.1 2020-05-11 [1] CRAN (R 4.0.0) +#> sessioninfo 1.1.1 2018-11-05 [1] CRAN (R 4.0.0) +#> stringi 1.4.6 2020-02-17 [1] CRAN (R 4.0.0) +#> stringr * 1.4.0 2019-02-10 [1] CRAN (R 4.0.0) +#> testthat * 2.3.2 2020-03-02 [1] CRAN (R 4.0.0) +#> tibble * 3.0.3 2020-07-10 [1] CRAN (R 4.0.2) +#> tidyr * 1.1.0 2020-05-20 [1] CRAN (R 4.0.0) +#> tidyselect 1.1.0 2020-05-11 [1] CRAN (R 4.0.0) +#> tidyverse * 1.3.0 2019-11-21 [1] CRAN (R 4.0.0) +#> usethis * 1.6.1 2020-04-29 [1] CRAN (R 4.0.2) +#> vctrs 0.3.2 2020-07-15 [1] CRAN (R 4.0.2) +#> withr 2.2.0 2020-04-20 [1] CRAN (R 4.0.0) +#> xfun 0.14 2020-05-20 [1] CRAN (R 4.0.0) +#> xml2 1.3.2 2020-04-23 [1] CRAN (R 4.0.0) +#> yaml 2.2.1 2020-02-01 [1] CRAN (R 4.0.0) #> -#> [1] /home/mauro/R/x86_64-pc-linux-gnu-library/4.0 -#> [2] /usr/local/lib/R/site-library -#> [3] /usr/lib/R/site-library -#> [4] /usr/lib/R/library +#> [1] /Library/Frameworks/R.framework/Versions/4.0/Resources/library ```
@@ -175,8 +178,8 @@ if (file_exists(expected_dataset)) { warn(glue("Removing existing file: {expected_dataset}")) file_delete(expected_dataset) } -#> Warning: Removing existing file: /home/mauro/git/PACTA_analysis/working_dir/ -#> 20_Raw_Inputs/TestPortfolio_Input.csv +#> Warning: Removing existing file: /Users/jacobkastl/Documents/git/ +#> PACTA_analysis/working_dir/20_Raw_Inputs/TestPortfolio_Input.csv file_copy(example_dataset, expected_dataset) @@ -200,9 +203,9 @@ ensure_empty_directory <- function(directory) { children <- c("30_Processed_Inputs", "40_Results", "50_Outputs") (paths <- here("working_dir", children)) -#> [1] "/home/mauro/git/PACTA_analysis/working_dir/30_Processed_Inputs" -#> [2] "/home/mauro/git/PACTA_analysis/working_dir/40_Results" -#> [3] "/home/mauro/git/PACTA_analysis/working_dir/50_Outputs" +#> [1] "/Users/jacobkastl/Documents/git/PACTA_analysis/working_dir/30_Processed_Inputs" +#> [2] "/Users/jacobkastl/Documents/git/PACTA_analysis/working_dir/40_Results" +#> [3] "/Users/jacobkastl/Documents/git/PACTA_analysis/working_dir/50_Outputs" walk(paths, ensure_empty_directory) ``` @@ -255,9 +258,9 @@ show_pattern_in_file <- function(file, pattern) { this_pattern <- "portfolio_name_ref_all.*<-" matched <- map(files, show_pattern_in_file, pattern = this_pattern) walk(matched, writeLines) -#> portfolio_name_ref_all <- c("TestPortfolio_Input") -#> portfolio_name_ref_all <- c("TestPortfolio_Input") -#> portfolio_name_ref_all <- c("TestPortfolio_Input") +#> portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml +#> portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml +#> portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml script_has_this_pattern <- grepl(this_pattern, matched) expect_true(all(script_has_this_pattern)) @@ -290,7 +293,7 @@ look_into <- function(path, n = -1L) { look_into(config_1) #> default: #> parameters: -#> portfolio_name_in: test_portfolio +#> portfolio_name_in: TestPortfolio_Input #> investor_name_in: Test ``` @@ -346,9 +349,9 @@ expect_true(all_paths_exist) look_into(config_2) #> default: #> paths: -#> project_location_ext: /home/mauro/git/PACTA_analysis/ -#> data_location_ext: /home/mauro/git/pacta-data/2019Q4/ -#> template_location: /home/mauro/git/create_interactive_report/ +#> project_location_ext: /Users/jacobkastl/Documents/git/PACTA_analysis/ +#> data_location_ext: /Users/jacobkastl/Documents/git/pacta-data/2019Q4/ +#> template_location: /Users/jacobkastl/Documents/git/create_interactive_report/ #> parameters: #> project_name: working_dir #> twodii_internal: FALSE @@ -372,8 +375,9 @@ out_1 <- path("working_dir", "30_Processed_Inputs") expect_false(dir_has_files(out_1)) source("web_tool_script_1.R") -#> Warning in read_file(paste0(file_location, "/fund_data.fst")): /home/mauro/git/ -#> pacta-data/2019Q4/cleaned_files/fund_data.fst does not exist +#> Warning in read_file(paste0(file_location, "/fund_data.fst")): / +#> Users/jacobkastl/Documents/git/pacta-data/2019Q4/cleaned_files/ +#> fund_data.fst does not exist #> [1] "No Equity in portfolio" expect_true(dir_has_files(out_1)) ``` @@ -385,12 +389,15 @@ out_2 <- path("working_dir", "40_Results") expect_false(dir_has_files(out_2)) source("web_tool_script_2.R") -#> Warning in dir.create(.x): '/home/mauro/git/PACTA_analysis/working_dir// -#> 30_Processed_Inputs/TestPortfolio_Input' already exists -#> Warning in dir.create(.x): '/home/mauro/git/PACTA_analysis/working_dir// -#> 40_Results/TestPortfolio_Input' already exists -#> Warning in dir.create(.x): '/home/mauro/git/PACTA_analysis/working_dir// -#> 50_Outputs/TestPortfolio_Input' already exists +#> Warning in dir.create(.x): '/Users/jacobkastl/Documents/ +#> git/PACTA_analysis/working_dir//30_Processed_Inputs/ +#> TestPortfolio_Input' already exists +#> Warning in dir.create(.x): '/Users/jacobkastl/Documents/git/ +#> PACTA_analysis/working_dir//40_Results/TestPortfolio_Input' +#> already exists +#> Warning in dir.create(.x): '/Users/jacobkastl/Documents/git/ +#> PACTA_analysis/working_dir//50_Outputs/TestPortfolio_Input' +#> already exists #> [1] "1: Test" expect_true(dir_has_files(out_2)) ``` @@ -433,11 +440,11 @@ look_into(index, n = 20L) #> #> #> -#> 1 Introduction: What to get out of this report and how to read it | Interactive Portfolio Report +#> Interactive Portfolio Report #> -#> +#> #> -#> +#> #> #> #> @@ -445,7 +452,7 @@ look_into(index, n = 20L) #> #> #> -#> +#> #> dir_tree(path(outputs, "TestPortfolio_Input"), recurse = FALSE) diff --git a/parameter_files/WebParameters_2dii.yml b/parameter_files/WebParameters_2dii.yml index f83f0907..a1a788c1 100644 --- a/parameter_files/WebParameters_2dii.yml +++ b/parameter_files/WebParameters_2dii.yml @@ -1,8 +1,8 @@ default: paths: - project_location_ext: /home/mauro/git/PACTA_analysis/ - data_location_ext: /home/mauro/git/pacta-data/2019Q4/ - template_location: /home/mauro/git/create_interactive_report/ + project_location_ext: /Users/jacobkastl/Documents/git/PACTA_analysis/ + data_location_ext: /Users/jacobkastl/Documents/git/pacta-data/2019Q4/ + template_location: /Users/jacobkastl/Documents/git/create_interactive_report/ parameters: project_name: working_dir twodii_internal: FALSE diff --git a/web_tool_script_1.R b/web_tool_script_1.R index c39c5417..736a87ab 100644 --- a/web_tool_script_1.R +++ b/web_tool_script_1.R @@ -16,6 +16,7 @@ library(r2dii.utils) library(fs) library(jsonlite) library(fst) +library(here) # source("0_portfolio_test.R") @@ -26,9 +27,15 @@ source("0_global_functions.R") source("0_web_functions.R") source("0_json_functions.R") -portfolio_name_ref_all <- c("TestPortfolio_Input") -working_location <- here::here() -set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +if (interactive()) { + portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml + working_location <- here::here() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +} else { + portfolio_name_ref_all = get_portfolio_name() + working_location <- getwd() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_docker.yml")) + } working_location <- paste0(working_location, "/") @@ -123,13 +130,6 @@ portfolio <- process_raw_portfolio(portfolio_raw, currencies, grouping_variables) -portfolio <- portfolio %>% - mutate(portfolio_name = ifelse(portfolio_name %>% unique() %>% length() > 1, - portfolio_name_ref_all, - portfolio_name) - ) - - portfolio <- add_revenue_split(has_revenue, portfolio, revenue_data) portfolio <- create_ald_flag(portfolio, comp_fin_data, debt_fin_data) @@ -172,16 +172,16 @@ emissions_totals <- calculate_portfolio_emissions(inc_emission_factors, # Identify the portfolios to save; # Subset and Save these files +create_portfolio_subfolders(portfolio_name_ref_all) file_names <- identify_portfolios(portfolio_total) -write_csv(file_names, paste0(proc_input_path, "/file_names.csv")) - -create_portfolio_subfolders(file_names, portfolio_name_ref_all) - portfolio_name <- file_names$portfolio_name proc_input_path_ <- paste0(proc_input_path, "/", portfolio_name_ref_all) + +write_csv(file_names, paste0(proc_input_path_,"/file_names.csv")) + export_audit_information_jsons(audit_file_ = audit_file %>% filter(portfolio_name == portfolio_name), portfolio_total_ = portfolio_total %>% filter(portfolio_name == portfolio_name), diff --git a/web_tool_script_2.R b/web_tool_script_2.R index c186e281..5553100c 100644 --- a/web_tool_script_2.R +++ b/web_tool_script_2.R @@ -14,14 +14,21 @@ library(r2dii.utils) library(fs) library(jsonlite) library(fst) +library(here) source("0_portfolio_test.R") source("0_global_functions.R") source("0_web_functions.R") -portfolio_name_ref_all <- c("TestPortfolio_Input") -working_location <- here::here() -set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +if (interactive()) { + portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml + working_location <- here::here() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +} else { + portfolio_name_ref_all = get_portfolio_name() + working_location <- getwd() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_docker.yml")) +} working_location <- paste0(working_location, "/") @@ -39,8 +46,8 @@ analysis_inputs_path <- set_analysis_inputs_path(twodii_internal, data_location_ unlink(paste0(results_path,"/",portfolio_name_ref_all,"/*"), force = TRUE, recursive = TRUE) # run again so output folders are available after deleting past results -file_names <- read_csv(paste0(proc_input_path, "/file_names.csv")) -create_portfolio_subfolders(file_names, portfolio_name_ref_all) +file_names <- read_csv(paste0(proc_input_path, "/", portfolio_name_ref_all, "/file_names.csv")) +create_portfolio_subfolders(portfolio_name_ref_all) port_col_types <- set_col_types(grouping_variables, "ddddccccddclc") @@ -120,12 +127,6 @@ if(file.exists(equity_input_file)){ if(data_check(port_all_eq)){write_rds(port_all_eq, paste0(pf_file_results_path, portfolio_name,"_Equity_results_portfolio.rda"))} if(has_map){if(data_check(map_eq)){write_rds(map_eq, paste0(pf_file_results_path, portfolio_name,"_Equity_results_map.rda"))}} - # investor_results_path <- paste0(results_path,"/", investor_name_select, "/") - # if(!dir.exists(investor_results_path)){dir.create(investor_results_path)} - # - # if(data_check(company_all_eq)){write_rds(company_all_eq, paste0(investor_results_path, "Equity_results_company.rda"))} - # if(data_check(port_all_eq)){write_rds(port_all_eq, paste0(investor_results_path, "Equity_results_portfolio.rda"))} - # if(has_map){if(data_check(map_eq)){write_rds(map_eq, paste0(investor_results_path, "Equity_results_map.rda"))}} } } @@ -203,12 +204,7 @@ if (file.exists(bonds_inputs_file)){ if(data_check(port_all_cb)){write_rds(port_all_cb, paste0(pf_file_results_path, portfolio_name,"_Bonds_results_portfolio.rda"))} if(has_map){if(data_check(map_cb)){write_rds(map_cb, paste0(pf_file_results_path, portfolio_name,"_Bonds_results_map.rda"))}} - # investor_results_path <- paste0(results_path,"/", investor_name_select, "/") - # if(!dir.exists(investor_results_path)){dir.create(investor_results_path)} - # - # if(data_check(company_all_cb)){ write_rds(company_all_cb, paste0(investor_results_path, "Bonds_results_company.rda"))} - # if(data_check(port_all_cb)){write_rds(port_all_cb, paste0(investor_results_path, "Bonds_results_portfolio.rda"))} - # if(has_map){if(data_check(map_cb)){write_rds(map_cb, paste0(investor_results_path, "Bonds_results_map.rda"))}} + } } diff --git a/web_tool_script_3.R b/web_tool_script_3.R index d84ba371..504fb480 100644 --- a/web_tool_script_3.R +++ b/web_tool_script_3.R @@ -16,12 +16,19 @@ library(r2dii.utils) library(fs) library(jsonlite) library(fst) +library(here) -portfolio_name_ref_all <- c("TestPortfolio_Input") -working_location <- here::here() -set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +if (interactive()) { + portfolio_name_ref_all <- c("TestPortfolio_Input") # must be the same name as in the _PortfolioParameters.yml + working_location <- here::here() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_2dii.yml")) +} else { + portfolio_name_ref_all = get_portfolio_name() + working_location <- getwd() + set_web_parameters(file_path = paste0(working_location,"/parameter_files/WebParameters_docker.yml")) +} -working_location <- paste0(here::here(), "/") +working_location <- paste0(working_location, "/") set_webtool_paths() @@ -38,7 +45,7 @@ source(paste0(template_path, "create_interactive_report.R")) -file_names <- read_csv(paste0(proc_input_path, "/file_names.csv")) +file_names <- read_csv(paste0(proc_input_path, "/", portfolio_name_ref_all, "/file_names.csv")) # TODO: Remove different start years diff --git a/working_dir/.DS_Store b/working_dir/.DS_Store index b3ac85b8..b1815c88 100644 Binary files a/working_dir/.DS_Store and b/working_dir/.DS_Store differ diff --git a/working_dir/10_Parameter_File/TestPortfolio_Input_PortfolioParameters.yml b/working_dir/10_Parameter_File/TestPortfolio_Input_PortfolioParameters.yml index 36d98fc1..71b01c58 100644 --- a/working_dir/10_Parameter_File/TestPortfolio_Input_PortfolioParameters.yml +++ b/working_dir/10_Parameter_File/TestPortfolio_Input_PortfolioParameters.yml @@ -1,5 +1,5 @@ default: parameters: - portfolio_name_in: test_portfolio + portfolio_name_in: TestPortfolio_Input investor_name_in: Test