From b221345689e4eb49f71656338904c1e6fd2c5e69 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 28 Aug 2026 11:50:28 -0400 Subject: [PATCH 01/11] Create blank templates for sis data in inst/ --- inst/resources/sis_assmt_template.csv | 75 +++++++++++++++++++++++++++ inst/resources/sis_ts_template.csv | 1 + 2 files changed, 76 insertions(+) create mode 100644 inst/resources/sis_assmt_template.csv create mode 100644 inst/resources/sis_ts_template.csv diff --git a/inst/resources/sis_assmt_template.csv b/inst/resources/sis_assmt_template.csv new file mode 100644 index 00000000..67fc2593 --- /dev/null +++ b/inst/resources/sis_assmt_template.csv @@ -0,0 +1,75 @@ +Variable,Explanation,Optional,Default,Value +model_identifier,"Argument used to distinguish between base model and a new, updated model sent in subsequent submission. Options: 'base', 'updated_model_1', 'updated_model_2', etc.",YES,base, +AS_POINT_OF_CONTACT,"The lead/corresponding author for a stock assessment, formatted as an email address.",NO,None, +AS_CATCH_DATA,"Categorical classification describing the availability of catch data for use in the stock assessment. This level should be based on the data that was actually used in the final version of the assessment model. Options: 0 (No quantitative catch data available), 1 (Some catch data, but major gaps for some fishery sectors or historical periods), 2 (Enough catch data to establish magnitude/trends for a major fishery sector for data-limited methods or closed fisheries), 3 (Catch data generally available for all sectors, but some gaps exist), 4 (No data gaps substantially impede assessment, but catch is not without uncertainty), 5 (Very complete knowledge of total catch).",NO,None, +AS_ABUNDANCE_DATA,"Categorical classification describing the availability of abundance data for use in the stock assessment. This level should be based on the data that was actually used in the final version of the assessment model. Options: 0 (No indicator of stock abundance/trend), 1 (Fishery-dependent CPUE available with high uncertainty, or expert opinion), 2 (Fishery-dependent CPUE sufficiently standardized for full assessments; no/insufficient fishery-independent data), 3 (Limited fishery-independent survey(s) provide relative abundance; limited spatiotemporal coverage or high variability), 4 (Complete fishery-independent survey(s) provide relative abundance covering large spatial extent over several years), 5 (Calibrated fishery-independent survey(s) or tag-recapture provide absolute abundance).",NO,None, +AS_BIOLOGICAL_DATA,"Categorical classification describing the availability of biological/life history data for use in the stock assessment. This level should be based on the data that was actually used in the final version of the assessment model. Options: 0 (No life history data), 1 (Most life history factors not based on empirical data; derived using proxies/meta-analyses/borrowed), 2 (Some factors based on empirical data, but at least one derived via proxies/meta-analyses/borrowed), 3 (Most factors based on stock-specific empirical data), 4 (Data sufficient to track changes over time in at least growth), 5 (No major gaps in life history knowledge).",NO,None, +AS_ECOSYSTEM_DATA,"Categorical classification describing the usage of ecosystem linkage data in the stock assessment. This level should be based on the data that was actually used in the final version of the assessment model. Options: 0 (No linkage/consideration of ecosystem dynamic/properties), 1 (Ecosystem-based hypotheses inform structure/inputs, but no explicit linkage to drivers), 2 (Includes some form of variability/effect to account for unidentified ecosystem dynamics), 3 (One or more features linked to dynamic data from environment, climate, habitat, or predator-prey), 4 (Linked to dynamic data supported directly by process studies), 5 (Configured to be coupled or linked with an ecosystem process).",NO,None, +AS_COMP_DATA,"Categorical classification describing the availability of size/age composition data for use in the stock assessment. This level should be based on the data that was actually used in the final version of the assessment model. Options: 0 (No composition data collected), 1 (Some collected, but major gaps and not used), 2 (Enough collected to enable data-limited approaches), 3 (Enough collected over sufficient time series to be informative in age/size structured models), 4 (Enough age composition collected over sufficient time series to enable age-structured methods), 5 (Very complete age and size composition data).",NO,None, +AS_MODEL_CAT,"Category of model used to complete the stock assessment (see Table 5.1; NOAA, 2018). Focuses on population dynamics structure, data requirements, and management advice types. If an ensemble approach was used, select the highest category describing one or more models in the ensemble. Options: 1 (Data-limited), 2 (Index-based), 3 (Aggregate Biomass Dynamics), 4 (Virtual Population Analysis), 5 (Statistical Catch-at-Length), 6 (Statistical Catch-at-Age).",NO,None, +AS_TYPE,"Type of stock assessment, with regards to approach, technique, effort level, and complexity (NOAA, 2018). Assigned automatically by SIS. Options: 'Research Stock Assessment' (development/revision of data type/method), 'Research/Operational Stock Assessment' (management advice + substantial revision), 'Operational Assessment' (scientific advice focusing on stock status/catch limits), 'Stock Monitoring Update' (stock-level advice between assessments without changes to methods/data).",NO,None, +AS_REVIEW_TYPE,"Final status of the assessment, chosen from a set of values found in the SIS manual. Options: 1 (Not Reviewed), 2 (Accept Previous Approach, Remand New Attempt), 3 (Full Acceptance), 4 (Partial Acceptance, Fishing Mortality Estimates), 5 (Partial Acceptance, Biomass Estimates), 6 (Partial Acceptance, Status Determinations Only), 7 (Reject, Data Insufficient for Assessment), 8 (Reject, Results Too Uncertain To Be Considered Accurate), 9 (Remand).",NO,None, +ASSESSMENT_ID,Unique numeric identifier assigned to all stock assessment records. Assigned automatically by SIS.,NO,None, +ENTITY_ID,Entity unique identifier value. Assigned automatically by SIS.,NO,None, +AS_YEAR,Year the assessment was completed. Assigned automatically by SIS.,NO,None, +AS_MONTH,Month the assessment was completed. Assigned automatically by SIS.,NO,None, +AS_LAST_DATA_YEAR,Year of the 'latest' data used in the assessment.,NO,Extracted as landings.end.year from key_quantities.csv, +AS_B_BASIS,"The basis of the biomass unit. Options: Spawning Stock Biomass, Total Stock Biomass, Survey-Estimated Biomass, Escapement, Stock Reproductive Output, Survey Index, Total Stock Abundance.",NO,None, +AS_F_BASIS,"The basis of the Fishing Mortality unit. Options: 1 (Max F at Age), 2 (F for Fully-Selected Fish), 3 (Catch / Biomass), 4 (Catch / Exploitable Biomass), 5 (Catch), 6 (Fishing Intensity), 7 (True F).",NO,None, +AS_FMSY,Estimated and/or calculated value of Fishing Mortality at MSY.,NO,Extracted as F.MSY.terminal from key_quantities.csv, +AS_F_BEST,"Best estimate of Fishing Mortality. Typically, Best F = Terminal F for the stock assessment unless transformed (e.g., averaging or retrospective adjustment).",NO,Extracted as F.terminal.est from key_quantities.csv, +AS_FLIMIT_BASIS,"Basis for the recommended fishing mortality limit, calculated or directly estimated. Only utilized in Alaska as assessments utilize catch projections in the current year. Most stocks utilize Flimit = Fmsy. Example: 'F from 2024 asmt corresponding to 2023 OFL'.",YES,NULL, +AS_B_YEAR,Year of the Biomass estimate for the stock.,NO,Extracted as B.terminal.year from key_quantities.csv, +AS_B_MAX,Maximum estimated value within the approved confidence interval of the Biomass estimate. Equivalent to the value of Best B Confidence Interval Upper estimate.,NO,Extracted as B.terminal.max from key_quantities.csv, +AS_BMSY,"Estimated stock size that would, on average, produce the maximum sustainable yield when fished at a level equal to FMSY.",NO,Extracted as B.msy from key_quantities.csv, +AS_B_BMSY_RATIO,Ratio of B / Bmsy. Automatically calculated by SIS.,YES,NULL, +AS_STOCK_LEVEL_BMSY,"Whether the stock is above, near, or below Bmsy based upon the value provided in the AS_B_BMSY_RATIO field. Options: 'Above', 'Near' (between 80% and 99%), 'Below' (<80%).",YES,NULL, +AS_B_MIN,Minimum estimated value within the approved confidence interval of the Biomass estimate. Equivalent to the value of Best B Confidence Interval Lower estimate.,NO,Extracted as B.terminal.max from key_quantities.csv, +AS_B_BEST,"Best estimate of Biomass. Typically, Best B = Terminal B for the stock assessment unless transformed (e.g., averaging or retrospective adjustment).",NO,Extracted as B.terminal.est from key_quantities.csv, +AS_BMSY_BASIS,Basis for the estimated BMSY value. Example: 'B35%'.,NO,None, +AS_FMSY_BASIS,"Estimated fishing mortality rate that, on average, would produce the maximum sustainable yield from a stock at BMSY. Example: 'F35% as proxy'.",NO,None, +AS_FLIMIT,"Recommended fishing mortality limit from the assessment, above which the stock would be considered to be experiencing overfishing.",NO,Extracted as F.limit from key_quantities.csv, +AS_F_YEAR,Terminal year estimate of stock Fishing Mortality. Always corresponds to the year of the Best estimate of Fishing Mortality (AS_F_BEST).,NO,Extracted as F.terminal.year from key_quantities.csv, +AS_F_UNIT,"Unit of measure corresponding to the fishing mortality estimate. Linked to F Basis selections. Options: 1 (Apical F = Max F at Age), 2 (Fully-selected F = F for Fully-Selected Fish), 3 (Exploitation Rate = Catch / Biomass), 4 (Relative F = Catch / Exploitable Biomass), 5 (Metric Tons = Catch), 6 (1 - SPR = Fishing Intensity), 7 (F = Z - M = True F).",NO,None, +AS_B_UNIT,"Unit of measure corresponding to the biomass estimate. Linked to B Basis selections. Options: 1 (Metric Tons = SSB / Total Biomass / Survey-Estimated Biomass), 2 (Thousand Metric Tons = SSB / Total Biomass / Survey-Estimated Biomass), 3 (Adult spawners - Natural & Hatchery - Escapement), 4 (Adult spawners - Hatchery - Escapement), 5 (Adult spawners - Natural - Escapement), 6 (Number of Eggs - Stock Reproductive Output), 7 (kg / tow - Survey Index), 8 (Number of Fish - Total Stock Abundance).",NO,None, +AS_MODEL,Model software package used to complete the final version of the assessment. Example: 'SS'.,NO,None, +AS_MODEL_VERSION,Version of the software package used to complete the final stock assessment. Example: '3.30.22'.,NO,None, +AS_ENSEMBLE_FLAG,"Whether the assessment was completed using an ensemble or multimodeling approach. Options: 'Y' (yes), 'N' (no).",NO,None, +AS_F_TRANSFORM,"Indicator identifying Fishing Mortality best estimates that include terminal year transformations (e.g., retrospective corrections or multi-year averaging). Options: 'Y' (yes), 'N' (no).",NO,None, +AS_B_RANGE_BASIS,"Approach used to calculate the confidence intervals provided for the stock assessment. Options: 'Asymptotic', 'Credible', 'Bootstrapped', user-specified.",YES,NULL, +AS_B_RANGE,Percentile range of the confidence intervals provided for the stock assessment.,YES,95, +AS_B_TRANSFORM,"Indicator identifying Biomass best estimates that include terminal year transformations (e.g., retrospective corrections or multi-year averaging). Options: 'Y' (yes), 'N' (no).",NO,None, +AS_F_MAX,Maximum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Best F CI Upper estimate.,NO,Extracted as F.terminal.max from key_quantities.csv, +AS_F_MIN,Minimum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Best F CI Lower estimate.,NO,Extracted as F.terminal.min from key_quantities.csv, +AS_F_RANGE_BASIS,"Approach used to calculate the confidence intervals provided for the stock assessment. Options: 'Asymptotic', 'Credible', 'Bootstrapped', user-specified.",YES,NULL, +AS_F_RANGE,Percentile range of the confidence intervals provided for the stock assessment.,YES,95, +AS_FMSY_MAX,Maximum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Fmsy CI Upper estimate.,NO,Extracted as F.MSY.terminal.max from key_quantities.csv, +AS_FMSY_MIN,Minimum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Fmsy CI Lower estimate.,NO,Extracted as F.MSY.terminal.min from key_quantities.csv, +AS_FMSY_RANGE_BASIS,"Approach used to calculate the confidence intervals provided for the stock assessment. Options: 'Asymptotic', 'Credible', 'Bootstrapped', user-specified.",YES,NULL, +AS_FMSY_RANGE,Percentile range of the confidence intervals provided for the stock assessment.,YES,95, +AS_FTARGET,Value of the Ftarget estimate produced by a stock assessment. Often used for stocks in a rebuilding plan.,NO,Extracted as F.target from key_quantities.csv, +AS_FTARGET_BASIS,Approach used to calculate the Ftarget estimate produced by a stock assessment.,NO,None, +AS_MSY,Value of the MSY estimated by the assessment.,NO,None, +AS_MSY_UNIT,"Unit associated with the MSY value. Options: Metric tons, Thousand metric tons, lbs, Thousand lbs, Number of fish.",NO,None, +AS_MSY_MAX,Maximum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to MSY CI Upper estimate.,NO,None, +AS_MSY_MIN,Minimum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to MSY CI Lower estimate.,NO,None, +AS_MSY_RANGE_BASIS,"Approach used to calculate the confidence intervals provided for the stock assessment. Options: 'Asymptotic', 'Credible', 'Bootstrapped', user-specified.",YES,NULL, +AS_MSY_RANGE,Percentile range of the confidence intervals provided for the stock assessment.,YES,95, +AS_BMSY_MAX,Maximum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Bmsy CI Upper estimate.,NO,Extracted as B.msy.max from key_quantities.csv, +AS_BMSY_MIN,Minimum estimated value within the approved confidence interval of the Fishing Mortality estimate. Equivalent to Bmsy CI Lower estimate.,NO,Extracted as B.msy.min from key_quantities.csv, +AS_BMSY_RANGE_BASIS,"Approach used to calculate the confidence intervals provided for the stock assessment. Options: 'Asymptotic', 'Credible', 'Bootstrapped', user-specified.",YES,NULL, +AS_BMSY_RANGE,Percentile range of the confidence intervals provided for the stock assessment.,YES,95, +AS_BLIMIT,"Stock size threshold, below which the stock is considered to be overfished.",NO,None, +AS_BLIMIT_BASIS,"Basis for the Blimit estimate. Examples: (0.7*Bmsy), B25%, etc.",NO,None, +AS_B_COMMENT,"Specific comments associated with the best estimate of biomass for this assessment. 1,000 character limit.",NO,None, +AS_F_COMMENT,"Specific comments associated with the best estimate of fishing mortality for this assessment. 1,000 character limit.",NO,None, +AS_IAS_FLIMIT,International commission F limit estimate.,YES,NULL, +AS_IAS_FLIMIT_BASIS,International commission estimate of Flimit estimation method. Example: 'msy'.,YES,NULL, +AS_IAS_FMSY,International commission estimate of Fmsy.,YES,NULL, +AS_IAS_FMSY_BASIS,International commission estimate of Fmsy estimation method.,YES,NULL, +AS_IAS_FTARGET,International commission estimate of Ftarget.,YES,NULL, +AS_IAS_FTARGET_BASIS,International commission estimate of Ftarget estimation method.,YES,NULL, +AS_IAS_BLIMIT,International commission biomass limit estimate.,YES,NULL, +AS_IAS_BLIMIT_BASIS,International commission estimate of Blimit estimation method.,YES,NULL, +AS_IAS_BMSY,International commission estimate of Bmsy.,YES,NULL, +AS_IAS_BMSY_BASIS,International commission estimate of Bmsy estimation method.,YES,NULL, \ No newline at end of file diff --git a/inst/resources/sis_ts_template.csv b/inst/resources/sis_ts_template.csv new file mode 100644 index 00000000..a74c1ad5 --- /dev/null +++ b/inst/resources/sis_ts_template.csv @@ -0,0 +1 @@ +Year,Category,Primary,Description,Unit,Value From 1249bcab5d8d239b246a6a6881b0479f1eba9c82 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 28 Aug 2026 11:52:18 -0400 Subject: [PATCH 02/11] Start extract_sis_data(), consisting of former code from asar::export_to_sis() --- NAMESPACE | 1 + R/extract_sis_data.R | 197 ++++++++++++++++++++++++++++++++++++++++ man/extract_sis_data.Rd | 41 +++++++++ 3 files changed, 239 insertions(+) create mode 100644 R/extract_sis_data.R create mode 100644 man/extract_sis_data.Rd diff --git a/NAMESPACE b/NAMESPACE index 687233f7..c6c98fc8 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ export(create_latex_table) export(create_rda) export(export_rda) export(extract_caps_alttext) +export(extract_sis_data) export(filter_data) export(plot_aa) export(plot_abundance_at_age) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R new file mode 100644 index 00000000..9e805078 --- /dev/null +++ b/R/extract_sis_data.R @@ -0,0 +1,197 @@ +#' Extract data from model results to send to SIS +#' +#' Semi-automate the extraction of key quantities from model results for eventual transmittance to SIS via `asar::export_to_sis()`. +#' +#' @param sis_data_dir Path. Location of the existing sis_assmt_template.csv and sis_ts_template.csv files, or, if absent, where new versions of the files should be saved. +#' +#' Default: The working directory. +#' +#' @param kq_dir Path. Location of the existing key_quantities.csv file, or, if absent, where a new version of the file should be saved. +#' +#' Default: The working directory. +#' +#' +#' @details This function acts within the following workflow: +#' +#' 1. When a stock assessment is scheduled to conclude, SIS will generate an +#' attachment or prompt containing metadata and identifiers. +#' 2. The user will open two csv files containing placeholders for all of the data required by SIS: sis_assmt_template.csv (assessment summary data) and sis_ts_template.csv (time series data). There are three ways to obtain these files: +#' 2a. Generate the files by running `asar::create_blank_sis()` +#' 2b. Locate the files in the "report" folder generated by running `asar::create_template()`. +#' 2c. Run `stockplotr::extract_sis_data()`, which will populate the templates with data originating from a converted model results file. +#' 3. The user will add the remaining necessary data into the csv files, ensuring that all required fields are completed. +#' 4. Run `export_to_sis()`, which will format and upload this data to a specific Google Drive folder. +#' 5. The uploaded contents will be resubmitted to SIS to finalize the record. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' extract_sis_data( +#' sis_data_dir = getwd() +#' ) +#' } +#' +extract_sis_data <- function(sis_data_dir = getwd(), + key_quantities_dir = getwd() + ) { + # Check if existing data files exist; if not, start from blank templates + if (!exists(fs::path(sis_data_dir, "sis_assmt_template.csv"))) { + assmt_dat <- read.csv(fs::path("inst/resources/sis_assmt_template.csv"), stringsAsFactors = FALSE) + cli::cli_alert_info("No existing sis_assmt_template.csv found in {sis_data_dir}. Using blank template.") + } else { + assmt_dat <- read.csv(fs::path(sis_data_dir, "sis_assmt_template.csv"), stringsAsFactors = FALSE) + cli::cli_alert_info("Found existing sis_assmt_template.csv in {sis_data_dir}.") + + } + + if (!exists(fs::path(sis_data_dir, "sis_ts_template.csv"))) { + ts_dat <- read.csv(fs::path("inst/resources/sis_ts_template.csv"), stringsAsFactors = FALSE) + cli::cli_alert_info("No existing sis_ts_template.csv found in {sis_data_dir}. Using blank template.") + } else { + ts_dat <- read.csv(fs::path(sis_data_dir, "sis_ts_template.csv"), stringsAsFactors = FALSE) + cli::cli_alert_info("Found existing sis_ts_template.csv in {sis_data_dir}.") + } + + # extract key quantities from csv and assign to variables + # TODO: make test to check if key_quantities.csv exists; if not, ask if they want to + # TODO: explain which variables are from which plots + # landings, biomass, fmort... + # export rdas for plot_fishing_mortality() and other relevant plots + kqs <- read.csv(fs::path(key_quantities_dir, + "key_quantities.csv"), + stringsAsFactors = FALSE) + + # AS_LAST_DATA_YEAR <- kqs |> + # dplyr::filter(key_quantity == "landings.end.year") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_FMSY <- kqs |> + # dplyr::filter(key_quantity == "F.MSY.terminal") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_BMSY <- kqs |> + # dplyr::filter(key_quantity == "B.msy") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_BMSY_MIN <- kqs |> + # dplyr::filter(key_quantity == "B.msy.min") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_BMSY_MAX <- kqs |> + # dplyr::filter(key_quantity == "B.msy.max") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_B_YEAR <- kqs |> + # dplyr::filter(key_quantity == "B.terminal.year") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_F_YEAR <- kqs |> + # dplyr::filter(key_quantity == "F.terminal.year") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_FTARGET <- kqs |> + # dplyr::filter(key_quantity == "F.target") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_FLIMIT <- kqs |> + # dplyr::filter(key_quantity == "F.limit") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_B_BEST <- kqs |> + # dplyr::filter(key_quantity == "B.terminal.est") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_F_BEST <- kqs |> + # dplyr::filter(key_quantity == "F.terminal.est") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_B_MIN <- kqs |> + # dplyr::filter(key_quantity == "B.terminal.min") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_B_MAX <- kqs |> + # dplyr::filter(key_quantity == "B.terminal.max") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_F_MIN <- kqs |> + # dplyr::filter(key_quantity == "F.terminal.min") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_F_MAX <- kqs |> + # dplyr::filter(key_quantity == "F.terminal.max") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_FMSY_MAX <- kqs |> + # dplyr::filter(key_quantity == "F.MSY.terminal.max") |> + # dplyr::select(value) |> + # as.numeric() + # + # AS_FMSY_MIN <- kqs |> + # dplyr::filter(key_quantity == "F.MSY.terminal.min") |> + # dplyr::select(value) |> + # as.numeric() + + # insert values into the sis_assmt_template.csv file + mapping <- tibble::tribble( + ~key_quantity, ~Variable, + "landings.end.year", "AS_LAST_DATA_YEAR", + "F.MSY.terminal", "AS_FMSY", + "B.msy", "AS_BMSY", + "B.msy.min", "AS_BMSY_MIN", + "B.msy.max", "AS_BMSY_MAX", + "B.terminal.year", "AS_B_YEAR", + "F.terminal.year", "AS_F_YEAR", + "F.target", "AS_FTARGET", + "F.limit", "AS_FLIMIT", + "B.terminal.est", "AS_B_BEST", + "F.terminal.est", "AS_F_BEST", + "B.terminal.min", "AS_B_MIN", + "B.terminal.max", "AS_B_MAX", + "F.terminal.min", "AS_F_MIN", + "F.terminal.max", "AS_F_MAX", + "F.MSY.terminal.max", "AS_FMSY_MAX", + "F.MSY.terminal.min", "AS_FMSY_MIN" + ) + + # Extract values and format into a key-value matching table + new_vals <- kqs |> + dplyr::inner_join(mapping, by = "key_quantity") |> + dplyr::mutate(value = as.numeric(value)) |> + dplyr::select(Variable, new_value = value) + + # Update assmt_dat + assmt_dat <- assmt_dat |> + dplyr::left_join(new_vals, by = "Variable") |> + dplyr::mutate(Value = ifelse(!is.na(new_value), new_value, Value)) |> + dplyr::select(-new_value) + + + + # At end: if assmt_dat$Value is NA and Default is 95, change it to Default + for (i in seq_len(nrow(assmt_dat))) { + if (is.na(assmt_dat$Value[i]) & assmt_dat$Default[i] == 95) { + assmt_dat$Value[i] <- assmt_dat$Default[i] + } + } + +} + +# if existing copies are present, check user wants to overwrite values; OR just add new ones? + + diff --git a/man/extract_sis_data.Rd b/man/extract_sis_data.Rd new file mode 100644 index 00000000..00db236f --- /dev/null +++ b/man/extract_sis_data.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/extract_sis_data.R +\name{extract_sis_data} +\alias{extract_sis_data} +\title{Extract data from model results to send to SIS} +\usage{ +extract_sis_data(sis_data_dir = getwd(), key_quantities_dir = getwd()) +} +\arguments{ +\item{sis_data_dir}{Path. Location of the existing sis_assmt_template.csv and sis_ts_template.csv files, or, if absent, where new versions of the files should be saved. + +Default: The working directory.} + +\item{kq_dir}{Path. Location of the existing key_quantities.csv file, or, if absent, where a new version of the file should be saved. + +Default: The working directory.} +} +\description{ +Semi-automate the extraction of key quantities from model results for eventual transmittance to SIS via `asar::export_to_sis()`. +} +\details{ +This function acts within the following workflow: + +1. When a stock assessment is scheduled to conclude, SIS will generate an + attachment or prompt containing metadata and identifiers. +2. The user will open two csv files containing placeholders for all of the data required by SIS: sis_assmt_template.csv (assessment summary data) and sis_ts_template.csv (time series data). There are three ways to obtain these files: +2a. Generate the files by running `asar::create_blank_sis()` +2b. Locate the files in the "report" folder generated by running `asar::create_template()`. +2c. Run `stockplotr::extract_sis_data()`, which will populate the templates with data originating from a converted model results file. +3. The user will add the remaining necessary data into the csv files, ensuring that all required fields are completed. +4. Run `export_to_sis()`, which will format and upload this data to a specific Google Drive folder. +5. The uploaded contents will be resubmitted to SIS to finalize the record. +} +\examples{ +\dontrun{ +extract_sis_data( + sis_data_dir = getwd() +) +} + +} From 66118634d06076a8f9bc8d0aa13563a4eb94aa92 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 28 Aug 2026 14:06:23 -0400 Subject: [PATCH 03/11] Continue extract_sis_data(), adding code to create figures but will remove it in next commit --- R/extract_sis_data.R | 213 ++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 125 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 9e805078..ec645eda 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -10,7 +10,13 @@ #' #' Default: The working directory. #' -#' +#' @param model_results Filepath to the standardized, converted model output +#' .rda file generated with `stockplotr::convert_output()`. If provided, will be +#' used to generate figures and key quantities used to populate the SIS templates +#' if key_quantities.csv does not exist. +#' +#' Default: NULL +#' #' @details This function acts within the following workflow: #' #' 1. When a stock assessment is scheduled to conclude, SIS will generate an @@ -33,7 +39,8 @@ #' } #' extract_sis_data <- function(sis_data_dir = getwd(), - key_quantities_dir = getwd() + key_quantities_dir = getwd(), + model_results = NULL ) { # Check if existing data files exist; if not, start from blank templates if (!exists(fs::path(sis_data_dir, "sis_assmt_template.csv"))) { @@ -42,7 +49,6 @@ extract_sis_data <- function(sis_data_dir = getwd(), } else { assmt_dat <- read.csv(fs::path(sis_data_dir, "sis_assmt_template.csv"), stringsAsFactors = FALSE) cli::cli_alert_info("Found existing sis_assmt_template.csv in {sis_data_dir}.") - } if (!exists(fs::path(sis_data_dir, "sis_ts_template.csv"))) { @@ -54,132 +60,89 @@ extract_sis_data <- function(sis_data_dir = getwd(), } # extract key quantities from csv and assign to variables - # TODO: make test to check if key_quantities.csv exists; if not, ask if they want to - # TODO: explain which variables are from which plots - # landings, biomass, fmort... - # export rdas for plot_fishing_mortality() and other relevant plots - kqs <- read.csv(fs::path(key_quantities_dir, + kqs_path <- fs::path(key_quantities_dir, "key_quantities.csv") + if (exists(kqs_path)){ + kqs <- read.csv(fs::path(key_quantities_dir, "key_quantities.csv"), stringsAsFactors = FALSE) + cli::cli_alert_info("Found existing key_quantities.csv in {key_quantities_dir}.") + } else { + cli::cli_alert_warning("No existing key_quantities.csv found in {key_quantities_dir}.") + cli::cli_alert_info("To obtain key quantities, run the following functions and specify `make_rda = TRUE`:") + cli::cli_bullets(c( + "*" = "plot_fishing_mortality()", + "*" = "plot_biomass()", + "*" = "plot_landings()" + )) + + # make_rdas_q <- readline("Do you want to create these plots now? (Y/N)") + # + # if (!interactive()) {make_rdas_q <- "n"} + # if (regexpr(make_rdas_q, "n", ignore.case = TRUE) == 1) { + # cli::cli_alert_danger("Fishing mortality, biomass, and landings plots will not be created.") + # } else if (regexpr(make_rdas_q, "y", ignore.case = TRUE) == 1) { + # if (is.null(model_results)) { + # cli::cli_alert_danger("No model results file provided. Plots will not be created.") + # } + # cli::cli_alert_info("Creating plots:") + # + # load(model_results) + # + # cli::cli_alert_info(" * Fishing mortality") + # plot_fishing_mortality(dat = out_new, + # make_rda = TRUE) + # + # cli::cli_alert_info(" * Biomass") + # plot_biomass(dat = out_new, + # make_rda = TRUE) + # + # cli::cli_alert_info(" * Landings") + # plot_landings(dat = out_new, + # make_rda = TRUE) + # + # cli::cli_alert_success("Plots created.") + # } else { + # cli::cli_alert_danger("Invalid input. Plots will not be created.") + # } + } - # AS_LAST_DATA_YEAR <- kqs |> - # dplyr::filter(key_quantity == "landings.end.year") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_FMSY <- kqs |> - # dplyr::filter(key_quantity == "F.MSY.terminal") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_BMSY <- kqs |> - # dplyr::filter(key_quantity == "B.msy") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_BMSY_MIN <- kqs |> - # dplyr::filter(key_quantity == "B.msy.min") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_BMSY_MAX <- kqs |> - # dplyr::filter(key_quantity == "B.msy.max") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_B_YEAR <- kqs |> - # dplyr::filter(key_quantity == "B.terminal.year") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_F_YEAR <- kqs |> - # dplyr::filter(key_quantity == "F.terminal.year") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_FTARGET <- kqs |> - # dplyr::filter(key_quantity == "F.target") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_FLIMIT <- kqs |> - # dplyr::filter(key_quantity == "F.limit") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_B_BEST <- kqs |> - # dplyr::filter(key_quantity == "B.terminal.est") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_F_BEST <- kqs |> - # dplyr::filter(key_quantity == "F.terminal.est") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_B_MIN <- kqs |> - # dplyr::filter(key_quantity == "B.terminal.min") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_B_MAX <- kqs |> - # dplyr::filter(key_quantity == "B.terminal.max") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_F_MIN <- kqs |> - # dplyr::filter(key_quantity == "F.terminal.min") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_F_MAX <- kqs |> - # dplyr::filter(key_quantity == "F.terminal.max") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_FMSY_MAX <- kqs |> - # dplyr::filter(key_quantity == "F.MSY.terminal.max") |> - # dplyr::select(value) |> - # as.numeric() - # - # AS_FMSY_MIN <- kqs |> - # dplyr::filter(key_quantity == "F.MSY.terminal.min") |> - # dplyr::select(value) |> - # as.numeric() - - # insert values into the sis_assmt_template.csv file - mapping <- tibble::tribble( - ~key_quantity, ~Variable, - "landings.end.year", "AS_LAST_DATA_YEAR", - "F.MSY.terminal", "AS_FMSY", - "B.msy", "AS_BMSY", - "B.msy.min", "AS_BMSY_MIN", - "B.msy.max", "AS_BMSY_MAX", - "B.terminal.year", "AS_B_YEAR", - "F.terminal.year", "AS_F_YEAR", - "F.target", "AS_FTARGET", - "F.limit", "AS_FLIMIT", - "B.terminal.est", "AS_B_BEST", - "F.terminal.est", "AS_F_BEST", - "B.terminal.min", "AS_B_MIN", - "B.terminal.max", "AS_B_MAX", - "F.terminal.min", "AS_F_MIN", - "F.terminal.max", "AS_F_MAX", - "F.MSY.terminal.max", "AS_FMSY_MAX", - "F.MSY.terminal.min", "AS_FMSY_MIN" - ) - - # Extract values and format into a key-value matching table - new_vals <- kqs |> - dplyr::inner_join(mapping, by = "key_quantity") |> - dplyr::mutate(value = as.numeric(value)) |> - dplyr::select(Variable, new_value = value) + if (exists(kqs_path)){ + # insert values into the sis_assmt_template.csv file + mapping <- tibble::tribble( + ~key_quantity, ~Variable, + "landings.end.year", "AS_LAST_DATA_YEAR", + "F.MSY.terminal", "AS_FMSY", + "B.msy", "AS_BMSY", + "B.msy.min", "AS_BMSY_MIN", + "B.msy.max", "AS_BMSY_MAX", + "B.terminal.year", "AS_B_YEAR", + "F.terminal.year", "AS_F_YEAR", + "F.target", "AS_FTARGET", + "F.limit", "AS_FLIMIT", + "B.terminal.est", "AS_B_BEST", + "F.terminal.est", "AS_F_BEST", + "B.terminal.min", "AS_B_MIN", + "B.terminal.max", "AS_B_MAX", + "F.terminal.min", "AS_F_MIN", + "F.terminal.max", "AS_F_MAX", + "F.MSY.terminal.max", "AS_FMSY_MAX", + "F.MSY.terminal.min", "AS_FMSY_MIN" + ) + + # Extract values and format into a key-value matching table + new_vals <- kqs |> + dplyr::inner_join(mapping, by = "key_quantity") |> + dplyr::mutate(value = as.numeric(value)) |> + dplyr::select(Variable, new_value = value) + + # Update assmt_dat + assmt_dat <- assmt_dat |> + dplyr::left_join(new_vals, by = "Variable") |> + dplyr::mutate(Value = ifelse(!is.na(new_value), new_value, Value)) |> + dplyr::select(-new_value) + } - # Update assmt_dat - assmt_dat <- assmt_dat |> - dplyr::left_join(new_vals, by = "Variable") |> - dplyr::mutate(Value = ifelse(!is.na(new_value), new_value, Value)) |> - dplyr::select(-new_value) + # obtain time series data From 8ea526f4161716ec87f8883ec6454f5586077431 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 28 Aug 2026 14:46:05 -0400 Subject: [PATCH 04/11] Add code to extract time series data from rdas --- R/extract_sis_data.R | 203 +++++++++++++++++++++++++++++++--------- man/extract_sis_data.Rd | 10 +- 2 files changed, 169 insertions(+), 44 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index ec645eda..9e8408b2 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -10,12 +10,10 @@ #' #' Default: The working directory. #' -#' @param model_results Filepath to the standardized, converted model output -#' .rda file generated with `stockplotr::convert_output()`. If provided, will be -#' used to generate figures and key quantities used to populate the SIS templates -#' if key_quantities.csv does not exist. -#' -#' Default: NULL +#' +#' @param figures_tables_dir Path. Location of the existing 'figures' and 'tables' directories. +#' +#' Default: The working directory. #' #' @details This function acts within the following workflow: #' @@ -40,10 +38,10 @@ #' extract_sis_data <- function(sis_data_dir = getwd(), key_quantities_dir = getwd(), - model_results = NULL + figures_tables_dir = getwd() ) { # Check if existing data files exist; if not, start from blank templates - if (!exists(fs::path(sis_data_dir, "sis_assmt_template.csv"))) { + if (!file.exists(fs::path(sis_data_dir, "sis_assmt_template.csv"))) { assmt_dat <- read.csv(fs::path("inst/resources/sis_assmt_template.csv"), stringsAsFactors = FALSE) cli::cli_alert_info("No existing sis_assmt_template.csv found in {sis_data_dir}. Using blank template.") } else { @@ -51,7 +49,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_alert_info("Found existing sis_assmt_template.csv in {sis_data_dir}.") } - if (!exists(fs::path(sis_data_dir, "sis_ts_template.csv"))) { + if (!file.exists(fs::path(sis_data_dir, "sis_ts_template.csv"))) { ts_dat <- read.csv(fs::path("inst/resources/sis_ts_template.csv"), stringsAsFactors = FALSE) cli::cli_alert_info("No existing sis_ts_template.csv found in {sis_data_dir}. Using blank template.") } else { @@ -61,52 +59,22 @@ extract_sis_data <- function(sis_data_dir = getwd(), # extract key quantities from csv and assign to variables kqs_path <- fs::path(key_quantities_dir, "key_quantities.csv") - if (exists(kqs_path)){ + if (file.exists(kqs_path)){ kqs <- read.csv(fs::path(key_quantities_dir, "key_quantities.csv"), stringsAsFactors = FALSE) cli::cli_alert_info("Found existing key_quantities.csv in {key_quantities_dir}.") } else { cli::cli_alert_warning("No existing key_quantities.csv found in {key_quantities_dir}.") - cli::cli_alert_info("To obtain key quantities, run the following functions and specify `make_rda = TRUE`:") + cli::cli_alert_info("To obtain key quantities relevant to the sis_assmt_template.csv, run the following functions and specify `make_rda = TRUE`:") cli::cli_bullets(c( "*" = "plot_fishing_mortality()", "*" = "plot_biomass()", "*" = "plot_landings()" )) - - # make_rdas_q <- readline("Do you want to create these plots now? (Y/N)") - # - # if (!interactive()) {make_rdas_q <- "n"} - # if (regexpr(make_rdas_q, "n", ignore.case = TRUE) == 1) { - # cli::cli_alert_danger("Fishing mortality, biomass, and landings plots will not be created.") - # } else if (regexpr(make_rdas_q, "y", ignore.case = TRUE) == 1) { - # if (is.null(model_results)) { - # cli::cli_alert_danger("No model results file provided. Plots will not be created.") - # } - # cli::cli_alert_info("Creating plots:") - # - # load(model_results) - # - # cli::cli_alert_info(" * Fishing mortality") - # plot_fishing_mortality(dat = out_new, - # make_rda = TRUE) - # - # cli::cli_alert_info(" * Biomass") - # plot_biomass(dat = out_new, - # make_rda = TRUE) - # - # cli::cli_alert_info(" * Landings") - # plot_landings(dat = out_new, - # make_rda = TRUE) - # - # cli::cli_alert_success("Plots created.") - # } else { - # cli::cli_alert_danger("Invalid input. Plots will not be created.") - # } } - if (exists(kqs_path)){ + if (file.exists(kqs_path)){ # insert values into the sis_assmt_template.csv file mapping <- tibble::tribble( ~key_quantity, ~Variable, @@ -143,8 +111,157 @@ extract_sis_data <- function(sis_data_dir = getwd(), } # obtain time series data + if (!dir.exists(fs::path(figures_tables_dir, "figures"))) { + cli::cli_alert_info("'figures' folder not found in {sis_data_dir}.") + cli::cli_alert_danger("Some time series data will not be extracted.") + } else { + fig_ts <- TRUE + # ABUNDANCE + tryCatch( + { + load(fs::path(figures_tables_dir, "figures", "abundance_at_age_figure.rda")) + aaa <- rda[["figure"]][["layers"]][["geom_line"]]$data + abundance <- aaa |> + dplyr::group_by(year) |> + dplyr::summarise(sum = sum(total_fish)) |> + dplyr::rename(Abundance = sum) + }, + error = function(e) { + cli::cli_alert_warning("The 'abundance_at_age_figure.rda' file was not found in the figures folder. Abundance data will not be extracted.") + abundance <<- NULL + } + ) + + # SPAWNERS + tryCatch({ + load(fs::path(figures_tables_dir, "figures", "spawning_biomass_figure.rda")) + sb <- rda[["figure"]][["layers"]][["geom_line"]]$data + spawning_biomass <- sb |> + dplyr::group_by(year) |> + dplyr::summarise(sum = sum(estimate)) |> + dplyr::rename(Spawners = sum) + }, + error = function(e) { + cli::cli_alert_warning("The 'spawning_biomass_figure.rda' file was not found in the figures folder. Spawning biomass data will not be extracted.") + spawning_biomass <<- NULL + } + ) + + # RECRUITMENT + tryCatch({ + load(fs::path(figures_tables_dir, "figures", "recruitment_figure.rda")) + rec <- rda[["figure"]][["layers"]][["geom_line"]]$data + recruitment <- rec |> + dplyr::group_by(year) |> + dplyr::summarise(sum = sum(predicted_recruitment)) |> + dplyr::rename(Recruitment = sum) + }, + error = function(e) { + cli::cli_alert_warning("The 'recruitment_figure.rda' file was not found in the figures folder. Recruitment data will not be extracted.") + recruitment <<- NULL + }) + + # FISHING MORTALITY + tryCatch({ + load(fs::path(figures_tables_dir, "figures", "fishing_mortality_figure.rda")) + fm <- rda[["figure"]][["layers"]][["geom_line"]]$data + fishing_mortality <- fm |> + dplyr::group_by(year) |> + dplyr::summarise(mean = mean(estimate)) |> + dplyr::rename(Fmort = mean) + }, + error = function(e) { + cli::cli_alert_warning("The 'fishing_mortality_figure.rda' file was not found in the figures folder. Fishing mortality data will not be extracted.") + fishing_mortality <<- NULL + }) + + # INDEX + tryCatch({ + load(fs::path(figures_tables_dir, "figures", "index_figure.rda")) + index <- rda[["figure"]][["layers"]][["geom_line"]]$data + index <- index |> + dplyr::group_by(year) |> + dplyr::summarise(mean = mean(estimate)) |> + dplyr::rename(Index = mean) + }, + error = function(e) { + cli::cli_alert_warning("The 'index_figure.rda' file was not found in the figures folder. Index data will not be extracted.") + index <<- NULL + }) + } + if (!dir.exists(fs::path(figures_tables_dir, "tables"))) { + cli::cli_alert_info("'tables' folder not found in {sis_data_dir}.") + cli::cli_alert_danger("Some time series data will not be extracted.") + } else { + table_ts <- TRUE + #tryCatch( + #{ + #TODO: update this once catch table released + # load(fs::path(figures_tables_dir, "tables", "catch_table.rda")) + # catch <- rda[["figure"]][["layers"]][["geom_line"]]$data + # catch <- aaa |> + # dplyr::group_by(year) |> + # dplyr::summarise(sum = sum(total_fish)) |> + # dplyr::rename(abundance = sum) + # }, + # error = function(e) { + # cli::cli_alert_warning("The 'catch_table.rda' file was not found in the figures folder. Catch data will not be extracted.") + # catch <<- NULL + # } + #) + } + + if(exists("fig_ts")){ + summaries <- c("abundance", "spawning_biomass", "recruitment", "fishing_mortality", "index") + + # join all summaries by year + all_summaries <- c() + for (i in seq_along(summaries)) { + if (i == 1) { + all_summaries <- get(summaries[i]) + } else { + all_summaries <- dplyr::full_join(all_summaries, + get(summaries[i]), + by = "year") + } + } + } + if (exists("table_ts") & exists("catch")){ + if (exists("all_summaries")){ + all_summaries <- dplyr::full_join(all_summaries, + get("catch"), + by = "year") + summaries <- c(summaries, "catch") + } else { + all_summaries <- get("catch") + } + } + ts_options <- summaries[!is.na(summaries)] + if (length(ts_options) > 1){ + cli::cli_alert_info("Multiple time series summaries were extracted: {ts_options}.") + primary <- readline("Which category should be designated as the Primary time series?") + + if (!interactive()) { + primary <- "abundance" + cli::cli_alert_info("Primary category set to 'abundance' by default in non-interactive mode.") + } + if (primary %notin% ts_options) { + cli::cli_abort("Invalid primary category specified. Please choose from: {ts_options}.") + } + } else if (length(ts_options) == 1) { + primary <- ts_options + cli::cli_alert_info("Only one time series summary was extracted ({primary}) and will be used as the Primary time series.") + } else { + cli::cli_abort("No time series summaries were extracted. Please check the figures and tables directories.") + } + + # TODO: add all_summaries to ts_dat, matching by year and variable name + test <- all_summaries |> + dplyr::rename("Year" = "year") |> + tidyr::pivot_longer(cols = -Year, names_to = "Category", values_to = "Value") |> + dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) # At end: if assmt_dat$Value is NA and Default is 95, change it to Default for (i in seq_len(nrow(assmt_dat))) { @@ -152,7 +269,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), assmt_dat$Value[i] <- assmt_dat$Default[i] } } - + #TODO: show an example of a filled-out template } # if existing copies are present, check user wants to overwrite values; OR just add new ones? diff --git a/man/extract_sis_data.Rd b/man/extract_sis_data.Rd index 00db236f..8017ab3f 100644 --- a/man/extract_sis_data.Rd +++ b/man/extract_sis_data.Rd @@ -4,13 +4,21 @@ \alias{extract_sis_data} \title{Extract data from model results to send to SIS} \usage{ -extract_sis_data(sis_data_dir = getwd(), key_quantities_dir = getwd()) +extract_sis_data( + sis_data_dir = getwd(), + key_quantities_dir = getwd(), + figures_tables_dir = getwd() +) } \arguments{ \item{sis_data_dir}{Path. Location of the existing sis_assmt_template.csv and sis_ts_template.csv files, or, if absent, where new versions of the files should be saved. Default: The working directory.} +\item{figures_tables_dir}{Path. Location of the existing 'figures' and 'tables' directories. + +Default: The working directory.} + \item{kq_dir}{Path. Location of the existing key_quantities.csv file, or, if absent, where a new version of the file should be saved. Default: The working directory.} From cffea402615f2edbab764ea5bc5c0e3426c86334 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Mon, 31 Aug 2026 15:27:50 -0400 Subject: [PATCH 05/11] add catch table --- R/extract_sis_data.R | 45 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 9e8408b2..64f02fd7 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -194,21 +194,32 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_alert_danger("Some time series data will not be extracted.") } else { table_ts <- TRUE - #tryCatch( - #{ - #TODO: update this once catch table released - # load(fs::path(figures_tables_dir, "tables", "catch_table.rda")) - # catch <- rda[["figure"]][["layers"]][["geom_line"]]$data - # catch <- aaa |> - # dplyr::group_by(year) |> - # dplyr::summarise(sum = sum(total_fish)) |> - # dplyr::rename(abundance = sum) - # }, - # error = function(e) { - # cli::cli_alert_warning("The 'catch_table.rda' file was not found in the figures folder. Catch data will not be extracted.") - # catch <<- NULL - # } - #) + tryCatch( + { + load(fs::path(figures_tables_dir, "tables", "total_catch_table.rda")) + catch <- rda[["table"]][["_data"]] + catch_cols <- colnames(catch) + cols_without_catch <- c("Sex", "Area", "Season", "Type") + if (any(cols_without_catch %in% catch_cols)) { + catch <- catch |> + dplyr::select(-dplyr::any_of(cols_without_catch)) + } + catch <- catch |> + # remove values in parentheses, if present + dplyr::mutate(dplyr::across(!Year, ~ stringr::str_remove_all(.x, "\\s*\\(.*?\\)"))) |> + dplyr::mutate(dplyr::across(!Year, ~ stringr::str_remove_all(.x, ","))) |> + dplyr::mutate(dplyr::across(!Year, ~ as.numeric(.x))) |> + # summarize non-Year rows + dplyr::rowwise() |> + dplyr::mutate(Catch = sum(c_across(!Year), na.rm = TRUE)) |> + dplyr::ungroup() |> + dplyr::select(Year, Catch) + }, + error = function(e) { + cli::cli_alert_warning("The 'total_catch_table.rda' file was not found in the figures folder. Catch data will not be extracted.") + catch <<- NULL + } + ) } if(exists("fig_ts")){ @@ -228,8 +239,10 @@ extract_sis_data <- function(sis_data_dir = getwd(), } if (exists("table_ts") & exists("catch")){ if (exists("all_summaries")){ + catch <- catch |> + dplyr::rename(year = Year) all_summaries <- dplyr::full_join(all_summaries, - get("catch"), + catch, by = "year") summaries <- c(summaries, "catch") } else { From a737163caa7bc470aae77c3ecb4a61bdb9ebcb4a Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Mon, 31 Aug 2026 17:04:32 -0400 Subject: [PATCH 06/11] Transform data into csv-ready dfs --- R/extract_sis_data.R | 54 +++++++++++++++++++++++++++++------------ man/extract_sis_data.Rd | 4 ++- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 64f02fd7..978d6389 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -2,7 +2,9 @@ #' #' Semi-automate the extraction of key quantities from model results for eventual transmittance to SIS via `asar::export_to_sis()`. #' -#' @param sis_data_dir Path. Location of the existing sis_assmt_template.csv and sis_ts_template.csv files, or, if absent, where new versions of the files should be saved. +#' @param sis_data_dir Path. Location of the existing sis_assmt_template.csv +#' file, or, if absent, where a new version of sis_assmt_template.csv and +#' sis_ts_template.csv should be saved. #' #' Default: The working directory. #' @@ -49,13 +51,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_alert_info("Found existing sis_assmt_template.csv in {sis_data_dir}.") } - if (!file.exists(fs::path(sis_data_dir, "sis_ts_template.csv"))) { - ts_dat <- read.csv(fs::path("inst/resources/sis_ts_template.csv"), stringsAsFactors = FALSE) - cli::cli_alert_info("No existing sis_ts_template.csv found in {sis_data_dir}. Using blank template.") - } else { - ts_dat <- read.csv(fs::path(sis_data_dir, "sis_ts_template.csv"), stringsAsFactors = FALSE) - cli::cli_alert_info("Found existing sis_ts_template.csv in {sis_data_dir}.") - } + ts_dat <- read.csv(fs::path("inst/resources/sis_ts_template.csv"), stringsAsFactors = FALSE) # extract key quantities from csv and assign to variables kqs_path <- fs::path(key_quantities_dir, "key_quantities.csv") @@ -270,21 +266,47 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_abort("No time series summaries were extracted. Please check the figures and tables directories.") } - # TODO: add all_summaries to ts_dat, matching by year and variable name - test <- all_summaries |> + ts_dat_filled <- all_summaries |> dplyr::rename("Year" = "year") |> tidyr::pivot_longer(cols = -Year, names_to = "Category", values_to = "Value") |> - dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) + dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) |> + dplyr::mutate(Description = dplyr::case_when( + Category == "Abundance" ~ "Total Abundance", + Category == "Spawners" ~ assmt_dat$Value[assmt_dat$Variable == "AS_B_BASIS"], + Category == "Recruitment" ~ "Recruits - Age 1", + Category == "Fmort" ~ assmt_dat$Value[assmt_dat$Variable == "AS_F_BASIS"], + Category == "Index" ~ "Estimated Index", + Category == "Catch" ~ "Estimated Total Catch", + TRUE ~ NA + )) |> + dplyr::mutate(Unit = dplyr::case_when( + Category == "Abundance" ~ "Number of Fish", + Category == "Spawners" ~ assmt_dat$Value[assmt_dat$Variable == "AS_B_UNIT"], + Category == "Recruitment" ~ ifelse(kqs$value[kqs$key_quantity == "recruitment.units"] == "mt", "Metric Tons", kqs$value[kqs$key_quantity == "recruitment.units"]), + Category == "Fmort" ~ assmt_dat$Value[assmt_dat$Variable == "AS_F_UNIT"], + Category == "Index" ~ "", + Category == "Catch" ~ ifelse(kqs$value[kqs$key_quantity == "tot.catch.units"] == " (mt)", "Metric Tons", kqs$value[kqs$key_quantity == "tot.catch.units"]), + TRUE ~ NA + )) |> + dplyr::relocate(Value, .after = Unit) - # At end: if assmt_dat$Value is NA and Default is 95, change it to Default + # Ensure ts_dat_filled has same cols as ts_dat + ifelse(colnames(ts_dat_filled) == colnames(ts_dat), + TRUE, + cli::cli_abort("Time series data does not match template structure.")) + + # if assmt_dat$Value is NA and Default is 95, change it to Default for (i in seq_len(nrow(assmt_dat))) { if (is.na(assmt_dat$Value[i]) & assmt_dat$Default[i] == 95) { assmt_dat$Value[i] <- assmt_dat$Default[i] } } + + # export files + # if existing copies are present, check user wants to overwrite values; OR just add new ones? + + write.csv(assmt_dat, fs::path(sis_data_dir, "sis_assmt_template.csv"), row.names = FALSE) + write.csv(ts_dat_filled, fs::path(sis_data_dir, "sis_ts_template.csv"), row.names = FALSE) + #TODO: show an example of a filled-out template } - -# if existing copies are present, check user wants to overwrite values; OR just add new ones? - - diff --git a/man/extract_sis_data.Rd b/man/extract_sis_data.Rd index 8017ab3f..abcd1992 100644 --- a/man/extract_sis_data.Rd +++ b/man/extract_sis_data.Rd @@ -11,7 +11,9 @@ extract_sis_data( ) } \arguments{ -\item{sis_data_dir}{Path. Location of the existing sis_assmt_template.csv and sis_ts_template.csv files, or, if absent, where new versions of the files should be saved. +\item{sis_data_dir}{Path. Location of the existing sis_assmt_template.csv +file, or, if absent, where a new version of sis_assmt_template.csv and +sis_ts_template.csv should be saved. Default: The working directory.} From d1751bf4fc79a44f7baf33c284de308777dd8bfd Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Thu, 3 Sep 2026 16:25:37 -0400 Subject: [PATCH 07/11] Manipulate data to enable exporting csvs --- R/extract_sis_data.R | 108 ++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 978d6389..949ac1fd 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -207,7 +207,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::mutate(dplyr::across(!Year, ~ as.numeric(.x))) |> # summarize non-Year rows dplyr::rowwise() |> - dplyr::mutate(Catch = sum(c_across(!Year), na.rm = TRUE)) |> + dplyr::mutate(Catch = sum(dplyr::c_across(!Year), na.rm = TRUE)) |> dplyr::ungroup() |> dplyr::select(Year, Catch) }, @@ -247,43 +247,89 @@ extract_sis_data <- function(sis_data_dir = getwd(), } ts_options <- summaries[!is.na(summaries)] + cli::cli_alert_info("The following time series summaries were extracted: {ts_options}.") + primary_options1 <- c("fishing_mortality", "recruitment", "catch") + primary_options2 <- c("spawning_biomass", "abundance") - if (length(ts_options) > 1){ - cli::cli_alert_info("Multiple time series summaries were extracted: {ts_options}.") - primary <- readline("Which category should be designated as the Primary time series?") - - if (!interactive()) { - primary <- "abundance" - cli::cli_alert_info("Primary category set to 'abundance' by default in non-interactive mode.") - } - if (primary %notin% ts_options) { - cli::cli_abort("Invalid primary category specified. Please choose from: {ts_options}.") - } + if (length(ts_options) == 0) { + primary <- NA + cli::cli_alert_info("Zero time series summaries were extracted. Please check the figures and tables directories.") } else if (length(ts_options) == 1) { primary <- ts_options cli::cli_alert_info("Only one time series summary was extracted ({primary}) and will be used as the Primary time series.") - } else { - cli::cli_abort("No time series summaries were extracted. Please check the figures and tables directories.") - } + } else if (length(ts_options) == 2) { + cli::cli_alert_info("Two time series summaries were extracted ({ts_options}).") + if (any(ts_options %in% primary_options1) & any(ts_options %in% primary_options2)) { + cli::cli_alert_info("These categories will be used as the Primary time series.") + primary <- ts_options + }} else { + cli::cli_alert_info("At most two categories can be chosen as Primary time series: and .") + if (interactive()) { + primary1 <- readline("Which category should be designated as Primary 1?") + primary2 <- readline("Which category should be designated as Primary 2?") + if (primary1 %notin% ts_options | primary2 %notin% ts_options) { + cli::cli_abort("Invalid Primary category specified. Please choose from: {ts_options}.") + } else { + primary <- c(primary1, primary2) + } + } else { + # choose Fmort as first primary if present, otherwise Recruitment, otherwise Catch; and choose Spawners if present, otherwise Biomass + cli::cli_alert_info("The following categories will be chosen, in order of preference, as Primary time series: and .") + primary1 <- ifelse("fishing_mortality" %in% ts_options, + "fishing_mortality", + ifelse("recruitment" %in% ts_options, + "recruitment", + ifelse("catch" %in% ts_options, + "catch", + NA))) + primary2 <- ifelse("spawning_biomass" %in% ts_options, + "spawning_biomass", + ifelse("abundance" %in% ts_options, + "abundance", + NA)) + primary <- c(primary1, primary2) + primary <- primary[!is.na(primary)] + if (length(primary) == 0) { + cli::cli_alert_danger("No valid Primary categories found.") + primary <- NA + } else { + cli::cli_alert_info("Primary categor{?y/ies} set to {primary} by default in non-interactive mode.") + } + } + } + + if (length(primary) == 0) {primary <- NA} + + category_pairs <- list( + "fishing_mortality" = "Fmort", + "recruitment" = "Recruitment", + "catch" = "Catch", + "spawning_biomass" = "Spawners", + "abundance" = "Abundance", # aka biomass + "index" = "Index" + ) + + primary <- unlist(lapply(primary, function(x) category_pairs[[x]])) ts_dat_filled <- all_summaries |> dplyr::rename("Year" = "year") |> tidyr::pivot_longer(cols = -Year, names_to = "Category", values_to = "Value") |> - dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) |> + dplyr::mutate(Primary = ifelse(tolower(Category) %in% tolower(primary), "Y", "")) |> + # dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) |> dplyr::mutate(Description = dplyr::case_when( Category == "Abundance" ~ "Total Abundance", - Category == "Spawners" ~ assmt_dat$Value[assmt_dat$Variable == "AS_B_BASIS"], + Category == "Spawners" ~ as.character(assmt_dat$Value[assmt_dat$Variable == "AS_B_BASIS"]), Category == "Recruitment" ~ "Recruits - Age 1", - Category == "Fmort" ~ assmt_dat$Value[assmt_dat$Variable == "AS_F_BASIS"], + Category == "Fmort" ~ as.character(assmt_dat$Value[assmt_dat$Variable == "AS_F_BASIS"]), Category == "Index" ~ "Estimated Index", Category == "Catch" ~ "Estimated Total Catch", TRUE ~ NA )) |> dplyr::mutate(Unit = dplyr::case_when( Category == "Abundance" ~ "Number of Fish", - Category == "Spawners" ~ assmt_dat$Value[assmt_dat$Variable == "AS_B_UNIT"], + Category == "Spawners" ~ as.character(assmt_dat$Value[assmt_dat$Variable == "AS_B_UNIT"]), Category == "Recruitment" ~ ifelse(kqs$value[kqs$key_quantity == "recruitment.units"] == "mt", "Metric Tons", kqs$value[kqs$key_quantity == "recruitment.units"]), - Category == "Fmort" ~ assmt_dat$Value[assmt_dat$Variable == "AS_F_UNIT"], + Category == "Fmort" ~ as.character(assmt_dat$Value[assmt_dat$Variable == "AS_F_UNIT"]), Category == "Index" ~ "", Category == "Catch" ~ ifelse(kqs$value[kqs$key_quantity == "tot.catch.units"] == " (mt)", "Metric Tons", kqs$value[kqs$key_quantity == "tot.catch.units"]), TRUE ~ NA @@ -291,9 +337,9 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::relocate(Value, .after = Unit) # Ensure ts_dat_filled has same cols as ts_dat - ifelse(colnames(ts_dat_filled) == colnames(ts_dat), - TRUE, - cli::cli_abort("Time series data does not match template structure.")) + if (isFALSE(any(colnames(ts_dat_filled) == colnames(ts_dat)))) { + cli::cli_abort("Time series data does not match template structure.") + } # if assmt_dat$Value is NA and Default is 95, change it to Default for (i in seq_len(nrow(assmt_dat))) { @@ -303,10 +349,20 @@ extract_sis_data <- function(sis_data_dir = getwd(), } # export files - # if existing copies are present, check user wants to overwrite values; OR just add new ones? + assmt_dat_path <- fs::path(sis_data_dir, "sis_assmt_template.csv") + ts_dat_path <- fs::path(sis_data_dir, "sis_ts_template.csv") - write.csv(assmt_dat, fs::path(sis_data_dir, "sis_assmt_template.csv"), row.names = FALSE) - write.csv(ts_dat_filled, fs::path(sis_data_dir, "sis_ts_template.csv"), row.names = FALSE) + if (file.exists(assmt_dat_path) | file.exists(ts_dat_path)) { + cli::cli_alert_info("Existing sis_assmt_template.csv or sis_ts_template.csv found in {sis_data_dir}.") + overwrite <- readline("Do you want to overwrite the existing files? (y/n): ") + if (tolower(overwrite) == "y") { + write.csv(assmt_dat, assmt_dat_path, row.names = FALSE) + write.csv(ts_dat_filled, ts_dat_path, row.names = FALSE) + cli::cli_alert_success("Files overwritten successfully.") + } else { + cli::cli_alert_info("Files not overwritten. Please rename the files, then rerun this function to save the data extracted in this function.") + } + } #TODO: show an example of a filled-out template } From 5d09a690c5c14b4c01a8843212ae8c3274ad9b29 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Thu, 3 Sep 2026 17:08:23 -0400 Subject: [PATCH 08/11] Improve messages; ensure files are exported --- R/extract_sis_data.R | 41 +++++++++++++++++++++++++++++------------ man/extract_sis_data.Rd | 4 +++- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 949ac1fd..bff33c1e 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -34,7 +34,9 @@ #' @examples #' \dontrun{ #' extract_sis_data( -#' sis_data_dir = getwd() +#' sis_data_dir = getwd(), +#' key_quantities_dir = "my_dir", +#' figures_tables_dir = "my_other_dir" #' ) #' } #' @@ -48,7 +50,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_alert_info("No existing sis_assmt_template.csv found in {sis_data_dir}. Using blank template.") } else { assmt_dat <- read.csv(fs::path(sis_data_dir, "sis_assmt_template.csv"), stringsAsFactors = FALSE) - cli::cli_alert_info("Found existing sis_assmt_template.csv in {sis_data_dir}.") + cli::cli_alert_success("Found existing sis_assmt_template.csv in {sis_data_dir}.") } ts_dat <- read.csv(fs::path("inst/resources/sis_ts_template.csv"), stringsAsFactors = FALSE) @@ -59,7 +61,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), kqs <- read.csv(fs::path(key_quantities_dir, "key_quantities.csv"), stringsAsFactors = FALSE) - cli::cli_alert_info("Found existing key_quantities.csv in {key_quantities_dir}.") + cli::cli_alert_success("Found existing key_quantities.csv in {key_quantities_dir}.") } else { cli::cli_alert_warning("No existing key_quantities.csv found in {key_quantities_dir}.") cli::cli_alert_info("To obtain key quantities relevant to the sis_assmt_template.csv, run the following functions and specify `make_rda = TRUE`:") @@ -123,7 +125,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::rename(Abundance = sum) }, error = function(e) { - cli::cli_alert_warning("The 'abundance_at_age_figure.rda' file was not found in the figures folder. Abundance data will not be extracted.") + cli::cli_alert_warning("Abundance data was not extracted from the 'abundance_at_age_figure.rda' file.") abundance <<- NULL } ) @@ -138,7 +140,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::rename(Spawners = sum) }, error = function(e) { - cli::cli_alert_warning("The 'spawning_biomass_figure.rda' file was not found in the figures folder. Spawning biomass data will not be extracted.") + cli::cli_alert_warning("Spawning biomass data was not extracted from the 'spawning_biomass_figure.rda' file.") spawning_biomass <<- NULL } ) @@ -153,7 +155,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::rename(Recruitment = sum) }, error = function(e) { - cli::cli_alert_warning("The 'recruitment_figure.rda' file was not found in the figures folder. Recruitment data will not be extracted.") + cli::cli_alert_warning("Recruitment data was not extracted from the 'recruitment_figure.rda' file.") recruitment <<- NULL }) @@ -167,7 +169,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::rename(Fmort = mean) }, error = function(e) { - cli::cli_alert_warning("The 'fishing_mortality_figure.rda' file was not found in the figures folder. Fishing mortality data will not be extracted.") + cli::cli_alert_warning("Fishing mortality data was not extracted from the 'fishing_mortality_figure.rda' file.") fishing_mortality <<- NULL }) @@ -181,7 +183,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::rename(Index = mean) }, error = function(e) { - cli::cli_alert_warning("The 'index_figure.rda' file was not found in the figures folder. Index data will not be extracted.") + cli::cli_alert_warning("Index data was not extracted from the 'index_figure.rda' file.") index <<- NULL }) } @@ -212,7 +214,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), dplyr::select(Year, Catch) }, error = function(e) { - cli::cli_alert_warning("The 'total_catch_table.rda' file was not found in the figures folder. Catch data will not be extracted.") + cli::cli_alert_warning("Catch data will not be extracted from the 'total_catch_table.rda' file.") catch <<- NULL } ) @@ -221,6 +223,8 @@ extract_sis_data <- function(sis_data_dir = getwd(), if(exists("fig_ts")){ summaries <- c("abundance", "spawning_biomass", "recruitment", "fishing_mortality", "index") + summaries <- summaries[sapply(summaries, function(x) !is.null(get(x)))] + # join all summaries by year all_summaries <- c() for (i in seq_along(summaries)) { @@ -247,7 +251,8 @@ extract_sis_data <- function(sis_data_dir = getwd(), } ts_options <- summaries[!is.na(summaries)] - cli::cli_alert_info("The following time series summaries were extracted: {ts_options}.") + cli::cli_alert_info("The following time series summaries were extracted:") + cli::cli_ul(ts_options) primary_options1 <- c("fishing_mortality", "recruitment", "catch") primary_options2 <- c("spawning_biomass", "abundance") @@ -263,7 +268,11 @@ extract_sis_data <- function(sis_data_dir = getwd(), cli::cli_alert_info("These categories will be used as the Primary time series.") primary <- ts_options }} else { - cli::cli_alert_info("At most two categories can be chosen as Primary time series: and .") + cli::cli_alert_info("At most, two categories can be chosen as Primary time series:") + cli::cli_ul(c( + "fishing_mortality OR recruitment OR catch", + "spawning_biomass OR abundance" + )) if (interactive()) { primary1 <- readline("Which category should be designated as Primary 1?") primary2 <- readline("Which category should be designated as Primary 2?") @@ -274,7 +283,11 @@ extract_sis_data <- function(sis_data_dir = getwd(), } } else { # choose Fmort as first primary if present, otherwise Recruitment, otherwise Catch; and choose Spawners if present, otherwise Biomass - cli::cli_alert_info("The following categories will be chosen, in order of preference, as Primary time series: and .") + cli::cli_alert_info("The following categories will be chosen, in order of preference, as Primary time series:") + cli::cli_ul(c( + "fishing_mortality OR recruitment OR catch, and", + "spawning_biomass OR abundance" + )) primary1 <- ifelse("fishing_mortality" %in% ts_options, "fishing_mortality", ifelse("recruitment" %in% ts_options, @@ -362,6 +375,10 @@ extract_sis_data <- function(sis_data_dir = getwd(), } else { cli::cli_alert_info("Files not overwritten. Please rename the files, then rerun this function to save the data extracted in this function.") } + } else { + write.csv(assmt_dat, assmt_dat_path, row.names = FALSE) + write.csv(ts_dat_filled, ts_dat_path, row.names = FALSE) + cli::cli_alert_success("Files saved successfully in {sis_data_dir}.") } #TODO: show an example of a filled-out template diff --git a/man/extract_sis_data.Rd b/man/extract_sis_data.Rd index abcd1992..a5826feb 100644 --- a/man/extract_sis_data.Rd +++ b/man/extract_sis_data.Rd @@ -44,7 +44,9 @@ This function acts within the following workflow: \examples{ \dontrun{ extract_sis_data( - sis_data_dir = getwd() + sis_data_dir = getwd(), + key_quantities_dir = "my_dir", + figures_tables_dir = "my_other_dir" ) } From 4990e80214ca71e6c1171fe1fdb16e2c35aae6f3 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 4 Sep 2026 10:58:06 -0400 Subject: [PATCH 09/11] update wordlist --- inst/WORDLIST | 1 + 1 file changed, 1 insertion(+) diff --git a/inst/WORDLIST b/inst/WORDLIST index 7062e059..cc30343c 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -19,6 +19,7 @@ Subseason aa alttext asar +assmt bam birthseas cha From 458bfd3307d45f915bd96f11d7b0288c2e5d94ba Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Fri, 4 Sep 2026 14:07:21 -0400 Subject: [PATCH 10/11] Address comments from review --- R/extract_sis_data.R | 15 +++++++-------- man/extract_sis_data.Rd | 16 ++++++++-------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index bff33c1e..27086233 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -2,17 +2,16 @@ #' #' Semi-automate the extraction of key quantities from model results for eventual transmittance to SIS via `asar::export_to_sis()`. #' -#' @param sis_data_dir Path. Location of the existing sis_assmt_template.csv -#' file, or, if absent, where a new version of sis_assmt_template.csv and -#' sis_ts_template.csv should be saved. +#' @param sis_data_dir Path. Location to save the sis_assmt_template.csv and +#' sis_ts_template.csv files or, if present, the location of the +#' existing sis_assmt_template.csv file. #' #' Default: The working directory. #' -#' @param kq_dir Path. Location of the existing key_quantities.csv file, or, if absent, where a new version of the file should be saved. +#' @param key_quantities_dir Path. Location of the existing key_quantities.csv file. #' #' Default: The working directory. #' -#' #' @param figures_tables_dir Path. Location of the existing 'figures' and 'tables' directories. #' #' Default: The working directory. @@ -22,9 +21,9 @@ #' 1. When a stock assessment is scheduled to conclude, SIS will generate an #' attachment or prompt containing metadata and identifiers. #' 2. The user will open two csv files containing placeholders for all of the data required by SIS: sis_assmt_template.csv (assessment summary data) and sis_ts_template.csv (time series data). There are three ways to obtain these files: -#' 2a. Generate the files by running `asar::create_blank_sis()` -#' 2b. Locate the files in the "report" folder generated by running `asar::create_template()`. -#' 2c. Run `stockplotr::extract_sis_data()`, which will populate the templates with data originating from a converted model results file. +#' 2a. Run `stockplotr::extract_sis_data()`, which will generate, populate, and export the templates with data originating from a converted model results file. +#' 2b. Generate blank files by running `asar::create_blank_sis()`. +#' 2c. Locate blank files in the "report" folder generated by running `asar::create_template()`. #' 3. The user will add the remaining necessary data into the csv files, ensuring that all required fields are completed. #' 4. Run `export_to_sis()`, which will format and upload this data to a specific Google Drive folder. #' 5. The uploaded contents will be resubmitted to SIS to finalize the record. diff --git a/man/extract_sis_data.Rd b/man/extract_sis_data.Rd index a5826feb..8ea31a73 100644 --- a/man/extract_sis_data.Rd +++ b/man/extract_sis_data.Rd @@ -11,17 +11,17 @@ extract_sis_data( ) } \arguments{ -\item{sis_data_dir}{Path. Location of the existing sis_assmt_template.csv -file, or, if absent, where a new version of sis_assmt_template.csv and -sis_ts_template.csv should be saved. +\item{sis_data_dir}{Path. Location to save the sis_assmt_template.csv and +sis_ts_template.csv files or, if present, the location of the +existing sis_assmt_template.csv file. Default: The working directory.} -\item{figures_tables_dir}{Path. Location of the existing 'figures' and 'tables' directories. +\item{key_quantities_dir}{Path. Location of the existing key_quantities.csv file. Default: The working directory.} -\item{kq_dir}{Path. Location of the existing key_quantities.csv file, or, if absent, where a new version of the file should be saved. +\item{figures_tables_dir}{Path. Location of the existing 'figures' and 'tables' directories. Default: The working directory.} } @@ -34,9 +34,9 @@ This function acts within the following workflow: 1. When a stock assessment is scheduled to conclude, SIS will generate an attachment or prompt containing metadata and identifiers. 2. The user will open two csv files containing placeholders for all of the data required by SIS: sis_assmt_template.csv (assessment summary data) and sis_ts_template.csv (time series data). There are three ways to obtain these files: -2a. Generate the files by running `asar::create_blank_sis()` -2b. Locate the files in the "report" folder generated by running `asar::create_template()`. -2c. Run `stockplotr::extract_sis_data()`, which will populate the templates with data originating from a converted model results file. +2a. Run `stockplotr::extract_sis_data()`, which will generate, populate, and export the templates with data originating from a converted model results file. +2b. Generate blank files by running `asar::create_blank_sis()`. +2c. Locate blank files in the "report" folder generated by running `asar::create_template()`. 3. The user will add the remaining necessary data into the csv files, ensuring that all required fields are completed. 4. Run `export_to_sis()`, which will format and upload this data to a specific Google Drive folder. 5. The uploaded contents will be resubmitted to SIS to finalize the record. From a9c028b84afd558fab066f7153c5492ac2dcf457 Mon Sep 17 00:00:00 2001 From: sbreitbart-NOAA Date: Tue, 22 Sep 2026 16:54:03 -0400 Subject: [PATCH 11/11] Updating extract_sis_data() to handle more edge cases and circumstances with different figures/tables available --- R/extract_sis_data.R | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/R/extract_sis_data.R b/R/extract_sis_data.R index 27086233..ce5ccc69 100644 --- a/R/extract_sis_data.R +++ b/R/extract_sis_data.R @@ -43,6 +43,11 @@ extract_sis_data <- function(sis_data_dir = getwd(), key_quantities_dir = getwd(), figures_tables_dir = getwd() ) { + # check if existing figures and tables folders exist; if both absent, throw an error + if (!dir.exists(fs::path(figures_tables_dir, "figures")) & !dir.exists(fs::path(figures_tables_dir, "tables"))) { + cli::cli_abort("Neither 'figures' nor 'tables' folders were found in {figures_tables_dir}. Please check the `figures_tables_dir` path, or export figures and tables, and then try again.") + } + # Check if existing data files exist; if not, start from blank templates if (!file.exists(fs::path(sis_data_dir, "sis_assmt_template.csv"))) { assmt_dat <- read.csv(fs::path("inst/resources/sis_assmt_template.csv"), stringsAsFactors = FALSE) @@ -116,7 +121,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), # ABUNDANCE tryCatch( { - load(fs::path(figures_tables_dir, "figures", "abundance_at_age_figure.rda")) + load(fs::path(figures_tables_dir, "figures", "abundance_at_age_figure.rda")) |> suppressWarnings() aaa <- rda[["figure"]][["layers"]][["geom_line"]]$data abundance <- aaa |> dplyr::group_by(year) |> @@ -131,7 +136,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), # SPAWNERS tryCatch({ - load(fs::path(figures_tables_dir, "figures", "spawning_biomass_figure.rda")) + load(fs::path(figures_tables_dir, "figures", "spawning_biomass_figure.rda")) |> suppressWarnings() sb <- rda[["figure"]][["layers"]][["geom_line"]]$data spawning_biomass <- sb |> dplyr::group_by(year) |> @@ -146,7 +151,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), # RECRUITMENT tryCatch({ - load(fs::path(figures_tables_dir, "figures", "recruitment_figure.rda")) + load(fs::path(figures_tables_dir, "figures", "recruitment_figure.rda")) |> suppressWarnings() rec <- rda[["figure"]][["layers"]][["geom_line"]]$data recruitment <- rec |> dplyr::group_by(year) |> @@ -160,7 +165,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), # FISHING MORTALITY tryCatch({ - load(fs::path(figures_tables_dir, "figures", "fishing_mortality_figure.rda")) + load(fs::path(figures_tables_dir, "figures", "fishing_mortality_figure.rda")) |> suppressWarnings() fm <- rda[["figure"]][["layers"]][["geom_line"]]$data fishing_mortality <- fm |> dplyr::group_by(year) |> @@ -174,7 +179,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), # INDEX tryCatch({ - load(fs::path(figures_tables_dir, "figures", "index_figure.rda")) + load(fs::path(figures_tables_dir, "figures", "index_figure.rda")) |> suppressWarnings() index <- rda[["figure"]][["layers"]][["geom_line"]]$data index <- index |> dplyr::group_by(year) |> @@ -189,11 +194,12 @@ extract_sis_data <- function(sis_data_dir = getwd(), if (!dir.exists(fs::path(figures_tables_dir, "tables"))) { cli::cli_alert_info("'tables' folder not found in {sis_data_dir}.") cli::cli_alert_danger("Some time series data will not be extracted.") + catch <- NULL } else { table_ts <- TRUE tryCatch( { - load(fs::path(figures_tables_dir, "tables", "total_catch_table.rda")) + load(fs::path(figures_tables_dir, "tables", "total_catch_table.rda")) |> suppressWarnings() catch <- rda[["table"]][["_data"]] catch_cols <- colnames(catch) cols_without_catch <- c("Sex", "Area", "Season", "Type") @@ -236,8 +242,8 @@ extract_sis_data <- function(sis_data_dir = getwd(), } } } - if (exists("table_ts") & exists("catch")){ - if (exists("all_summaries")){ + if (exists("table_ts") & !is.null(catch)){ + if (!is.null(all_summaries) & exists("fig_ts")){ catch <- catch |> dplyr::rename(year = Year) all_summaries <- dplyr::full_join(all_summaries, @@ -246,6 +252,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), summaries <- c(summaries, "catch") } else { all_summaries <- get("catch") + summaries <- "catch" } } @@ -324,7 +331,7 @@ extract_sis_data <- function(sis_data_dir = getwd(), primary <- unlist(lapply(primary, function(x) category_pairs[[x]])) ts_dat_filled <- all_summaries |> - dplyr::rename("Year" = "year") |> + dplyr::rename_with(~ "Year", .cols = matches("^year$")) |> tidyr::pivot_longer(cols = -Year, names_to = "Category", values_to = "Value") |> dplyr::mutate(Primary = ifelse(tolower(Category) %in% tolower(primary), "Y", "")) |> # dplyr::mutate(Primary = ifelse(tolower(Category) == primary, "Y", "")) |>