Merge pull request #343 from USEPA/cm

Cm

DESCRIPTION

@@ -9,21 +9,38 @@ Authors@R:
            role = c("aut", "cre"),
            email = "[email protected]",
            comment = c(ORCID = "0000-0002-0615-6087")),
-    person(given = "Michelle",
-           family = "Thawley",
-           role = "aut"),
     person(given = "Jacob",
            family = "Greif",
            role = "aut"),
-    person(given = "Laura",
-           family = "Shumway",
-           role = "aut"),
     person(given = "Elise",
            family = "Hinman",
            role = "aut"), 
     person(given = "Kathleen",
            family = "Healy",
-           role = "aut")
+           role = "aut"), 
+    person(given = "Justin", 
+           family = "Bousquin",
+           role = "aut"),
+    person(given = "Laura",
+           family = "Decicco", 
+           role = "ctb"),
+    person(given = "Brad",
+           family = "Cooper", 
+           role = "ctr"),
+    person(given = "Florian",
+           family = "Rupprecht",
+           role = "ctb"),
+    person(given = "Michelle",
+           family = "Thawley",
+           role = "ctb"),
+    person(given = "Zachary", 
+           family = "Smith", 
+           role = "ctb"),
+    person(given = "Laura",
+           family = "Shumway",
+           role = "ctb"), 
+    person(family = "TADA Working Group", 
+           role = "ctb")
     )
 Description: Assists data partners in performing water quality analyses with
     data from the Water Quality Portal.

R/Figures.R

@@ -542,7 +542,14 @@ TADA_Scatterplot <- function(.data, id_cols = c("TADA.ComparableDataIdentifier")
   TADA_CheckColumns(.data, id_cols)
 
   # check .data has required columns
-  TADA_CheckColumns(.data, c("TADA.ResultMeasureValue", "TADA.ResultMeasure.MeasureUnitCode", "ActivityRelativeDepthName", "ActivityStartDate", "ActivityStartDateTime"))
+  TADA_CheckColumns(.data, c("TADA.ResultDepthHeightMeasure.MeasureValue",
+                             "TADA.ResultDepthHeightMeasure.MeasureUnitCode",
+                             "TADA.CharacteristicName",
+                             "ActivityStartDate",
+                             "ActivityStartDateTime",
+                             "TADA.ResultMeasureValue",
+                             "TADA.ResultMeasure.MeasureUnitCode",
+                             "TADA.ComparableDataIdentifier"))
 
   start <- dim(.data)[1]
 

R/GenerateRefTables.R

@@ -38,6 +38,7 @@ TADA_GetWQXCharValRef <- function() {
   }
 
   # Try to download up-to-date raw data
+
   raw.data <- tryCatch(
     {
       # read raw csv from url

R/ResultFlagsDependent.R

@@ -546,15 +546,18 @@ TADA_FindQCActivities <- function(.data, clean = FALSE, flaggedonly = FALSE) {
 
 #' AutoFilter
 #'
-#' This function Auto-Filters a dataframe so it only includes results that are
-#' ready to use for analyses. Ideally, this function should be run after data
-#' other data cleaning, QA/QC, and harmonization steps are completed using other
-#' TADA package functions, or manually.
+#' This function removes rows where the result value is not numeric to 
+#' prepare a dataframe for quantitative analyses. Ideally, this function should
+#' be run after other data cleaning, QA/QC, and harmonization steps are 
+#' completed using other TADA package functions, or manually. Specifically, .
+#' this function removes rows with "Text","Coerced to NA", and "ND or NA" 
+#' in the TADA.ResultMeasureValueDataTypes.Flag column, or NA in the 
+#' TADA.ResultMeasureValue column.
 #'
 #' @param .data TADA dataframe OR TADA sites dataframe
 #'
 #' @return .data with rows removed where result values are not quantitative (NA or text),
-#' the media is not water, or the results have other issues that are not dealt with elsewhere.
+#' or the results have other issues that are not dealt with elsewhere.
 #'
 #' @export
 #'
@@ -578,8 +581,8 @@ TADA_AutoFilter <- function(.data) {
   autofilter <- dplyr::filter(.data, TADA.ResultMeasureValueDataTypes.Flag != "ND or NA" &
     TADA.ResultMeasureValueDataTypes.Flag != "Text" &
     TADA.ResultMeasureValueDataTypes.Flag != "Coerced to NA" &
-    !is.na(TADA.ResultMeasureValue) &
-    TADA.ActivityMediaName == "WATER")
+    !is.na(TADA.ResultMeasureValue))# &
+    #TADA.ActivityMediaName == "WATER")
 
   # filter out QA/QC ActivityTypeCode's
   autofilter <- TADA_FindQCActivities(autofilter, clean = TRUE)

R/Transformations.R

@@ -407,7 +407,16 @@ TADA_CalculateTotalNP <- function(.data, sum_ref, daily_agg = c("max", "min", "m
   # create nutrient groups by site, date, and depth
   depths <- names(.data)[grepl("DepthHeightMeasure", names(.data))]
   depths <- depths[grepl("TADA.", depths)]
-  grpcols <- c("ActivityStartDate", "MonitoringLocationIdentifier", "TADA.LongitudeMeasure", "TADA.LatitudeMeasure", "ActivityMediaSubdivisionName", "TADA.ComparableDataIdentifier", "TADA.ResultMeasure.MeasureUnitCode", depths)
+  grpcols <- c("ActivityStartDate", 
+               "ActivityStartDateTime",
+               "ActivityRelativeDepthName",
+               "MonitoringLocationIdentifier",
+               "TADA.LongitudeMeasure",
+               "TADA.LatitudeMeasure",
+               "ActivityMediaSubdivisionName",
+               "TADA.ComparableDataIdentifier",
+               "TADA.ResultMeasure.MeasureUnitCode",
+               depths)
 
   dat <- suppressMessages(TADA_AggregateMeasurements(.data, grouping_cols = grpcols, agg_fun = daily_agg, clean = TRUE))
 

R/Utilities.R

@@ -1047,7 +1047,7 @@ TADA_UpdateExampleData <- function() {
   )
   print("Data_Nutrients_UT")
   print(dim(Data_Nutrients_UT))
-  save(Data_Nutrients_UT, file = "Data_Nutrients_UT.rda")
+  save(Data_Nutrients_UT, file = "data/Data_Nutrients_UT.rda")
   rm(Data_Nutrients_UT)
 
   Data_6Tribes_5y <- TADA_DataRetrieval(
@@ -1063,7 +1063,7 @@ TADA_UpdateExampleData <- function() {
   )
   print("Data_6Tribes_5y:")
   print(dim(Data_6Tribes_5y))
-  save(Data_6Tribes_5y, file = "inst/extdata/Data_6Tribes_5y.rda")
+  save(Data_6Tribes_5y, file = "data/Data_6Tribes_5y.rda")
 
   y <- subset(Data_6Tribes_5y, Data_6Tribes_5y$TADA.ActivityMediaName %in% c("WATER"))
   y <- TADA_RunKeyFlagFunctions(Data_6Tribes_5y)
@@ -1088,7 +1088,7 @@ TADA_UpdateExampleData <- function() {
   Data_6Tribes_5y_Harmonized <- TADA_HarmonizeSynonyms(y)
   print("Data_6Tribes_5y_Harmonized:")
   print(dim(Data_6Tribes_5y_Harmonized))
-  save(Data_6Tribes_5y_Harmonized, file = "inst/extdata/Data_6Tribes_5y_Harmonized.rda")
+  save(Data_6Tribes_5y_Harmonized, file = "data/Data_6Tribes_5y_Harmonized.rda")
   rm(Data_6Tribes_5y_Harmonized)
 
   Data_NCTCShepherdstown_HUC12 <- TADA::TADA_DataRetrieval(
@@ -1108,7 +1108,7 @@ TADA_UpdateExampleData <- function() {
   )
   print("Data_NCTCShepherdstown_HUC12:")
   print(dim(Data_NCTCShepherdstown_HUC12))
-  save(Data_NCTCShepherdstown_HUC12, file = "inst/extdata/Data_NCTCShepherdstown_HUC12.rda")
+  save(Data_NCTCShepherdstown_HUC12, file = "data/Data_NCTCShepherdstown_HUC12.rda")
   rm(Data_NCTCShepherdstown_HUC12)
 }
 

inst/CITATION

@@ -4,19 +4,20 @@ bibentry(bibtype = "Manual",
  author = personList(
               as.person("Cristina A. Mullin"),
               as.person("Jacob Greif"),
-              as.person("Michelle Thawley"),
-              as.person("Laura Shumway"), 
+              as.person("Elise Hinman"),
+              as.person("Katie Healy"),
               as.person("Justin Bousquin"), 
-              as.person("Laura Decicco"), 
-              as.person("Katie Healy"), 
-              as.person("Zach Smith"), 
-              as.person("Florian Rupprecht"), 
-              as.person("Brad Cooper")
+              as.person("Laura Decicco"),
+              as.person("Brad Cooper"),
+              as.person("Florian Rupprecht"),
+              as.person("Michelle Thawley"),
+              as.person("Zach Smith"),
+              as.person("Laura Shumway")
               ),
   title = "Water Quality Portal (WQP) Tools for Automated Data Analysis (TADA)",
   address="Washington, DC",
   institution = "U.S. Environmental Protection Agency",
-  year = 2022,
+  year = 2023,
   url = "https://github.com/USEPA/TADA",
-  textVersion = "Mullin et al., 2022, Water Quality Portal (WQP) Tools for Automated Data Analysis (TADA), U.S. Environmental Protection Agency, Washington, DC"
+  textVersion = "Mullin et al., 2023, Water Quality Portal (WQP) Tools for Automated Data Analysis (TADA), U.S. Environmental Protection Agency, Washington, DC"
 )

inst/extdata/WQXCharacteristicRef.csv

@@ -4730,7 +4730,9 @@ d","Deprecated","SULFLURAMID"
 "2,3,6-Trimethylnaphthalene","Accepted",""
 "2,3,7,8,-Tetrachlorodibenzo-p-dioxin","Accepted",""
 "2,3,7,8-Tetrachlorodibenzo-p-dioxin","Accepted",""
+"2,3,7,8-Tetrachlorodibenzo-p-dioxin, TEC","Accepted",""
 "2,3,7,8-Tetrachlorodibenzo-p-dioxin, TEQ","Accepted",""
+"2,3,7,8-Tetrachlorodibenzo-p-dioxin, TTE","Accepted",""
 "2,3,7,8-Tetrachlorodibenzo-p-dioxin-C13","Accepted",""
 "2,3,7,8-Tetrachlorodibenzo-p-dioxin-Cl37","Accepted",""
 "2,3,7,8-Tetrachlorodibenzofuran","Accepted",""
@@ -8951,6 +8953,7 @@ d","Deprecated","SULFLURAMID"
 "A'-Neogammacerane, (17.alpha.)-","Accepted",""
 "A'-Neogammacerane, (21.beta.)-","Accepted",""
 "AB-PINACA 4-hydroxypentyl","Accepted",""
+"ACET","Accepted",""
 "AD-67 antidote","Accepted",""
 "AHTN","Accepted",""
 "AMPA (glyphosate metabolite)***retired***use Aminomethylphosphonic acid","Deprecated","AMINOMETHYLPHOSPHONIC ACID"
@@ -9665,6 +9668,7 @@ d","Deprecated","SULFLURAMID"
 "Atorvastatin-d5","Accepted",""
 "Atraton","Accepted",""
 "Atrazine","Accepted",""
+"Atrazine + N-Dealkylated Metabolites","Accepted",""
 "Atrazine-13C3","Accepted",""
 "Atrazine-d5","Accepted",""
 "Atrazine-d5 (ethyl-d5)","Accepted",""
@@ -10426,6 +10430,7 @@ d","Deprecated","SULFLURAMID"
 "Benzo[a]fluoranthene","Accepted",""
 "Benzo[a]pyrene","Accepted",""
 "Benzo[a]pyrene-d12","Accepted",""
+"Benzo[b,j,k]fluoranthene","Accepted",""
 "Benzo[b,j]fluoranthene","Accepted",""
 "Benzo[b,k]fluoranthene","Accepted",""
 "Benzo[b]chrysene","Accepted",""
@@ -10465,6 +10470,7 @@ d","Deprecated","SULFLURAMID"
 "Benzoic acid, 2,3,4,5-tetrachloro-6-[[[3-[[(1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,8-heptadecafluorooctyl)sulfonyl]oxy]phenyl]amino]carbonyl]-, potassium salt (1:1)","Accepted",""
 "Benzoic acid, 2-(methylsulfonyl)-4-(trifluoromethyl)-, methyl ester","Accepted",""
 "Benzoic acid, 2-(phenylthio)-, 2,2,2-trifluoro-1-phenyl-1-(trifluoromethyl)ethyl ester+E955","Accepted",""
+"Benzoic acid, 2-amino-4-(methylsulfonyl)-","Accepted",""
 "Benzoic acid, 2-chloro-4-fluoro-5-(2-methoxy-6-oxo-4-(pentafluoroethyl)-1(6H)-pyrimidinyl)-, 1-methyl ethyl ester","Accepted",""
 "Benzoic acid, 2-hydroxy-4-(2,2,3,3-tetrafluoropropoxy)-","Accepted",""
 "Benzoic acid, 2-hydroxy-4-(2,2,3,3-tetrafluoropropoxy)-, methyl ester","Accepted",""
@@ -12128,6 +12134,8 @@ d","Deprecated","SULFLURAMID"
 "Data-logger operating voltage","Accepted",""
 "Daunomycin","Accepted",""
 "Daunorubicin-13C-d3","Accepted",""
+"Day, ice off","Accepted",""
+"Day, ice on","Accepted",""
 "Ddifluorotris(pentafluoroethyl)phosphorane","Accepted",""
 "Dead","Accepted",""
 "Dead centric diatoms","Accepted",""
@@ -12767,6 +12775,7 @@ d","Deprecated","SULFLURAMID"
 "Docosanoic acid, methyl ester","Accepted",""
 "Docosapentaenoic","Accepted",""
 "Docosatetraenoic acid","Accepted",""
+"Docosatetraenylethanolamide","Accepted",""
 "Dodecafluoro-3H-4,8-dioxanonanoate","Accepted",""
 "Dodecafluorohexane-1,6-disulfonyl difluoride","Accepted",""
 "Dodecafluorooctanedinitrile","Accepted",""
@@ -12924,7 +12933,7 @@ d","Deprecated","SULFLURAMID"
 "Erythromycin-anhydro","Accepted",""
 "Escherichia","Accepted",""
 "Escherichia coli","Accepted",""
-"Escherichia coli DNA","Accepted",""
+"Escherichia coli DNA marker","Accepted",""
 "Escherichia coli proportional contribution by human (%)","Accepted",""
 "Escherichia coli proportional contribution by livestock (%)","Accepted",""
 "Escherichia coli proportional contribution by pet (%)","Accepted",""
@@ -14212,6 +14221,7 @@ d","Deprecated","SULFLURAMID"
 "Horse copies","Accepted",""
 "Horse host %","Accepted",""
 "Hostachem hoe-2838","Accepted",""
+"Human Enteric viruses","Accepted",""
 "Human Influence - Buildings (choice list)","Accepted",""
 "Human Influence - Landfill/Trash (choice list)","Accepted",""
 "Human Influence - Logging Operations (choice list)","Accepted",""
@@ -14416,7 +14426,6 @@ d","Deprecated","SULFLURAMID"
 "Inorganic monomeric aluminum","Accepted",""
 "Inorganic nitrogen (NO2, NO3, & NH3)","Accepted","INORGANIC NITROGEN (NO2, NO3, & NH3)"
 "Inorganic nitrogen (ammonia, nitrate and nitrite)***retired***use Inorganic nitrogen (NO2, NO3, & NH3)","Deprecated","INORGANIC NITROGEN (NO2, NO3, & NH3)"
-"Inorganic nitrogen (nitrate and nitrite and ammonia)***retired***use Inorganic nitrogen (NO2, NO3, & NH3)","Deprecated","INORGANIC NITROGEN (NO2, NO3, & NH3)"
 "Inorganic nitrogen (nitrate and nitrite) ***retired***use Nitrate + Nitrite","Deprecated","NITRATE + NITRITE"
 "Inorganic nitrogen***retired***use Inorganic nitrogen (NO2, NO3, & NH3)","Deprecated","INORGANIC NITROGEN"
 "Inorganic phosphorus","Accepted","ORTHOPHOSPHATE"
@@ -17828,7 +17837,6 @@ d","Deprecated","SULFLURAMID"
 "Perfluoroheptanes (branched and linear)","Accepted",""
 "Perfluoroheptanesulfonate","Accepted",""
 "Perfluoroheptanesulfonic Acid***retired***use 1-Heptanesulfonic acid, 1,1,2,2,3,3,4,4,5,5,6,6,7,7,7-pentadecafluoro-","Deprecated","1-HEPTANESULFONIC ACID, 1,1,2,2,3,3,4,4,5,5,6,6,7,7,7-PENTADECAFLUORO-"
-"Perfluoroheptanesulfonic acid","Accepted",""
 "Perfluoroheptanoate","Accepted",""
 "Perfluoroheptanoate (anionic form)","Accepted",""
 "Perfluoroheptanoic acid","Accepted",""
@@ -20990,6 +20998,7 @@ d","Deprecated","SULFLURAMID"
 "Total Suspended Particulate Matter","Accepted",""
 "Total Tetrachloro Biphenyls","Accepted",""
 "Total Trichloro Biphenyls","Accepted",""
+"Total Volatile Hydrocarbons, C6-C10","Accepted",""
 "Total beta radioactivity","Accepted",""
 "Total carbon","Accepted",""
 "Total dissolved solids","Accepted",""
@@ -22138,7 +22147,7 @@ d","Deprecated","SULFLURAMID"
 "pH","Accepted",""
 "pH Saturated Paste","Accepted",""
 "pH, lab***retired***use pH","Deprecated","PH"
-"pH, saturated***retired***use pH","Deprecated","PH"
+"pH, saturated***retired***use pH","Deprecated",""
 "perfluorooctyl methacrylate","Accepted",""
 "phosphatase (corrected for carbon content)","Accepted",""
 "potassium pentafluoroethane-1-sulfonate","Accepted",""

inst/extdata/WQXMeasureQualifierCodeRef.csv

@@ -121,6 +121,7 @@
 "Result Measure Qualifier(MeasureQualifierCode)",1,"J","Estimated: The analyte was positively identified and the associated numerical value is the approximate concentration of the analyte in the sample.","8/11/2006 11:50:00 PM","Pass"
 "Result Measure Qualifier(MeasureQualifierCode)",7,"J+","Estimated: The analyte was positively identified and the associated numerical value... +++.","5/7/2009 1:08:20 PM","Pass"
 "Result Measure Qualifier(MeasureQualifierCode)",57,"J-","The analyte was positively identified; the associated numerical value is the approximate concentration of the analyte in the sample, and may have a potential negative bias.","2/19/2016 8:34:06 AM","Pass"
+"Result Measure Qualifier(MeasureQualifierCode)",68,"J-1","Estimated value. A full explanation is presented in the narrative.","9/28/2023 2:48:59 PM","Not Reviewed"
 "Result Measure Qualifier(MeasureQualifierCode)",64,"J-R","Approximate value result is below the reporting level but greater than the method detection limit","8/23/2021 11:04:25 AM","Pass"
 "Result Measure Qualifier(MeasureQualifierCode)",161,"JCN","Sample Container Damaged, no sample lost","3/16/2016 1:04:25 PM","Pass"
 "Result Measure Qualifier(MeasureQualifierCode)",162,"JCW","Sample Container Damaged, sample lost","3/16/2016 1:04:25 PM","Suspect"