Fix for column type mismatches in Athena/parquet output (#96)

ccao-data · Dec 20, 2023 · 63fb10e · 63fb10e
1 parent 3a9aad9
commit 63fb10e
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/pipeline/02-assess.R b/pipeline/02-assess.R
@@ -529,7 +529,11 @@ assessment_pin_data_final %>%
     type = "short",
     as_factor = FALSE
   ) %>%
-  mutate(meta_complex_id = as.numeric(meta_complex_id)) %>%
+  # Coerce columns to their expected Athena output type
+  mutate(
+    land_rate_per_pin = as.numeric(land_rate_per_pin),
+    meta_complex_id = as.numeric(meta_complex_id)
+  ) %>%
   # Reorder columns into groups by prefix
   select(
     starts_with(c("meta_", "loc_")), char_yrblt, char_total_bldg_sf,

diff --git a/pipeline/03-evaluate.R b/pipeline/03-evaluate.R
@@ -68,7 +68,9 @@ gen_agg_stats <- function(data, truth, estimate, bldg_sqft,
                           rsn_col, rsf_col, triad, geography,
                           class, col_dict, min_n) {
   # Helper function to return NA when sale sample size is too small
-  gte_n <- \(n_sales, min_n, fn) ifelse(sum(!is.na(n_sales)) >= min_n, fn, NA)
+  gte_n <- \(n_sales, min_n, fn) {
+    ifelse(sum(!is.na(n_sales)) >= min_n, fn, NA_real_)
+  }
 
   # List of summary stat/performance functions applied within summarize() below
   # Each function is listed on the right while the name of the function is on