Fix broken test.

Optimization had assumed the only way to be null is to have a nullish value. A Parse failure could be null.
mchav · Dec 29, 2024 · f3b6aab · f3b6aab
1 parent 029464d
commit f3b6aab
Showing 1 changed file with 10 additions and 9 deletions.
diff --git a/src/Data/DataFrame/Operations.hs b/src/Data/DataFrame/Operations.hs
@@ -720,7 +720,7 @@ parseDefault safeRead (Just (BoxedColumn (c :: V.Vector a))) =
                 nullish = S.fromList ["nan", "NULL", "null", "", " "]
                 emptyToNothing v = if S.member v nullish then Nothing else Just v
                 safeVector = V.map emptyToNothing c
-                hasNulls = isJust $ V.find (`S.member` nullish) c
+                hasNulls = V.foldl' (\acc v -> if isNothing v then acc || True else acc) False safeVector
               in Just $ if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c
             Nothing -> Just $ BoxedColumn c
         Just Refl ->
@@ -729,18 +729,19 @@ parseDefault safeRead (Just (BoxedColumn (c :: V.Vector a))) =
               emptyToNothing v = if S.member v nullish then Nothing else Just v
            in case readInt example of
                 Just _ ->
-                  let
-                      hasNulls = isJust $ V.find (`S.member` nullish) c
-                   in Just (if safeRead && hasNulls then BoxedColumn (V.map ((=<<) readInt . emptyToNothing) c) else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . readInt . (c V.!))))
+                  let safeVector = V.map ((=<<) readInt . emptyToNothing) c
+                      hasNulls = V.elem Nothing safeVector
+                   in Just $ if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0  . (safeVector V.!)))
                 Nothing -> case readDouble example of
                   Just _ ->
-                    let
-                        hasNulls = isJust $ V.find (`S.member` nullish) c
-                     in Just $ if safeRead && hasNulls then BoxedColumn (V.map ((=<<) readDouble . emptyToNothing) c) else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . readDouble . (c V.!)))
+                    let safeVector = V.map ((=<<) readDouble . emptyToNothing) c
+                        hasNulls = V.elem Nothing safeVector
+                     in Just $ if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))
                   Nothing -> case parseTimeOpt example of
                     Just d -> let
-                        hasNulls = isJust $ V.find (`S.member` nullish) c
-                      in Just $ if safeRead && hasNulls then BoxedColumn (V.map ((=<<) parseTimeOpt . emptyToNothing) c) else BoxedColumn (V.map unsafeParseTime c)
+                        safeVector = V.map ((=<<) parseTimeOpt . emptyToNothing) c
+                        hasNulls = V.elem Nothing safeVector
+                      in Just $ if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn (V.map unsafeParseTime c)
                     Nothing -> let
                         safeVector = V.map emptyToNothing c
                         hasNulls = isJust $ V.find (`S.member` nullish) c