Strip white space from string when parsing

mchav · Jan 18, 2025 · 8d2f64d · 8d2f64d
1 parent a9986b6
commit 8d2f64d
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/Data/DataFrame/IO/CSV.hs b/src/Data/DataFrame/IO/CSV.hs
@@ -69,7 +69,7 @@ readSeparated :: Char -> ReadOptions -> String -> IO DataFrame
 readSeparated c opts path = do
     totalRows <- countRows path
     withFile path ReadMode $ \handle -> do
-        firstRow <- map decodeUtf8Lenient . parseSep c <$> C.hGetLine handle
+        firstRow <- map (decodeUtf8Lenient . C.strip) . parseSep c <$> C.hGetLine handle
         let columnNames = if hasHeader opts
                         then map (T.filter (/= '\"')) firstRow
                         else map (T.singleton . intToDigit) [0..(length firstRow - 1)]
@@ -78,7 +78,7 @@ readSeparated c opts path = do
 
         -- Initialize mutable vectors for each column
         let numColumns = length columnNames
-        dataRow <- map decodeUtf8Lenient . parseSep c <$> C.hGetLine handle
+        dataRow <- map (decodeUtf8Lenient . C.strip) . parseSep c <$> C.hGetLine handle
         let actualRows = if hasHeader opts then totalRows - 1 else totalRows
         nullIndices <- VM.unsafeNew numColumns
         VM.set nullIndices []

diff --git a/src/Data/DataFrame/Internal/Column.hs b/src/Data/DataFrame/Internal/Column.hs
@@ -338,7 +338,7 @@ zipColumns (UnboxedColumn column) (UnboxedColumn other) = UnboxedColumn (VU.gene
 -- Clean this up.
 writeColumn :: Int -> C.ByteString -> Column -> IO (Either T.Text Bool)
 writeColumn i value' (MutableBoxedColumn (col :: VBM.IOVector a)) = let
-    value = decodeUtf8Lenient value'
+    value = (decodeUtf8Lenient . C.strip) value'
   in case testEquality (typeRep @a) (typeRep @T.Text) of
       Just Refl -> (if isNullish value
                     then VBM.unsafeWrite col i "" >> return (Left value)