fix all issues and code clean up

microsoft · Jan 13, 2025 · 7b7913a · 7b7913a
1 parent 429e0a7
commit 7b7913a
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 939 deletions.
diff --git a/operators/tokenizer/bpe_tokenizer_model.hpp b/operators/tokenizer/bpe_tokenizer_model.hpp
@@ -152,7 +152,7 @@ class BpeModel {
         } else {
           vocab_map_[line] = id;
         }
-        special_tokens_.Add(std::move(line_32), id);
+        ORTX_RETURN_IF_ERROR(special_tokens_.Add(std::move(line_32), id));
       }
     }