Tokenizer's APIs Update (#7128)

* Tokenizer's APIs Update * Address the feedback * Address the feedback and use the new TestTokenizers package
dotnet · Apr 19, 2024 · 72cfdf6 · 72cfdf6
1 parent fac1e10
commit 72cfdf6
Show file tree

Hide file tree

Showing 31 changed files with 2,803 additions and 1,114 deletions.
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -41,7 +41,7 @@
     <MicrosoftDotNetInteractiveVersion>1.0.0-beta.23509.3</MicrosoftDotNetInteractiveVersion>
     <MicrosoftMLOnnxRuntimeVersion>1.16.3</MicrosoftMLOnnxRuntimeVersion>
     <MlNetMklDepsVersion>0.0.0.12</MlNetMklDepsVersion>
-    <!--  
+    <!--
 @("inteltbb.devel", "win", "2021.7.1.15305")
     -->
     <OneDalPkgVersion Condition="'$(OS)' == 'Windows_NT'">2023.0.0.23189</OneDalPkgVersion>
@@ -87,6 +87,7 @@
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
     <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
     <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
+    <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24218.2</MicrosoftMLTestTokenizersVersion>
     <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
     <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
     <XunitCombinatorialVersion>1.2.7</XunitCombinatorialVersion>

diff --git a/src/Microsoft.ML.Tokenizers/EncodingResult.cs b/src/Microsoft.ML.Tokenizers/EncodingResult.cs