Fix(LLMLingua): fix the assert issue due to tokenization (#139)

* fix tokenizer bug * exclude macros-latest on python3.9 unit test for environment conflict
microsoft · Apr 25, 2024 · 40ac969 · 40ac969
1 parent a411a3f
commit 40ac969
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
@@ -19,6 +19,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-2019]
         python-version: ["3.9", "3.10", "3.11"]
+        exclude:
+          - os: macos-latest
+            python-version: '3.9'
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

diff --git a/llmlingua/prompt_compressor.py b/llmlingua/prompt_compressor.py
@@ -992,7 +992,6 @@ def get_prefix_length(self, prefix: str, text: str):
             cur_prefix = self.tokenizer.decode(full_input_ids[:i])
             if cur_prefix == prefix:
                 break
-        assert self.tokenizer.decode(full_input_ids[i:]) == text[:100]
         return i
 
     def get_condition_ppl(