chore: Remove unused codE

PacktPublishing · Aug 7, 2024 · a34016a · a34016a
1 parent 82fa3fa
commit a34016a
Showing 1 changed file with 0 additions and 16 deletions.
diff --git a/llm_engineering/application/dataset/generation.py b/llm_engineering/application/dataset/generation.py
@@ -1,5 +1,3 @@
-from typing import Generator
-
 import tiktoken
 from langchain_core.exceptions import OutputParserException
 from langchain_core.language_models.fake import FakeListLLM
@@ -67,20 +65,6 @@ def get_prompts(cls, documents: list[CleanedDocument]) -> dict[DataCategory, lis
 
         return grouped_prompts
 
-    @classmethod
-    def _batch_by_category(
-        cls, category: DataCategory, documents: list[CleanedDocument]
-    ) -> Generator[list, None, None]:
-        match category:
-            case DataCategory.ARTICLES:
-                return splits.misc.batch(documents, size=1)
-            case DataCategory.POSTS:
-                return splits.misc.batch(documents, size=5)
-            case DataCategory.REPOSITORIES:
-                return splits.misc.batch(documents, size=1)
-            case _:
-                raise ValueError(f"Unsupported category: {category}")
-
     @classmethod
     def get_prompt(cls, document: CleanedDocument) -> GenerateDatasetSamplesPrompt:
         data_category = document.get_category()