+
+
+ ' + + '' + + _("Hide Search Matches") + + "
" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms") + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; + if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/genindex.html b/genindex.html new file mode 100644 index 0000000..d2be6bc --- /dev/null +++ b/genindex.html @@ -0,0 +1,611 @@ + + + + + +
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..cac2d97
--- /dev/null
+++ b/index.html
@@ -0,0 +1,398 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Index
+ +
+ C
+ | D
+ | E
+ | G
+ | H
+ | I
+ | L
+ | M
+ | N
+ | O
+ | P
+ | Q
+ | R
+ | S
+ | T
+ | U
+ | V
+
+
+C
+| + | + |
D
+| + | + |
E
+| + | + |
G
+| + |
H
+| + | + |
I
+| + | + |
L
+| + |
M
+
|
+
|
+
N
+| + | + |
O
+| + | + |
P
+| + | + |
Q
+| + | + |
R
+| + | + |
S
+| + | + |
T
+| + |
U
+| + | + |
V
+| + |
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.html b/magnet.html
new file mode 100644
index 0000000..847dd1d
--- /dev/null
+++ b/magnet.html
@@ -0,0 +1,389 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + +Welcome to the 🧲 docs
+
+Packages & useful classes
+-
+
- magnet.ic package + +
- magnet.ize package
-
+
- Submodules +
- magnet.ize.filings module
-
+
Processor
+
+ - magnet.ize.memory module
-
+
Embedder
+
+
+ - magnet.ron package + +
- magnet.utils package
-
+
- Subpackages + +
- Submodules +
- magnet.utils.data_classes module + +
- magnet.utils.globals module
-
+
Utils
+reversal()
+
+ - magnet.utils.huggingface module
-
+
InferenceAPI
+
+ - magnet.utils.local module +
- magnet.utils.markdown module
-
+
m_print()
+
+ - magnet.utils.milvus module
-
+
MilvusDB
+
+ - magnet.utils.prompts module
-
+
Prompts
+
+
+
don’t forget to check out the helpful 💻 examples
+the small distributed language model toolkit
+
⚡️ fine-tune state-of-the-art LLMs anywhere, rapidly ⚡️
+🧬 Installation
+
pip install llm-magnet
+or
+python3 setup.py install
+🔮 features
+
-
+
⚡️ It’s Fast
+-
+
fast on consumer hardware
+very fast on Apple Silicon
+extremely fast on ROCm/CUDA
+
+🫵 Automatic or your way
+-
+
rely on established transformer patterns to let
magnetdo the +work
+keep your existing data processing functions, bring them to +
magnet!
+
+🛰️ 100% Distributed
+-
+
processing, embedding, storage, retrieval, querying, or inference +from anywhere
+as much or as little compute as you need
+
+🧮 Choose Inference Method
+-
+
HuggingFace
+vLLM node
+GPU
+mlx
+
+🌎 Huge Volumes
+-
+
handle gigantic amounts of data inexpensively
+fault-tolerant by design
+decentralized workloads
+
+🔐 Secure
+-
+
JWT
+Basic
+
+🪵 World-Class Comprehension
+-
+
magnetoptionally logs its own code as it’s executed (yes, +really)
+build a self-aware system and allow it to learn from itself
+emojis are the future
+
+
learn more about how Prismadic uses 🧲
+subscribe to our substack
+
Indices and tables
+
-
+
- +
- +
- +
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.ic.html b/magnet.ic.html
new file mode 100644
index 0000000..0fcb539
--- /dev/null
+++ b/magnet.ic.html
@@ -0,0 +1,384 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ magnet package
+Subpackages
+
+
+-
+
- magnet.ic package + +
- magnet.ize package + +
- magnet.ron package + +
- magnet.utils package
-
+
- Subpackages + +
- Submodules +
- magnet.utils.data_classes module
-
+
EmbeddingPayload+
+GeneratedPayload+
+MistralArgs+
+Payload+
+
+ - magnet.utils.globals module + +
- magnet.utils.huggingface module
-
+
InferenceAPI+
+
+ - magnet.utils.local module +
- magnet.utils.markdown module
-
+
m_print()
+
+ - magnet.utils.milvus module + +
- magnet.utils.prompts module
-
+
Prompts+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.ize.html b/magnet.ize.html
new file mode 100644
index 0000000..4057842
--- /dev/null
+++ b/magnet.ize.html
@@ -0,0 +1,488 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ magnet.ic package
+Submodules
+magnet.ic.field module
+-
+
- +class magnet.ic.field.Charge(server)[source] +
Bases:
+objectThe Charge class is responsible for connecting to a NATS server, managing streams and categories, and publishing data to the server.
+-
+
- Args:
server (str): The NATS server URL.
+
+- Attributes:
server (str): The NATS server URL. +category (str): The current category. +stream (str): The current stream. +nc: The NATS connection object. +js: The JetStream API object.
+
+
-
+
- +async emp(name=None)[source] +
Deletes the specified stream if the name matches the current stream, or prints an error message if the name doesn’t match or the stream doesn’t exist.
+-
+
- Args:
name (str, optional): The name of the stream to delete. Defaults to None.
+
+
-
+
- +async on(category: str = 'no_category', stream: str = 'documents', create: bool = False)[source] +
Connects to the NATS server, creates a stream and category if they don’t exist, and prints a success message.
+-
+
- Args:
category (str, optional): The category to connect to. Defaults to ‘no_category’. +stream (str, optional): The stream to connect to. Defaults to ‘documents’. +create (bool, optional): Whether to create the stream and category if they don’t exist. Defaults to False.
+
+
-
+
- +class magnet.ic.field.Resonator(server: str)[source] +
Bases:
+objectThe Resonator class is responsible for connecting to a NATS server, subscribing to a specific category in a stream, and consuming messages from that category.
+-
+
- Args:
server (str): The address of the NATS server.
+
+- Attributes:
server (str): The address of the NATS server. +category (str): The category to subscribe to. +stream (str): The stream to subscribe to. +session (str): The session name for durable subscriptions. +nc (nats.aio.client.Client): The NATS client. +js (nats.aio.client.JetStream): The JetStream client. +sub (nats.aio.client.Subscription): The subscription.
+
+- Methods:
__init__(self, server: str) +on(self, category: str = ‘no_category’, stream: str = ‘documents’, cb=print, session=’magnet’) -> None +info(self, session: str = None) -> None +off(self) -> None
+
+
-
+
- +async info(session: str = None)[source] +
Retrieves information about a consumer in a JetStream stream.
+-
+
- Parameters: +
session – A string representing the session name of the consumer. If not provided, information about all consumers in the stream will be retrieved.
+
+- Returns: +
None
+
+
-
+
- +async off()[source] +
Unsubscribes from the category and stream and disconnects from the NATS server.
+-
+
- Returns: +
None
+
+
-
+
- +async on(category: str = 'no_category', stream: str = 'documents', cb=<built-in function print>, session='magnet')[source] +
Connects to a NATS server, subscribes to a specific category in a stream, and consumes messages from that category.
+-
+
- Args:
category (str, optional): The category to subscribe to. Defaults to ‘no_category’. +stream (str, optional): The stream to subscribe to. Defaults to ‘documents’. +cb (function, optional): The callback function to process the received messages. Defaults to print. +session (str, optional): The session name for durable subscriptions. Defaults to ‘magnet’.
+
+- Returns:
None
+
+- Raises:
TimeoutError: If connection to the NATS server times out. +Exception: If there is an error in consuming the message or processing the callback function.
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.ron.html b/magnet.ron.html
new file mode 100644
index 0000000..b81859b
--- /dev/null
+++ b/magnet.ron.html
@@ -0,0 +1,252 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ magnet.ize package
+Submodules
+magnet.ize.filings module
+-
+
- +class magnet.ize.filings.Processor[source] +
Bases:
+objectThe Processor class is responsible for loading, processing, and saving data. It provides methods for loading data from different file formats, splitting the data into smaller chunks, and saving the processed data to a file.
+-
+
- Example Usage:
# Initialize the Processor class object +processor = Processor()
+# Load data into the processor +processor.load(raw_data, text_column=”clean”, id_column=”id”)
+# Process and split the loaded data into smaller chunks +await processor.process(path=”processed_data.csv”, splitter=None, nlp=True)
+# Save the processed data to a file +processor.save(filename=”processed_data.csv”, raw=processor.df)
+
+- Main functionalities:
-
+
Loading data from different file formats (csv, json, xlsx, parquet) or a pandas DataFrame
+Splitting the loaded data into smaller chunks either by sentences using natural language processing (NLP) or by a fixed character length
+Saving the processed data to a file in different formats (csv, json, xlsx, parquet)
+
+- Methods:
-
+
__init__(): Initializes the Processor class object and sets the df attribute to None and creates an instance of the Utils class.
+save(filename: str = None, raw: pd.DataFrame = None): Saves the pandas DataFrame to a file with the specified filename and file format.
+load(raw: str | pd.DataFrame = None, text_column: str = “clean”, id_column: str = ‘id’): Loads data into the df attribute of the Processor class.
+process(path: str = None, splitter: any = None, nlp=True): Processes and splits the loaded data into smaller chunks.
+default_splitter(data, window_size=768, overlap=76, nlp=True): Splits the given input data into smaller chunks either by sentences or by a fixed character length.
+
+- Fields:
-
+
df: A pandas DataFrame that stores the loaded data.
+utils: An instance of the Utils class that provides utility functions.
+
+
-
+
- +async create_charge(field=None)[source] +
Process and send data to a field.
+-
+
- Args:
field (optional): The field object to which the data will be sent.
+
+- Returns:
None
+
+- Raises:
ValueError: If no data is loaded. +ValueError: If no field is loaded.
+
+- Example Usage:
# Initialize the Processor class object +processor = Processor()
+# Load data into the processor +processor.load(raw_data, text_column=”clean”, id_column=”id”)
+# Initialize the field object +field = Field()
+# Create charges and send data to the field +await processor.create_charge(field)
+
+
-
+
- +default_splitter(data, window_size=768, overlap=76, nlp=True)[source] +
Splits the given input data into smaller chunks either by sentences or by a fixed character length.
+-
+
- Args:
data (str): The input data to be split. +window_size (int, optional): The size of each chunk when splitting by a fixed character length. Defaults to 768. +overlap (int, optional): The number of characters to overlap between each chunk when splitting by a fixed character length. Defaults to 76. +nlp (bool, optional): A flag indicating whether to split the data into sentences using natural language processing (NLP) or by a fixed character length. Defaults to True.
+
+- Returns:
list: If nlp is True, returns a list of sentences extracted from the input data. If nlp is False, returns a list of chunks obtained by splitting the input data into fixed-sized chunks.
+
+
-
+
- +load(raw: str | DataFrame = None, text_column: str = 'clean', id_column: str = 'id')[source] +
Load data into the df attribute of the Processor class.
+-
+
- Args:
raw (str | pd.DataFrame): The input data to be loaded. It can be either a file path (str) or a pandas DataFrame. +text_column (str): The name of the column in the input data that contains the text data. Default is “clean”. +id_column (str): The name of the column in the input data that contains the unique identifier for each data entry. Default is “id”.
+
+- Raises:
ValueError: If the file extension is not supported or an exception occurs during the loading process.
+
+- Returns:
None: If an error occurs during the loading process. +str: Success message if the data is successfully loaded.
+
+
-
+
- +async process(path: str = None, splitter: any = None, nlp=True)[source] +
Process and split the loaded data into smaller chunks.
+-
+
- Args:
path (str): The path to save the processed data. +splitter (function, optional): A custom function to split the text into chunks. If not provided, the default splitter function in the Processor class will be used. +nlp (bool, optional): A flag indicating whether to split the data into sentences using natural language processing (NLP) or by a fixed character length.
+
+- Returns:
None: If there is no data loaded or if an error occurs during the processing. +The processed data saved to the specified file path.
+
+
-
+
- +save(filename: str = None, raw: DataFrame = None)[source] +
Save the pandas DataFrame to a file with the specified filename and file format.
+-
+
- Args:
filename (str): The name of the file to save the data to. +raw (pd.DataFrame): The pandas DataFrame containing the data to be saved.
+
+- Raises:
ValueError: If the data format is unsupported or an error occurs during the saving process.
+
+- Returns:
None: If an error occurs during the saving process. +str: Success message if the data is successfully saved to the specified file.
+
+
magnet.ize.memory module
+-
+
- +class magnet.ize.memory.Embedder(config, create=False)[source] +
Bases:
+objectThe Embedder class is responsible for embedding text using a pre-trained sentence transformer model and storing or sending the embeddings for further processing. It utilizes the Milvus database for storing and searching the embeddings.
+-
+
- Args:
config (dict): A dictionary containing the configuration parameters for the Embedder class, including the model name, index name, and index parameters. +create (bool, optional): If set to True, a connection to the Milvus database will be created. Defaults to False.
+
+- Attributes:
config (dict): A dictionary containing the configuration parameters for the Embedder class, including the model name, index name, and index parameters. +model (SentenceTransformer): An instance of the SentenceTransformer class from the sentence_transformers library, used for text embedding. +db (MilvusDB): An instance of the MilvusDB class from the magnet.utils.milvus module, used for connecting to the Milvus database.
+
+- Methods:
embed_and_store(self, payload, verbose=False, field=None): Embeds the given payload using a pre-trained sentence transformer model and stores it in the Milvus database. +embed_and_charge(self, payload, verbose=False, field=None): Embeds the given payload using a pre-trained sentence transformer model and sends it to a specified field for further processing.
+
+
-
+
- +delete(name=None)[source] +
Delete an index from the Milvus database.
+-
+
- Args:
name (str, optional): The name of the index to be deleted. If not provided, no index will be deleted.
+
+- Returns:
None
+
+- Raises:
Exception: If an error occurs during the deletion process.
+
+- Example Usage:
-
+
- config = {
‘MODEL’: ‘bert-base-nli-mean-tokens’, +‘INDEX’: ‘my_index’, +‘INDEX_PARAMS’: {‘nprobe’: 16}
+
+
} +embedder = Embedder(config) +embedder.delete(‘my_index’)
+
+
-
+
- +async embed_and_charge(payload, verbose=False, field=None)[source] +
Embeds the given payload using a pre-trained sentence transformer model and sends it to a specified field for further processing.
+-
+
- Args:
payload (object): The payload to be embedded and charged. It should contain the text and document attributes. +verbose (bool, optional): If set to True, additional information will be logged during the embedding process. Defaults to False. +field (object): The field to which the encoded payload will be sent for further processing.
+
+- Raises:
ValueError: If the field parameter is not provided.
+
+- Returns:
None
+
+- Example Usage:
-
+
- config = {
‘MODEL’: ‘bert-base-nli-mean-tokens’, +‘INDEX’: ‘my_index’, +‘INDEX_PARAMS’: {‘nprobe’: 16}
+
+
} +embedder = Embedder(config) +await embedder.embed_and_charge(payload, verbose=True, field=my_field)
+
+
-
+
- +async embed_and_store(payload, verbose=False, field=None)[source] +
Embeds the given payload using a pre-trained sentence transformer model and stores it in a Milvus database.
+-
+
- Args:
payload (object): An object containing the text and document attributes to be embedded and stored. +verbose (bool, optional): A boolean indicating whether additional information should be logged during the embedding process. Defaults to False. +field (object, optional): An object representing the field to which the encoded payload will be sent for further processing.
+
+- Returns:
None
+
+- Raises:
Exception: If an error occurs during the embedding or storing process.
+
+
-
+
- +search(payload, limit=100, cb=None)[source] +
Search for relevant passages based on a given input payload.
+-
+
- Args:
payload (str): The search query or input payload. +limit (int, optional): The maximum number of results to return. Defaults to 100. +cb (function, optional): A callback function to process the results. Defaults to None.
+
+- Returns:
list: A list of dictionaries, each containing the text, document, and distance of a relevant passage.
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.utils.html b/magnet.utils.html
new file mode 100644
index 0000000..4cedbfe
--- /dev/null
+++ b/magnet.utils.html
@@ -0,0 +1,639 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/magnet.utils.mlx.html b/magnet.utils.mlx.html
new file mode 100644
index 0000000..92c6cfc
--- /dev/null
+++ b/magnet.utils.mlx.html
@@ -0,0 +1,252 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ magnet.utils package
+Subpackages
+ +Submodules
+magnet.utils.data_classes module
+-
+
- +class magnet.utils.data_classes.EmbeddingPayload(document: str, embedding: list, text: list, model: str)[source] +
Bases:
+objectRepresents a payload for embedding text data.
+-
+
- Attributes:
document (str): The document associated with the text data. +embedding (list): The embedding of the text data. +text (list): The text of the data. +model (str): The model used for embedding the text data.
+
+
-
+
- +document: str +
-
+
- +embedding: list +
-
+
- +model: str +
-
+
- +text: list +
-
+
- +class magnet.utils.data_classes.GeneratedPayload(query: str, prompt: str, context: list, result: str, model: str)[source] +
Bases:
+objectRepresents a payload generated by a system.
+-
+
- Args:
query (str): The query associated with the payload. +prompt (str): The prompt associated with the payload. +context (list): The context associated with the payload. +result (str): The result generated by the system. +model (str): The model used to generate the payload.
+
+
-
+
- +context: list +
-
+
- +model: str +
-
+
- +prompt: str +
-
+
- +query: str +
-
+
- +result: str +
-
+
- +class magnet.utils.data_classes.MistralArgs(dim: int, n_layers: int, head_dim: int, hidden_dim: int, n_heads: int, n_kv_heads: int, norm_eps: float, vocab_size: int)[source] +
Bases:
+objectRepresents a set of arguments for the Mistral model.
+-
+
- Args:
dim (int): The dimensionality of the model. +n_layers (int): The number of layers in the model. +head_dim (int): The dimensionality of each attention head. +hidden_dim (int): The dimensionality of the hidden layer in the feed-forward network. +n_heads (int): The number of attention heads. +n_kv_heads (int): The number of attention heads used for key-value attention. +norm_eps (float): The epsilon value used for numerical stability in layer normalization. +vocab_size (int): The size of the vocabulary used in the model.
+
+
-
+
- +dim: int +
-
+
- +head_dim: int +
-
+
+
-
+
- +n_heads: int +
-
+
- +n_kv_heads: int +
-
+
- +n_layers: int +
-
+
- +norm_eps: float +
-
+
- +vocab_size: int +
-
+
- +class magnet.utils.data_classes.Payload(text: str, document: str)[source] +
Bases:
+objectRepresents a payload with two main fields: text and document.
+-
+
- Args:
text (str): The text associated with the payload. +document (str): The document associated with the payload.
+
+
-
+
- +document: str +
-
+
- +text: str +
magnet.utils.globals module
+-
+
- +class magnet.utils.globals.Utils[source] +
Bases:
+objectThe Utils class provides various utility functions for tasks such as checking CUDA availability, normalizing text, and uploading files to Amazon S3.
+-
+
- Example Usage:
# Create an instance of the Utils class +utils = Utils()
+# Check if CUDA is available +utils.check_cuda()
+# Normalize a text string +normalized_text = utils.normalize_text(” This is a sample text. “)
+# Upload a file to Amazon S3 +utils.upload_to_s3(file_or_dir=”path/to/file.txt”, keys=(“YOUR_ACCESS_KEY”, “YOUR_SECRET_KEY”), bucket=”my-bucket”, bucket_path=”data”)
+
+
Main functionalities: +- Checking if CUDA is available on the machine +- Normalizing text by performing various cleaning operations +- Uploading files to Amazon S3
+Methods: +- __init__(): Initializes the Utils class and sets up the spaCy English language model with a sentence tokenizer. +- check_cuda(): Checks if CUDA is available on the machine and provides additional information if it is. +- normalize_text(_): Cleans a text string by removing whitespace, replacing characters, and removing curly braces. +- upload_to_s3(file_or_dir, keys, bucket, bucket_path): Uploads a file or directory to an Amazon S3 bucket using the provided access keys and bucket information.
+Fields: +The Utils class does not have any fields.
+-
+
- +check_cuda()[source] +
The function checks if CUDA is available on the machine and provides additional information if +it is.
+
-
+
- +normalize_text(_)[source] +
The clean function takes a string as input and performs various cleaning operations on it, +such as removing whitespace, replacing characters, and removing curly braces.
+-
+
- Parameters: +
_ – The parameter “_” is a placeholder for the input string that needs to be cleaned
+
+- Returns: +
a cleaned version of the input string.
+
+
-
+
- +upload_to_s3(file_or_dir: str = None, keys: tuple = ('YOUR_ACCESS_KEY', 'YOUR_SECRET_KEY'), bucket: str = None, bucket_path: str = None)[source] +
Uploads a file or directory to an Amazon S3 bucket.
+-
+
- Args:
file_or_dir (str): The path of the file or directory to be uploaded. +keys (tuple): A tuple containing the access keys (access key ID and secret access key) required to access the Amazon S3 bucket. Default is (“YOUR_ACCESS_KEY”, “YOUR_SECRET_KEY”). +bucket (str): The name of the Amazon S3 bucket where the file or directory will be uploaded. +bucket_path (str): The path within the bucket where the file or directory will be uploaded.
+
+- Returns:
None
+
+- Raises:
None
+
+- Example Usage:
# Create an instance of the Utils class +utils = Utils()
+# Upload a file to Amazon S3 +utils.upload_to_s3(file_or_dir=”path/to/file.txt”, keys=(“YOUR_ACCESS_KEY”, “YOUR_SECRET_KEY”), bucket=”my-bucket”, bucket_path=”data”)
+
+
magnet.utils.huggingface module
+-
+
- +class magnet.utils.huggingface.InferenceAPI(token)[source] +
Bases:
+objectA class that provides a convenient way to make HTTP POST requests to an inference API endpoint.
+-
+
- Attributes:
token (str): A string representing the token used for authorization.
+
+
-
+
- +invoke(payload)[source] +
Makes an HTTP POST request to an inference API endpoint and returns the response.
+-
+
- Args:
payload (str): A JSON string representing the payload to be sent to the inference API. It should contain the model name and input data.
+
+- Returns:
str: A JSON string representing the response from the inference API.
+
+
magnet.utils.local module
+magnet.utils.markdown module
+ + +magnet.utils.milvus module
+-
+
- +class magnet.utils.milvus.MilvusDB(config)[source] +
Bases:
+objectA class that provides a high-level interface for interacting with a Milvus database.
+-
+
- Args:
config (dict): A configuration dictionary that contains the necessary parameters for connecting to the Milvus server and creating a collection.
+
+- Attributes:
config (dict): A configuration dictionary that contains the necessary parameters for connecting to the Milvus server and creating a collection. +fields (list): A list of FieldSchema objects that define the fields of the collection. +schema (CollectionSchema): A CollectionSchema object that represents the schema of the collection. +index_params (dict): A dictionary that contains the parameters for creating an index on the collection. +connection: The connection object to the Milvus server. +collection: The collection object in MilvusDB.
+
+
-
+
- +create(overwrite=False)[source] +
Create a collection in MilvusDB and create an index on a specific field of the collection.
+-
+
- Args:
overwrite (bool, optional): A boolean flag indicating whether to overwrite the existing collection with the same name. Default is False.
+
+- Returns:
None
+
+- Raises:
Exception: If an error occurs during the creation of the collection or index.
+
+- Example Usage:
-
+
- config = {
‘MILVUS_URI’: ‘localhost’, +‘INDEX’: ‘my_collection’, +‘DIMENSION’: 128, +‘INDEX_PARAMS’: {‘index_type’: ‘IVF_FLAT’, ‘nlist’: 100}
+
+
} +milvus_db = MilvusDB(config) +milvus_db.create(overwrite=True)
+
+
magnet.utils.prompts module
+-
+
- +class magnet.utils.prompts.Prompts[source] +
Bases:
+objectThe Prompts class is used to generate a formatted prompt for generating an answer based on a given question and a list of documents.
+-
+
- +qa_ref(docs: list = [], q: str = None) str[source] +
Generates a formatted prompt for generating an answer based on the given documents and question.
+-
+
- Args:
docs (list): A list of dictionaries representing the documents. Each dictionary should have a “title” key and a “content” key. +q (str): The question for which an answer is to be generated.
+
+- Returns:
str: A formatted string representing a prompt for generating an answer based on the given documents and question.
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/modules.html b/modules.html
new file mode 100644
index 0000000..e2a7a73
--- /dev/null
+++ b/modules.html
@@ -0,0 +1,386 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/objects.inv b/objects.inv
new file mode 100644
index 0000000..0f1a939
Binary files /dev/null and b/objects.inv differ
diff --git a/py-modindex.html b/py-modindex.html
new file mode 100644
index 0000000..cd5023c
--- /dev/null
+++ b/py-modindex.html
@@ -0,0 +1,305 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ magnet
+
+
+-
+
- magnet.ic package + +
- magnet.ize package + +
- magnet.ron package + +
- magnet.utils package
-
+
- Subpackages + +
- Submodules +
- magnet.utils.data_classes module
-
+
EmbeddingPayload+
+GeneratedPayload+
+MistralArgs+
+Payload+
+
+ - magnet.utils.globals module + +
- magnet.utils.huggingface module
-
+
InferenceAPI+
+
+ - magnet.utils.local module +
- magnet.utils.markdown module
-
+
m_print()
+
+ - magnet.utils.milvus module + +
- magnet.utils.prompts module
-
+
Prompts+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/search.html b/search.html
new file mode 100644
index 0000000..9d50f40
--- /dev/null
+++ b/search.html
@@ -0,0 +1,260 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Python Module Index
+ +
+ m
+
+
+ | + m | ||
| + |
+ magnet | + |
| + |
+ magnet.ic.field | + |
| + |
+ magnet.ize.filings | + |
| + |
+ magnet.ize.memory | + |
| + |
+ magnet.utils.data_classes | + |
| + |
+ magnet.utils.globals | + |
| + |
+ magnet.utils.huggingface | + |
| + |
+ magnet.utils.markdown | + |
| + |
+ magnet.utils.milvus | + |
| + |
+ magnet.utils.prompts | + |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/searchindex.js b/searchindex.js
new file mode 100644
index 0000000..9852289
--- /dev/null
+++ b/searchindex.js
@@ -0,0 +1 @@
+Search.setIndex({"docnames": ["index", "magnet", "magnet.ic", "magnet.ize", "magnet.ron", "magnet.utils", "magnet.utils.mlx", "modules", "setup"], "filenames": ["index.rst", "magnet.rst", "magnet.ic.rst", "magnet.ize.rst", "magnet.ron.rst", "magnet.utils.rst", "magnet.utils.mlx.rst", "modules.rst", "setup.rst"], "titles": ["\ud83e\uddec Installation", "magnet package", "magnet.ic package", "magnet.ize package", "magnet.ron package", "magnet.utils package", "magnet.utils.mlx package", "magnet", "setup module"], "terms": {"welcom": 0, "doc": [0, 5], "magnet": 0, "ic": [0, 1, 7], "submodul": [0, 1, 7], "field": [0, 1, 3, 5, 7], "modul": [0, 1, 7], "charg": [0, 1, 2, 3, 7], "reson": [0, 1, 2, 7], "iz": [0, 1, 7], "file": [0, 1, 5, 7], "processor": [0, 1, 3, 7], "memori": [0, 1, 7], "embedd": [0, 1, 3, 7], "ron": [0, 1, 7], "llm": [0, 1, 7], "util": [0, 1, 3, 7], "subpackag": [0, 7], "mlx": [0, 1, 5, 7], "data_class": [0, 1, 7], "embeddingpayload": [0, 1, 5, 7], "generatedpayload": [0, 1, 5, 7], "mistralarg": [0, 1, 5, 7], "payload": [0, 1, 2, 3, 5, 7], "global": [0, 1, 7], "revers": [0, 1, 5, 7], "huggingfac": [0, 1, 7], "inferenceapi": [0, 1, 5, 7], "local": [0, 1, 7], "markdown": [0, 1, 7], "m_print": [0, 1, 5, 7], "milvu": [0, 1, 3, 7], "milvusdb": [0, 1, 3, 5, 7], "prompt": [0, 1, 7], "don": [0, 2], "t": [0, 2], "forget": 0, "check": [0, 5], "out": [0, 2], "help": 0, "exampl": [0, 3, 5], "small": 0, "distribut": 0, "languag": [0, 3, 5], "model": [0, 1, 3, 5, 7], "toolkit": 0, "fine": 0, "tune": 0, "state": 0, "art": 0, "anywher": 0, "rapidli": 0, "pip": 0, "python3": 0, "setup": 0, "py": 0, "It": [0, 3, 5], "": 0, "fast": 0, "consum": [0, 2], "hardwar": 0, "veri": 0, "appl": 0, "silicon": 0, "extrem": 0, "rocm": 0, "cuda": [0, 5], "automat": 0, "your": 0, "wai": [0, 5], "reli": 0, "establish": [0, 5], "transform": [0, 3], "pattern": 0, "let": 0, "do": 0, "work": 0, "keep": 0, "exist": [0, 2, 5], "data": [0, 2, 3, 5], "process": [0, 1, 2, 3, 7], "function": [0, 2, 3, 5], "bring": 0, "them": 0, "100": [0, 3, 5], "embed": [0, 1, 3, 5, 7], "storag": 0, "retriev": [0, 2], "queri": [0, 1, 3, 5, 7], "infer": [0, 5], "from": [0, 2, 3, 5], "much": 0, "littl": 0, "comput": 0, "you": 0, "need": [0, 5], "choos": 0, "method": [0, 2, 3, 5], "vllm": 0, "node": 0, "gpu": 0, "huge": 0, "volum": 0, "handl": 0, "gigant": 0, "amount": 0, "inexpens": 0, "fault": 0, "toler": 0, "design": 0, "decentr": 0, "workload": 0, "secur": 0, "jwt": 0, "basic": 0, "world": 0, "comprehens": 0, "option": [0, 2, 3, 5], "log": [0, 3], "its": 0, "own": 0, "code": 0, "execut": 0, "ye": 0, "realli": 0, "build": 0, "self": [0, 2, 3], "awar": 0, "system": [0, 5], "allow": 0, "learn": 0, "itself": 0, "emoji": 0, "ar": 0, "futur": 0, "more": 0, "about": [0, 2], "how": 0, "prismad": 0, "subscrib": [0, 2], "our": 0, "substack": 0, "index": [0, 3, 5], "search": [0, 1, 3, 7], "page": 0, "emp": [1, 2, 7], "off": [1, 2, 5, 7], "puls": [1, 2, 7], "reset": [1, 2, 7], "info": [1, 2, 3, 7], "create_charg": [1, 3, 7], "default_splitt": [1, 3, 7], "load": [1, 3, 7], "save": [1, 3, 7], "delet": [1, 2, 3, 5, 7], "disconnect": [1, 2, 3, 5, 7], "embed_and_charg": [1, 3, 7], "embed_and_stor": [1, 3, 7], "mistral": [1, 5, 7], "document": [1, 2, 3, 5, 7], "text": [1, 3, 5, 7], "context": [1, 5, 7], "result": [1, 3, 5, 7], "dim": [1, 5, 7], "head_dim": [1, 5, 7], "hidden_dim": [1, 5, 7], "n_head": [1, 5, 7], "n_kv_head": [1, 5, 7], "n_layer": [1, 5, 7], "norm_ep": [1, 5, 7], "vocab_s": [1, 5, 7], "check_cuda": [1, 5, 7], "normalize_text": [1, 5, 7], "upload_to_s3": [1, 5, 7], "invok": [1, 5, 7], "creat": [1, 2, 3, 5, 7], "delete_index": [1, 5, 7], "qa_ref": [1, 5, 7], "class": [2, 3, 5], "server": [2, 5], "sourc": [2, 3, 5], "base": [2, 3, 5], "object": [2, 3, 5], "The": [2, 3, 5], "i": [2, 3, 5], "respons": [2, 3, 5], "connect": [2, 3, 5], "nat": 2, "manag": 2, "stream": 2, "categori": 2, "publish": 2, "arg": [2, 3, 5], "str": [2, 3, 5], "url": 2, "attribut": [2, 3, 5], "current": 2, "nc": 2, "j": 2, "jetstream": 2, "api": [2, 5], "async": [2, 3], "name": [2, 3, 5], "none": [2, 3, 5], "specifi": [2, 3], "match": 2, "print": 2, "an": [2, 3, 5], "error": [2, 3, 5], "messag": [2, 3], "doesn": 2, "default": [2, 3, 5], "warn": 2, "no_categori": 2, "bool": [2, 3, 5], "fals": [2, 3, 5], "thei": 2, "success": [2, 3], "whether": [2, 3, 5], "us": [2, 3, 5], "dict": [2, 3, 5], "purg": 2, "specif": [2, 5], "address": 2, "session": 2, "durabl": 2, "subscript": 2, "aio": 2, "client": 2, "sub": 2, "__init__": [2, 3, 5], "cb": [2, 3], "inform": [2, 3, 5], "paramet": [2, 3, 5], "A": [2, 3, 5], "string": [2, 5], "repres": [2, 3, 5], "If": [2, 3, 5], "provid": [2, 3, 5], "all": 2, "return": [2, 3, 5], "unsubscrib": 2, "built": 2, "callback": [2, 3], "receiv": 2, "rais": [2, 3, 5], "timeouterror": 2, "time": 2, "except": [2, 3, 5], "differ": 3, "format": [3, 5], "split": 3, "smaller": 3, "chunk": 3, "usag": [3, 5], "initi": [3, 5], "raw_data": 3, "text_column": 3, "clean": [3, 5], "id_column": 3, "id": [3, 5], "await": 3, "path": [3, 5], "processed_data": 3, "csv": 3, "splitter": 3, "nlp": 3, "true": [3, 5], "filenam": 3, "raw": 3, "df": 3, "main": [3, 5], "json": [3, 5], "xlsx": 3, "parquet": 3, "panda": 3, "datafram": 3, "either": 3, "sentenc": [3, 5], "natur": 3, "fix": 3, "charact": [3, 5], "length": 3, "set": [3, 5], "instanc": [3, 5], "pd": 3, "ani": [3, 5], "window_s": 3, "768": 3, "overlap": 3, "76": 3, "given": [3, 5], "input": [3, 5], "store": 3, "send": 3, "which": [3, 5], "sent": [3, 5], "valueerror": 3, "int": [3, 5], "size": [3, 5], "each": [3, 5], "when": 3, "number": [3, 5], "between": 3, "flag": [3, 5], "indic": [3, 5], "list": [3, 5], "extract": 3, "obtain": 3, "can": 3, "column": 3, "contain": [3, 5], "uniqu": 3, "identifi": 3, "entri": 3, "extens": 3, "support": 3, "occur": [3, 5], "dure": [3, 5], "successfulli": 3, "custom": 3, "unsupport": 3, "config": [3, 5], "pre": 3, "train": 3, "further": 3, "databas": [3, 5], "dictionari": [3, 5], "configur": [3, 5], "includ": 3, "sentencetransform": 3, "sentence_transform": 3, "librari": 3, "db": 3, "verbos": 3, "emb": 3, "bert": 3, "nli": 3, "mean": 3, "token": [3, 5], "my_index": 3, "index_param": [3, 5], "nprobe": 3, "16": 3, "should": [3, 5], "addit": [3, 5], "encod": 3, "my_field": 3, "boolean": [3, 5], "limit": 3, "relev": 3, "passag": 3, "maximum": 3, "distanc": 3, "associ": 5, "gener": 5, "float": 5, "argument": 5, "dimension": 5, "layer": 5, "attent": 5, "head": 5, "hidden": 5, "feed": 5, "forward": 5, "network": 5, "kei": 5, "valu": 5, "epsilon": 5, "numer": 5, "stabil": 5, "normal": 5, "vocabulari": 5, "two": 5, "variou": 5, "task": 5, "avail": 5, "upload": 5, "amazon": 5, "s3": 5, "normalized_text": 5, "thi": 5, "sampl": 5, "file_or_dir": 5, "txt": 5, "your_access_kei": 5, "your_secret_kei": 5, "bucket": 5, "my": 5, "bucket_path": 5, "machin": 5, "perform": 5, "oper": 5, "up": 5, "spaci": 5, "english": 5, "_": 5, "remov": 5, "whitespac": 5, "replac": 5, "curli": 5, "brace": 5, "directori": 5, "access": 5, "doe": 5, "have": 5, "take": 5, "placehold": 5, "version": 5, "tupl": 5, "secret": 5, "requir": 5, "where": 5, "within": 5, "conveni": 5, "make": 5, "http": 5, "post": 5, "request": 5, "endpoint": 5, "author": 5, "high": 5, "level": 5, "interfac": 5, "interact": 5, "necessari": 5, "collect": 5, "fieldschema": 5, "defin": 5, "schema": 5, "collectionschema": 5, "overwrit": 5, "same": 5, "creation": 5, "milvus_uri": 5, "localhost": 5, "my_collect": 5, "dimens": 5, "128": 5, "index_typ": 5, "ivf_flat": 5, "nlist": 5, "milvus_db": 5, "answer": 5, "question": 5, "q": 5, "titl": 5, "content": 5, "packag": 7}, "objects": {"magnet.ic": [[2, 0, 0, "-", "field"]], "magnet.ic.field": [[2, 1, 1, "", "Charge"], [2, 1, 1, "", "Resonator"]], "magnet.ic.field.Charge": [[2, 2, 1, "", "emp"], [2, 2, 1, "", "off"], [2, 2, 1, "", "on"], [2, 2, 1, "", "pulse"], [2, 2, 1, "", "reset"]], "magnet.ic.field.Resonator": [[2, 2, 1, "", "info"], [2, 2, 1, "", "off"], [2, 2, 1, "", "on"]], "magnet.ize": [[3, 0, 0, "-", "filings"], [3, 0, 0, "-", "memory"]], "magnet.ize.filings": [[3, 1, 1, "", "Processor"]], "magnet.ize.filings.Processor": [[3, 2, 1, "", "create_charge"], [3, 2, 1, "", "default_splitter"], [3, 2, 1, "", "load"], [3, 2, 1, "", "process"], [3, 2, 1, "", "save"]], "magnet.ize.memory": [[3, 1, 1, "", "Embedder"]], "magnet.ize.memory.Embedder": [[3, 2, 1, "", "delete"], [3, 2, 1, "", "disconnect"], [3, 2, 1, "", "embed_and_charge"], [3, 2, 1, "", "embed_and_store"], [3, 2, 1, "", "info"], [3, 2, 1, "", "search"]], "magnet.utils": [[5, 0, 0, "-", "data_classes"], [5, 0, 0, "-", "globals"], [5, 0, 0, "-", "huggingface"], [5, 0, 0, "-", "markdown"], [5, 0, 0, "-", "milvus"], [5, 0, 0, "-", "prompts"]], "magnet.utils.data_classes": [[5, 1, 1, "", "EmbeddingPayload"], [5, 1, 1, "", "GeneratedPayload"], [5, 1, 1, "", "MistralArgs"], [5, 1, 1, "", "Payload"]], "magnet.utils.data_classes.EmbeddingPayload": [[5, 3, 1, "", "document"], [5, 3, 1, "", "embedding"], [5, 3, 1, "", "model"], [5, 3, 1, "", "text"]], "magnet.utils.data_classes.GeneratedPayload": [[5, 3, 1, "", "context"], [5, 3, 1, "", "model"], [5, 3, 1, "", "prompt"], [5, 3, 1, "", "query"], [5, 3, 1, "", "result"]], "magnet.utils.data_classes.MistralArgs": [[5, 3, 1, "", "dim"], [5, 3, 1, "", "head_dim"], [5, 3, 1, "", "hidden_dim"], [5, 3, 1, "", "n_heads"], [5, 3, 1, "", "n_kv_heads"], [5, 3, 1, "", "n_layers"], [5, 3, 1, "", "norm_eps"], [5, 3, 1, "", "vocab_size"]], "magnet.utils.data_classes.Payload": [[5, 3, 1, "", "document"], [5, 3, 1, "", "text"]], "magnet.utils.globals": [[5, 1, 1, "", "Utils"], [5, 4, 1, "", "reversal"]], "magnet.utils.globals.Utils": [[5, 2, 1, "", "check_cuda"], [5, 2, 1, "", "normalize_text"], [5, 2, 1, "", "upload_to_s3"]], "magnet.utils.huggingface": [[5, 1, 1, "", "InferenceAPI"]], "magnet.utils.huggingface.InferenceAPI": [[5, 2, 1, "", "invoke"]], "magnet.utils.markdown": [[5, 4, 1, "", "m_print"]], "magnet.utils.milvus": [[5, 1, 1, "", "MilvusDB"]], "magnet.utils.milvus.MilvusDB": [[5, 2, 1, "", "create"], [5, 2, 1, "", "delete_index"], [5, 2, 1, "", "off"], [5, 2, 1, "", "on"]], "magnet.utils.prompts": [[5, 1, 1, "", "Prompts"]], "magnet.utils.prompts.Prompts": [[5, 2, 1, "", "qa_ref"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:attribute", "4": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "function", "Python function"]}, "titleterms": {"packag": [0, 1, 2, 3, 4, 5, 6], "us": 0, "class": 0, "instal": 0, "featur": 0, "indic": 0, "tabl": 0, "magnet": [1, 2, 3, 4, 5, 6, 7], "subpackag": [1, 5], "ic": 2, "submodul": [2, 3, 4, 5, 6], "field": 2, "modul": [2, 3, 4, 5, 6, 8], "iz": 3, "file": 3, "memori": 3, "ron": 4, "llm": 4, "util": [5, 6], "data_class": 5, "global": 5, "huggingfac": 5, "local": 5, "markdown": 5, "milvu": 5, "prompt": 5, "mlx": 6, "mistral": 6, "setup": 8}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.todo": 2, "sphinx.ext.viewcode": 1, "sphinx": 60}, "alltitles": {"Packages & useful classes": [[0, null]], "\ud83e\uddec Installation": [[0, "installation"]], "\ud83d\udd2e features": [[0, "features"]], "Indices and tables": [[0, "indices-and-tables"]], "magnet package": [[1, "magnet-package"]], "Subpackages": [[1, "subpackages"], [5, "subpackages"]], "magnet.ic package": [[2, "magnet-ic-package"]], "Submodules": [[2, "submodules"], [3, "submodules"], [4, "submodules"], [5, "submodules"], [6, "submodules"]], "magnet.ic.field module": [[2, "module-magnet.ic.field"]], "magnet.ize package": [[3, "magnet-ize-package"]], "magnet.ize.filings module": [[3, "module-magnet.ize.filings"]], "magnet.ize.memory module": [[3, "module-magnet.ize.memory"]], "magnet.ron package": [[4, "magnet-ron-package"]], "magnet.ron.llm module": [[4, "magnet-ron-llm-module"]], "magnet.utils package": [[5, "magnet-utils-package"]], "magnet.utils.data_classes module": [[5, "module-magnet.utils.data_classes"]], "magnet.utils.globals module": [[5, "module-magnet.utils.globals"]], "magnet.utils.huggingface module": [[5, "module-magnet.utils.huggingface"]], "magnet.utils.local module": [[5, "magnet-utils-local-module"]], "magnet.utils.markdown module": [[5, "module-magnet.utils.markdown"]], "magnet.utils.milvus module": [[5, "module-magnet.utils.milvus"]], "magnet.utils.prompts module": [[5, "module-magnet.utils.prompts"]], "magnet.utils.mlx package": [[6, "magnet-utils-mlx-package"]], "magnet.utils.mlx.mistral module": [[6, "magnet-utils-mlx-mistral-module"]], "magnet": [[7, "magnet"]], "setup module": [[8, "setup-module"]]}, "indexentries": {"charge (class in magnet.ic.field)": [[2, "magnet.ic.field.Charge"]], "resonator (class in magnet.ic.field)": [[2, "magnet.ic.field.Resonator"]], "emp() (magnet.ic.field.charge method)": [[2, "magnet.ic.field.Charge.emp"]], "info() (magnet.ic.field.resonator method)": [[2, "magnet.ic.field.Resonator.info"]], "magnet.ic.field": [[2, "module-magnet.ic.field"]], "module": [[2, "module-magnet.ic.field"], [3, "module-magnet.ize.filings"], [3, "module-magnet.ize.memory"], [5, "module-magnet.utils.data_classes"], [5, "module-magnet.utils.globals"], [5, "module-magnet.utils.huggingface"], [5, "module-magnet.utils.markdown"], [5, "module-magnet.utils.milvus"], [5, "module-magnet.utils.prompts"]], "off() (magnet.ic.field.charge method)": [[2, "magnet.ic.field.Charge.off"]], "off() (magnet.ic.field.resonator method)": [[2, "magnet.ic.field.Resonator.off"]], "on() (magnet.ic.field.charge method)": [[2, "magnet.ic.field.Charge.on"]], "on() (magnet.ic.field.resonator method)": [[2, "magnet.ic.field.Resonator.on"]], "pulse() (magnet.ic.field.charge method)": [[2, "magnet.ic.field.Charge.pulse"]], "reset() (magnet.ic.field.charge method)": [[2, "magnet.ic.field.Charge.reset"]], "embedder (class in magnet.ize.memory)": [[3, "magnet.ize.memory.Embedder"]], "processor (class in magnet.ize.filings)": [[3, "magnet.ize.filings.Processor"]], "create_charge() (magnet.ize.filings.processor method)": [[3, "magnet.ize.filings.Processor.create_charge"]], "default_splitter() (magnet.ize.filings.processor method)": [[3, "magnet.ize.filings.Processor.default_splitter"]], "delete() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.delete"]], "disconnect() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.disconnect"]], "embed_and_charge() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.embed_and_charge"]], "embed_and_store() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.embed_and_store"]], "info() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.info"]], "load() (magnet.ize.filings.processor method)": [[3, "magnet.ize.filings.Processor.load"]], "magnet.ize.filings": [[3, "module-magnet.ize.filings"]], "magnet.ize.memory": [[3, "module-magnet.ize.memory"]], "process() (magnet.ize.filings.processor method)": [[3, "magnet.ize.filings.Processor.process"]], "save() (magnet.ize.filings.processor method)": [[3, "magnet.ize.filings.Processor.save"]], "search() (magnet.ize.memory.embedder method)": [[3, "magnet.ize.memory.Embedder.search"]], "embeddingpayload (class in magnet.utils.data_classes)": [[5, "magnet.utils.data_classes.EmbeddingPayload"]], "generatedpayload (class in magnet.utils.data_classes)": [[5, "magnet.utils.data_classes.GeneratedPayload"]], "inferenceapi (class in magnet.utils.huggingface)": [[5, "magnet.utils.huggingface.InferenceAPI"]], "milvusdb (class in magnet.utils.milvus)": [[5, "magnet.utils.milvus.MilvusDB"]], "mistralargs (class in magnet.utils.data_classes)": [[5, "magnet.utils.data_classes.MistralArgs"]], "payload (class in magnet.utils.data_classes)": [[5, "magnet.utils.data_classes.Payload"]], "prompts (class in magnet.utils.prompts)": [[5, "magnet.utils.prompts.Prompts"]], "utils (class in magnet.utils.globals)": [[5, "magnet.utils.globals.Utils"]], "check_cuda() (magnet.utils.globals.utils method)": [[5, "magnet.utils.globals.Utils.check_cuda"]], "context (magnet.utils.data_classes.generatedpayload attribute)": [[5, "magnet.utils.data_classes.GeneratedPayload.context"]], "create() (magnet.utils.milvus.milvusdb method)": [[5, "magnet.utils.milvus.MilvusDB.create"]], "delete_index() (magnet.utils.milvus.milvusdb method)": [[5, "magnet.utils.milvus.MilvusDB.delete_index"]], "dim (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.dim"]], "document (magnet.utils.data_classes.embeddingpayload attribute)": [[5, "magnet.utils.data_classes.EmbeddingPayload.document"]], "document (magnet.utils.data_classes.payload attribute)": [[5, "magnet.utils.data_classes.Payload.document"]], "embedding (magnet.utils.data_classes.embeddingpayload attribute)": [[5, "magnet.utils.data_classes.EmbeddingPayload.embedding"]], "head_dim (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.head_dim"]], "hidden_dim (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.hidden_dim"]], "invoke() (magnet.utils.huggingface.inferenceapi method)": [[5, "magnet.utils.huggingface.InferenceAPI.invoke"]], "m_print() (in module magnet.utils.markdown)": [[5, "magnet.utils.markdown.m_print"]], "magnet.utils.data_classes": [[5, "module-magnet.utils.data_classes"]], "magnet.utils.globals": [[5, "module-magnet.utils.globals"]], "magnet.utils.huggingface": [[5, "module-magnet.utils.huggingface"]], "magnet.utils.markdown": [[5, "module-magnet.utils.markdown"]], "magnet.utils.milvus": [[5, "module-magnet.utils.milvus"]], "magnet.utils.prompts": [[5, "module-magnet.utils.prompts"]], "model (magnet.utils.data_classes.embeddingpayload attribute)": [[5, "magnet.utils.data_classes.EmbeddingPayload.model"]], "model (magnet.utils.data_classes.generatedpayload attribute)": [[5, "magnet.utils.data_classes.GeneratedPayload.model"]], "n_heads (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.n_heads"]], "n_kv_heads (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.n_kv_heads"]], "n_layers (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.n_layers"]], "norm_eps (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.norm_eps"]], "normalize_text() (magnet.utils.globals.utils method)": [[5, "magnet.utils.globals.Utils.normalize_text"]], "off() (magnet.utils.milvus.milvusdb method)": [[5, "magnet.utils.milvus.MilvusDB.off"]], "on() (magnet.utils.milvus.milvusdb method)": [[5, "magnet.utils.milvus.MilvusDB.on"]], "prompt (magnet.utils.data_classes.generatedpayload attribute)": [[5, "magnet.utils.data_classes.GeneratedPayload.prompt"]], "qa_ref() (magnet.utils.prompts.prompts method)": [[5, "magnet.utils.prompts.Prompts.qa_ref"]], "query (magnet.utils.data_classes.generatedpayload attribute)": [[5, "magnet.utils.data_classes.GeneratedPayload.query"]], "result (magnet.utils.data_classes.generatedpayload attribute)": [[5, "magnet.utils.data_classes.GeneratedPayload.result"]], "reversal() (in module magnet.utils.globals)": [[5, "magnet.utils.globals.reversal"]], "text (magnet.utils.data_classes.embeddingpayload attribute)": [[5, "magnet.utils.data_classes.EmbeddingPayload.text"]], "text (magnet.utils.data_classes.payload attribute)": [[5, "magnet.utils.data_classes.Payload.text"]], "upload_to_s3() (magnet.utils.globals.utils method)": [[5, "magnet.utils.globals.Utils.upload_to_s3"]], "vocab_size (magnet.utils.data_classes.mistralargs attribute)": [[5, "magnet.utils.data_classes.MistralArgs.vocab_size"]]}})
\ No newline at end of file
diff --git a/setup.html b/setup.html
new file mode 100644
index 0000000..d739a31
--- /dev/null
+++ b/setup.html
@@ -0,0 +1,244 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+