Add fire finetuning

.github/workflows/cpu-basic-install-prepare-train-inf-test.yml

@@ -0,0 +1,35 @@
+name: Basic Pytorch Installation, Data Prep, CPU Training, CPU Inference
+on: [push, pull_request]
+jobs:
+  Install-Dependencies_Data-Prep_CPU-Training_CPU-Inference:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "Currently on ${{ github.ref }} branch"
+      - name: ls of directory
+        run: |
+          ls ${{ github.workspace }}
+             # Caching pip dependencies
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install CPU Dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
+          python3 -m pip install -r requirements_cpu.txt
+      - name: Run Small Network on CPU
+        run: |
+          python3 data/shakespeare_char/prepare.py
+          python3 train.py --out_dir=out --device=cpu --eval_interval=2 --log_interval=1 --block_size=2 --batch_size=2 --n_layer=2 --n_head=2 --n_kv_group=2 --n_embd=16 --max_iters=3 --lr_decay_iters=2 --dropout=0.0
+      - name: Run CPU Inference
+        run: |
+          python3 sample.py --device=cpu --out_dir="out"
+

.github/workflows/cpu-test-all-activation.yml

@@ -0,0 +1,33 @@
+name: Install Then Test All activations
+on: [push, pull_request]
+jobs:
+  Install-And-Test-Activations:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "Currently on ${{ github.ref }} branch"
+      - name: ls of directory
+        run: |
+          ls ${{ github.workspace }}
+             # Caching pip dependencies
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install CPU Dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
+          python3 -m pip install -r requirements_cpu.txt
+      - name: Test all activation variations
+        run: |
+          python3 data/shakespeare_char/prepare.py
+          cd tests
+          source test_all_activation_variations_cpu.sh
+

.github/workflows/cpu-test-all-softmax.yml

@@ -0,0 +1,33 @@
+name: Install Then Test All Softmaxes
+on: [push, pull_request]
+jobs:
+  Install-And-Test-Softmax:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "Currently on ${{ github.ref }} branch"
+      - name: ls of directory
+        run: |
+          ls ${{ github.workspace }}
+             # Caching pip dependencies
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install CPU Dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
+          python3 -m pip install -r requirements_cpu.txt
+      - name: Test all softmax variations
+        run: |
+          python3 data/shakespeare_char/prepare.py
+          cd tests
+          source test_all_softmax_variations_cpu.sh
+

.github/workflows/cpu-test-gqa.yml

@@ -0,0 +1,33 @@
+name: Install Then Test GQA Variations
+on: [push, pull_request]
+jobs:
+  Install-And-Test-GQA:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "Currently on ${{ github.ref }} branch"
+      - name: ls of directory
+        run: |
+          ls ${{ github.workspace }}
+             # Caching pip dependencies
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install CPU Dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
+          python3 -m pip install -r requirements_cpu.txt
+      - name: Test all softmax variations
+        run: |
+          python3 data/shakespeare_char/prepare.py
+          cd tests
+          source test_gqa_variations_cpu.sh
+

.github/workflows/cpu-test-run-exp.yml

@@ -0,0 +1,32 @@
+name: Install Then Test Run Experiments script
+on: [push, pull_request]
+jobs:
+  Install-And-Test-Run-Exp:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "Currently on ${{ github.ref }} branch"
+      - name: ls of directory
+        run: |
+          ls ${{ github.workspace }}
+             # Caching pip dependencies
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install CPU Dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
+          python3 -m pip install -r requirements_cpu.txt
+      - name: Test all softmax variations
+        run: |
+          python3 data/shakespeare_char/prepare.py
+          python3 run_experiments.py --config explorations/config_cpu.json
+

.gitignore

@@ -1,10 +1,19 @@
-.DS_Store
-.idea
-.ipynb_checkpoints/
-.vscode
+# folders
 __pycache__/
-*.bin
+logs/
+csv_logs/
+
+# file extensions
 *.pkl
-*.pt
-*.pyc
-input.txt
+*.bin
+*.txt
+
+# audio file extensions
+*.wav
+*.mp3
+
+# checkpoint directories
+out*/
+.aider*
+
+venv/*

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "data/template/whisper.cpp"]
+	path = data/template/whisper.cpp
+	url = https://github.com/ggerganov/whisper.cpp.git
+[submodule "modules/nanoGPT"]
+	path = modules/nanoGPT
+	url = https://github.com/karpathy/nanoGPT
+[submodule "modules/llm.c"]
+	path = modules/llm.c
+	url = https://github.com/karpathy/llm.c.git

Contributing_Features.md

@@ -0,0 +1,116 @@
+# How to add new features
+
+This is a guide for adding a new feature to the search space.
+
+# TOC
+
+* [Step 1 Add new variation](#step-1-add-new-variation)
+* [Step 2 Adjust model.py](#step-2-adjust-modelpy)
+* [Step 3 Add a config within the model.py](#step-3-add-a-config-within-the-modelpy)
+* [Step 4 Add an argparse argument for the train.py](#step-4-add-an-argparse-argument-for-the-trainpy)
+* [Step 5 Create configuration json in exploration folder](#step-5-create-configuration-json-in-exploration-folder)
+* [Other Parameter Groups](#other-parameter-groups)
+* [Ideas](#ideas)
+
+## Step 1 Add new variation
+
+
+If the variation is in the following categories, add to the appropriate file or
+create a new file in the variations folder:
+
+```
+variations/
+├── activation_variations.py
+├── normalization_variations.py
+├── position_encoding_variations.py
+└── softmax_variations.py
+```
+Some variations, such as orderings of the network, may need to be made directly
+to the `model.py` file.
+
+## Step 2 Adjust model.py
+
+Import the new variation:
+```
+from variations.softmax_variations import YourSoftmaxVariation
+```
+
+And add to the model.py in appropriate section:
+```
+    if self.softmax_variant_attn == "yournewvariation":
+      self.softmax_layer = YourNewVariation(config)
+```
+
+## Step 3 Add a config within the model.py
+
+Open up `model.py` and add your new configuration within the `GPTConfig`
+dataclass:
+
+```python
+@dataclass
+class GPTConfig:
+    block_size: int = 1024
+    vocab_size: int = 50304 # GPT-2 vocab_size of 50257, padded up to nearest m
+    n_layer: int = 12
+    n_head: int = 12
+    n_embd: int = 768
+    dropout: float = 0.0
+
+    # Your New Setting
+    new_variation_setting: bool = True
+```
+
+## Step 4 Add an argparse argument for the train.py
+
+
+Open up `train.py` and add your new feature to the model group inside `parse_args` function,
+depending on the type:
+
+For boolean values:
+```python
+model_group.add_argument('--use_faster_inference', default=True, action=argparse.BooleanOptionalAction)
+```
+
+For string values (e.g. for selection between several types of a module):
+```python
+model_group.add_argument("--softmax_variant", type=str, default="softermax", choices=["constantmax", "polymax", "strongermax", "softermax", "sigsoftmax", "sigsoftmax_base2"])
+```
+
+For numeric values:
+```python
+model_group.add_argument("--block_size", type=int, default=256)
+```
+
+## Step 5 Create configuration json in exploration folder
+
+`cd` into the exploration folder and copy a template for a new exploration sweep.
+
+Run the sweep with `run_experiments.py` from the repo root specifying our
+config file.
+
+```
+python3 run_experiments.py --config explorations/config.json --output_dir out_test
+```
+
+This will automatically timestamp and apply labels to your tensorboard logs,
+create direct csv logs, and save output checkpoints into a specified folder.
+
+## Other Parameter Groups
+
+`train.py` is parameterized with argparse into three groups:
+
+1. `model_group` - these are automatically added to a config and sent to model.py
+2. `training_group` - only used by train.py
+3. `logging_group` - also only used by train.py (specifically for logging)
+
+Adding to the model group will have it sent into model.py, making it really just
+a two step process for adding a new feature.
+
+## Ideas
+
+In addition to scanning perplexity results from for different settings:
+
+- Reinforcement Loops - adding gymnasium to optimize parameters
+- Training Loop - Generating output sample.py, augmenting, then feeding back as training data.
+- Monitoring of hyperparameters - e.g. gamma and beta values for constantmax
+

HW/SA/Makefile

@@ -0,0 +1,30 @@
+TESTBENCH = ../SA/tb/tb.sv
+SIM_FILES = ../SA/define.vh ../SA/verilog/fadd.sv ../SA/verilog/fmul.sv ../SA/verilog/PE.sv ../SA/verilog/SA.sv
+
+VV         = vcs
+VVOPTS     = +v2k +vc -sverilog -timescale=1ns/1ps +vcs+lic+wait +multisource_int_delays  +lint=TFIPC-L                   \
+	       	+neg_tchk +incdir+$(VERIF) +plusarg_save +overlap +warn=noSDFCOM_UHICD,noSDFCOM_IWSBA,noSDFCOM_IANE,noSDFCOM_PONF -full64 -cc gcc +libext+.v+.vlib+.vh 
+
+ifdef WAVES
+VVOPTS += +define+DUMP_VCD=1 +memcbk +vcs+dumparrays +sdfverbose
+endif
+
+ifdef GUI
+VVOPTS += -gui
+endif
+
+all: clean sim
+
+clean:
+	rm -f ucli.key
+	rm -f sim
+	rm -f sim_synth
+	rm -fr sim.daidir
+	rm -rf *.log
+	rm -fr csrc
+
+sim: clean
+	$(VV) -o $@  $(VVOPTS) -debug_access+all $(SIM_FILES) $(TESTBENCH) -kdb -R -gui | tee sim_result.txt
+
+dve:	$(SIM_FILES)  $(TESTBENCH)
+	$(VV) $(VVOPTS) -lncurses $^ -debug_access+all -kdb -o $@ -R -gui

HW/SA/define.vh

@@ -0,0 +1,16 @@
+`ifndef _DEFINE_SVH_
+`define _DEFINE_SVH_ 
+package DEFINE_PKG;
+
+`define DIMENSION 4
+`define M_W     23   
+`define EXP_W   8
+`define BIT_W   32
+`define MULT_W  `M_W+`M_W+2
+`define EXP_MAX  2**(`EXP_W-1)+2**(`EXP_W)-3
+
+`define N_TESTS 100000
+
+endpackage
+
+`endif

HW/SA/tb/TestCasesGeneratorMultiplication.py

@@ -0,0 +1,22 @@
+import bitstring
+import random 
+
+span = 10000000
+iteration = 100000
+
+def ieee754(flt):
+    b = bitstring.BitArray(float=flt, length=32)
+    return b
+
+with open("TestVectorMultiply", "w") as f:
+
+    for i in range(iteration):
+        a = ieee754(random.uniform(-span, span))
+        b = ieee754(random.uniform(-span, span))
+        ab = ieee754(a.float * b.float)
+
+        f.write(a.hex +"_" +  b.hex  +  "_" + ab.hex + "\n")
+
+
+
+##############END OF PROGRAM###########################################################