Upload 14 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data.txt filter=lfs diff=lfs merge=lfs -text

chat.py

@@ -0,0 +1,37 @@
+import torch
+from model import TransformerModel
+from tokenizer import SimpleTokenizer
+
+# Load tokenizer
+tokenizer = SimpleTokenizer("vocab.pth")
+
+# Use same values from train.py
+vocab_size = len(tokenizer.char_to_idx)
+embed_size = 64
+num_heads = 2
+hidden_dim = 128
+num_layers = 2
+max_len = 32
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Create the same model and load weights
+model = TransformerModel(vocab_size, embed_size, num_heads, hidden_dim, num_layers, max_len).to(device)
+model.load_state_dict(torch.load("model.pth", map_location=device))
+model.eval()
+
+# Chat loop
+while True:
+    user_input = input("You: ")
+    if user_input.lower() in ["quit", "exit"]:
+        break
+
+    input_ids = tokenizer.encode(user_input)
+    input_tensor = torch.tensor([input_ids], dtype=torch.long).to(device)
+
+    with torch.no_grad():
+        output = model(input_tensor)[0]  # shape: [seq_len, vocab_size]
+        prediction = torch.argmax(output, dim=-1).squeeze().tolist()
+
+    response = tokenizer.decode(prediction)
+    print("AI:", response)

data.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9feec5f917db5f960b189c720479a443afcd4a6c51f5f98cb370f747f4a7b6b
+size 401944371

feedforward.py

@@ -0,0 +1,14 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class FeedForward(nn.Module):
+    def __init__(self, d_model, ff_dim=2048):
+        super(FeedForward, self).__init__()
+        self.linear1 = nn.Linear(d_model, ff_dim)
+        self.linear2 = nn.Linear(ff_dim, d_model)
+
+    def forward(self, x):
+        x = F.relu(self.linear1(x))
+        x = self.linear2(x)
+        return x

model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398b4803f6b42d46f2da3ec4d07dfcf0349da443e5555321d7c85e1fcb364489
+size 15764984

model.py

@@ -0,0 +1,42 @@
+import torch
+import torch.nn as nn
+
+class TransformerBlock(nn.Module):
+    def __init__(self, embed_size, heads, ff_hidden_dim, dropout):
+        super().__init__()
+        self.attention = nn.MultiheadAttention(embed_dim=embed_size, num_heads=heads, batch_first=True)
+        self.norm1 = nn.LayerNorm(embed_size)
+        self.norm2 = nn.LayerNorm(embed_size)
+        self.ff = nn.Sequential(
+            nn.Linear(embed_size, ff_hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(ff_hidden_dim, embed_size)
+        )
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        attn_output, _ = self.attention(x, x, x)
+        x = self.norm1(x + self.dropout(attn_output))
+        ff_output = self.ff(x)
+        x = self.norm2(x + self.dropout(ff_output))
+        return x
+
+class TransformerModel(nn.Module):
+    def __init__(self, vocab_size, embed_size=512, num_heads=8, hidden_dim=2048, num_layers=6, max_len=512, dropout=0.1):
+        super().__init__()
+        self.embedding = nn.Embedding(vocab_size, embed_size)
+        self.pos_embedding = nn.Parameter(torch.zeros(1, max_len, embed_size))
+        self.transformer_blocks = nn.Sequential(
+            *[TransformerBlock(embed_size, num_heads, hidden_dim, dropout) for _ in range(num_layers)]
+        )
+        self.norm = nn.LayerNorm(embed_size)
+        self.output = nn.Linear(embed_size, vocab_size)
+
+    def forward(self, x):
+        seq_len = x.size(1)
+        positions = self.pos_embedding[:, :seq_len, :]
+        x = self.embedding(x) + positions
+        x = self.transformer_blocks(x)
+        x = self.norm(x)
+        return self.output(x)

multi_head_attention.py

@@ -0,0 +1,42 @@
+import torch
+import torch.nn as nn
+import math
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_model, n_heads):
+        super(MultiHeadAttention, self).__init__()
+
+        self.d_model = d_model
+        self.n_heads = n_heads
+
+        assert d_model % self.n_heads == 0
+        self.head_dim = d_model // n_heads
+
+        self.query = nn.Linear(d_model, d_model)
+        self.key = nn.Linear(d_model, d_model)
+        self.value = nn.Linear(d_model, d_model)
+
+        self.fc_out = nn.Linear(d_model, d_model)
+
+    def forward(self, query, key, value, mask=None):
+        N = query.shape[0]
+        Q = self.query(query)
+        K = self.key(key)
+        V = self.value(value)
+
+        Q = Q.view(N, -1, self.n_heads, self.head_dim).transpose(1, 2)
+        K = K.view(N, -1, self.n_heads, self.head_dim).transpose(1, 2)
+        V = V.view(N, -1, self.n_heads, self.head_dim).transpose(1, 2)
+
+        energy = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim)
+
+        if mask is not None:
+            energy = energy.masked_fill(mask == 0, float('-1e20'))
+
+        attention = torch.softmax(energy, dim=-1)
+        out = torch.matmul(attention, V)
+
+        out = out.transpose(1, 2).contiguous().view(N, -1, self.n_heads * self.head_dim)
+        out = self.fc_out(out)
+
+        return out

positional_encoding.py

@@ -0,0 +1,19 @@
+import torch
+import math
+import torch.nn as nn
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, max_len=5000):
+        super(PositionalEncoding, self).__init__()
+
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model))
+
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+
+        self.register_buffer('pe', pe.unsqueeze(0))
+
+    def forward(self, x):
+        return x + self.pe[:, :x.size(1)]

tiny_llama_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5761ad26bb663d61e9af274845b0122825705cecfcd9ac9aeb1140b100fca102
+size 28985127

tokenizer.py

@@ -0,0 +1,29 @@
+import torch
+
+class SimpleTokenizer:
+    def __init__(self, vocab_path):
+        self.char_to_idx = torch.load(vocab_pth)
+
+        # Add <unk> if not in vocab
+        if '<unk>' not in self.char_to_idx:
+            self.char_to_idx['<unk>'] = max(self.char_to_idx.values()) + 1
+
+        self.idx_to_char = {i: c for c, i in self.char_to_idx.items()}
+
+
+    def encode(self, text):
+        return [self.char_to_idx.get(c, self.char_to_idx.get('<unk>', 0)) for c in text]
+
+    def decode(self, indices):
+        return ''.join([self.idx_to_char.get(i, '') for i in indices])
+
+# Example usage
+vocab_path = 'vocab.pth'  # Replace with the actual path to your vocab file
+tokenizer = SimpleTokenizer(vocab_path)
+
+text = "Hello, world!"
+tokens = tokenizer.encode(text)  # Use the encode method here
+print(tokens)
+
+decoded_text = tokenizer.decode(tokens)
+print(decoded_text)

train.log

@@ -0,0 +1 @@
+-sh: nohup: command not found

train.py

@@ -0,0 +1,111 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import psutil
+from tqdm import tqdm
+import time
+
+# Load your data
+def load_data(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return f.read()
+
+# Tokenizer
+class SimpleTokenizer:
+    def __init__(self, vocab_path):
+        self.char_to_idx = torch.load(vocab_path)
+        self.idx_to_char = {i: c for c, i in self.char_to_idx.items()}
+
+    def encode(self, text):
+        return [self.char_to_idx.get(c, self.char_to_idx.get('<unk>', 0)) for c in text]
+
+    def decode(self, indices):
+        return ''.join([self.idx_to_char.get(i, '') for i in indices])
+
+# Model
+class TransformerModel(nn.Module):
+    def __init__(self, vocab_size, emb_size=256, num_heads=4, num_layers=4, ff_hid_dim=1024):
+        super().__init__()
+        self.embedding = nn.Embedding(vocab_size, emb_size)
+        self.pos_embedding = nn.Parameter(torch.zeros(1, 512, emb_size))
+        self.transformer_blocks = nn.ModuleList([
+            nn.TransformerEncoderLayer(d_model=emb_size, nhead=num_heads, dim_feedforward=ff_hid_dim)
+            for _ in range(num_layers)
+        ])
+        self.output = nn.Linear(emb_size, vocab_size)
+
+    def forward(self, x):
+        x = self.embedding(x) + self.pos_embedding[:, :x.size(1), :]
+        for block in self.transformer_blocks:
+            x = block(x)
+        return self.output(x)
+
+# Batching
+def get_batches(data, batch_size, seq_length):
+    inputs, targets = [], []
+    for i in range(0, len(data) - seq_length - 1, seq_length):
+        x = data[i:i + seq_length]
+        y = data[i + 1:i + 1 + seq_length]
+        if len(x) == seq_length and len(y) == seq_length:
+            inputs.append(x)
+            targets.append(y)
+        if len(inputs) == batch_size:
+            yield (
+                torch.tensor(inputs, dtype=torch.long),
+                torch.tensor(targets, dtype=torch.long)
+            )
+            inputs, targets = [], []
+
+# Memory
+def show_memory():
+    process = psutil.Process()
+    mem_info = process.memory_info()
+    return f"{mem_info.rss / 1024**2:.2f} MB"
+
+# Training
+def train():
+    vocab_size = 30000
+    batch_size = 64
+    seq_length = 64
+    num_epochs = 3
+    lr = 0.001
+    vocab_path = 'vocab.pth'
+    data_path = 'data.txt'
+
+    text = load_data(data_path)
+    tokenizer = SimpleTokenizer(vocab_path)
+    tokens = tokenizer.encode(text)
+    model = TransformerModel(vocab_size)
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model.to(device)
+
+    for epoch in range(num_epochs):
+        batches = list(get_batches(tokens, batch_size, seq_length))
+        total = len(batches)
+        total_loss = 0
+        print(f"\n🧠 Epoch {epoch+1}/{num_epochs} — {total} batches")
+
+        with tqdm(total=total, desc="Training", bar_format="{l_bar}{bar} [ time left: {remaining} ]") as pbar:
+            for step, (x, y) in enumerate(batches):
+                x, y = x.to(device), y.to(device)
+                optimizer.zero_grad()
+                output = model(x)
+                loss = criterion(output.view(-1, vocab_size), y.view(-1))
+                loss.backward()
+                optimizer.step()
+
+                total_loss += loss.item()
+                avg_loss = total_loss / (step + 1)
+
+                if step % 10 == 0:
+                    pbar.set_description(f"Loss: {loss.item():.4f} | RAM: {show_memory()}")
+                pbar.update(1)
+
+        torch.save(model.state_dict(), f"model_epoch_{epoch+1}.pth")
+        print(f"💾 Model saved: model_epoch_{epoch+1}.pth")
+
+if __name__ == "__main__":
+    train()

transformer_block.py

@@ -0,0 +1,26 @@
+import torch
+import torch.nn as nn
+from multi_head_attention import MultiHeadAttention  # Add this import
+from feedforward import FeedForward
+
+class TransformerBlock(nn.Module):
+    def __init__(self, d_model, n_heads, ff_dim):
+        super(TransformerBlock, self).__init__()
+
+        self.attention = MultiHeadAttention(d_model, n_heads)
+        self.ffn = FeedForward(d_model, ff_dim)
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(0.1)
+        self.dropout2 = nn.Dropout(0.1)
+
+    def forward(self, x, mask=None):
+        # Multi-head attention
+        attn_out = self.attention(x, x, x, mask)
+        x = self.norm1(x + self.dropout1(attn_out))
+
+        # Feedforward network
+        ff_out = self.ffn(x)
+        x = self.norm2(x + self.dropout2(ff_out))
+
+        return x

transformer_model.py

@@ -0,0 +1,37 @@
+import torch.nn as nn
+import torch
+
+class TransformerBlock(nn.Module):
+    def __init__(self, d_model, n_heads, ff_dim):
+        super().__init__()
+        self.attention = nn.MultiheadAttention(d_model, n_heads, batch_first=True)
+        self.ff = nn.Sequential(
+            nn.Linear(d_model, ff_dim),
+            nn.ReLU(),
+            nn.Linear(ff_dim, d_model),
+        )
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+
+    def forward(self, x):
+        attn_output, _ = self.attention(x, x, x)
+        x = self.norm1(x + attn_output)
+        x = self.norm2(x + self.ff(x))
+        return x
+
+class TransformerModel(nn.Module):
+    def __init__(self, vocab_size, d_model, n_heads, n_layers, max_len):
+        super().__init__()
+        self.embedding = nn.Embedding(vocab_size, d_model)
+        self.pos_embedding = nn.Parameter(torch.randn(1, max_len, d_model))
+        self.transformer_blocks = nn.ModuleList([
+            TransformerBlock(d_model, n_heads, ff_dim=4*d_model)
+            for _ in range(n_layers)
+        ])
+        self.output = nn.Linear(d_model, vocab_size)
+
+    def forward(self, x):
+        x = self.embedding(x) + self.pos_embedding[:, :x.size(1), :]
+        for block in self.transformer_blocks:
+            x = block(x)
+        return self.output(x)

vocab.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17ec8f094e0274e43a931d387fbfb59caa5df051cbbe95d1f0b30584b1082d6a
+size 696472