diff --git a/.gitignore b/.gitignore
index 71e28f4..882a3ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,5 @@ website/out/
# MCP server build artifacts
mcp-server/node_modules/
mcp-server/dist/
+
+__pycache__
\ No newline at end of file
diff --git a/llm/Sinusoidal-Positional-Embedding/problem.toml b/llm/Sinusoidal-Positional-Embedding/problem.toml
index 713ea82..4fbb37d 100644
--- a/llm/Sinusoidal-Positional-Embedding/problem.toml
+++ b/llm/Sinusoidal-Positional-Embedding/problem.toml
@@ -11,8 +11,8 @@ tracks = ["advanced", "llm-path"]
# GENERATED from the grader spec — edit python/torchleet/problems/*.py, then run: generate.py hints
hints = [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position — register it as a buffer, not a parameter.",
]
@@ -21,7 +21,7 @@ question = "sinusoidal-q7-Question.ipynb"
solution = "sinusoidal-q7.ipynb"
[grading]
-tier = "C" # A=auto-graded B=partial C=not auto-graded (SEEDED, confirm)
+tier = "A" # A=auto-graded B=partial C=not auto-graded (SEEDED, confirm)
entries = ["SinusoidalPositionalEmbedding"]
extras = []
device = "cpu"
diff --git a/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7-Question.ipynb b/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7-Question.ipynb
index 3d9dbfb..00c6b15 100644
--- a/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7-Question.ipynb
+++ b/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7-Question.ipynb
@@ -14,11 +14,11 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Implement Attention from Scratch\n",
+ "# Implement Sinusoidal Embeddings\n",
"### Problem Statement\n",
"Transformers are order-agnostic — they see tokens like goldfish: no sense of sequence. To inject **position awareness** into the model, we use **Sinusoidal Positional Embeddings**, where each position in the sequence gets a unique deterministic vector. These vectors are computed using sine and cosine waves at different frequencies.\n",
"\n",
- "Your task is to implement the sinusoidal position encoding mechanism from scratch using PyTorch — no cheating with built-ins from `fairseq` or Hugging Face.\n",
+ "Your task is to implement the sinusoidal position encoding from \"Attention Is All You Need\" using PyTorch. Do not use a positional-encoding module from Fairseq or Hugging Face.\n",
"\n",
"---\n",
"\n",
@@ -26,29 +26,32 @@
"\n",
"1. **Define the Sinusoidal Embedding Class**\n",
" - Implement a `SinusoidalPositionalEmbedding` class inheriting from `nn.Module`.\n",
- " - Initialize with `max_seq_len` and `d_model`.\n",
- " - Create a tensor `pe` of shape `(max_seq_len, d_model)` filled with sine and cosine encodings:\n",
- " - `sin(position * ω)` for even indices\n",
- " - `cos(position * ω)` for odd indices\n",
+ " - Initialize with `max_seq_len` and `d_model`, in that order.\n",
+ " - Create a tensor `pe` of shape `(max_seq_len, d_model)`. Positions start at 0. `d_model` is even. For each frequency index `i = 0, 1, ..., d_model/2 - 1`, set `ω_i = 1 / 10000**(2i/d_model)`:\n",
+ " - dimension `2i` gets `sin(pos * ω_i)`\n",
+ " - dimension `2i+1` gets `cos(pos * ω_i)`\n",
+ " - That same `ω_i` is shared by the pair. `i` is not the raw dimension index.\n",
"\n",
"2. **Register as Buffer**\n",
" - Use `self.register_buffer(\"pe\", pe)` to store `pe` without treating it as a trainable parameter.\n",
"\n",
"3. **Generate Encodings**\n",
- " - On calling `forward(x)`, return the slice of positional encodings matching the sequence length of `x`.\n",
+ " - `forward(x)` receives `x` of shape `(batch_size, seq_len, d_model)`.\n",
+ " - Use only `seq_len` (`x.shape[1]`). Ignore the values in `x`.\n",
+ " - Return the first `seq_len` rows of `pe` with a leading size-1 batch dimension, shape `(1, seq_len, d_model)`, so the result broadcasts across the batch.\n",
"\n",
"4. **Test the Embeddings**\n",
- " - Initialize the embedding class with `max_seq_len = 100` and `d_model = 64`.\n",
- " - Pass a sequence of length 50 to verify the returned shape is `(1, 50, 64)`.\n",
+ " - Construct `SinusoidalPositionalEmbedding(max_seq_len, d_model)` with `max_seq_len = 100` and `d_model = 64`.\n",
+ " - Pass an input of shape `(1, 50, 64)` and check that the output shape is `(1, 50, 64)`.\n",
"\n",
"---\n",
"\n",
"### Constraints\n",
"\n",
- "- ✅ Do not use Hugging Face, Fairseq, or built-in PyTorch modules for position encoding.\n",
+ "- ✅ Do not use Hugging Face, Fairseq, or any prebuilt positional-encoding module.\n",
"- ✅ Ensure the `pe` tensor is not a trainable parameter.\n",
"- ✅ Support any sequence length up to `max_seq_len`.\n",
- "- ❌ Do not inject these embeddings directly into token embeddings yet — this is just the embedding module.\n",
+ "- ❌ Do not add these embeddings to token embeddings yet — this module only returns the positional table.\n",
"\n",
"---\n",
"\n",
@@ -56,9 +59,9 @@
" 💡 Hint\n",
"\n",
" - Use `torch.arange(0, max_seq_len).unsqueeze(1)` to create position indices.\n",
- " - Compute frequencies with `torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))`.\n",
- " - Alternate `sin` and `cos` values for even and odd embedding dimensions.\n",
- " - When returning the embedding in `forward`, use `.unsqueeze(0)` to broadcast over the batch dimension.\n",
+ " - Compute the shared frequencies with `torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))`. `arange(0, d_model, 2)` is `2i`, so each value is the `ω_i` for one sin/cos pair.\n",
+ " - Write those sines into `pe[:, 0::2]` and the cosines into `pe[:, 1::2]`.\n",
+ " - In `forward`, slice `self.pe[:seq_len]` and call `.unsqueeze(0)` so the batch dimension is 1.\n",
"\n",
""
]
@@ -71,33 +74,9 @@
"source": [
"import torch\n",
"import torch.nn as nn\n",
- "import torch.optim as optim\n",
- "import torch.nn.functional as F\n",
"import math"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Synthetic data\n",
- "torch.manual_seed(42)\n",
- "batch_size = 3\n",
- "seq_len = 4\n",
- "d_model = 8\n",
- "num_heads = 2\n",
- "\n",
- "q = torch.rand(batch_size, seq_len, d_model)\n",
- "k = torch.rand(batch_size, seq_len, d_model)\n",
- "v = torch.rand(batch_size, seq_len, d_model)\n",
- "print(q.shape)\n",
- "\n",
- "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
- "device = \"cpu\""
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -120,10 +99,11 @@
" Returns the positional embedding for a given input tensor.\n",
" \n",
" Args:\n",
- " x (Tensor): Input tensor of shape (batch_size, seq_len, d_model).\n",
+ " x (Tensor): Input of shape (batch_size, seq_len, d_model).\n",
+ " Only seq_len is used; the values in x are ignored.\n",
" \n",
" Returns:\n",
- " Tensor: Positional embeddings of shape (batch_size, seq_len, d_model).\n",
+ " Tensor: Positional embeddings of shape (1, seq_len, d_model).\n",
" \"\"\"\n",
" ..."
]
@@ -134,24 +114,17 @@
"metadata": {},
"outputs": [],
"source": [
- "# from fairseq.modules.sinusoidal_positional_embedding import SinusoidalPositionalEmbedding\n",
- "\n",
"max_seq_len = 100\n",
"d_model = 64\n",
- "\n",
- "# Fairseq's implementation requires the number of embeddings (seq length) and embedding dim\n",
- "# pos_emb = SinusoidalPositionalEmbedding(d_model, max_seq_len, padding_idx=None)\n",
- "\n",
- "# Generate embeddings for a sequence of length 50\n",
"seq_len = 50\n",
- "positions = torch.arange(seq_len).unsqueeze(0) # Shape: (1, seq_len)\n",
- "# positional_encoding = pos_emb(positions) # Shape: (1, seq_len, d_model)\n",
"\n",
- "custom_pos_emb = SinusoidalPositionalEmbedding(d_model, max_seq_len)\n",
+ "custom_pos_emb = SinusoidalPositionalEmbedding(max_seq_len, d_model)\n",
"\n",
- "positional_encoding_custom = custom_pos_emb(positions)\n",
+ "x = torch.zeros(1, seq_len, d_model)\n",
+ "positional_encoding = custom_pos_emb(x)\n",
"\n",
- "print(positional_encoding_custom.shape) # (1, 50, 64)\n"
+ "print(positional_encoding.shape) # (1, 50, 64)\n",
+ "assert positional_encoding.shape == (1, 50, 64)"
]
}
],
diff --git a/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7.ipynb b/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7.ipynb
index 62422ae..a6572a2 100644
--- a/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7.ipynb
+++ b/llm/Sinusoidal-Positional-Embedding/sinusoidal-q7.ipynb
@@ -4,11 +4,11 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Implement Attention from Scratch\n",
+ "# Implement Sinusoidal Embeddings\n",
"### Problem Statement\n",
"Transformers are order-agnostic — they see tokens like goldfish: no sense of sequence. To inject **position awareness** into the model, we use **Sinusoidal Positional Embeddings**, where each position in the sequence gets a unique deterministic vector. These vectors are computed using sine and cosine waves at different frequencies.\n",
"\n",
- "Your task is to implement the sinusoidal position encoding mechanism from scratch using PyTorch — no cheating with built-ins from `fairseq` or Hugging Face.\n",
+ "Your task is to implement the sinusoidal position encoding from \"Attention Is All You Need\" using PyTorch. Do not use a positional-encoding module from Fairseq or Hugging Face.\n",
"\n",
"---\n",
"\n",
@@ -16,29 +16,32 @@
"\n",
"1. **Define the Sinusoidal Embedding Class**\n",
" - Implement a `SinusoidalPositionalEmbedding` class inheriting from `nn.Module`.\n",
- " - Initialize with `max_seq_len` and `d_model`.\n",
- " - Create a tensor `pe` of shape `(max_seq_len, d_model)` filled with sine and cosine encodings:\n",
- " - `sin(position * ω)` for even indices\n",
- " - `cos(position * ω)` for odd indices\n",
+ " - Initialize with `max_seq_len` and `d_model`, in that order.\n",
+ " - Create a tensor `pe` of shape `(max_seq_len, d_model)`. Positions start at 0. `d_model` is even. For each frequency index `i = 0, 1, ..., d_model/2 - 1`, set `ω_i = 1 / 10000**(2i/d_model)`:\n",
+ " - dimension `2i` gets `sin(pos * ω_i)`\n",
+ " - dimension `2i+1` gets `cos(pos * ω_i)`\n",
+ " - That same `ω_i` is shared by the pair. `i` is not the raw dimension index.\n",
"\n",
"2. **Register as Buffer**\n",
" - Use `self.register_buffer(\"pe\", pe)` to store `pe` without treating it as a trainable parameter.\n",
"\n",
"3. **Generate Encodings**\n",
- " - On calling `forward(x)`, return the slice of positional encodings matching the sequence length of `x`.\n",
+ " - `forward(x)` receives `x` of shape `(batch_size, seq_len, d_model)`.\n",
+ " - Use only `seq_len` (`x.shape[1]`). Ignore the values in `x`.\n",
+ " - Return the first `seq_len` rows of `pe` with a leading size-1 batch dimension, shape `(1, seq_len, d_model)`, so the result broadcasts across the batch.\n",
"\n",
"4. **Test the Embeddings**\n",
- " - Initialize the embedding class with `max_seq_len = 100` and `d_model = 64`.\n",
- " - Pass a sequence of length 50 to verify the returned shape is `(1, 50, 64)`.\n",
+ " - Construct `SinusoidalPositionalEmbedding(max_seq_len, d_model)` with `max_seq_len = 100` and `d_model = 64`.\n",
+ " - Pass an input of shape `(1, 50, 64)` and check that the output shape is `(1, 50, 64)`.\n",
"\n",
"---\n",
"\n",
"### Constraints\n",
"\n",
- "- ✅ Do not use Hugging Face, Fairseq, or built-in PyTorch modules for position encoding.\n",
+ "- ✅ Do not use Hugging Face, Fairseq, or any prebuilt positional-encoding module.\n",
"- ✅ Ensure the `pe` tensor is not a trainable parameter.\n",
"- ✅ Support any sequence length up to `max_seq_len`.\n",
- "- ❌ Do not inject these embeddings directly into token embeddings yet — this is just the embedding module.\n",
+ "- ❌ Do not add these embeddings to token embeddings yet — this module only returns the positional table.\n",
"\n",
"---\n",
"\n",
@@ -46,56 +49,24 @@
" 💡 Hint\n",
"\n",
" - Use `torch.arange(0, max_seq_len).unsqueeze(1)` to create position indices.\n",
- " - Compute frequencies with `torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))`.\n",
- " - Alternate `sin` and `cos` values for even and odd embedding dimensions.\n",
- " - When returning the embedding in `forward`, use `.unsqueeze(0)` to broadcast over the batch dimension.\n",
+ " - Compute the shared frequencies with `torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))`. `arange(0, d_model, 2)` is `2i`, so each value is the `ω_i` for one sin/cos pair.\n",
+ " - Write those sines into `pe[:, 0::2]` and the cosines into `pe[:, 1::2]`.\n",
+ " - In `forward`, slice `self.pe[:seq_len]` and call `.unsqueeze(0)` so the batch dimension is 1.\n",
"\n",
""
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
- "import torch.optim as optim\n",
- "import torch.nn.functional as F\n",
"import math"
]
},
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "torch.Size([3, 4, 8])\n"
- ]
- }
- ],
- "source": [
- "# Synthetic data\n",
- "torch.manual_seed(42)\n",
- "batch_size = 3\n",
- "seq_len = 4\n",
- "d_model = 8\n",
- "num_heads = 2\n",
- "\n",
- "q = torch.rand(batch_size, seq_len, d_model)\n",
- "k = torch.rand(batch_size, seq_len, d_model)\n",
- "v = torch.rand(batch_size, seq_len, d_model)\n",
- "print(q.shape)\n",
- "\n",
- "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
- "device = \"cpu\""
- ]
- },
{
"cell_type": "code",
"execution_count": 5,
@@ -134,46 +105,40 @@
" Returns the positional embedding for a given input tensor.\n",
" \n",
" Args:\n",
- " x (Tensor): Input tensor of shape (batch_size, seq_len, d_model).\n",
+ " x (Tensor): Input of shape (batch_size, seq_len, d_model).\n",
+ " Only seq_len is used; the values in x are ignored.\n",
" \n",
" Returns:\n",
- " Tensor: Positional embeddings of shape (batch_size, seq_len, d_model).\n",
+ " Tensor: Positional embeddings of shape (1, seq_len, d_model).\n",
" \"\"\"\n",
" return self.pe[:x.shape[1], :].unsqueeze(0) # Shape: (1, seq_len, d_model)"
]
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "torch.Size([1, 50, 100])\n"
+ "torch.Size([1, 50, 64])\n"
]
}
],
"source": [
- "# from fairseq.modules.sinusoidal_positional_embedding import SinusoidalPositionalEmbedding\n",
- "\n",
"max_seq_len = 100\n",
"d_model = 64\n",
- "\n",
- "# Fairseq's implementation requires the number of embeddings (seq length) and embedding dim\n",
- "# pos_emb = SinusoidalPositionalEmbedding(d_model, max_seq_len, padding_idx=None)\n",
- "\n",
- "# Generate embeddings for a sequence of length 50\n",
"seq_len = 50\n",
- "positions = torch.arange(seq_len).unsqueeze(0) # Shape: (1, seq_len)\n",
- "# positional_encoding = pos_emb(positions) # Shape: (1, seq_len, d_model)\n",
"\n",
- "custom_pos_emb = SinusoidalPositionalEmbedding(d_model, max_seq_len)\n",
+ "custom_pos_emb = SinusoidalPositionalEmbedding(max_seq_len, d_model)\n",
"\n",
- "positional_encoding_custom = custom_pos_emb(positions)\n",
+ "x = torch.zeros(1, seq_len, d_model)\n",
+ "positional_encoding = custom_pos_emb(x)\n",
"\n",
- "print(positional_encoding_custom.shape) # (1, 50, 64)\n"
+ "print(positional_encoding.shape) # (1, 50, 64)\n",
+ "assert positional_encoding.shape == (1, 50, 64)"
]
}
],
diff --git a/mcp-server/src/data/hints.generated.ts b/mcp-server/src/data/hints.generated.ts
index 7bac22e..3ca5a19 100644
--- a/mcp-server/src/data/hints.generated.ts
+++ b/mcp-server/src/data/hints.generated.ts
@@ -274,13 +274,13 @@ export const authoredHints: Record = {
"With num_query_groups == num_heads GQA is ordinary multi-head attention."
],
"sinusoidal-positional-embedding": [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position — register it as a buffer, not a parameter."
],
"v2-10": [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position — register it as a buffer, not a parameter."
],
"rotary-positional-embedding": [
diff --git a/problems.json b/problems.json
index 5c2d3ce..a332029 100644
--- a/problems.json
+++ b/problems.json
@@ -1001,8 +1001,8 @@
"SinusoidalPositionalEmbedding"
],
"hints": [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position \u2014 register it as a buffer, not a parameter."
]
},
diff --git a/python/torchleet/problems/sinusoidal_positional_embedding.py b/python/torchleet/problems/sinusoidal_positional_embedding.py
index ed2f08e..0059317 100644
--- a/python/torchleet/problems/sinusoidal_positional_embedding.py
+++ b/python/torchleet/problems/sinusoidal_positional_embedding.py
@@ -1,12 +1,13 @@
"""sinusoidal-positional-embedding."""
+import math
import torch
ENTRIES = ["SinusoidalPositionalEmbedding"]
DEVICE = "cpu"
EXTRAS = []
HINTS = [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position — register it as a buffer, not a parameter.",
]
@@ -56,5 +57,36 @@ def check_positions_distinct(ns):
assert float(d.min()) > 1e-6, "two positions have identical embeddings"
+def check_sinusoidal_values(ns):
+ """Even dims are sin(pos * ω_i), odd dims are cos(pos * ω_i), with the
+ paper's ω_i = 1/10000**(2i/d_model) shared by dimension pair (2i, 2i+1)."""
+ t = _table(_pe(ns))
+ assert t.dim() == 2 and t.shape[0] >= MAX_LEN and t.shape[-1] == D_MODEL, \
+ f"expected a table of shape (position, {D_MODEL}), got {tuple(t.shape)}"
+ t = t[:MAX_LEN, :D_MODEL].float()
+ omega = torch.exp(
+ torch.arange(0, D_MODEL, 2).float() * (-math.log(10000.0) / D_MODEL))
+ pos = torch.arange(MAX_LEN, dtype=torch.float).unsqueeze(1)
+ assert torch.allclose(t[:, 0::2], torch.sin(pos * omega), atol=1e-5), \
+ "even dimensions 2i should be sin(pos / 10000**(2i/d_model)); " \
+ "i is the pair index 0, 1, ..., d_model/2 - 1"
+ assert torch.allclose(t[:, 1::2], torch.cos(pos * omega), atol=1e-5), \
+ "odd dimensions 2i+1 should be cos of that same angle, not a new frequency"
+
+
+def check_forward_slices_seq_len(ns):
+ m = _pe(ns)
+ seq, batch = 7, 4
+ out = m(torch.zeros(batch, seq, D_MODEL))
+ assert tuple(out.shape) == (1, seq, D_MODEL), \
+ "forward(x) should return shape (1, seq_len, d_model) so it broadcasts " \
+ f"over the batch; got {tuple(out.shape)}"
+ table = _table(m)[:seq, :D_MODEL].float()
+ assert torch.allclose(out[0].float(), table, atol=1e-5), \
+ "forward should return the first seq_len rows of the positional table " \
+ "and ignore the values in x"
+
+
CHECKS = [check_table_shape, check_values_bounded, check_deterministic,
- check_positions_distinct]
+ check_positions_distinct, check_sinusoidal_values,
+ check_forward_slices_seq_len]
diff --git a/space/problems.json b/space/problems.json
index 5c2d3ce..a332029 100644
--- a/space/problems.json
+++ b/space/problems.json
@@ -1001,8 +1001,8 @@
"SinusoidalPositionalEmbedding"
],
"hints": [
- "Even dimensions get sin, odd dimensions get cos.",
- "The frequency for dimension i is 1 / 10000**(2i/d_model).",
+ "Even dimensions 2i get sin and odd dimensions 2i+1 get cos. The pair shares one frequency.",
+ "i is 0, 1, ..., d_model/2 - 1, not the raw dimension. The frequency is 1 / 10000**(2i/d_model).",
"It is a fixed function of position \u2014 register it as a buffer, not a parameter."
]
},