From 32fc105652bc563b4b6e9df8bcfbe6f919f711e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sun, 25 Jan 2026 02:36:08 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat(dynamodb):=20add=20DynamoD?= =?UTF-8?q?B=20design=20patterns=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expert guidance on DynamoDB schema design based on Rick Houlihan's AWS re:Invent talks and Alex DeBrie's DynamoDBGuide.com. Skills: - dynamodb-patterns: Core NoSQL design patterns and anti-patterns - schema-designer: Guided workflow for new schema design - schema-audit: Interactive audit of existing schemas Agent: - schema-auditor: Deep analysis for DynamoDB code review Co-Authored-By: Claude Opus 4.5 --- plugins/dynamodb/.claude-plugin/plugin.json | 12 + plugins/dynamodb/README.md | 72 +++++ plugins/dynamodb/agents/schema-auditor.md | 110 +++++++ .../skills/dynamodb-patterns/SKILL.md | 69 +++++ .../dynamodb-patterns/references/examples.md | 149 +++++++++ .../dynamodb-patterns/references/patterns.md | 286 ++++++++++++++++++ plugins/dynamodb/skills/schema-audit/SKILL.md | 90 ++++++ .../dynamodb/skills/schema-designer/SKILL.md | 87 ++++++ 8 files changed, 875 insertions(+) create mode 100644 plugins/dynamodb/.claude-plugin/plugin.json create mode 100644 plugins/dynamodb/README.md create mode 100644 plugins/dynamodb/agents/schema-auditor.md create mode 100644 plugins/dynamodb/skills/dynamodb-patterns/SKILL.md create mode 100644 plugins/dynamodb/skills/dynamodb-patterns/references/examples.md create mode 100644 plugins/dynamodb/skills/dynamodb-patterns/references/patterns.md create mode 100644 plugins/dynamodb/skills/schema-audit/SKILL.md create mode 100644 plugins/dynamodb/skills/schema-designer/SKILL.md diff --git a/plugins/dynamodb/.claude-plugin/plugin.json b/plugins/dynamodb/.claude-plugin/plugin.json new file mode 100644 index 0000000..975a79b --- /dev/null +++ b/plugins/dynamodb/.claude-plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "name": "dynamodb", + "version": "0.1.0", + "description": "DynamoDB advanced design patterns expertise based on Rick Houlihan's AWS re:Invent talks", + "author": { + "name": "ZeroAE", + "url": "https://github.com/zeroae" + }, + "repository": "https://github.com/zeroae/.claude", + "license": "MIT", + "keywords": ["dynamodb", "nosql", "aws", "data-modeling", "single-table-design"] +} diff --git a/plugins/dynamodb/README.md b/plugins/dynamodb/README.md new file mode 100644 index 0000000..7acdbd2 --- /dev/null +++ b/plugins/dynamodb/README.md @@ -0,0 +1,72 @@ +# DynamoDB Plugin + +Advanced DynamoDB design patterns and data modeling expertise for Claude Code, based on Rick Houlihan's AWS re:Invent talks. + +## Overview + +Expert guidance on DynamoDB schema design using NoSQL-native patterns: + +- Design efficient single-table schemas +- Optimize access patterns +- Review and audit existing schemas +- Apply advanced patterns (GSI overloading, adjacency lists, write sharding) + +## Skills + +| Skill | Triggers When | +|-------|---------------| +| `dynamodb-patterns` | Asking about DynamoDB design, access patterns, single-table design, GSIs | +| `schema-designer` | "Help me design a DynamoDB schema for X", "Model my data for DynamoDB" | +| `schema-audit` | "Audit my DynamoDB schema", "Review this table design", "Check for issues" | + +## Agents + +| Agent | Triggers When | +|-------|---------------| +| `schema-auditor` | User shares DynamoDB code (CDK/CloudFormation/Terraform) and asks for feedback | + +## Usage Examples + +### Design a new schema +``` +"Help me design a DynamoDB schema for a task management app with users, projects, and tasks" +``` + +### Audit existing schema +``` +"Can you audit my DynamoDB schema? Here's my CDK code..." +``` + +### Quick review (agent triggers automatically) +``` +"Here's my DynamoDB table definition, what do you think?" +[paste code] +``` + +### Ask questions (patterns skill triggers) +``` +"How do I model many-to-many relationships in DynamoDB?" +"What's the best partition key for time-series data?" +"Should I use single-table design?" +``` + +## Patterns Covered + +- **Single-Table Design** - Multiple entities in one table +- **Composite Keys** - Hierarchical data with sort key prefixes +- **GSI Overloading** - Multiple access patterns from one GSI +- **Adjacency Lists** - M:N relationships +- **Write Sharding** - Distribute hot partition writes +- **Sparse GSIs** - Efficient rare attribute filtering + +## References + +Based on Rick Houlihan's AWS re:Invent talks: + +- [2017: DAT403-R](https://www.youtube.com/watch?v=jzeKPKpucS0) +- [2018: DAT401](https://www.youtube.com/watch?v=HaEPXoXVf2k) +- [2019: DAT403-R1](https://www.youtube.com/watch?v=6yqfmXiZTlM) + +## License + +MIT diff --git a/plugins/dynamodb/agents/schema-auditor.md b/plugins/dynamodb/agents/schema-auditor.md new file mode 100644 index 0000000..6a1d8f7 --- /dev/null +++ b/plugins/dynamodb/agents/schema-auditor.md @@ -0,0 +1,110 @@ +--- +name: schema-auditor +description: Deep DynamoDB schema analysis specialist. Use proactively when user shares DynamoDB table definitions (CDK, CloudFormation, Terraform, or code) and asks for feedback, review, or optimization. Performs thorough multi-dimensional analysis beyond interactive workflows. +model: sonnet +color: orange +tools: + - Read + - Glob + - Grep +whenToUse: | + + Context: User shares CDK code defining DynamoDB table + user: "Here's my DynamoDB table, what do you think?" + assistant: [Uses schema-auditor agent for deep analysis] + + + Context: User asks for schema optimization + user: "Can you review this DynamoDB schema and suggest improvements?" + assistant: [Uses schema-auditor agent] + + + Context: User has performance issues with DynamoDB + user: "My DynamoDB queries are slow, here's my table definition..." + assistant: [Uses schema-auditor agent to identify issues] + +--- + +# DynamoDB Schema Auditor + +Perform deep, multi-dimensional analysis of DynamoDB schemas. Go beyond surface-level review to identify subtle issues and optimization opportunities. + +## Analysis Dimensions + +### 1. Access Pattern Coverage +- Are ALL access patterns served by Query? +- Identify any Scan operations (critical issue) +- Find patterns requiring multiple round trips + +### 2. Key Design Quality +- Partition key cardinality assessment +- Sort key range query effectiveness +- Composite key hierarchy evaluation +- Single-table design opportunities + +### 3. GSI Efficiency +- Overloading opportunities (consolidate GSIs) +- Sparse GSI candidates +- Unused or redundant GSIs +- Projection optimization + +### 4. Hot Partition Risk +- Identify potential hot keys +- Write sharding needs +- Time-series partition distribution + +### 5. Data Model Assessment +- Normalization anti-patterns +- Denormalization opportunities +- Item size concerns (400KB limit) +- Adjacency list pattern applicability + +### 6. Cost Optimization +- Provisioned vs on-demand recommendation +- GSI capacity sizing +- TTL opportunities for expiring data +- Storage class considerations + +## Output Format + +```markdown +## Schema Audit Report + +### Summary +- Critical: X | Warnings: Y | Suggestions: Z + +### Access Pattern Matrix +| Pattern | Implementation | Status | +|---------|---------------|--------| +| ... | ... | ✅/⚠️/❌ | + +### Critical Issues +[Issues that will cause problems at scale] + +### Warnings +[Suboptimal but functional] + +### Suggestions +[Nice-to-have improvements] + +### Recommended Changes +[Specific fixes with before/after] +``` + +## Severity Criteria + +**Critical** (must fix): +- Scan for common queries +- Hot partition without sharding +- Missing index for frequent pattern +- Application-layer joins + +**Warning** (should fix): +- Over-indexing (>5 GSIs) +- Large items (>100KB) +- Inefficient key structure + +**Suggestion** (consider): +- GSI consolidation +- TTL for cost savings +- Naming improvements diff --git a/plugins/dynamodb/skills/dynamodb-patterns/SKILL.md b/plugins/dynamodb/skills/dynamodb-patterns/SKILL.md new file mode 100644 index 0000000..05f1ebe --- /dev/null +++ b/plugins/dynamodb/skills/dynamodb-patterns/SKILL.md @@ -0,0 +1,69 @@ +--- +name: dynamodb-patterns +description: DynamoDB advanced design patterns and data modeling expertise. Use when users ask about DynamoDB schema design, single-table design, access patterns, partition keys, sort keys, GSIs, adjacency lists, NoSQL data modeling, or migrating from relational to DynamoDB. +--- + +# DynamoDB Advanced Design Patterns + +Apply NoSQL-native thinking. **Unlearn relational normalization** - it's the hardest but most important step when switching to DynamoDB. + +## Core Principles + +1. **Access patterns drive schema design** - Identify ALL patterns upfront, design keys to serve them with Query (never Scan) +2. **DynamoDB is for OLTP, not OLAP** - Real-time transactional queries only. Analytics/reporting belong in a data warehouse. +3. **Denormalization is correct** - Duplicate data to avoid joins. Storage is cheap, latency is expensive. + +## Modeling Workflow + +1. **List ALL access patterns** - Every query the app will make +2. **Filter to OLTP only** - Remove analytics, reporting, ad-hoc queries (use Athena/Redshift for those) +3. **Identify entities** - Users, Orders, Products, etc. +4. **Identify relationships** - 1:1, 1:N, M:N +5. **Design keys for primary patterns** - PK/SK combinations +6. **Add GSIs for secondary patterns** - Overload when possible +7. **Validate with sample data** - Write out example items +8. **Iterate until 95% confident** - Don't go to production until design is solid + +## Pattern Index + +| Pattern | When to Use | Reference | +|---------|-------------|-----------| +| Single-Table Design | Most apps - multiple entities in one table | [patterns.md#single-table](references/patterns.md) | +| Composite Keys | Hierarchical data, time-series | [patterns.md#composite-keys](references/patterns.md) | +| GSI Overloading | Multiple secondary access patterns | [patterns.md#gsi-overloading](references/patterns.md) | +| Adjacency List | Many-to-many relationships | [patterns.md#adjacency-list](references/patterns.md) | +| Write Sharding | Hot keys (counters, leaderboards) | [patterns.md#write-sharding](references/patterns.md) | +| Sparse GSI | Filter rare attributes efficiently | [patterns.md#sparse-gsi](references/patterns.md) | + +## Quick Reference + +**Single-table key pattern:** +``` +PK: ENTITY# SK: TYPE# +``` + +**Hierarchical sort key (for `begins_with` queries):** +``` +SK: STATE##CITY##ZIP# +``` + +**GSI overloading:** +``` +GSI1PK: GSI1SK: +``` + +## Anti-Patterns + +- **Normalized tables** - Don't create separate tables and join in app +- **Scan operations** - If you need Scan, redesign your keys +- **Hot partitions** - Use write sharding for high-traffic keys +- **OLAP in DynamoDB** - Use streams to replicate to analytics systems +- **Ad-hoc queries** - DynamoDB requires known access patterns + +## Resources + +- **Detailed patterns**: See [references/patterns.md](references/patterns.md) +- **Real-world examples**: See [references/examples.md](references/examples.md) +- **Rick Houlihan talks**: [2017](https://youtube.com/watch?v=jzeKPKpucS0) | [2018](https://youtube.com/watch?v=HaEPXoXVf2k) | [2019](https://youtube.com/watch?v=6yqfmXiZTlM) +- **Alex DeBrie's guide**: [DynamoDBGuide.com](https://www.dynamodbguide.com) +- **Jeremy Daly's migration guide**: [20 Steps to Switch from RDBMS](https://www.jeremydaly.com/how-to-switch-from-rdbms-to-dynamodb-in-20-easy-steps/) diff --git a/plugins/dynamodb/skills/dynamodb-patterns/references/examples.md b/plugins/dynamodb/skills/dynamodb-patterns/references/examples.md new file mode 100644 index 0000000..3f80a67 --- /dev/null +++ b/plugins/dynamodb/skills/dynamodb-patterns/references/examples.md @@ -0,0 +1,149 @@ +# Real-World DynamoDB Examples + +Case studies from Rick Houlihan's AWS re:Invent talks demonstrating pattern application. + +## Table of Contents +- [Audible WhisperSync](#audible) +- [Delivery Service](#delivery-service) + +--- + +## Audible WhisperSync {#audible} + +Syncs reading/listening position across Kindle and Audible apps. + +### Access Patterns + +1. Get user's library (all books) +2. Get reading position for a specific book +3. Update reading position +4. Get book metadata +5. Find all users reading a specific book (analytics) + +### Entity Analysis + +- **User**: Has library of books +- **Book**: Has metadata, owned by many users +- **UserBook**: Junction - reading position per user per book + +### Single-Table Design + +``` +PK | SK | GSI1PK | Attributes +----------------|-----------------|---------------|------------------ +USER#12345 | BOOK#asin-001 | ASIN#asin-001 | position, lastRead, ... +USER#12345 | BOOK#asin-002 | ASIN#asin-002 | position, lastRead, ... +USER#67890 | BOOK#asin-001 | ASIN#asin-001 | position, lastRead, ... +BOOK#asin-001 | META | - | title, author, duration, ... +BOOK#asin-002 | META | - | title, author, duration, ... +``` + +### Query Implementation + +| Access Pattern | Implementation | +|---------------|----------------| +| User's library | `PK=USER#12345, SK begins_with BOOK#` | +| Reading position | `PK=USER#12345, SK=BOOK#asin-001` | +| Update position | `UpdateItem PK=USER#12345, SK=BOOK#asin-001` | +| Book metadata | `PK=BOOK#asin-001, SK=META` | +| Users reading book | GSI1: `GSI1PK=ASIN#asin-001` | + +### Key Insights + +- **User-centric primary key**: Most queries are "for this user, get X" +- **GSI for inverse query**: Find users by book (analytics use case) +- **Metadata separate**: Book info stored once, not duplicated per user +- **Position updates are single-item**: Fast, no transactions needed + +--- + +## Delivery Service {#delivery-service} + +Complex logistics app with customers, orders, drivers, deliveries. + +### Access Patterns + +1. Get customer profile +2. Get customer's orders +3. Get order details with items +4. Get driver's current deliveries +5. Get delivery status +6. Update delivery location +7. Find orders by status (for dispatch) + +### Entity Analysis + +- **Customer**: Places orders +- **Order**: Contains items, assigned to delivery +- **OrderItem**: Product in an order +- **Driver**: Handles deliveries +- **Delivery**: Links order to driver with status/location + +### Relational Model (What NOT to Do) + +``` +Customers Orders OrderItems Drivers Deliveries +--------- ------ ---------- ------- ---------- +id id id id id +name customer_id order_id name order_id +email status product_id vehicle driver_id + total quantity status + price location +``` + +This requires JOINs and multiple queries. Don't do this in DynamoDB. + +### Single-Table Design + +``` +PK | SK | GSI1PK | GSI1SK | Attributes +----------------|----------------------|-----------------|------------------|------------ +CUST#c001 | PROFILE | - | - | name, email, ... +CUST#c001 | ORDER#o001 | STATUS#pending | DATE#2024-01-15 | total, ... +CUST#c001 | ORDER#o002 | STATUS#delivered| DATE#2024-01-14 | total, ... +ORDER#o001 | ITEM#prod-a | - | - | qty, price, ... +ORDER#o001 | ITEM#prod-b | - | - | qty, price, ... +ORDER#o001 | DELIVERY | DRIVER#d001 | STATUS#in_transit| location, ... +DRIVER#d001 | PROFILE | - | - | name, vehicle, ... +DRIVER#d001 | DELIVERY#ORDER#o001 | - | - | (reference) +``` + +### Query Implementation + +| Access Pattern | Implementation | +|---------------|----------------| +| Customer profile | `PK=CUST#c001, SK=PROFILE` | +| Customer orders | `PK=CUST#c001, SK begins_with ORDER#` | +| Order with items | `PK=ORDER#o001` (returns items + delivery) | +| Driver's deliveries | `PK=DRIVER#d001, SK begins_with DELIVERY#` | +| Delivery status | `PK=ORDER#o001, SK=DELIVERY` | +| Update location | `UpdateItem PK=ORDER#o001, SK=DELIVERY` | +| Orders by status | GSI1: `GSI1PK=STATUS#pending` | + +### Key Insights + +- **Multiple entry points**: Same data accessible via customer, order, or driver +- **Denormalization**: Delivery info stored under both ORDER# and DRIVER# +- **GSI for status queries**: Dispatch needs to find pending orders quickly +- **Composite SK**: `DELIVERY#ORDER#o001` under driver enables range queries + +### Design Trade-offs + +| Decision | Rationale | +|----------|-----------| +| Duplicate delivery under driver | Enables "driver's deliveries" without GSI | +| Status in GSI | Dispatch queries are frequent, need efficiency | +| Items under ORDER# PK | Always fetched with order, natural grouping | +| Customer as PK prefix | Most queries are customer-initiated | + +--- + +## Pattern Application Summary + +Both examples demonstrate: + +1. **Access patterns first**: Schema designed around queries, not entities +2. **Single table**: All entities in one table with key prefixes +3. **GSI for secondary access**: When primary key doesn't serve a pattern +4. **Strategic denormalization**: Duplicate data to avoid joins +5. **Composite keys**: Enable range queries and hierarchical access diff --git a/plugins/dynamodb/skills/dynamodb-patterns/references/patterns.md b/plugins/dynamodb/skills/dynamodb-patterns/references/patterns.md new file mode 100644 index 0000000..e12f2ae --- /dev/null +++ b/plugins/dynamodb/skills/dynamodb-patterns/references/patterns.md @@ -0,0 +1,286 @@ +# DynamoDB Design Patterns + +Detailed reference for each pattern. Load only the sections needed. + +## Table of Contents +- [Single-Table Design](#single-table) +- [Composite Keys](#composite-keys) +- [GSI Overloading](#gsi-overloading) +- [Adjacency List](#adjacency-list) +- [Write Sharding](#write-sharding) +- [Sparse GSI](#sparse-gsi) +- [Transactions](#transactions) +- [DynamoDB Streams](#streams) + +--- + +## Single-Table Design {#single-table} + +Store multiple entity types in ONE table. + +**Why:** +- Retrieve related data in one Query (no round trips) +- Enable transactions across entity types +- Simplify operations (one table to manage) + +**How:** +- Use generic attribute names: `PK`, `SK` +- Encode entity type in key values: `PK=USER#123`, `SK=PROFILE` +- Use composite sort keys for hierarchical queries + +**Example - E-commerce:** +``` +PK | SK | Data +----------------|---------------------|------------------ +USER#123 | PROFILE | {name, email, ...} +USER#123 | ORDER#2024-001 | {total, status, ...} +USER#123 | ORDER#2024-002 | {total, status, ...} +ORDER#2024-001 | ITEM#SKU-A | {qty, price, ...} +ORDER#2024-001 | ITEM#SKU-B | {qty, price, ...} +``` + +**Queries:** +- User profile: `PK=USER#123, SK=PROFILE` +- User's orders: `PK=USER#123, SK begins_with ORDER#` +- Order items: `PK=ORDER#2024-001, SK begins_with ITEM#` + +--- + +## Composite Keys {#composite-keys} + +Encode hierarchy or time in sort key for range queries. + +### Hierarchical Data + +Use `#` delimiter to encode hierarchy levels. Query with `begins_with()` at any level. + +**Example - Store Locations (from DynamoDB Guide):** + +``` +PK (StoreNumber) | Country | StateCityZip +-----------------|---------|------------------ +STORE#001 | USA | NE#OMAHA#68144 +STORE#002 | USA | NE#OMAHA#68102 +STORE#003 | USA | CA#SF#94102 +STORE#004 | USA | CA#LA#90001 +``` + +**GSI (StoreLocationIndex):** +- PK: Country +- SK: StateCityZip (composite: `STATE#CITY#ZIP`) + +**Queries using `begins_with()`:** +- All US stores: `Country=USA` +- Stores in Nebraska: `Country=USA, SK begins_with NE#` +- Stores in Omaha: `Country=USA, SK begins_with NE#OMAHA#` +- Stores in zip 68144: `Country=USA, SK begins_with NE#OMAHA#68144` + +**Key insight:** One GSI supports 4+ access patterns via hierarchical sort key. + +### Time-Series Data + +``` +PK = DEVICE#sensor-1 +SK = READING#2024-01-15T10:30:00Z +``` + +**Query last 24 hours:** +``` +SK between READING#2024-01-14T10:30:00Z and READING#2024-01-15T10:30:00Z +``` + +--- + +## GSI Overloading {#gsi-overloading} + +Use generic GSI attribute names to support multiple access patterns from one index. + +**Base table with overloaded GSI:** +``` +PK | SK | GSI1PK | GSI1SK +--------|-------------|-----------------|---------------- +USER#1 | PROFILE | EMAIL#a@b.com | USER#1 +USER#1 | ORDER#001 | STATUS#pending | DATE#2024-01-15 +PROD#A | INFO | CAT#electronics | PRICE#299 +``` + +**GSI1 supports:** +- Find user by email: `GSI1PK=EMAIL#a@b.com` +- Find pending orders: `GSI1PK=STATUS#pending` +- Products by category sorted by price: `GSI1PK=CAT#electronics` + +**Key insight:** Different entity types use the same GSI with different key prefixes. + +--- + +## Adjacency List {#adjacency-list} + +Model many-to-many relationships by storing edges as items. + +**Pattern:** Store both the node and its edges under the same partition key. + +``` +PK | SK | Data +------------|-------------|------------------ +USER#alice | USER#alice | {type: "user", name: "Alice"} +USER#alice | GROUP#devs | {type: "membership", role: "admin"} +USER#alice | GROUP#ops | {type: "membership", role: "member"} +GROUP#devs | GROUP#devs | {type: "group", name: "Developers"} +GROUP#devs | USER#alice | {type: "membership", role: "admin"} +GROUP#devs | USER#bob | {type: "membership", role: "member"} +``` + +**Access patterns:** +- User and all their groups: `PK=USER#alice` +- Group and all its members: `PK=GROUP#devs` +- Bidirectional traversal in single table + +**Note:** Edges are duplicated (stored under both nodes) for bidirectional queries. + +--- + +## Write Sharding {#write-sharding} + +Distribute writes across partitions for hot keys. + +**Problem:** Single partition key limited to ~1000 writes/second. + +**Solution:** Add random suffix to partition key. + +```python +NUM_SHARDS = 10 +shard = random.randint(0, NUM_SHARDS - 1) +pk = f"COUNTER#{shard}" +``` + +### Scatter-Gather Pattern + +To read sharded data, query all shards and aggregate in application: + +```python +# SCATTER: Query each shard +results = [] +for shard in range(NUM_SHARDS): + response = table.query( + KeyConditionExpression=Key('PK').eq(f'LEADERBOARD#{shard}'), + ScanIndexForward=False, # Descending + Limit=10 # Top 10 per shard + ) + results.extend(response['Items']) + +# GATHER: Sort and take top N +top_10 = sorted(results, key=lambda x: x['score'], reverse=True)[:10] +``` + +**Trade-offs:** +- More shards = better write distribution, but more read overhead +- Only use when write traffic genuinely exceeds single-partition capacity +- Consider caching aggregated results + +**Use for:** Vote counters, view counts, leaderboards, any high-write single logical key. + +--- + +## Sparse GSI {#sparse-gsi} + +Only items with the GSI attribute appear in the index. + +``` +PK | SK | GSI_PK (sparse) +--------|---------|---------------- +USER#1 | PROFILE | (not set) +USER#2 | PROFILE | ADMIN +USER#3 | PROFILE | (not set) +USER#4 | PROFILE | ADMIN +``` + +**Result:** GSI contains ONLY admin users (2 items, not 4). + +**Use for:** +- Featured/promoted items +- Flagged content requiring review +- Items in specific workflow states +- Any rare attribute that needs efficient filtering + +**Benefit:** Small, focused index with low storage and write costs. + +--- + +## Transactions {#transactions} + +ACID transactions across up to 100 items in same region. + +**TransactWriteItems example:** +```python +client.transact_write_items( + TransactItems=[ + { + 'Update': { + 'TableName': 'MyTable', + 'Key': {'PK': {'S': 'ACCT#A'}}, + 'UpdateExpression': 'SET balance = balance - :amt', + 'ExpressionAttributeValues': {':amt': {'N': '100'}} + } + }, + { + 'Update': { + 'TableName': 'MyTable', + 'Key': {'PK': {'S': 'ACCT#B'}}, + 'UpdateExpression': 'SET balance = balance + :amt', + 'ExpressionAttributeValues': {':amt': {'N': '100'}} + } + }, + { + 'Put': { + 'TableName': 'MyTable', + 'Item': { + 'PK': {'S': 'TXN#123'}, + 'SK': {'S': 'LOG'}, + 'amount': {'N': '100'}, + 'from': {'S': 'ACCT#A'}, + 'to': {'S': 'ACCT#B'} + } + } + } + ] +) +``` + +**Use for:** +- Financial transfers +- Inventory reservations +- Any operation requiring all-or-nothing semantics + +**Limits:** +- 100 items max per transaction +- 4MB total size +- Items must be in same region +- 2x capacity consumption + +--- + +## DynamoDB Streams {#streams} + +Capture item-level changes for event-driven architectures. + +**Use cases:** +- **OLAP offloading**: Replicate to Redshift/Athena for analytics +- **Aggregations**: Maintain counters, leaderboards without hot partitions +- **Cross-region replication**: Global Tables use streams internally +- **Audit trails**: Capture all changes for compliance +- **Event triggers**: Lambda functions on data changes + +**Architecture for OLAP:** +``` +DynamoDB (OLTP) → Stream → Lambda → Kinesis Firehose → S3 → Athena (OLAP) +``` + +**Key insight:** Keep DynamoDB for real-time OLTP. Use streams to feed analytics systems for reporting and ad-hoc queries. + +**Stream record types:** +- `KEYS_ONLY`: Just the key attributes +- `NEW_IMAGE`: Item after modification +- `OLD_IMAGE`: Item before modification +- `NEW_AND_OLD_IMAGES`: Both (for change detection) + +**Best practice:** Use `NEW_AND_OLD_IMAGES` when you need to compute deltas (e.g., increment aggregates by the difference). diff --git a/plugins/dynamodb/skills/schema-audit/SKILL.md b/plugins/dynamodb/skills/schema-audit/SKILL.md new file mode 100644 index 0000000..811585a --- /dev/null +++ b/plugins/dynamodb/skills/schema-audit/SKILL.md @@ -0,0 +1,90 @@ +--- +name: schema-audit +description: DynamoDB schema audit workflow. Use when users ask to audit their DynamoDB schema, review their table design, check for issues, or optimize their existing DynamoDB setup. Triggers on "audit my DynamoDB", "review this schema", "check my table design". +--- + +# DynamoDB Schema Audit + +Structured workflow to audit an existing DynamoDB schema. + +## Step 1: Locate Schema + +Search for DynamoDB definitions or ask user: + +``` +Glob: **/*dynamodb*.ts, **/cdk*.ts, **/*.tf, **/template.yaml +Grep: "dynamodb.Table" or "AWS::DynamoDB" or "aws_dynamodb_table" +``` + +If not found, ask user to provide or describe their schema. + +## Step 2: Parse Schema + +Extract and confirm: +- Table name +- Primary key (PK + SK) +- GSI definitions +- Billing mode + +"I found table X with PK=Y, SK=Z, and N GSIs. Correct?" + +## Step 3: Gather Access Patterns + +Ask user: +"What queries does your application perform? List each: +- Get/List/Find [what] by/for [condition]" + +## Step 4: Map Patterns to Schema + +For each access pattern, identify: +1. Which table/index serves it +2. Query or Scan operation +3. Filter expressions needed + +Present mapping for review. + +## Step 5: Analyze Against Best Practices + +**Key Design** +- [ ] High-cardinality partition key? +- [ ] Effective sort key for ranges? +- [ ] Single-table design where appropriate? + +**Access Efficiency** +- [ ] All patterns use Query? +- [ ] Filters doing key's work? + +**GSI Health** +- [ ] Overloading used? +- [ ] Sparse GSI opportunities? + +**Scalability** +- [ ] Hot partition risks? +- [ ] Write sharding needed? + +## Step 6: Present Findings + +``` +## Audit Results + +### Summary +Critical: X | Warnings: Y | Suggestions: Z + +### Access Pattern Analysis +| Pattern | Method | Status | +|---------|--------|--------| + +### Issues (prioritized) +1. [Critical] ... +2. [Warning] ... + +### Recommended Actions +1. ... +``` + +## Step 7: Offer Next Steps + +"Would you like me to: +1. Generate the improved schema? +2. Explain a specific recommendation? +3. Show migration approach?" diff --git a/plugins/dynamodb/skills/schema-designer/SKILL.md b/plugins/dynamodb/skills/schema-designer/SKILL.md new file mode 100644 index 0000000..4922577 --- /dev/null +++ b/plugins/dynamodb/skills/schema-designer/SKILL.md @@ -0,0 +1,87 @@ +--- +name: schema-designer +description: Guided DynamoDB schema design workflow. Use when users want to design a new DynamoDB schema, model their data for DynamoDB, create a table structure from access patterns, or ask "help me design a DynamoDB schema for X". +--- + +# DynamoDB Schema Designer + +Guide users through designing a new DynamoDB schema from access patterns. + +## Workflow + +### Step 1: Gather Access Patterns + +Ask user to list ALL queries their application will perform: + +"List every query your application needs: +- Get [entity] by [key] +- List [entities] for [parent] +- Find [entities] where [condition] + +Example: Get user by ID, List orders for user, Find orders by status" + +### Step 2: Identify Entities and Relationships + +From access patterns, document: +- **Entities**: User, Order, Product, etc. +- **Relationships**: 1:1, 1:N, M:N + +### Step 3: Design Primary Keys + +Apply single-table design: + +``` +Entity: User → PK: USER# SK: PROFILE +Entity: Order → PK: USER# SK: ORDER# +Entity: OrderItem → PK: ORDER# SK: ITEM# +``` + +Validate each access pattern can use Query (not Scan). + +### Step 4: Design GSIs + +For patterns not served by primary key: +- Use GSI overloading (GSI1PK, GSI1SK) +- Consider sparse GSIs for rare attributes +- Minimize GSI count (each costs write capacity) + +### Step 5: Create Sample Data + +Write example items to validate design: + +``` +| PK | SK | GSI1PK | GSI1SK | +|-------------|--------------|----------------|-----------------| +| USER#123 | PROFILE | EMAIL#a@b.com | USER#123 | +| USER#123 | ORDER#001 | STATUS#pending | DATE#2024-01-15 | +| ORDER#001 | ITEM#SKU-A | | | +``` + +### Step 6: Verify All Patterns + +Map each pattern to its query: + +| Access Pattern | Table/Index | Query | +|---------------|-------------|-------| +| Get user | Base | PK=USER#id, SK=PROFILE | +| User's orders | Base | PK=USER#id, SK begins_with ORDER# | +| Orders by status | GSI1 | GSI1PK=STATUS#pending | + +**Every pattern must use Query, never Scan.** + +### Step 7: Generate Schema + +Output in user's preferred format: +- CDK (TypeScript/Python) +- CloudFormation YAML +- Terraform +- AWS CLI JSON + +## Checklist + +Before completing: +- [ ] All access patterns documented +- [ ] Single-table design applied +- [ ] Every pattern uses Query +- [ ] GSIs overloaded where possible +- [ ] Sample data validates design From aacde712d71d25906a5c5c4e47b8453084ddb738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sun, 25 Jan 2026 02:40:32 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8=20feat(vidref):=20add=20video=20r?= =?UTF-8?q?eference=20extraction=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLI wrapper around yt-dlp and ffmpeg to extract transcripts and screenshots from videos for use as Claude Skill reference material. Skills: - extract: Guided workflow for video extraction - review: Review extracted reference material with slides Scripts: - vidref: Bash script for extraction pipeline - vidref.py: Python helper for reference.md generation Co-Authored-By: Claude Opus 4.5 --- plugins/vidref/.claude-plugin/plugin.json | 13 ++ plugins/vidref/README.md | 68 ++++++ plugins/vidref/scripts/vidref | 168 ++++++++++++++ plugins/vidref/skills/extract/SKILL.md | 70 ++++++ plugins/vidref/skills/review/SKILL.md | 48 ++++ plugins/vidref/vidref.py | 266 ++++++++++++++++++++++ 6 files changed, 633 insertions(+) create mode 100644 plugins/vidref/.claude-plugin/plugin.json create mode 100644 plugins/vidref/README.md create mode 100755 plugins/vidref/scripts/vidref create mode 100644 plugins/vidref/skills/extract/SKILL.md create mode 100644 plugins/vidref/skills/review/SKILL.md create mode 100755 plugins/vidref/vidref.py diff --git a/plugins/vidref/.claude-plugin/plugin.json b/plugins/vidref/.claude-plugin/plugin.json new file mode 100644 index 0000000..0b98ee4 --- /dev/null +++ b/plugins/vidref/.claude-plugin/plugin.json @@ -0,0 +1,13 @@ +{ + "name": "vidref", + "version": "0.1.0", + "description": "Extract transcripts and screenshots from videos for Claude Skill reference material", + "author": { + "name": "ZeroAE", + "url": "https://github.com/zeroae" + }, + "repository": "https://github.com/zeroae/.claude", + "license": "MIT", + "keywords": ["video", "transcript", "screenshots", "yt-dlp", "reference-material"], + "skills": "./skills/" +} diff --git a/plugins/vidref/README.md b/plugins/vidref/README.md new file mode 100644 index 0000000..e9034f6 --- /dev/null +++ b/plugins/vidref/README.md @@ -0,0 +1,68 @@ +# vidref + +Extract transcripts and screenshots from videos for Claude Skill reference material. + +## Overview + +`vidref` is a simple CLI wrapper around `yt-dlp` and `ffmpeg` that: + +1. Downloads video transcripts (auto-generated or manual subtitles) +2. Extracts chapter metadata +3. Captures screenshots every 30 seconds throughout the video +4. Generates a combined markdown reference with slides organized by chapter + +## Installation + +Ensure you have the required dependencies installed: + +```bash +# macOS +brew install yt-dlp ffmpeg jq + +# Ubuntu/Debian +sudo apt install yt-dlp ffmpeg jq +``` + +## Usage + +```bash +# Basic usage +vidref "https://www.youtube.com/watch?v=VIDEO_ID" + +# Custom output directory +vidref "https://www.youtube.com/watch?v=VIDEO_ID" ./my-output +``` + +## Output + +``` +output/ +├── transcript.vtt # Timestamped transcript +├── chapters.json # Chapter metadata +├── video.info.json # Full video metadata from yt-dlp +├── video.mp4 # Downloaded video +├── slides/ +│ ├── 00_00_00.png # Screenshot every 30 seconds +│ ├── 00_00_30.png +│ ├── 00_01_00.png +│ └── ... +└── reference.md # Combined markdown reference +``` + +## Skills + +| Skill | Description | +|-------|-------------| +| `extract` | Guided workflow for extracting video references | +| `review` | Review extracted reference material with Claude | + +## Dependencies + +- `yt-dlp` - Video/metadata download (supports YouTube, Vimeo, and 1000+ sites) +- `ffmpeg` - Screenshot extraction +- `jq` - JSON processing +- `python3` - Markdown generation + +## License + +MIT diff --git a/plugins/vidref/scripts/vidref b/plugins/vidref/scripts/vidref new file mode 100755 index 0000000..c181c5a --- /dev/null +++ b/plugins/vidref/scripts/vidref @@ -0,0 +1,168 @@ +#!/bin/bash +set -euo pipefail + +# vidref - Extract transcripts and screenshots from videos +# Usage: vidref [output-dir] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$(dirname "$SCRIPT_DIR")" + +show_help() { + cat << EOF +vidref - Video to Reference Material + +Usage: vidref [output-dir] + +Arguments: + video-url Video URL (YouTube, Vimeo, etc. - any yt-dlp supported site) + output-dir Output directory (default: ./output) + +Output: + output/ + ├── transcript.vtt # Timestamped transcript + ├── chapters.json # Chapter metadata + ├── slides/ + │ ├── 00_00_00.png # Screenshots every 30s + │ └── ... + └── reference.md # Combined markdown + +Dependencies: yt-dlp, ffmpeg, jq, python3 +EOF +} + +if [[ $# -lt 1 ]] || [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + show_help + exit 0 +fi + +URL="$1" +OUTPUT_DIR="${2:-output}" + +# Create output directories +mkdir -p "$OUTPUT_DIR/slides" + +echo "==> Downloading metadata and transcript..." +yt-dlp --write-auto-sub --sub-lang en --skip-download \ + --write-info-json -o "$OUTPUT_DIR/video" "$URL" 2>&1 || true + +# Check if we got subtitles, if not try other formats +if ! ls "$OUTPUT_DIR"/video.*.vtt 2>/dev/null; then + echo "==> Auto-subs not available, trying manual subs..." + yt-dlp --write-sub --sub-lang en --skip-download \ + -o "$OUTPUT_DIR/video" "$URL" 2>&1 || true +fi + +# Rename VTT file to transcript.vtt if found +VTT_FILE=$(ls "$OUTPUT_DIR"/video.*.vtt 2>/dev/null | head -1 || true) +if [[ -n "$VTT_FILE" ]]; then + mv "$VTT_FILE" "$OUTPUT_DIR/transcript.vtt" + echo "==> Transcript saved to $OUTPUT_DIR/transcript.vtt" +else + echo "==> Warning: No transcript found" +fi + +# Extract chapters from info.json +INFO_JSON="$OUTPUT_DIR/video.info.json" +if [[ -f "$INFO_JSON" ]]; then + jq '{ + title: .title, + duration: .duration, + chapters: (.chapters // []) + }' "$INFO_JSON" > "$OUTPUT_DIR/chapters.json" + echo "==> Chapters saved to $OUTPUT_DIR/chapters.json" +else + echo "==> Error: Failed to download video metadata" + exit 1 +fi + +# Check if video has chapters +CHAPTER_COUNT=$(jq '.chapters | length' "$OUTPUT_DIR/chapters.json") +if [[ "$CHAPTER_COUNT" -eq 0 ]]; then + echo "==> Video has no chapters, creating timestamps at regular intervals..." + DURATION=$(jq '.duration' "$OUTPUT_DIR/chapters.json") + # Create chapters every 5 minutes or at least 5 chapters + INTERVAL=300 + if (( DURATION < 1500 )); then + INTERVAL=$((DURATION / 5)) + fi + + # Generate synthetic chapters + python3 << EOF +import json +duration = $DURATION +interval = max($INTERVAL, 60) +chapters = [] +t = 0 +i = 0 +while t < duration: + chapters.append({ + "start_time": t, + "end_time": min(t + interval, duration), + "title": f"Segment {i + 1}" + }) + t += interval + i += 1 + +with open("$OUTPUT_DIR/chapters.json", "r") as f: + data = json.load(f) +data["chapters"] = chapters +with open("$OUTPUT_DIR/chapters.json", "w") as f: + json.dump(data, f, indent=2) +EOF + CHAPTER_COUNT=$(jq '.chapters | length' "$OUTPUT_DIR/chapters.json") +fi + +echo "==> Found $CHAPTER_COUNT chapters" + +# Download video for screenshot extraction +echo "==> Downloading video for screenshot extraction..." +yt-dlp -f "bestvideo[height<=1080]+bestaudio/best[height<=1080]/best" \ + --merge-output-format mp4 \ + -o "$OUTPUT_DIR/video.mp4" "$URL" 2>&1 + +if [[ ! -f "$OUTPUT_DIR/video.mp4" ]]; then + echo "==> Error: Failed to download video" + exit 1 +fi + +# Extract screenshots at regular intervals (every 30 seconds) +DURATION=$(jq '.duration' "$OUTPUT_DIR/chapters.json") +INTERVAL=30 +SCREENSHOT_COUNT=$((${DURATION%.*} / INTERVAL + 1)) + +echo "==> Extracting $SCREENSHOT_COUNT screenshots (every ${INTERVAL}s)..." +for ((i=0; i/dev/null +done + +echo "==> Extracted $SCREENSHOT_COUNT screenshots" + +echo "==> Screenshots saved to $OUTPUT_DIR/slides/" + +# Generate reference.md using Python helper +echo "==> Generating reference.md..." +python3 "$PLUGIN_DIR/vidref.py" "$OUTPUT_DIR" + +echo "==> Done! Output saved to $OUTPUT_DIR/" +echo "" +echo "Files created:" +ls -la "$OUTPUT_DIR/" +echo "" +echo "Slides:" +ls -la "$OUTPUT_DIR/slides/" 2>/dev/null || echo " (no slides)" diff --git a/plugins/vidref/skills/extract/SKILL.md b/plugins/vidref/skills/extract/SKILL.md new file mode 100644 index 0000000..ee74dec --- /dev/null +++ b/plugins/vidref/skills/extract/SKILL.md @@ -0,0 +1,70 @@ +--- +name: extract +description: Extract transcripts and screenshots from videos for reference material. Use when user wants to create skill references from video tutorials or presentations. +--- + +# Video Reference Extraction + +Extract transcripts and screenshots from videos for use as Claude Skill reference material. + +## Usage + +```bash +# Basic extraction +vidref "https://www.youtube.com/watch?v=VIDEO_ID" + +# Custom output directory +vidref "https://www.youtube.com/watch?v=VIDEO_ID" ./my-output +``` + +## What This Does + +1. Downloads video metadata and transcript using `yt-dlp` +2. Extracts chapter information (or creates segments for videos without chapters) +3. Downloads the video and extracts screenshots every 30 seconds +4. Generates a combined `reference.md` with slides and transcript segments + +## Output Structure + +``` +output/ +├── transcript.vtt # Timestamped transcript (VTT format) +├── chapters.json # Chapter metadata +├── video.info.json # Full video metadata +├── video.mp4 # Downloaded video (can be deleted after) +├── slides/ +│ ├── 00_00_00.png # Screenshots every 30 seconds +│ ├── 00_00_30.png +│ └── ... +└── reference.md # Combined markdown reference +``` + +## Workflow + +When asked to extract reference material from a video: + +1. **Run the extraction:** + ```bash + ${CLAUDE_PLUGIN_ROOT}/scripts/vidref "VIDEO_URL" output_directory + ``` + +2. **Review the output:** + - Check `reference.md` for the combined output + - Verify slides are captured at the right moments + - Check transcript quality + +3. **Clean up (optional):** + - Delete `video.mp4` if not needed + - Delete `video.info.json` if not needed + +4. **Use in skills:** + - Copy relevant content to skill references + - Edit transcript for clarity if needed + - Organize slides as needed + +## Tips + +- Videos with chapters produce better organized output +- Screenshots are taken every 30 seconds for comprehensive coverage +- Auto-generated transcripts may have errors - manual review recommended +- Supports any site that yt-dlp supports (YouTube, Vimeo, etc.) diff --git a/plugins/vidref/skills/review/SKILL.md b/plugins/vidref/skills/review/SKILL.md new file mode 100644 index 0000000..5c67eaf --- /dev/null +++ b/plugins/vidref/skills/review/SKILL.md @@ -0,0 +1,48 @@ +--- +name: review +description: Review extracted video reference material with slides and transcript. Use when user wants to understand or discuss a video that was extracted with vidref. +--- + +# Review Video Reference + +Load and review extracted video reference material including slides and transcript. + +## Usage + +When asked to review a video reference: + +1. **Read the reference.md** to get the structure, chapters, and transcript: + ``` + Read output/reference.md + ``` + +2. **Read slides for visual context** - The slides are PNG images that Claude can view: + ``` + Read output/slides/00_00_00.png # First slide + Read output/slides/00_05_00.png # 5 minutes in + ``` + +3. **For comprehensive review**, read multiple key slides: + - Read slides at chapter boundaries + - Read slides where transcript mentions diagrams, code, or visual elements + +## Batch Reading Slides + +To efficiently review the video, read slides in parallel at key timestamps: + +``` +# Read 6 slides spanning the video +Read output/slides/00_00_00.png +Read output/slides/00_10_00.png +Read output/slides/00_20_00.png +Read output/slides/00_30_00.png +Read output/slides/00_40_00.png +Read output/slides/00_49_00.png +``` + +## Tips + +- Start by reading reference.md to understand the video structure +- Read slides where the transcript mentions visual content ("as you can see", "this diagram", "on screen") +- For technical videos, slides often contain code, architecture diagrams, or key points +- The transcript has repetition from auto-captions - focus on the visual slides for clarity diff --git a/plugins/vidref/vidref.py b/plugins/vidref/vidref.py new file mode 100755 index 0000000..ba03c2e --- /dev/null +++ b/plugins/vidref/vidref.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +""" +yt2ref.py - Generate reference markdown from YouTube video artifacts + +This script processes the output from yt-dlp (transcript, chapters, screenshots) +and generates a combined reference.md file suitable for Claude Skill references. +""" + +import json +import re +import sys +from pathlib import Path +from datetime import timedelta + + +def parse_vtt_timestamp(ts: str) -> float: + """Parse VTT timestamp (HH:MM:SS.mmm) to seconds.""" + parts = ts.replace(',', '.').split(':') + if len(parts) == 3: + h, m, s = parts + return int(h) * 3600 + int(m) * 60 + float(s) + elif len(parts) == 2: + m, s = parts + return int(m) * 60 + float(s) + return float(parts[0]) + + +def format_timestamp(seconds: float) -> str: + """Format seconds as H:MM:SS or MM:SS.""" + td = timedelta(seconds=int(seconds)) + total_seconds = int(td.total_seconds()) + hours, remainder = divmod(total_seconds, 3600) + minutes, secs = divmod(remainder, 60) + if hours > 0: + return f"{hours}:{minutes:02d}:{secs:02d}" + return f"{minutes}:{secs:02d}" + + +def parse_vtt(vtt_path: Path) -> list[dict]: + """Parse VTT file into list of {start, end, text} segments.""" + segments = [] + if not vtt_path.exists(): + return segments + + content = vtt_path.read_text(encoding='utf-8') + + # Remove WEBVTT header and metadata + lines = content.split('\n') + + current_segment = None + timestamp_pattern = re.compile(r'(\d{1,2}:)?(\d{2}):(\d{2})[.,](\d{3})\s*-->\s*(\d{1,2}:)?(\d{2}):(\d{2})[.,](\d{3})') + + for line in lines: + line = line.strip() + + # Skip empty lines, WEBVTT header, and numeric cue identifiers + if not line or line.startswith('WEBVTT') or line.isdigit(): + if current_segment and current_segment.get('text'): + segments.append(current_segment) + current_segment = None + continue + + # Check for timestamp line + match = timestamp_pattern.match(line) + if match: + if current_segment and current_segment.get('text'): + segments.append(current_segment) + + # Parse start time + start_h = int(match.group(1)[:-1]) if match.group(1) else 0 + start_m = int(match.group(2)) + start_s = int(match.group(3)) + start_ms = int(match.group(4)) + start = start_h * 3600 + start_m * 60 + start_s + start_ms / 1000 + + # Parse end time + end_h = int(match.group(5)[:-1]) if match.group(5) else 0 + end_m = int(match.group(6)) + end_s = int(match.group(7)) + end_ms = int(match.group(8)) + end = end_h * 3600 + end_m * 60 + end_s + end_ms / 1000 + + current_segment = {'start': start, 'end': end, 'text': ''} + continue + + # Skip lines that look like metadata (contain -->) + if '-->' in line: + continue + + # Text content - strip VTT tags like + clean_text = re.sub(r'<[^>]+>', '', line) + if current_segment is not None and clean_text: + if current_segment['text']: + current_segment['text'] += ' ' + current_segment['text'] += clean_text + + # Don't forget the last segment + if current_segment and current_segment.get('text'): + segments.append(current_segment) + + return segments + + +def get_transcript_for_range(segments: list[dict], start: float, end: float) -> str: + """Extract transcript text for a time range.""" + texts = [] + seen = set() # Deduplicate repeated lines (common in auto-subs) + + for seg in segments: + # Check if segment overlaps with our range + if seg['end'] >= start and seg['start'] < end: + text = seg['text'].strip() + # Simple deduplication + if text and text not in seen: + seen.add(text) + texts.append(text) + + return ' '.join(texts) + + +def get_slides_sorted(slides_dir: Path) -> list[tuple[float, str]]: + """Get all slides sorted by timestamp, returns list of (seconds, filename).""" + slides = [] + pattern = re.compile(r'^(\d{2})_(\d{2})_(\d{2})\.png$') + + for slide in slides_dir.glob('*.png'): + match = pattern.match(slide.name) + if match: + hours, minutes, seconds = int(match.group(1)), int(match.group(2)), int(match.group(3)) + total_seconds = hours * 3600 + minutes * 60 + seconds + slides.append((total_seconds, slide.name)) + + return sorted(slides, key=lambda x: x[0]) + + +def get_chapter_for_timestamp(chapters: list[dict], timestamp: float, duration: float) -> dict | None: + """Find which chapter a timestamp belongs to.""" + for i, chapter in enumerate(chapters): + start = chapter['start_time'] + end = chapter.get('end_time', chapters[i + 1]['start_time'] if i + 1 < len(chapters) else duration) + if start <= timestamp < end: + return chapter + return None + + +def generate_reference(output_dir: Path) -> str: + """Generate reference markdown from processed video artifacts.""" + chapters_file = output_dir / 'chapters.json' + transcript_file = output_dir / 'transcript.vtt' + slides_dir = output_dir / 'slides' + info_file = output_dir / 'video.info.json' + + # Load metadata + with open(chapters_file) as f: + chapters_data = json.load(f) + + title = chapters_data.get('title', 'Video Reference') + duration = chapters_data.get('duration', 0) + chapters = chapters_data.get('chapters', []) + + # Get video URL from info.json if available + video_url = "" + if info_file.exists(): + with open(info_file) as f: + info = json.load(f) + video_url = info.get('webpage_url', info.get('original_url', '')) + + # Parse transcript + segments = parse_vtt(transcript_file) + + # Get all slides + slides = get_slides_sorted(slides_dir) + interval = 30 # Default interval between screenshots + + # Build markdown + lines = [ + f"# {title}", + "", + ] + + if video_url: + lines.extend([ + f"**Source:** {video_url}", + "", + ]) + + lines.extend([ + f"**Duration:** {format_timestamp(duration)}", + f"**Slides:** {len(slides)} screenshots (every {interval}s)", + "", + "---", + "", + ]) + + # Track current chapter to show headers + current_chapter_title = None + + # Process each slide with its transcript segment + for i, (timestamp, slide_name) in enumerate(slides): + # Determine end time for transcript + if i + 1 < len(slides): + end_time = slides[i + 1][0] + else: + end_time = duration + + # Check if we're entering a new chapter + chapter = get_chapter_for_timestamp(chapters, timestamp, duration) + if chapter: + chapter_title = chapter.get('title', '') + if chapter_title != current_chapter_title: + current_chapter_title = chapter_title + lines.append(f"## {chapter_title}") + lines.append("") + + # Add slide + lines.append(f"### {format_timestamp(timestamp)}") + lines.append("") + lines.append(f"![{format_timestamp(timestamp)}](slides/{slide_name})") + lines.append("") + + # Add transcript for this segment + transcript = get_transcript_for_range(segments, timestamp, end_time) + if transcript: + # Wrap transcript, breaking into paragraphs + paragraphs = [] + words = transcript.split() + current = [] + for word in words: + current.append(word) + if len(current) >= 50: # ~50 words per paragraph + paragraphs.append(' '.join(current)) + current = [] + if current: + paragraphs.append(' '.join(current)) + + for para in paragraphs: + lines.append(f"> {para}") + lines.append(">") + lines.append("") + + lines.append("---") + lines.append("") + + return '\n'.join(lines) + + +def main(): + if len(sys.argv) < 2: + print("Usage: yt2ref.py ") + sys.exit(1) + + output_dir = Path(sys.argv[1]) + + if not output_dir.exists(): + print(f"Error: Output directory {output_dir} does not exist") + sys.exit(1) + + reference_md = generate_reference(output_dir) + + output_file = output_dir / 'reference.md' + output_file.write_text(reference_md, encoding='utf-8') + print(f"Reference saved to {output_file}") + + +if __name__ == '__main__': + main()