transcription

What this skill does

# Transcription

Real-time and batch speech-to-text powered by [Deepgram](https://deepgram.com). Provides live streaming transcription for video rooms via Deepgram's WebSocket API, batch transcription for recorded audio, a scrolling real-time transcript panel with speaker labels, and a full post-meeting transcript viewer with search and speaker filtering. All transcripts are persisted to Postgres via Drizzle.

## Prerequisites

- Next.js app with `src/` directory and App Router
- `video-room` skill installed (LiveKit room infrastructure)
- `db` skill installed (Drizzle ORM with PostgreSQL)
- `env-config` skill installed (`src/env.ts`)
- shadcn/ui initialized

## Installation

```bash
bun add @deepgram/sdk
```

## Environment Variables

Add to `.env.local`:

```env
# Deepgram
DEEPGRAM_API_KEY=your-deepgram-api-key-here
```

### Update `src/env.ts`

Add to the `server` object:

```typescript
  server: {
    // ... existing variables
    DEEPGRAM_API_KEY: z.string().min(1),
  },
```

Add to the `runtimeEnv` object:

```typescript
  runtimeEnv: {
    // ... existing variables
    DEEPGRAM_API_KEY: process.env.DEEPGRAM_API_KEY,
  },
```

## What Gets Created

```
src/
├── lib/
│   ├── video/
│   │   ├── transcription.ts              # Server-side Deepgram client, batch + streaming
│   │   ├── types-transcription.ts        # TranscriptSegment, Transcript, Speaker types
│   │   └── use-transcription.ts          # "use client" hook for live transcript state
│   └── db/
│       └── schema/
│           └── transcripts.ts            # Drizzle: transcripts + transcript_segments tables
├── components/
│   └── video/
│       ├── live-transcript.tsx           # Scrolling real-time transcript panel
│       └── transcript-viewer.tsx         # Post-meeting full transcript with search
└── app/
    └── api/
        └── video/
            └── transcripts/
                ├── route.ts              # GET list / POST start transcription
                └── [id]/
                    └── route.ts          # GET full transcript with segments
```

## Database

After applying this skill, push the schema to create the `transcripts` and `transcript_segments` tables:

```bash
bunx drizzle-kit push
```

## Setup Steps

### Step 1: Create `src/lib/video/types-transcription.ts`

```typescript
export type Speaker = {
  id: number;
  name: string;
};

export type TranscriptSegment = {
  speaker: number;
  text: string;
  startTime: number;
  endTime: number;
  confidence: number;
};

export type Transcript = {
  id: string;
  roomName: string;
  segments: TranscriptSegment[];
  speakers: Speaker[];
  duration: number;
  createdAt: Date;
};

export type LiveTranscriptEvent = {
  type: "transcript";
  segment: TranscriptSegment;
  isFinal: boolean;
};

export type TranscriptionStatus = "idle" | "connecting" | "transcribing" | "error" | "stopped";
```

### Step 2: Create `src/lib/video/transcription.ts`

```typescript
import { createClient, LiveTranscriptionEvents } from "@deepgram/sdk";
import type { TranscriptSegment, Speaker } from "./types-transcription";

/**
 * Create a configured Deepgram client.
 * Must be called server-side only — uses DEEPGRAM_API_KEY.
 */
export function createDeepgramClient() {
  const apiKey = process.env.DEEPGRAM_API_KEY;
  if (!apiKey) {
    throw new Error("DEEPGRAM_API_KEY is not set");
  }
  return createClient(apiKey);
}

/**
 * Transcribe an audio buffer using Deepgram's batch (pre-recorded) API.
 * Supports speaker diarization and punctuation.
 *
 * @param audioBuffer - Raw audio data (WAV, MP3, OGG, etc.)
 * @param mimetype - MIME type of the audio (e.g., "audio/wav")
 * @returns Transcript segments with speaker labels and timestamps.
 */
export async function transcribeAudio(
  audioBuffer: Buffer,
  mimetype = "audio/wav"
): Promise<{ segments: TranscriptSegment[]; speakers: Speaker[]; duration: number }> {
  const client = createDeepgramClient();

  const { result } = await client.listen.prerecorded.transcribeFile(audioBuffer, {
    model: "nova-3",
    smart_format: true,
    diarize: true,
    punctuate: true,
    utterances: true,
    mime_type: mimetype,
  });

  const segments: TranscriptSegment[] = [];
  const speakerSet = new Set<number>();

  const utterances = result?.results?.utterances ?? [];
  for (const utterance of utterances) {
    const speakerId = utterance.speaker ?? 0;
    speakerSet.add(speakerId);

    segments.push({
      speaker: speakerId,
      text: utterance.transcript,
      startTime: utterance.start,
      endTime: utterance.end,
      confidence: utterance.confidence,
    });
  }

  const speakers: Speaker[] = Array.from(speakerSet).map((id) => ({
    id,
    name: `Speaker ${id + 1}`,
  }));

  const duration = result?.metadata?.duration ?? 0;

  return { segments, speakers, duration };
}

type LiveTranscriptionCallbacks = {
  onSegment: (segment: TranscriptSegment, isFinal: boolean) => void;
  onError: (error: Error) => void;
  onClose: () => void;
};

type LiveTranscriptionHandle = {
  send: (audioData: Buffer) => void;
  close: () => void;
};

/**
 * Create a live (streaming) transcription session via Deepgram's WebSocket API.
 * Accumulates transcript segments with speaker labels and timestamps in real time.
 *
 * @param roomName - The video room name (used for logging/context).
 * @param callbacks - Handlers for transcript segments, errors, and close events.
 * @returns A handle with `send(audioData)` to push audio chunks and `close()` to end the session.
 */
export function createLiveTranscription(
  roomName: string,
  callbacks: LiveTranscriptionCallbacks
): LiveTranscriptionHandle {
  const client = createDeepgramClient();

  const connection = client.listen.live({
    model: "nova-3",
    smart_format: true,
    diarize: true,
    punctuate: true,
    interim_results: true,
    utterance_end_ms: 1000,
    encoding: "linear16",
    sample_rate: 16000,
    channels: 1,
  });

  connection.on(LiveTranscriptionEvents.Open, () => {
    console.log(`[transcription] Live session opened for room: ${roomName}`);
  });

  connection.on(LiveTranscriptionEvents.Transcript, (data) => {
    const alternatives = data.channel?.alternatives;
    if (!alternatives || alternatives.length === 0) return;

    const alternative = alternatives[0];
    const transcript = alternative.transcript;
    if (!transcript || transcript.trim().length === 0) return;

    const isFinal = data.is_final === true;
    const speakerId = data.channel?.alternatives?.[0]?.words?.[0]?.speaker ?? 0;
    const words = alternative.words ?? [];

    const startTime = words.length > 0 ? (words[0].start ?? 0) : 0;
    const lastWord = words.length > 0 ? words[words.length - 1] : undefined;
    const endTime = lastWord?.end ?? startTime;

    const segment: TranscriptSegment = {
      speaker: speakerId,
      text: transcript,
      startTime,
      endTime,
      confidence: alternative.confidence ?? 0,
    };

    callbacks.onSegment(segment, isFinal);
  });

  connection.on(LiveTranscriptionEvents.Error, (error) => {
    console.error(`[transcription] Error for room ${roomName}:`, error);
    callbacks.onError(error instanceof Error ? error : new Error(String(error)));
  });

  connection.on(LiveTranscriptionEvents.Close, () => {
    console.log(`[transcription] Live session closed for room: ${roomName}`);
    callbacks.onClose();
  });

  return {
    send(audioData: Buffer) {
      connection.send(audioData.buffer.slice(audioData.byteOffset, audioData.byteOffset + audioData.byteLength));
    },
    close() {
      connection.requestClose();
    },
  };
}
```

### Step 3: Create `src/lib/video/use-transcription.ts`

```typescript
"use client";

import { useState, useCallback, useEffect, useRef } from "react";
import type { TranscriptSegment, TranscriptionStatus } from "./types-transcription";

type UseTranscriptionOptions = {
  /** The LiveKit data channel topic to subscribe to for transcript events */
  dataTopic?: string;
};

type TranscriptMess