video-messaging

What this skill does

# Video Messaging

Asynchronous video messaging with browser-based recording (screen + camera + audio), MUX-hosted playback, automatic transcription, AI-generated summaries, and threaded conversations. Think Loom-style video messages with rich post-processing.

## Prerequisites

- Next.js app with `src/` directory and App Router
- `stream-mux` skill applied (MUX upload + asset management)
- `video-player` skill applied (MUX Player React components)
- `db` skill applied (Drizzle ORM + Postgres)
- `auth` skill applied (better-auth for user sessions)
- `queue` skill applied (Inngest for background transcription + AI summary jobs)
- `storage` skill applied (for supplementary file storage)

## Installation

No additional packages required. This skill uses:

- `MediaRecorder` browser API (built into all modern browsers)
- `navigator.mediaDevices.getDisplayMedia` for screen capture
- `navigator.mediaDevices.getUserMedia` for camera + audio
- `@mux/mux-node` (from `stream-mux` dependency)
- `@mux/mux-player-react` (from `video-player` dependency)

## What Gets Created

```
src/
├── lib/
│   ├── video/
│   │   ├── video-message.ts       # Server: CRUD for video messages
│   │   ├── use-recorder.ts        # Client hook: screen + camera recording
│   │   └── types-messaging.ts     # VideoMessage, VideoThread types
│   └── db/
│       └── schema/
│           └── video-messages.ts   # Drizzle schema: video_messages table
├── components/
│   └── video/
│       ├── video-recorder.tsx     # Record UI with camera preview + controls
│       ├── video-message-card.tsx # Message card with summary + transcript
│       └── video-thread.tsx       # Threaded conversation of video messages
└── app/
    └── api/
        └── video/
            └── messages/
                ├── route.ts       # POST create, GET list
                └── [id]/
                    └── route.ts   # GET message with details, GET replies
```

## Setup Steps

### Step 1: Create `src/lib/video/types-messaging.ts`

```typescript
export type VideoMessageStatus =
  | "recording"
  | "uploading"
  | "processing"
  | "transcribing"
  | "ready"
  | "errored";

export type VideoMessage = {
  id: string;
  userId: string;
  title: string;
  muxAssetId: string | null;
  muxPlaybackId: string | null;
  transcriptId: string | null;
  meetingNoteId: string | null;
  parentId: string | null;
  duration: number | null;
  status: VideoMessageStatus;
  createdAt: Date;
};

export type VideoMessageWithDetails = VideoMessage & {
  transcript: string | null;
  summary: string | null;
  actionItems: string[] | null;
  senderName: string | null;
  senderEmail: string | null;
  senderImage: string | null;
  replyCount: number;
};

export type VideoThread = {
  message: VideoMessageWithDetails;
  replies: VideoMessageWithDetails[];
};

export type RecorderOptions = {
  screen: boolean;
  camera: boolean;
  audio: boolean;
};

export type RecorderState = {
  isRecording: boolean;
  isPaused: boolean;
  duration: number;
  previewStream: MediaStream | null;
  cameraStream: MediaStream | null;
  recordedBlob: Blob | null;
  error: string | null;
};
```

### Step 2: Create `src/lib/video/use-recorder.ts`

```typescript
"use client";

import { useState, useCallback, useRef, useEffect } from "react";
import type { RecorderOptions, RecorderState } from "@/lib/video/types-messaging";

type UseRecorderReturn = RecorderState & {
  startRecording: (options: RecorderOptions) => Promise<void>;
  stopRecording: () => void;
  pauseRecording: () => void;
  resumeRecording: () => void;
  resetRecording: () => void;
};

function useRecorder(): UseRecorderReturn {
  const [isRecording, setIsRecording] = useState(false);
  const [isPaused, setIsPaused] = useState(false);
  const [duration, setDuration] = useState(0);
  const [previewStream, setPreviewStream] = useState<MediaStream | null>(null);
  const [cameraStream, setCameraStream] = useState<MediaStream | null>(null);
  const [recordedBlob, setRecordedBlob] = useState<Blob | null>(null);
  const [error, setError] = useState<string | null>(null);

  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
  const chunksRef = useRef<Blob[]>([]);
  const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
  const startTimeRef = useRef<number>(0);
  const pausedDurationRef = useRef<number>(0);

  // Clean up timer on unmount
  useEffect(() => {
    return () => {
      if (timerRef.current) {
        clearInterval(timerRef.current);
      }
    };
  }, []);

  const stopAllTracks = useCallback((stream: MediaStream | null) => {
    if (stream) {
      for (const track of stream.getTracks()) {
        track.stop();
      }
    }
  }, []);

  const startTimer = useCallback(() => {
    startTimeRef.current = Date.now() - pausedDurationRef.current * 1000;
    timerRef.current = setInterval(() => {
      const elapsed = (Date.now() - startTimeRef.current) / 1000;
      setDuration(Math.floor(elapsed));
    }, 100);
  }, []);

  const stopTimer = useCallback(() => {
    if (timerRef.current) {
      clearInterval(timerRef.current);
      timerRef.current = null;
    }
  }, []);

  const startRecording = useCallback(
    async (options: RecorderOptions) => {
      setError(null);
      setRecordedBlob(null);
      chunksRef.current = [];
      pausedDurationRef.current = 0;

      try {
        const tracks: MediaStreamTrack[] = [];
        let screenStream: MediaStream | null = null;
        let camStream: MediaStream | null = null;

        // Get screen capture if requested
        if (options.screen) {
          screenStream = await navigator.mediaDevices.getDisplayMedia({
            video: {
              width: { ideal: 1920 },
              height: { ideal: 1080 },
              frameRate: { ideal: 30 },
            },
            audio: options.audio,
          });
          for (const track of screenStream.getVideoTracks()) {
            tracks.push(track);
          }
          // Use screen audio if available
          if (options.audio) {
            for (const track of screenStream.getAudioTracks()) {
              tracks.push(track);
            }
          }
        }

        // Get camera + microphone if requested
        if (options.camera || (options.audio && !options.screen)) {
          const constraints: MediaStreamConstraints = {};
          if (options.camera) {
            constraints.video = {
              width: { ideal: 640 },
              height: { ideal: 480 },
              facingMode: "user",
            };
          }
          if (options.audio && !screenStream?.getAudioTracks().length) {
            constraints.audio = {
              echoCancellation: true,
              noiseSuppression: true,
            };
          }

          camStream = await navigator.mediaDevices.getUserMedia(constraints);
          setCameraStream(camStream);

          // If we have screen capture, only add audio from camera (not video)
          if (options.screen) {
            if (options.audio) {
              for (const track of camStream.getAudioTracks()) {
                tracks.push(track);
              }
            }
          } else {
            for (const track of camStream.getTracks()) {
              tracks.push(track);
            }
          }
        }

        if (tracks.length === 0) {
          throw new Error("No media tracks available. Enable screen, camera, or audio.");
        }

        const combinedStream = new MediaStream(tracks);
        setPreviewStream(combinedStream);

        // Determine best MIME type
        const mimeType = MediaRecorder.isTypeSupported("video/webm;codecs=vp9,opus")
          ? "video/webm;codecs=vp9,opus"
          : MediaRecorder.isTypeSupported("video/webm;codecs=vp8,opus")
            ? "video/webm;codecs=vp8,opus"
            : "video/webm";

        const recorder = new MediaRecorder(combinedStream, {
          mimeType,
          videoBitsPerSecond: 2_500_000,
        });

        recorder.ondataavailable