knowledge-sync

What this skill does

# Knowledge Sync

Fetches knowledge items from an external REST API, maps them into the RAG document pipeline, and indexes them for vector search. Supports on-demand sync via API call and tracks sync history with per-item status.

Designed to bridge an existing knowledge base (e.g. a voice agent's knowledge items) with the RAG-powered chat assistant so both systems draw from the same source of truth.

## Prerequisites

- Next.js app with `src/` directory and App Router
- `ai-rag-ingest` skill installed (`document` + `documentPage` tables in `@/db/schema/rag`)
- `ai-rag-vectors` skill installed (`indexDocument()` at `@/lib/rag/embeddings`)
- `auth` skill installed (`withAuth` at `@/lib/auth-guard`)
- `docker` skill installed (PostgreSQL running)

## Installation

No additional packages required. Uses existing `drizzle-orm` and `ai` packages.

## Environment Variables

Add to `.env.local`:

```env
# Knowledge Sync
KNOWLEDGE_API_URL=https://api.example.com/v1/knowledge
KNOWLEDGE_API_KEY=your-api-key-here
```

### Update `src/env.ts`

Add to the `server` object:

```typescript
  server: {
    // ... existing variables
    KNOWLEDGE_API_URL: z.string().url(),
    KNOWLEDGE_API_KEY: z.string(),
  },
```

Add to the `runtimeEnv` object:

```typescript
  runtimeEnv: {
    // ... existing variables
    KNOWLEDGE_API_URL: process.env.KNOWLEDGE_API_URL,
    KNOWLEDGE_API_KEY: process.env.KNOWLEDGE_API_KEY,
  },
```

## What Gets Created

```
src/
├── db/
│   └── schema/
│       └── knowledge-sync.ts                  # syncJob + syncItem tables
├── lib/
│   └── knowledge/
│       ├── fetcher.ts                         # Fetch items from external API
│       ├── mapper.ts                          # Map external items → RAG documents
│       └── sync.ts                            # Orchestrate fetch → ingest → index
└── app/
    └── api/
        └── knowledge/
            ├── sync/
            │   └── route.ts                   # POST — trigger sync, GET — list sync jobs
            └── sources/
                └── route.ts                   # GET — list synced knowledge sources
```

## Database

After applying this skill, push the schema:

```bash
bunx drizzle-kit push
```

## Setup Steps

### Step 1: Create `src/db/schema/knowledge-sync.ts`

```typescript
import {
  pgTable,
  text,
  timestamp,
  uuid,
  integer,
  pgEnum,
} from "drizzle-orm/pg-core";

export const syncStatusEnum = pgEnum("sync_status", [
  "pending",
  "running",
  "completed",
  "failed",
]);

export const syncItemStatusEnum = pgEnum("sync_item_status", [
  "pending",
  "ingested",
  "indexed",
  "error",
]);

export const syncJob = pgTable("sync_job", {
  id: uuid("id").defaultRandom().primaryKey(),
  userId: text("user_id").notNull(),
  status: syncStatusEnum("status").notNull().default("pending"),
  totalItems: integer("total_items").notNull().default(0),
  processedItems: integer("processed_items").notNull().default(0),
  failedItems: integer("failed_items").notNull().default(0),
  errorMessage: text("error_message"),
  startedAt: timestamp("started_at", { withTimezone: true }),
  completedAt: timestamp("completed_at", { withTimezone: true }),
  createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
});

export const syncItem = pgTable("sync_item", {
  id: uuid("id").defaultRandom().primaryKey(),
  syncJobId: uuid("sync_job_id")
    .notNull()
    .references(() => syncJob.id, { onDelete: "cascade" }),
  externalId: text("external_id").notNull(),
  title: text("title").notNull(),
  documentId: uuid("document_id"),
  status: syncItemStatusEnum("status").notNull().default("pending"),
  errorMessage: text("error_message"),
  createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
  updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(),
});

export type SyncJob = typeof syncJob.$inferSelect;
export type SyncItem = typeof syncItem.$inferSelect;
```

### Step 2: Add export to `src/db/schema/index.ts`

Find the last `export * from` line and add:

```typescript
export * from "./knowledge-sync";
```

### Step 3: Create `src/lib/knowledge/fetcher.ts`

```typescript
type ExternalKnowledgeItem = {
  id: string;
  title: string;
  content: string;
  category?: string;
  updatedAt?: string;
};

type FetchKnowledgeResult = {
  items: ExternalKnowledgeItem[];
  total: number;
};

/**
 * Fetch knowledge items from the external API.
 *
 * Adjust the response parsing to match your API's shape.
 * This implementation expects: { items: [...], total: number }
 */
export async function fetchKnowledgeItems(): Promise<FetchKnowledgeResult> {
  const apiUrl = process.env.KNOWLEDGE_API_URL;
  const apiKey = process.env.KNOWLEDGE_API_KEY;

  if (!apiUrl || !apiKey) {
    throw new Error("KNOWLEDGE_API_URL and KNOWLEDGE_API_KEY must be set");
  }

  const response = await fetch(apiUrl, {
    headers: {
      Authorization: `Bearer ${apiKey}`,
      "Content-Type": "application/json",
    },
  });

  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(
      `Knowledge API error ${response.status}: ${errorText}`
    );
  }

  const data = (await response.json()) as FetchKnowledgeResult;

  return {
    items: data.items,
    total: data.total,
  };
}

export type { ExternalKnowledgeItem };
```

### Step 4: Create `src/lib/knowledge/mapper.ts`

```typescript
import type { ExternalKnowledgeItem } from "./fetcher";

type MappedDocument = {
  externalId: string;
  title: string;
  pageTexts: string[];
  metadata: Record<string, string>;
};

/**
 * Map an external knowledge item into the shape expected by the RAG pipeline.
 *
 * Each item becomes a single "document" with one "page" containing the full text.
 * For longer content, this splits into multiple pages of ~4000 chars to improve
 * chunking and retrieval quality.
 */
export function mapToDocument(item: ExternalKnowledgeItem): MappedDocument {
  const MAX_PAGE_SIZE = 4000;
  const content = item.content.trim();

  // Split long content into pages
  const pageTexts: string[] = [];

  if (content.length <= MAX_PAGE_SIZE) {
    pageTexts.push(content);
  } else {
    // Split on paragraph boundaries
    const paragraphs = content.split(/\n\n+/);
    let currentPage = "";

    for (const paragraph of paragraphs) {
      if (currentPage.length + paragraph.length + 2 > MAX_PAGE_SIZE && currentPage) {
        pageTexts.push(currentPage.trim());
        currentPage = paragraph;
      } else {
        currentPage = currentPage ? `${currentPage}\n\n${paragraph}` : paragraph;
      }
    }

    if (currentPage.trim()) {
      pageTexts.push(currentPage.trim());
    }
  }

  const metadata: Record<string, string> = {
    source: "knowledge-sync",
    externalId: item.id,
  };

  if (item.category) {
    metadata.category = item.category;
  }

  if (item.updatedAt) {
    metadata.externalUpdatedAt = item.updatedAt;
  }

  return {
    externalId: item.id,
    title: item.title,
    pageTexts,
    metadata,
  };
}
```

### Step 5: Create `src/lib/knowledge/sync.ts`

```typescript
import { eq } from "drizzle-orm";
import { db } from "@/db";
import { document, documentPage } from "@/db/schema/rag";
import { syncJob, syncItem } from "@/db/schema/knowledge-sync";
import { indexDocument } from "@/lib/rag/embeddings";
import { fetchKnowledgeItems } from "./fetcher";
import { mapToDocument } from "./mapper";

/**
 * Run a full knowledge sync:
 * 1. Fetch items from external API
 * 2. Map each to a RAG document
 * 3. Insert document + pages into Postgres
 * 4. Generate embeddings and index
 * 5. Track status per item
 */
export async function runKnowledgeSync(userId: string): Promise<string> {
  // Create sync job
  const [job] = await db
    .insert(syncJob)
    .values({
      userId,
      status: "running",
      startedAt: new Date(),
    })
    .returning();

  try {
    // Fetch external items
    const { items } = await fetchKnowledgeItems();

    await db