bknd-bulk-operations

What this skill does

# Bulk Operations

Perform large-scale insert, update, and delete operations efficiently using Bknd's bulk APIs.

## Prerequisites

- Bknd project running (local or deployed)
- Entity exists (use `bknd-create-entity` first)
- SDK configured or API endpoint known

## When to Use UI Mode

UI mode not recommended for bulk operations. Use Admin Panel only for:
- Importing small CSV files (<100 records)
- Manual data cleanup

## When to Use Code Mode

- Migrating data from another system
- Seeding large datasets
- Batch updates (publish all drafts, archive old records)
- Mass deletion (cleanup, GDPR requests)
- ETL pipelines

## Code Approach

### Bulk Insert: createMany

```typescript
import { Api } from "bknd";

const api = new Api({ host: "http://localhost:7654" });

// Insert multiple records in one call
const { ok, data, error } = await api.data.createMany("products", [
  { name: "Product A", price: 10, stock: 100 },
  { name: "Product B", price: 20, stock: 50 },
  { name: "Product C", price: 15, stock: 75 },
]);

if (ok) {
  console.log(`Created ${data.length} products`);
  // data contains array of created records with IDs
}
```

### Bulk Update: updateMany

```typescript
// Update all records matching where clause
const { ok, data } = await api.data.updateMany(
  "posts",
  { status: { $eq: "draft" } },     // where clause (required)
  { status: "archived" }             // update data
);

// Archive old posts
await api.data.updateMany(
  "posts",
  {
    status: { $eq: "published" },
    created_at: { $lt: "2024-01-01" },
  },
  { status: "archived" }
);

// Increment view count for multiple posts
await api.data.updateMany(
  "posts",
  { id: { $in: [1, 2, 3, 4, 5] } },
  { featured: true }
);
```

### Bulk Delete: deleteMany

```typescript
// Delete all records matching where clause
const { ok, data } = await api.data.deleteMany("logs", {
  created_at: { $lt: "2023-01-01" },  // Delete old logs
});

// Delete by IDs
await api.data.deleteMany("temp_files", {
  id: { $in: [10, 11, 12, 13] },
});

// Delete archived items
await api.data.deleteMany("posts", {
  status: { $eq: "archived" },
  deleted_at: { $isnull: false },
});
```

**Warning:** `where` clause is required - prevents accidental delete-all.

## Chunked Processing

For large datasets, process in chunks to avoid timeouts and memory issues:

### Basic Chunking

```typescript
async function bulkInsertChunked(
  api: Api,
  entity: string,
  items: object[],
  chunkSize = 100
): Promise<object[]> {
  const results: object[] = [];

  for (let i = 0; i < items.length; i += chunkSize) {
    const chunk = items.slice(i, i + chunkSize);
    const { ok, data, error } = await api.data.createMany(entity, chunk);

    if (!ok) {
      throw new Error(`Chunk ${i / chunkSize + 1} failed: ${error.message}`);
    }

    results.push(...data);
  }

  return results;
}

// Usage
const products = generateProducts(5000);  // Large dataset
const created = await bulkInsertChunked(api, "products", products);
console.log(`Created ${created.length} products`);
```

### With Progress Callback

```typescript
type ProgressCallback = (done: number, total: number, chunk: number) => void;

async function bulkInsertWithProgress(
  api: Api,
  entity: string,
  items: object[],
  onProgress?: ProgressCallback,
  chunkSize = 100
): Promise<{ success: object[]; failed: object[] }> {
  const success: object[] = [];
  const failed: object[] = [];
  const totalChunks = Math.ceil(items.length / chunkSize);

  for (let i = 0; i < items.length; i += chunkSize) {
    const chunkNum = Math.floor(i / chunkSize) + 1;
    const chunk = items.slice(i, i + chunkSize);

    const { ok, data, error } = await api.data.createMany(entity, chunk);

    if (ok) {
      success.push(...data);
    } else {
      failed.push(...chunk);
      console.warn(`Chunk ${chunkNum} failed:`, error.message);
    }

    onProgress?.(Math.min(i + chunkSize, items.length), items.length, chunkNum);
  }

  return { success, failed };
}

// Usage with progress
await bulkInsertWithProgress(
  api,
  "products",
  products,
  (done, total, chunk) => {
    const percent = Math.round((done / total) * 100);
    console.log(`Progress: ${percent}% (chunk ${chunk})`);
  }
);
```

### Parallel Chunk Processing

Process multiple chunks concurrently (use with caution):

```typescript
async function bulkInsertParallel(
  api: Api,
  entity: string,
  items: object[],
  chunkSize = 100,
  concurrency = 3
): Promise<object[]> {
  const chunks: object[][] = [];
  for (let i = 0; i < items.length; i += chunkSize) {
    chunks.push(items.slice(i, i + chunkSize));
  }

  const results: object[] = [];

  // Process in batches of concurrent requests
  for (let i = 0; i < chunks.length; i += concurrency) {
    const batch = chunks.slice(i, i + concurrency);
    const promises = batch.map((chunk) =>
      api.data.createMany(entity, chunk)
    );

    const responses = await Promise.all(promises);
    for (const { ok, data } of responses) {
      if (ok) results.push(...data);
    }
  }

  return results;
}
```

## REST API Approach

### Bulk Insert

```bash
curl -X POST http://localhost:7654/api/data/products \
  -H "Content-Type: application/json" \
  -d '[
    {"name": "Product A", "price": 10},
    {"name": "Product B", "price": 20}
  ]'
```

### Bulk Update

```bash
curl -X PATCH http://localhost:7654/api/data/posts \
  -H "Content-Type: application/json" \
  -d '{
    "where": {"status": {"$eq": "draft"}},
    "data": {"status": "archived"}
  }'
```

### Bulk Delete

```bash
curl -X DELETE http://localhost:7654/api/data/logs \
  -H "Content-Type: application/json" \
  -d '{"where": {"created_at": {"$lt": "2023-01-01"}}}'
```

## Server-Side Seeding

For initial data population, use the seed function:

```typescript
import { App, em, entity, text, number } from "bknd";

const schema = em({
  products: entity("products", {
    name: text().required(),
    price: number().required(),
    stock: number({ default_value: 0 }),
  }),
});

new App({
  ...schema,
  options: {
    seed: async (ctx) => {
      // Check if already seeded
      const { data } = await ctx.em.repo("products").count();
      if (data.count > 0) return;

      // Bulk insert via mutator
      await ctx.em.mutator("products").insertMany([
        { name: "Widget", price: 9.99, stock: 100 },
        { name: "Gadget", price: 19.99, stock: 50 },
        { name: "Gizmo", price: 14.99, stock: 75 },
      ]);

      console.log("Seeded products");
    },
  },
});
```

## Error Handling Strategies

### All-or-Nothing (Fail Fast)

Stop on first error:

```typescript
async function bulkInsertStrict(api: Api, entity: string, items: object[]) {
  for (let i = 0; i < items.length; i += 100) {
    const chunk = items.slice(i, i + 100);
    const { ok, error } = await api.data.createMany(entity, chunk);

    if (!ok) {
      throw new Error(`Failed at chunk ${i / 100 + 1}: ${error.message}`);
    }
  }
}
```

### Best Effort (Continue on Error)

Collect failures, continue processing:

```typescript
async function bulkInsertBestEffort(api: Api, entity: string, items: object[]) {
  const results = { success: [] as object[], failed: [] as object[] };

  for (let i = 0; i < items.length; i += 100) {
    const chunk = items.slice(i, i + 100);
    const { ok, data } = await api.data.createMany(entity, chunk);

    if (ok) {
      results.success.push(...data);
    } else {
      results.failed.push(...chunk);
    }
  }

  return results;
}
```

### Individual Fallback

Fall back to individual inserts on chunk failure:

```typescript
async function bulkInsertWithFallback(api: Api, entity: string, items: object[]) {
  const success: object[] = [];
  const failed: object[] = [];

  for (let i = 0; i < items.length; i += 100) {
    const chunk = items.slice(i, i + 100);
    const { ok, data } = await api.data.createMany(entity, chunk);

    if (ok) {
      success.push(...data);
    } else {
      // Fall back to individual inserts