github-harvester

What this skill does

# GitHub Harvester Skill

> Extract and ingest content from GitHub repositories into RAG.

## Overview

GitHub repositories contain valuable documentation, code examples, and discussions. This skill covers:
- README and documentation extraction
- Code example mining
- Issue and discussion harvesting
- Wiki content extraction
- Release notes and changelogs

## Prerequisites

```bash
# GitHub CLI (recommended)
brew install gh  # macOS
# or: https://cli.github.com/

# Python libraries
pip install PyGithub httpx
```

## Authentication

```bash
# Authenticate with GitHub CLI
gh auth login

# Or set token for API access
export GITHUB_TOKEN="ghp_..."
```

## Extraction Methods

### Method 1: GitHub CLI (Recommended)

Best for quick extraction and authenticated access.

```bash
#!/bin/bash
# Extract repo content using gh CLI

REPO="$1"  # owner/repo format

# Clone with depth 1 for content only
gh repo clone "$REPO" -- --depth 1

# Get repo info
gh repo view "$REPO" --json name,description,readme

# Get issues
gh issue list --repo "$REPO" --limit 100 --json title,body,comments

# Get discussions (if enabled)
gh api "repos/$REPO/discussions" --paginate

# Get releases
gh release list --repo "$REPO" --limit 20
```

### Method 2: PyGithub API

Better for programmatic access and complex queries.

```python
#!/usr/bin/env python3
"""GitHub content extraction using PyGithub."""

from github import Github
from typing import Dict, List, Optional
import base64
import os

class GitHubExtractor:
    """Extract content from GitHub repositories."""

    def __init__(self, token: str = None):
        self.token = token or os.getenv("GITHUB_TOKEN")
        self.github = Github(self.token) if self.token else Github()

    def get_repo(self, repo_name: str):
        """Get repository object."""
        return self.github.get_repo(repo_name)

    def get_readme(self, repo_name: str) -> Dict:
        """Extract README content."""
        repo = self.get_repo(repo_name)

        try:
            readme = repo.get_readme()
            content = base64.b64decode(readme.content).decode('utf-8')

            return {
                "content": content,
                "path": readme.path,
                "size": readme.size,
                "url": readme.html_url
            }
        except Exception as e:
            return {"error": str(e)}

    def get_docs(self, repo_name: str) -> List[Dict]:
        """Extract documentation files."""
        repo = self.get_repo(repo_name)
        docs = []

        # Common doc locations
        doc_paths = ['docs', 'doc', 'documentation', '.github']

        for path in doc_paths:
            try:
                contents = repo.get_contents(path)
                docs.extend(self._extract_dir(repo, contents))
            except Exception:
                continue

        # Also get root markdown files
        try:
            root_contents = repo.get_contents("")
            for item in root_contents:
                if item.type == "file" and item.name.endswith('.md'):
                    content = base64.b64decode(item.content).decode('utf-8')
                    docs.append({
                        "path": item.path,
                        "content": content,
                        "url": item.html_url
                    })
        except Exception:
            pass

        return docs

    def _extract_dir(self, repo, contents) -> List[Dict]:
        """Recursively extract directory contents."""
        docs = []

        if not isinstance(contents, list):
            contents = [contents]

        for item in contents:
            if item.type == "dir":
                sub_contents = repo.get_contents(item.path)
                docs.extend(self._extract_dir(repo, sub_contents))
            elif item.type == "file":
                if item.name.endswith(('.md', '.rst', '.txt')):
                    try:
                        content = base64.b64decode(item.content).decode('utf-8')
                        docs.append({
                            "path": item.path,
                            "content": content,
                            "url": item.html_url
                        })
                    except Exception:
                        pass

        return docs

    def get_code_examples(
        self,
        repo_name: str,
        patterns: List[str] = None
    ) -> List[Dict]:
        """Extract code examples from repository."""
        repo = self.get_repo(repo_name)
        examples = []

        if patterns is None:
            patterns = ['examples', 'samples', 'demo', 'tutorials']

        for pattern in patterns:
            try:
                contents = repo.get_contents(pattern)
                examples.extend(self._extract_code(repo, contents))
            except Exception:
                continue

        return examples

    def _extract_code(self, repo, contents) -> List[Dict]:
        """Extract code files."""
        code = []
        code_extensions = ['.py', '.js', '.ts', '.go', '.rs', '.java', '.rb']

        if not isinstance(contents, list):
            contents = [contents]

        for item in contents:
            if item.type == "dir":
                sub = repo.get_contents(item.path)
                code.extend(self._extract_code(repo, sub))
            elif item.type == "file":
                if any(item.name.endswith(ext) for ext in code_extensions):
                    try:
                        content = base64.b64decode(item.content).decode('utf-8')
                        code.append({
                            "path": item.path,
                            "content": content,
                            "language": self._detect_language(item.name),
                            "url": item.html_url
                        })
                    except Exception:
                        pass

        return code

    def _detect_language(self, filename: str) -> str:
        """Detect programming language from filename."""
        ext_map = {
            '.py': 'python',
            '.js': 'javascript',
            '.ts': 'typescript',
            '.go': 'go',
            '.rs': 'rust',
            '.java': 'java',
            '.rb': 'ruby',
            '.sh': 'bash',
        }
        for ext, lang in ext_map.items():
            if filename.endswith(ext):
                return lang
        return 'unknown'

    def get_issues(
        self,
        repo_name: str,
        state: str = "all",
        limit: int = 100
    ) -> List[Dict]:
        """Extract issues with comments."""
        repo = self.get_repo(repo_name)
        issues = []

        for issue in repo.get_issues(state=state)[:limit]:
            issue_data = {
                "number": issue.number,
                "title": issue.title,
                "body": issue.body or "",
                "state": issue.state,
                "labels": [l.name for l in issue.labels],
                "created_at": issue.created_at.isoformat(),
                "url": issue.html_url,
                "comments": []
            }

            # Get comments
            for comment in issue.get_comments():
                issue_data["comments"].append({
                    "body": comment.body,
                    "author": comment.user.login,
                    "created_at": comment.created_at.isoformat()
                })

            issues.append(issue_data)

        return issues

    def get_discussions(self, repo_name: str, limit: int = 50) -> List[Dict]:
        """Extract discussions using GraphQL API."""
        # Note: Requires GraphQL query, simplified version here
        query = """
        query($owner: String!, $name: String!, $first: Int!) {
          repository(owner: $owner, name: $name) {
            discussions(first: $first) {
              nodes {
                title
                body
                url
                category { name }
                comments(first: 10) {
                  nod