Build an AI Test Generator with Python and Claude

pip install anthropic

export ANTHROPIC_API_KEY="sk-your-ezai-key"
export ANTHROPIC_BASE_URL="https://ezaiapi.com"

import sys, os, re
import anthropic

def generate_tests(source_path: str) -> str:
    """Read source file and generate pytest tests via Claude."""

    with open(source_path) as f:
        source_code = f.read()

    module_name = os.path.basename(source_path).replace(".py", "")

    client = anthropic.Anthropic(
        base_url=os.getenv("ANTHROPIC_BASE_URL", "https://ezaiapi.com"),
    )

    prompt = f"""Generate a complete pytest test file for this Python module.

SOURCE FILE: {source_path}
```python
{source_code}
```

REQUIREMENTS:
- Import the module as: from {module_name} import *
- Test every public function and class method
- Include: happy path, edge cases (empty input, None, boundary values), error cases
- Use pytest.raises for expected exceptions
- Use @pytest.mark.parametrize where it reduces duplication
- Use descriptive test names: test_<function>_<scenario>
- Add brief docstrings explaining what each test verifies
- Mock external dependencies (network, file I/O, databases)
- Return ONLY the Python code, no markdown fences

TARGET: 80%+ code coverage with meaningful assertions."""

    message = client.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=4096,
        messages=[{"role": "user", "content": prompt}],
    )

    test_code = message.content[0].text

    # Strip markdown fences if present
    test_code = re.sub(r"^```python\n?", "", test_code)
    test_code = re.sub(r"\n?```$", "", test_code)

    return test_code

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python testgen.py <source.py> [output.py]")
        sys.exit(1)

    source = sys.argv[1]
    output = sys.argv[2] if len(sys.argv) > 2 else f"test_{os.path.basename(source)}"

    print(f"🧪 Generating tests for {source}...")
    tests = generate_tests(source)

    with open(output, "w") as f:
        f.write(tests)

    print(f"✅ Tests written to {output}")
    print(f"   Run: pytest {output} -v")

python testgen.py utils.py
# 🧪 Generating tests for utils.py...
# ✅ Tests written to test_utils.py
#    Run: pytest test_utils.py -v

import glob

def batch_generate(src_dir: str, test_dir: str = "tests"):
    """Generate tests for all Python files in a directory."""
    os.makedirs(test_dir, exist_ok=True)

    sources = glob.glob(os.path.join(src_dir, "**/*.py"), recursive=True)
    sources = [s for s in sources if not s.endswith("__init__.py")]

    for source in sources:
        basename = os.path.basename(source)
        test_file = os.path.join(test_dir, f"test_{basename}")

        if os.path.exists(test_file):
            print(f"⏭  Skipping {basename} (test exists)")
            continue

        print(f"🧪 {basename} → {test_file}")
        try:
            tests = generate_tests(source)
            with open(test_file, "w") as f:
                f.write(tests)
        except Exception as e:
            print(f"❌ Failed: {e}")

    print(f"\nDone. Run: pytest {test_dir}/ -v")

import subprocess

def validate_and_fix(source_path: str, test_path: str, max_retries: int = 2):
    """Run tests and auto-fix failures using Claude."""
    client = anthropic.Anthropic(
        base_url=os.getenv("ANTHROPIC_BASE_URL", "https://ezaiapi.com"),
    )

    for attempt in range(max_retries):
        result = subprocess.run(
            ["python", "-m", "pytest", test_path, "-v", "--tb=short"],
            capture_output=True, text=True, timeout=30
        )

        if result.returncode == 0:
            print(f"✅ All tests pass!")
            return True

        print(f"🔧 Attempt {attempt + 1}: fixing {result.stdout.count('FAILED')} failures")

        with open(source_path) as f:
            source = f.read()
        with open(test_path) as f:
            tests = f.read()

        fix_prompt = f"""Fix these failing tests. Return the COMPLETE corrected test file.

SOURCE CODE:
```python
{source}
```

CURRENT TESTS:
```python
{tests}
```

PYTEST OUTPUT:
```
{result.stdout[-2000:]}
{result.stderr[-1000:]}
```

Fix all failures. Return ONLY Python code, no markdown."""

        msg = client.messages.create(
            model="claude-sonnet-4-5",
            max_tokens=4096,
            messages=[{"role": "user", "content": fix_prompt}],
        )

        fixed = msg.content[0].text
        fixed = re.sub(r"^```python\n?", "", fixed)
        fixed = re.sub(r"\n?```$", "", fixed)

        with open(test_path, "w") as f:
            f.write(fixed)

    return False

name: AI Test Generation
on:
  pull_request:
    paths: ['src/**/*.py']

jobs:
  generate-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - run: pip install anthropic pytest
      - run: python testgen.py src/ --batch
        env:
          ANTHROPIC_API_KEY: ${{ secrets.EZAI_API_KEY }}
          ANTHROPIC_BASE_URL: https://ezaiapi.com
      - run: pytest tests/ -v --tb=short