# evaluation/tasks.py
TASKS = [
    # Category: New function implementation
    {
        "id": "T001",
        "category": "implementation",
        "difficulty": "easy",
        "prompt": """Add a function `calculate_discount(price: float, tier: str) -> float`
        to pricing.py. Tier can be 'bronze' (5%), 'silver' (10%), 'gold' (15%), 'platinum' (25%).
        Raise ValueError for unknown tiers. Match the style of existing functions in that file.""",
        "test_file": "tests/test_pricing.py",
        "test_function": "test_calculate_discount",
        "reference_files": ["pricing.py"],  # Files to include as context
    },

    # Category: Bug fix
    {
        "id": "T002",
        "category": "bug_fix",
        "difficulty": "medium",
        "prompt": """Fix the bug in `UserRepository.get_by_email()`.
        The function returns None for valid emails when the email contains uppercase letters.
        The test `test_email_case_insensitive_lookup` is failing.""",
        "test_file": "tests/test_user_repository.py",
        "test_function": "test_email_case_insensitive_lookup",
        "reference_files": ["repositories/user_repository.py", "models/user.py"],
    },

    # Category: Refactoring
    {
        "id": "T003",
        "category": "refactoring",
        "difficulty": "hard",
        "prompt": """Refactor `OrderProcessor.process()` to use the Strategy pattern.
        The current if-elif chain for payment types (credit_card, paypal, crypto) should
        become separate strategy classes. Maintain the same public interface and ensure
        all existing tests still pass.""",
        "test_file": "tests/test_order_processor.py",
        "test_function": None,  # All tests must pass
        "reference_files": ["processors/order_processor.py", "tests/test_order_processor.py"],
    },
]

# evaluation/runner.py
import subprocess
import anthropic
import openai
from pathlib import Path
import json
import time

def run_evaluation(tasks, model_config):
    """Run all tasks against a specific model and return results."""
    results = []

    for task in tasks:
        # Load reference files as context
        context = "\n\n".join(
            f"=== {f} ===\n{Path(f).read_text()}"
            for f in task.get("reference_files", [])
        )

        prompt = f"{context}\n\n{task['prompt']}" if context else task["prompt"]

        # Get model response
        start = time.time()
        generated_code = get_model_response(prompt, model_config)
        latency = time.time() - start

        # Apply the generated code to the codebase
        # (In practice, write to a temp branch and run tests)
        test_result = run_tests(task, generated_code)

        results.append({
            "task_id": task["id"],
            "category": task["category"],
            "difficulty": task["difficulty"],
            "passed": test_result["passed"],
            "test_output": test_result["output"],
            "latency_seconds": latency,
            "tokens_used": test_result.get("tokens", 0),
        })

    return results

def get_model_response(prompt, config):
    if config["provider"] == "anthropic":
        client = anthropic.Anthropic()
        response = client.messages.create(
            model=config["model"],
            max_tokens=2048,
            messages=[{"role": "user", "content": prompt}]
        )
        return response.content[0].text

    elif config["provider"] == "openai":
        client = openai.OpenAI()
        response = client.chat.completions.create(
            model=config["model"],
            messages=[{"role": "user", "content": prompt}],
            max_tokens=2048
        )
        return response.choices[0].message.content

def run_tests(task, generated_code):
    """Apply generated code and run the relevant test."""
    # Parse code from markdown blocks if present
    if "```python" in generated_code:
 code = generated_code.split("```python")[1].split("```")[0].strip()
 else:
 code = generated_code

 # Identify which file to update based on task
 target_file = task["reference_files"][0]
 original = Path(target_file).read_text()

 try:
 Path(target_file).write_text(code)
 test_cmd = ["python", "-m", "pytest", task["test_file"], "-v", "--tb=short"]
 if task.get("test_function"):
 test_cmd.extend([f"-k", task["test_function"]])

 result = subprocess.run(test_cmd, capture_output=True, text=True, timeout=30)
 return {
 "passed": result.returncode == 0,
 "output": result.stdout + result.stderr
 }
 finally:
 # Restore original file
 Path(target_file).write_text(original)

def calculate_metrics(results):
 total = len(results)
 passed = sum(1 for r in results if r["passed"])

 # Break down by category
 by_category = {}
 for cat in set(r["category"] for r in results):
 cat_results = [r for r in results if r["category"] == cat]
 by_category[cat] = {
 "pass_rate": sum(1 for r in cat_results if r["passed"]) / len(cat_results),
 "count": len(cat_results)
 }

 # Break down by difficulty
 by_difficulty = {}
 for diff in ["easy", "medium", "hard"]:
 diff_results = [r for r in results if r["difficulty"] == diff]
 if diff_results:
 by_difficulty[diff] = {
 "pass_rate": sum(1 for r in diff_results if r["passed"]) / len(diff_results),
 "count": len(diff_results)
 }

 return {
 "overall_pass_rate": passed / total,
 "by_category": by_category,
 "by_difficulty": by_difficulty,
 "avg_latency": sum(r["latency_seconds"] for r in results) / total,
 "total_tasks": total,
 }

def create_blind_comparison(task, responses_by_model):
 """Shuffle responses and remove model identifiers."""
 import random

 shuffled = list(responses_by_model.items())
 random.shuffle(shuffled)

 return {
 "task": task["prompt"],
 "options": {
 chr(65 + i): code # A, B, C...
 for i, (model, code) in enumerate(shuffled)
 },
 "_mapping": {chr(65 + i): model for i, (model, _) in enumerate(shuffled)}
 }

def measure_context_utilization(task, generated_code):
 """Check how many identifiers from reference files appear in the output."""
 import ast
 import re

 reference_identifiers = set()
 for ref_file in task.get("reference_files", []):
 try:
 source = Path(ref_file).read_text()
 tree = ast.parse(source)
 for node in ast.walk(tree):
 if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Name)):
 name = getattr(node, 'name', None) or getattr(node, 'id', None)
 if name and len(name) > 3: # skip short names
 reference_identifiers.add(name)
 except Exception:
 # Fall back to regex extraction for non-Python files
 reference_identifiers.update(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]{3,}\b', source))

 if not reference_identifiers:
 return None

 used = sum(1 for ident in reference_identifiers if ident in generated_code)
 return {
 "total_reference_identifiers": len(reference_identifiers),
 "used_in_output": used,
 "utilization_rate": used / len(reference_identifiers)
 }

def cost_adjusted_metrics(results, pricing_per_million_tokens):
 """Compare models by cost per successfully completed task."""
 total_tokens = sum(r.get("tokens_used", 0) for r in results)
 total_cost = (total_tokens / 1_000_000) * pricing_per_million_tokens
 correct_tasks = sum(1 for r in results if r["passed"])

 return {
 "total_cost_usd": round(total_cost, 4),
 "correct_tasks": correct_tasks,
 "cost_per_correct_task": round(total_cost / correct_tasks, 4) if correct_tasks else None,
 "cost_per_task_attempted": round(total_cost / len(results), 4)
 }