Benchmark Guide

# benchmark/repos.yaml
schema_version: "1.0.0"
last_updated: "2025-01-15"
description: "Curated dataset for evaluating AI detection accuracy"
 
targets:
  accuracy: 0.90
  precision: 0.85
  recall: 0.95
  f1_score: 0.90
 
repositories:
  # Known AI repository
  - url: "https://github.com/huggingface/transformers"
    name: "transformers"
    labels:
      contains_ai: true
      ai_type: ["deep_learning", "nlp"]
      sector: "general_ml"
      risk_category: "minimal_risk"
      frameworks: ["pytorch", "tensorflow"]
    notes: "HuggingFace Transformers library"
 
  # Known non-AI repository
  - url: "https://github.com/pallets/flask"
    name: "flask"
    labels:
      contains_ai: false
      sector: "web_development"
      risk_category: null
    notes: "Python web framework - no AI components"

# Run benchmark with default dataset
aigovhub benchmark
 
# Use custom dataset
aigovhub benchmark --dataset my-repos.yaml
 
# Output results to specific directory
aigovhub benchmark --output results/run-001/

#!/bin/bash
# benchmark_run.sh
 
REPOS_FILE="benchmark/repos.yaml"
RESULTS_DIR="benchmark/results/$(date +%Y%m%d_%H%M%S)"
mkdir -p "$RESULTS_DIR"
 
# Parse repos from YAML and scan each
# (Requires yq or similar YAML parser)
 
for repo in $(cat repos.txt); do
    echo "Scanning $repo..."
 
    # Clone if not cached
    if [ ! -d "benchmark/cached/$repo" ]; then
        git clone "https://github.com/$repo" "benchmark/cached/$repo"
    fi
 
    # Scan
    aigovhub scan "benchmark/cached/$repo" \
        --output "$RESULTS_DIR/$repo.yaml" \
        --no-llm
 
done
 
# Evaluate results
python scripts/evaluate_benchmark.py "$RESULTS_DIR"

"""benchmark_evaluate.py - Evaluate benchmark results."""
 
import yaml
from pathlib import Path
from dataclasses import dataclass
 
@dataclass
class BenchmarkResult:
    true_positives: int = 0
    true_negatives: int = 0
    false_positives: int = 0
    false_negatives: int = 0
 
    @property
    def accuracy(self) -> float:
        total = self.tp + self.tn + self.fp + self.fn
        return (self.tp + self.tn) / total if total else 0
 
    @property
    def precision(self) -> float:
        return self.tp / (self.tp + self.fp) if (self.tp + self.fp) else 0
 
    @property
    def recall(self) -> float:
        return self.tp / (self.tp + self.fn) if (self.tp + self.fn) else 0
 
    @property
    def f1_score(self) -> float:
        p, r = self.precision, self.recall
        return 2 * p * r / (p + r) if (p + r) else 0
 
def evaluate(repos_file: Path, results_dir: Path) -> BenchmarkResult:
    """Evaluate benchmark results against ground truth."""
    result = BenchmarkResult()
 
    # Load ground truth
    with open(repos_file) as f:
        dataset = yaml.safe_load(f)
 
    for repo in dataset["repositories"]:
        name = repo["name"]
        expected_ai = repo["labels"]["contains_ai"]
 
        # Load scan result
        result_file = results_dir / f"{name}.yaml"
        if not result_file.exists():
            print(f"Warning: No result for {name}")
            continue
 
        with open(result_file) as f:
            scan_result = yaml.safe_load(f)
 
        detected_ai = len(scan_result.get("ai_systems", [])) > 0
 
        # Update metrics
        if expected_ai and detected_ai:
            result.true_positives += 1
        elif not expected_ai and not detected_ai:
            result.true_negatives += 1
        elif not expected_ai and detected_ai:
            result.false_positives += 1
            print(f"False positive: {name}")
        else:  # expected_ai and not detected_ai
            result.false_negatives += 1
            print(f"False negative: {name}")
 
    return result
 
if __name__ == "__main__":
    import sys
    results_dir = Path(sys.argv[1])
    result = evaluate(Path("benchmark/repos.yaml"), results_dir)
 
    print(f"\nBenchmark Results:")
    print(f"  Accuracy:  {result.accuracy:.1%}")
    print(f"  Precision: {result.precision:.1%}")
    print(f"  Recall:    {result.recall:.1%}")
    print(f"  F1 Score:  {result.f1_score:.1%}")

# Template for new repository
- url: "https://github.com/org/repo"
  name: "repo"
  labels:
    contains_ai: true|false
    ai_type: []              # If contains_ai: true
    sector: ""
    risk_category: null      # or specific category
    frameworks: []
  notes: "Brief description"

# Find common patterns in false negatives
for repo in false_negatives:
    print(f"Missed: {repo['name']}")
    print(f"  AI Types: {repo['labels']['ai_type']}")
    print(f"  Frameworks: {repo['labels']['frameworks']}")

# benchmark/results/summary.yaml
runs:
  - date: "2025-01-15"
    version: "0.1.0"
    config:
      threshold: 0.7
      llm: false
    results:
      accuracy: 0.92
      precision: 0.88
      recall: 0.96
      f1: 0.92
    notes: "Added transformers detection"
 
  - date: "2025-01-20"
    version: "0.1.1"
    # ...

# 1. Ensure dataset is up to date
cat benchmark/repos.yaml | head -20
 
# 2. Clone repositories (first time only)
./scripts/clone_benchmark_repos.sh
 
# 3. Run benchmark
aigovhub benchmark --output benchmark/results/run-001/
 
# 4. Review results
cat benchmark/results/run-001/summary.json
 
# 5. Analyze failures
python scripts/analyze_failures.py benchmark/results/run-001/
 
# 6. Update detection if needed
# Edit src/aigovhub/core/constants.py
# Add new signal detectors
 
# 7. Re-run and compare
aigovhub benchmark --output benchmark/results/run-002/