AI Testing & Evaluation Framework

// Example: Testing a summarization prompt
describe('Summarization Prompt', () => {
  it('should summarize long text correctly', async () => {
    const longText = "..."; // Your long text
    const result = await summarize(longText);
    
    // Check length
    expect(result.length).toBeLessThan(200);
    
    // Check key points present
    expect(result).toContain('main point');
    
    // Check semantic similarity to expected
    const similarity = await calculateSemanticSimilarity(
      result,
      expectedSummary
    );
    expect(similarity).toBeGreaterThan(0.8);
  });
  
  it('should handle edge cases', async () => {
    expect(await summarize('')).toBe('');
    expect(await summarize('short')).toBe('short');
  });
});

// Store golden examples
const goldenExamples = [
  { input: 'What is AI?', expectedKeywords: ['artificial', 'intelligence'] },
  { input: 'Explain RAG', expectedKeywords: ['retrieval', 'generation'] },
];

async function runRegressionTests() {
  const results = [];
  
  for (const example of goldenExamples) {
    const response = await llm.call(example.input);
    const relevance = calculateKeywordRelevance(
      response,
      example.expectedKeywords
    );
    
    results.push({
      input: example.input,
      passed: relevance >= 0.8,
      relevance,
    });
  }
  
  const passRate = results.filter(r => r.passed).length / results.length;
  console.log(`Pass rate: ${(passRate * 100).toFixed(1)}%`);
  
  return passRate >= 0.95; // 95% threshold
}

interface ABTestResult {
  variantA: { accuracy: number; cost: number; latency: number };
  variantB: { accuracy: number; cost: number; latency: number };
  winner: 'A' | 'B' | 'tie';
}

async function abTest(
  testCases: Array<{ input: string; expected: string }>,
  variantA: LLMConfig,
  variantB: LLMConfig
): Promise<ABTestResult> {
  const resultsA = await runTests(testCases, variantA);
  const resultsB = await runTests(testCases, variantB);
  
  // Calculate winner based on weighted score
  const scoreA = resultsA.accuracy * 0.6 + (1 - resultsA.cost / 100) * 0.2 + (1 - resultsA.latency / 1000) * 0.2;
  const scoreB = resultsB.accuracy * 0.6 + (1 - resultsB.cost / 100) * 0.2 + (1 - resultsB.latency / 1000) * 0.2;
  
  return {
    variantA: resultsA,
    variantB: resultsB,
    winner: scoreA > scoreB ? 'A' : scoreB > scoreA ? 'B' : 'tie',
  };
}

interface HumanEvaluation {
  responseId: string;
  ratings: {
    accuracy: number;    // 1-5
    relevance: number;   // 1-5
    helpfulness: number; // 1-5
    clarity: number;     // 1-5
  };
  feedback: string;
}

async function collectHumanEvaluations(
  responses: Array<{ id: string; question: string; answer: string }>
): Promise<HumanEvaluation[]> {
  // Present to human evaluators via UI
  // Store ratings in database
  // Calculate inter-rater reliability
  
  return evaluations;
}

function analyzeHumanEvals(evals: HumanEvaluation[]) {
  const avgRatings = {
    accuracy: average(evals.map(e => e.ratings.accuracy)),
    relevance: average(evals.map(e => e.ratings.relevance)),
    helpfulness: average(evals.map(e => e.ratings.helpfulness)),
    clarity: average(evals.map(e => e.ratings.clarity)),
  };
  
  return avgRatings;
}