Embedding Model Mastery

123456789101112
from openai import OpenAI
import numpy as np

client = OpenAI()

def get_embedding(text: str, dimensions: int = 1024) -> list[float]:
    """Get embedding with native dimension reduction.
    
    Supported dimensions for text-embedding-3-large: 256, 512, 1024, 3072
    Supported dimensions for text-embedding-3-small: 512, 1536
    """
    response = client.embeddings.create(

123456789101112
from openai import OpenAI
import numpy as np
from typing import List, Tuple

client = OpenAI()

class AdaptiveEmbedder:
    def __init__(self, model: str = "text-embedding-3-large"):
        self.model = model
        self.dimension_tiers = {
            "fast": 256,      # Initial retrieval
            "balanced": 512,  # Standard use

123456789101112
from sentence_transformers import SentenceTransformer
import torch

class InstructionEmbedder:
    def __init__(self):
        self.model = SentenceTransformer('intfloat/e5-large-instruct')
        
        # Define task-specific instructions
        self.instructions = {
            'support_query': 'Represent this customer support question for finding relevant help documentation:',
            'support_doc': 'Represent this help article for retrieval by customer questions:',
            'code_query': 'Represent this programming question for finding relevant code examples:',

123456789101112
import numpy as np
from typing import List, Dict, Tuple
from dataclasses import dataclass
import time
import json

@dataclass
class BenchmarkResult:
    model_name: str
    mrr_at_10: float
    ndcg_at_10: float
    recall_at_100: float

123456789101112
from enum import Enum
from typing import Optional, Callable
import re
import random

class QueryCategory(Enum):
    SHORT_FACTUAL = "short_factual"
    COMPLEX_ANALYTICAL = "complex_analytical"
    MULTILINGUAL = "multilingual"
    CODE_RELATED = "code_related"
    GENERAL = "general"

123456789101112
from typing import List, Dict, Tuple
from collections import defaultdict
import numpy as np

class HybridSearch:
    def __init__(self, semantic_weight: float = 0.6, k_constant: int = 60):
        self.semantic_weight = semantic_weight
        self.keyword_weight = 1 - semantic_weight
        self.k = k_constant  # RRF constant, typically 60
        
    def reciprocal_rank_fusion(self, 
                               semantic_results: List[Tuple[str, float]],