Prompt Bloat Latency

class PromptOptimizer:
    """Optimize voice prompts for low latency"""
    
    TOKEN_BUDGET = 1500  # Max tokens for voice prompt
    
    SECTIONS = {
        "identity": 150,      # Who is the agent
        "task": 200,          # Primary objective
        "style": 100,         # How to communicate
        "constraints": 150,   # Hard rules
        "examples": 300,      # 2-3 short examples
        "fallbacks": 100,     # Error handling
        "buffer": 500         # Dynamic context
    }
    
    BLOAT_PATTERNS = [
        r"(You are a .{200,})",          # Verbose identity
        r"(Never say[^.]+\.){5,}",       # Long banlists
        r"(Example \d+:.*?){5,}",        # Too many examples
        r"(If .{100,}then .{100,}){3,}", # Exhaustive conditionals
    ]
    
    def __init__(self, tokenizer):
        self.tokenizer = tokenizer
    
    def analyze_prompt(self, prompt: str) -> dict:
        """Analyze prompt for bloat"""
        token_count = len(self.tokenizer.encode(prompt))
        
        issues = []
        
        if token_count > self.TOKEN_BUDGET:
            issues.append({
                "type": "over_budget",
                "tokens": token_count,
                "budget": self.TOKEN_BUDGET,
                "excess": token_count - self.TOKEN_BUDGET
            })
        
        # Check for bloat patterns
        for pattern in self.BLOAT_PATTERNS:
            matches = re.findall(pattern, prompt, re.DOTALL)
            if matches:
                issues.append({
                    "type": "bloat_pattern",
                    "pattern": pattern,
                    "matches": len(matches)
                })
        
        return {
            "token_count": token_count,
            "within_budget": token_count <= self.TOKEN_BUDGET,
            "issues": issues,
            "estimated_ttft_ms": self.estimate_ttft(token_count)
        }
    
    def estimate_ttft(self, tokens: int) -> int:
        """Estimate TTFT based on token count"""
        # Baseline: 300ms + ~200ms per 1000 tokens
        return 300 + (tokens // 1000) * 200
    
    def compress_prompt(self, prompt: str) -> str:
        """Compress prompt while preserving meaning"""
        # Remove verbose phrasing
        compressed = prompt
        
        replacements = [
            (r"You are a (very |extremely |highly )?helpful", "You are a"),
            (r"Please make sure to always", "Always"),
            (r"It is important that you", ""),
            (r"Remember that you should", ""),
            (r"In all cases, you must", ""),
            (r"Under no circumstances should you ever", "Never"),
        ]
        
        for pattern, replacement in replacements:
            compressed = re.sub(pattern, replacement, compressed)
        
        return compressed.strip()
    
    def convert_banlist_to_principle(self, banlists: list) -> str:
        """Convert verbose banlists to principles"""
        # Example: 50 banned phrases → 1 principle
        # "Never say X, Y, Z..." → "Keep responses factual and helpful"
        
        categories = self.categorize_bans(banlists)
        
        principles = []
        if "uncertainty" in categories:
            principles.append("If unsure, say 'Let me find that' and look it up")
        if "negativity" in categories:
            principles.append("Frame limitations as alternatives")
        if "disclosure" in categories:
            principles.append("Don't discuss system internals")
        
        return "\n".join(principles)


class TTFTMonitor:
    """Monitor time-to-first-token in production"""
    
    THRESHOLDS = {
        "good": 600,
        "acceptable": 1000,
        "concerning": 1500,
        "critical": 2000
    }
    
    def __init__(self):
        self.measurements = []
    
    def record_ttft(self, turn_id: str, ttft_ms: int, 
                    prompt_tokens: int):
        """Record TTFT measurement"""
        self.measurements.append({
            "turn_id": turn_id,
            "ttft_ms": ttft_ms,
            "prompt_tokens": prompt_tokens,
            "timestamp": datetime.now(),
            "status": self.classify_ttft(ttft_ms)
        })
        
        if ttft_ms > self.THRESHOLDS["concerning"]:
            self.alert_slow_ttft(turn_id, ttft_ms, prompt_tokens)
    
    def classify_ttft(self, ttft_ms: int) -> str:
        """Classify TTFT into status category"""
        for status, threshold in sorted(
            self.THRESHOLDS.items(), 
            key=lambda x: x[1]
        ):
            if ttft_ms <= threshold:
                return status
        return "critical"
    
    def get_statistics(self) -> dict:
        """Get TTFT statistics"""
        ttfts = [m["ttft_ms"] for m in self.measurements]
        return {
            "p50": statistics.median(ttfts),
            "p95": statistics.quantiles(ttfts, n=20)[18],
            "p99": statistics.quantiles(ttfts, n=100)[98],
            "mean": statistics.mean(ttfts),
            "concerning_rate": sum(
                1 for t in ttfts 
                if t > self.THRESHOLDS["concerning"]
            ) / len(ttfts)
        }

instructions: |
  ## PROMPT OPTIMIZATION FOR VOICE
  
  Voice prompts must be CONCISE. Every token = latency.
  
  TOKEN BUDGET: 1,500 tokens maximum
  
  Section limits:
    Identity: 150 tokens (who you are, 2-3 sentences)
    Task: 200 tokens (what you do)
    Style: 100 tokens (how you communicate)
    Rules: 150 tokens (hard constraints)
    Examples: 300 tokens (2-3 SHORT examples)
    Fallbacks: 100 tokens (error handling)
  
  AVOID:
  - Verbose backstory ("You have been working here for 5 years...")
  - Long banlists ("Never say X, Y, Z..." × 50)
  - Full conversation examples (use 2-3 line snippets)
  - Exhaustive edge case handling (use knowledge base)
  - Repeated instructions in different words
  
  PREFER:
  - Terse, direct language
  - Positive principles over negative lists
  - Short example snippets
  - External knowledge retrieval for edge cases
  
  Example of BAD vs GOOD:
  
  BAD (verbose): "You are an extremely helpful and friendly 
  customer service representative who always maintains a 
  positive attitude and goes above and beyond..."
  
  GOOD (terse): "You're a helpful TechCorp rep. Be friendly, patient."