Overview

This document provides concrete, ready-to-use examples of KPI implementations for different types of GenAI applications using the Pay-i Python SDK. These examples are drawn from real-world implementations and can be adapted for your specific use cases.

For a conceptual understanding of KPIs, see the Use Case KPIs guide. For general implementation guidance, see Working with KPIs.

Chat Bot KPIs

For conversational AI applications, common KPIs include:

# Step 1: Define the KPIs for the Chat-Bot use case type (done once)
client.use_cases.definitions.kpis.create(
    use_case_name="Chat-Bot",
    kpi_name="Deflection Rate",
    description="Tracks when AI resolves issues without human intervention",
    kpi_type="boolean",
    goal=0.25  # Goal: 25% of queries resolved by AI
)

client.use_cases.definitions.kpis.create(
    use_case_name="Chat-Bot",
    kpi_name="User Satisfaction",
    description="Measures if users were satisfied with the interaction",
    kpi_type="boolean",
    goal=0.75  # Goal: 75% of users satisfied
)

# Step 2: Record values for specific chat sessions (done per instance)
chat_session = client.use_cases.create(use_case_name="Chat-Bot")
chat_session_id = chat_session.use_case_id

# Track whether the AI successfully handled the query without human intervention
client.use_cases.kpis.create(
    kpi_name="Deflection Rate",
    use_case_id=chat_session_id,
    score=1.0  # Boolean: 1.0 = True (deflected), 0.0 = False (escalated)
)

# Record user satisfaction with the AI interaction
client.use_cases.kpis.create(
    kpi_name="User Satisfaction",
    use_case_id=chat_session_id,
    score=1.0  # Boolean: 1.0 = True (satisfied), 0.0 = False (unsatisfied)
)

Image-Ad Generation KPIs

For AI-generated advertising content, typical KPIs include:

# Step 1: Define the KPIs for the Image-Ad use case type (done once)
client.use_cases.definitions.kpis.create(
    use_case_name="Image-Ad",
    kpi_name="Clickthrough Rate",
    description="Tracks when users click on AI-generated content",
    kpi_type="boolean",
    goal=0.05  # Goal: 5% clickthrough rate
)

client.use_cases.definitions.kpis.create(
    use_case_name="Image-Ad",
    kpi_name="Close Rate",
    description="Measures conversion after engaging with AI content",
    kpi_type="boolean",
    goal=0.01  # Goal: 1% conversion rate
)

client.use_cases.definitions.kpis.create(
    use_case_name="Image-Ad",
    kpi_name="Conversion $ Value",
    description="Measures monetary value of conversions",
    kpi_type="number",
    goal=10.00  # Goal: $10 per conversion
)

client.use_cases.definitions.kpis.create(
    use_case_name="Image-Ad",
    kpi_name="Customer Satisfaction",
    description="User rating of content quality",
    kpi_type="likert5",
    goal=3.5  # Goal: 3.5 out of 5 rating
)

# Step 2: Record values for specific ad campaigns (done per instance)
ad_campaign = client.use_cases.create(use_case_name="Image-Ad")
ad_campaign_id = ad_campaign.use_case_id

# Track user engagement with the generated ad
client.use_cases.kpis.create(
    kpi_name="Clickthrough Rate",
    use_case_id=ad_campaign_id,
    score=1.0  # Boolean: 1.0 = True (clicked), 0.0 = False (not clicked)
)

# Record if the interaction led to conversion
client.use_cases.kpis.create(
    kpi_name="Close Rate",
    use_case_id=ad_campaign_id,
    score=1.0  # Boolean: 1.0 = True (converted), 0.0 = False (not converted)
)

# Quantify the financial value of the conversion
client.use_cases.kpis.create(
    kpi_name="Conversion $ Value",
    use_case_id=ad_campaign_id,
    score=10.14  # Numeric value representing dollars
)

# Measure user satisfaction with the generated content
client.use_cases.kpis.create(
    kpi_name="Customer Satisfaction",
    use_case_id=ad_campaign_id,
    score=3.89  # Likert5 scale (1-5)
)

Document Processing KPIs

For AI-powered document processing applications, accuracy and efficiency KPIs are essential:

# Step 1: Define the KPIs for the Document-Processor use case type
client.use_cases.definitions.kpis.create(
    use_case_name="Document-Processor",
    kpi_name="Extraction Accuracy",
    description="Percentage of correctly extracted fields",
    kpi_type="number",  # Stored as decimal (0.0-1.0)
    goal=0.98  # 98% accuracy target
)

client.use_cases.definitions.kpis.create(
    use_case_name="Document-Processor",
    kpi_name="Processing Time",
    description="Seconds to process document",
    kpi_type="number",
    goal=3.0  # 3 seconds target
)

client.use_cases.definitions.kpis.create(
    use_case_name="Document-Processor",
    kpi_name="Error Detection",
    description="Whether the AI correctly identified errors in document",
    kpi_type="boolean",
    goal=0.90  # 90% detection rate target
)

# Step 2: Record values for specific document processing runs
document_process = client.use_cases.create(use_case_name="Document-Processor")
document_id = document_process.use_case_id

# Record extraction accuracy for this document processing run
client.use_cases.kpis.create(
    kpi_name="Extraction Accuracy",
    use_case_id=document_id,
    score=0.95  # 95% of fields extracted correctly
)

# Record processing time in seconds
client.use_cases.kpis.create(
    kpi_name="Processing Time",
    use_case_id=document_id,
    score=2.7  # 2.7 seconds to process
)

# Record whether errors were correctly detected
client.use_cases.kpis.create(
    kpi_name="Error Detection",
    use_case_id=document_id,
    score=1.0  # Boolean: 1.0 = True (correctly detected errors)
)

Text Generation KPIs

For AI-powered text generation applications like content generators or copywriting tools:

# Step 1: Define the KPIs for the Text-Generator use case type
client.use_cases.definitions.kpis.create(
    use_case_name="Text-Generator",
    kpi_name="Content Quality",
    description="Expert rating of generated content quality",
    kpi_type="likert5",
    goal=4.2  # 4.2/5.0 quality target
)

client.use_cases.definitions.kpis.create(
    use_case_name="Text-Generator",
    kpi_name="Brand Compliance",
    description="Whether content meets brand guidelines",
    kpi_type="boolean",
    goal=0.95  # 95% compliance target
)

client.use_cases.definitions.kpis.create(
    use_case_name="Text-Generator",
    kpi_name="Revision Count",
    description="Number of revisions needed before approval",
    kpi_type="number",
    goal=1.5  # Target: 1.5 or fewer revisions on average
)

# Step 2: Record values for specific content generation runs
content_gen = client.use_cases.create(use_case_name="Text-Generator")
content_id = content_gen.use_case_id

# Record content quality rating from expert review
client.use_cases.kpis.create(
    kpi_name="Content Quality",
    use_case_id=content_id,
    score=4.5  # 4.5/5.0 quality rating
)

# Record brand compliance assessment
client.use_cases.kpis.create(
    kpi_name="Brand Compliance",
    use_case_id=content_id,
    score=1.0  # Boolean: 1.0 = True (compliant with brand guidelines)
)

# Record how many revisions were needed
client.use_cases.kpis.create(
    kpi_name="Revision Count",
    use_case_id=content_id,
    score=1.0  # Only one revision needed
)

Combining KPIs with Use Case Properties

For more context around your KPIs, combine them with use case properties:

# Define the KPI for your use case type
client.use_cases.definitions.kpis.create(
    use_case_name="Content-Generator",
    kpi_name="Response Quality",
    description="Quality rating of generated content",
    kpi_type="likert5",
    goal=4.0
)

# Create a use case instance
content_generation = client.use_cases.create(use_case_name="Content-Generator")
use_case_id = content_generation.use_case_id

# Add context to the use case instance
client.use_cases.properties.create(
    property_name="model_version",
    use_case_id=use_case_id,
    value="gpt-4o"
)

client.use_cases.properties.create(
    property_name="prompt_template",
    use_case_id=use_case_id,
    value="structured_blog_post_v2"
)

# Then add KPI value for that specific instance
client.use_cases.kpis.create(
    kpi_name="Response Quality",
    use_case_id=use_case_id,
    score=4.8
)

This combination allows you to analyze KPI performance across different conditions, models, or configurations.

Funnel Conversion KPI Implementation

For tracking multi-step user journeys, implementing funnel conversion KPIs helps track progression through each stage:

# Step 1: Define KPIs for each stage of the recommendation funnel
funnel_stages = [
    ("Recommendation Viewed", "User viewed AI-generated recommendations", 0.90),
    ("Recommendation Clicked", "User clicked on at least one recommendation", 0.20),
    ("Added to Cart", "User added recommended item to cart", 0.05),
    ("Purchased", "User completed purchase of recommended item", 0.01)
]

for kpi_name, description, goal in funnel_stages:
    client.use_cases.definitions.kpis.create(
        use_case_name="Product-Recommender",
        kpi_name=kpi_name,
        description=description,
        kpi_type="boolean",
        goal=goal
    )

# Step 2: Create a function to track a user's journey through the funnel
def track_recommendation_funnel(recommendation_id, stages_completed):
    """
    Track a user's progression through the recommendation funnel
    
    Args:
        recommendation_id: Identifier for the recommendation session
        stages_completed: Dictionary with funnel stages as keys and boolean values
    """
    # Create a use case instance for this recommendation session
    recommender = client.use_cases.create(
        use_case_name="Product-Recommender",
        properties={"recommendation_id": recommendation_id}
    )
    use_case_id = recommender.use_case_id
    
    # Record which stages were completed
    for stage, completed in stages_completed.items():
        client.use_cases.kpis.create(
            kpi_name=stage,
            use_case_id=use_case_id,
            score=1.0 if completed else 0.0
        )
    
    return use_case_id

# Example usage: Track a user journey through the funnel
user_journey = {
    "Recommendation Viewed": True,
    "Recommendation Clicked": True,
    "Added to Cart": True,
    "Purchased": False
}

track_recommendation_funnel("user_123_session_456", user_journey)

Error Handling Best Practices

When implementing KPIs in production environments, include proper error handling:

def record_kpi_safely(kpi_name, use_case_id, score):
    """Safely record a KPI value with error handling"""
    try:
        client.use_cases.kpis.create(
            kpi_name=kpi_name,
            use_case_id=use_case_id,
            score=score
        )
        return True
    except Exception as e:
        # Log the error but don't crash your application
        logger.error(f"Failed to record KPI {kpi_name} for use case {use_case_id}: {str(e)}")
        
        # Optionally queue for retry
        retry_queue.append({
            "kpi_name": kpi_name,
            "use_case_id": use_case_id,
            "score": score
        })
        return False

Related Resources

Use Case KPIs - Core concepts of using KPIs in Pay-i
Working with KPIs - Practical guide for implementing KPIs
Python SDK KPIs - SDK implementation patterns and code samples
Python SDK KPIs API Reference - Detailed API documentation
KPI Best Practices - Recommendations for effective KPI implementation