Basic BootstrapFewShot Optimization
Objective: Learn to use BootstrapFewShot to improve a simple QA system.
Problem
You have a basic question-answering system that needs improvement. Use BootstrapFewShot to optimize it with provided training data.
import dspy
from dspy.teleprompt import BootstrapFewShot
class BasicQA(dspy.Module):
def __init__(self):
super().__init__()
self.generate_answer = dspy.Predict("question -> answer")
def forward(self, question):
return self.generate_answer(question=question)
# Training data
trainset = [
dspy.Example(question="What is 2+2?", answer="4").with_inputs("question"),
dspy.Example(question="Capital of France?", answer="Paris").with_inputs("question"),
dspy.Example(question="Who wrote Romeo and Juliet?", answer="Shakespeare").with_inputs("question"),
dspy.Example(question="What is H2O?", answer="Water").with_inputs("question"),
dspy.Example(question="How many continents?", answer="7").with_inputs("question"),
]
# Test data
testset = [
dspy.Example(question="What is 3+3?", answer="6").with_inputs("question"),
dspy.Example(question="Capital of Spain?", answer="Madrid").with_inputs("question"),
]
# TODO: Implement this function
def bootstrap_optimize(program, trainset, max_demos=4):
"""Optimize the program using BootstrapFewShot."""
passTasks
KNNFewShot for Context-Aware Selection
Objective: Implement KNNFewShot to select relevant examples dynamically based on query similarity.
Problem
Build a context-aware classifier that selects different examples based on the input text's topic.
import dspy
from dspy.teleprompt import KNNFewShot
class TopicClassifier(dspy.Module):
def __init__(self):
super().__init__()
self.classify = dspy.Predict("text -> topic")
def forward(self, text):
return self.classify(text=text)
# Diverse training data
trainset = [
dspy.Example(
text="The company's stock increased after earnings",
topic="finance"
).with_inputs("text"),
dspy.Example(
text="New study reveals vaccine effectiveness",
topic="healthcare"
).with_inputs("text"),
dspy.Example(
text="Court ruled in favor of the plaintiff",
topic="legal"
).with_inputs("text"),
dspy.Example(
text="Quarterback threw a touchdown pass",
topic="sports"
).with_inputs("text"),
]
# TODO: Implement these functions
def create_knn_optimizer(k=3):
"""Create KNNFewShot optimizer."""
pass
def evaluate_classifier(classifier, testset):
"""Evaluate classifier accuracy."""
passTasks
MIPRO for Complex Reasoning
Objective: Use MIPRO to optimize a Chain of Thought program for mathematical reasoning.
Problem
Improve a mathematical problem solver that requires step-by-step reasoning.
import dspy
from dspy.teleprompt import MIPRO
class MathSolver(dspy.Module):
def __init__(self):
super().__init__()
self.solve = dspy.ChainOfThought("problem -> answer")
def forward(self, problem):
result = self.solve(problem=problem)
return dspy.Prediction(
steps=result.rationale,
answer=result.answer
)
# Math problems
trainset = [
dspy.Example(
problem="A rope is 12m long. Cut into 3 equal pieces. Length of each?",
answer="4 meters"
).with_inputs("problem"),
dspy.Example(
problem="Train travels 60km in 1 hour. How far in 3 hours?",
answer="180 km"
).with_inputs("problem"),
dspy.Example(
problem="Box has 8 rows of 5 apples each. Total apples?",
answer="40 apples"
).with_inputs("problem"),
]
# TODO: Implement these functions
def create_math_metric():
"""Create metric for math problems."""
pass
def mipro_optimize(program, trainset, num_candidates=10):
"""Optimize using MIPRO."""
passTasks
Optimizer Comparison Benchmark
Objective: Compare different optimizers on the same task to understand trade-offs.
Problem
Build a sentiment analyzer and optimize with BootstrapFewShot, KNNFewShot, and MIPRO. Compare results.
import dspy
import time
from dspy.teleprompt import BootstrapFewShot, KNNFewShot, MIPRO
class SentimentAnalyzer(dspy.Module):
def __init__(self):
super().__init__()
self.analyze = dspy.Predict("text -> sentiment, confidence")
def forward(self, text):
return self.analyze(text=text)
trainset = [
dspy.Example(text="I love this!", sentiment="positive", confidence="high").with_inputs("text"),
dspy.Example(text="Terrible quality.", sentiment="negative", confidence="high").with_inputs("text"),
dspy.Example(text="Works as expected.", sentiment="neutral", confidence="medium").with_inputs("text"),
# ... more examples
]
# TODO: Implement benchmark function
def benchmark_optimizers(program, trainset, testset):
"""Compare multiple optimizers."""
results = {}
# Test each optimizer
# Record: accuracy, compile time, inference speed
return resultsTasks
Progressive Optimization Pipeline
Objective: Design a progressive optimization strategy that automatically selects the best optimizer.
Problem
Create a system that starts with simple optimization and progressively tries more advanced optimizers until reaching the target accuracy.
import dspy
from dspy.teleprompt import BootstrapFewShot, KNNFewShot, MIPRO
def progressive_optimize(program, trainset, valset, target_accuracy=0.85):
"""
Progressively optimize until target accuracy is reached.
Strategy:
1. Start with BootstrapFewShot (fast)
2. If not good enough, try KNNFewShot
3. If still not good enough, try MIPRO
4. Return best model that meets target, or best overall
"""
# TODO: Implement progressive optimization
pass
def analyze_optimization_roi(results):
"""
Analyze return on investment for each optimization step.
Calculate: accuracy gain per minute of optimization time
"""
# TODO: Implement ROI analysis
pass