Stats
Actions
Tags
From linkedin-audience-simulator
Review calibration metrics, accuracy trends, and model confidence levels
How this skill is triggered — by the user, by Claude, or both
Slash command
/linkedin-audience-simulator:prediction-accuracyThe summary Claude sees in its skill listing — used to decide when to auto-load this skill
**Purpose:** Show how accurate predictions have become and identify where the model needs more data.
Purpose: Show how accurate predictions have become and identify where the model needs more data.
{
"min_predictions_for_calibration": "integer (optional, default 10)"
}
{
"overall_status": {
"total_scored_posts": 18,
"total_tracked_results": 4,
"overall_mae": 4.2,
"overall_rmse": 5.1,
"global_calibration": {
"slope": 0.98,
"intercept": 0.3
},
"confidence_level": "low",
"next_milestone": "Need 10 predictions per segment for medium confidence"
},
"confidence_by_segment": [
{
"segment": "Technical Leaders",
"predictions_tracked": 2,
"confidence": "low",
"mae": 3.1,
"rmse": 3.8,
"calibration_slope": 1.02,
"calibration_intercept": -0.5,
"interpretation": "Good performance so far, but need 8 more tracked predictions"
}
],
"topic_calibration_details": [
{
"topic": "AI/ML",
"predictions_tracked": 3,
"mae": 2.8,
"rmse": 3.1,
"segments_with_data": ["Technical Leaders", "Growth & Sales"],
"status": "good_confidence_for_tech"
}
],
"calibration_matrix": {
"Technical Leaders": {
"AI/ML": {
"count": 2,
"mae": 3.1,
"status": "need_8_more"
}
}
},
"accuracy_trends": {
"posts_1_to_5": {
"mae": 5.2,
"improvement": "baseline"
},
"posts_6_to_10": {
"mae": 4.1,
"improvement": "+21%"
}
},
"build_plan": [
{
"priority": "high",
"recommendation": "Post 5+ more about Leadership + Executives",
"rationale": "0 tracked predictions. High affinity topic but no calibration."
}
],
"recommendations": [
"You're on track. After 10-15 total posts with tracked results, medium confidence kicks in.",
"Post about: Leadership (0 data), Growth Hacking (0 data), Founder Stories (0 data)"
]
}
def get_prediction_accuracy(min_predictions=10):
"""Review prediction calibration and accuracy."""
conn = get_db_connection()
try:
cursor = conn.cursor()
# Step 1: Overall status
cursor.execute('SELECT COUNT(*) as count FROM predictions')
total_scored = cursor.fetchone()['count']
cursor.execute('SELECT COUNT(*) as count FROM predictions WHERE actual_reactions IS NOT NULL')
total_tracked = cursor.fetchone()['count']
# Calculate overall MAE/RMSE
cursor.execute('''
SELECT
AVG(ABS(predicted_reactions - actual_reactions)) as mae,
SQRT(AVG((predicted_reactions - actual_reactions) * (predicted_reactions - actual_reactions))) as rmse
FROM predictions
WHERE actual_reactions IS NOT NULL
''')
accuracy_row = cursor.fetchone()
overall_mae = accuracy_row['mae'] if accuracy_row['mae'] else 0.0
overall_rmse = accuracy_row['rmse'] if accuracy_row['rmse'] else 0.0
# Global calibration
cursor.execute('''
SELECT
AVG(calibration_slope) as avg_slope,
AVG(calibration_intercept) as avg_intercept
FROM prediction_accuracy
WHERE prediction_count >= ?
''', (min_predictions,))
calib_row = cursor.fetchone()
global_slope = calib_row['avg_slope'] if calib_row and calib_row['avg_slope'] else 1.0
global_intercept = calib_row['avg_intercept'] if calib_row and calib_row['avg_intercept'] else 0.0
# Overall confidence
avg_predictions = total_tracked / max(4, total_scored / 4) if total_scored > 0 else 0
if avg_predictions < min_predictions / 4:
confidence_level = 'low'
elif avg_predictions < min_predictions / 2:
confidence_level = 'medium'
else:
confidence_level = 'high'
overall_status = {
'total_scored_posts': total_scored,
'total_tracked_results': total_tracked,
'overall_mae': round(overall_mae, 1),
'overall_rmse': round(overall_rmse, 1),
'global_calibration': {
'slope': round(global_slope, 2),
'intercept': round(global_intercept, 2)
},
'confidence_level': confidence_level,
'next_milestone': f"Need {min_predictions - (total_tracked // 4)} more tracked predictions per segment"
}
# Step 2: Confidence by segment
cursor.execute('''
SELECT segment_name, prediction_count, mean_absolute_error, rmse,
calibration_slope, calibration_intercept
FROM prediction_accuracy
GROUP BY segment_name
ORDER BY prediction_count DESC
''')
confidence_by_segment = []
for row in cursor.fetchall():
if row['prediction_count'] == 0:
confidence = 'no_data'
elif row['prediction_count'] < min_predictions / 2:
confidence = 'low'
elif row['prediction_count'] < min_predictions:
confidence = 'medium'
else:
confidence = 'high'
interpretation = get_segment_interpretation(row['prediction_count'], min_predictions)
confidence_by_segment.append({
'segment': row['segment_name'],
'predictions_tracked': row['prediction_count'],
'confidence': confidence,
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'rmse': round(row['rmse'] or 0.0, 1),
'calibration_slope': round(row['calibration_slope'] or 1.0, 2),
'calibration_intercept': round(row['calibration_intercept'] or 0.0, 2),
'interpretation': interpretation
})
# Step 3: Topic calibration details
cursor.execute('''
SELECT topic_name, prediction_count, mean_absolute_error, rmse
FROM prediction_accuracy
WHERE topic_name IS NOT NULL
ORDER BY prediction_count DESC
LIMIT 10
''')
topic_calibration = []
for row in cursor.fetchall():
segments_with_data = get_segments_for_topic(cursor, row['topic_name'])
status = 'need_data'
if row['prediction_count'] >= min_predictions:
status = 'high_confidence'
elif row['prediction_count'] >= min_predictions / 2:
status = 'medium_confidence'
elif row['prediction_count'] > 0:
status = 'building_confidence'
topic_calibration.append({
'topic': row['topic_name'],
'predictions_tracked': row['prediction_count'],
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'rmse': round(row['rmse'] or 0.0, 1),
'segments_with_data': segments_with_data,
'status': status
})
# Step 4: Calibration matrix
cursor.execute('''
SELECT segment_name, topic_name, prediction_count, mean_absolute_error
FROM prediction_accuracy
WHERE segment_name IS NOT NULL AND topic_name IS NOT NULL
ORDER BY segment_name, prediction_count DESC
''')
calibration_matrix = {}
for row in cursor.fetchall():
seg = row['segment_name']
topic = row['topic_name']
if seg not in calibration_matrix:
calibration_matrix[seg] = {}
need_more = max(0, min_predictions - row['prediction_count'])
calibration_matrix[seg][topic] = {
'count': row['prediction_count'],
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'status': f"need_{need_more}_more" if need_more > 0 else 'confident'
}
# Step 5: Accuracy trends
cursor.execute('''
SELECT
CASE
WHEN rowid <= 5 THEN 'posts_1_to_5'
WHEN rowid <= 10 THEN 'posts_6_to_10'
WHEN rowid <= 15 THEN 'posts_11_to_15'
ELSE 'posts_16+'
END as period,
AVG(ABS(predicted_reactions - actual_reactions)) as mae
FROM predictions
WHERE actual_reactions IS NOT NULL
GROUP BY period
ORDER BY period
''')
trends = {}
baseline_mae = None
for row in cursor.fetchall():
mae = round(row['mae'] or 0.0, 1)
if baseline_mae is None:
baseline_mae = mae
improvement = 'baseline'
else:
improvement = f"+{int((baseline_mae - mae) / baseline_mae * 100)}%"
trends[row['period']] = {
'mae': mae,
'improvement': improvement
}
# Step 6: Build plan
build_plan = generate_build_plan(cursor, min_predictions)
return {
'overall_status': overall_status,
'confidence_by_segment': confidence_by_segment,
'topic_calibration_details': topic_calibration,
'calibration_matrix': calibration_matrix,
'accuracy_trends': trends,
'build_plan': build_plan,
'recommendations': generate_recommendations(confidence_by_segment, total_tracked, min_predictions)
}
except Exception as e:
return {'error': f'Error: {str(e)}'}
finally:
conn.close()
def get_segment_interpretation(count, min_needed):
"""Interpret prediction count for a segment."""
if count == 0:
return f"No data yet. Need {min_needed} predictions to build calibration."
elif count < min_needed / 2:
return f"Early stage. Have {count}, need {min_needed - count} more."
elif count < min_needed:
return f"Getting there. Have {count}, need {min_needed - count} more for high confidence."
else:
return f"High confidence! Predictions are reliable ({count} predictions tracked)."
def get_segments_for_topic(cursor, topic_name):
"""Get segments that have prediction data for this topic."""
cursor.execute('''
SELECT DISTINCT segment_name
FROM prediction_accuracy
WHERE topic_name = ? AND prediction_count > 0
''', (topic_name,))
return [row['segment_name'] for row in cursor.fetchall()]
def generate_build_plan(cursor, min_predictions):
"""Generate a prioritized list of what to post next."""
cursor.execute('''
SELECT segment_name, topic_name, prediction_count
FROM prediction_accuracy
WHERE prediction_count < ?
ORDER BY prediction_count ASC
LIMIT 5
''', (min_predictions,))
plan = []
for row in cursor.fetchall():
need_more = min_predictions - row['prediction_count']
if need_more >= 8:
priority = 'high'
rationale = f"{row['prediction_count']} tracked. High affinity topic but no calibration."
elif need_more >= 5:
priority = 'medium'
rationale = f"{row['prediction_count']} tracked. Need more data for confidence."
else:
priority = 'low'
rationale = f"{row['prediction_count']} tracked. Almost there."
plan.append({
'priority': priority,
'segment': row['segment_name'],
'topic': row['topic_name'],
'recommendation': f"Post about {row['topic_name']} for {row['segment_name']}",
'need_more': need_more,
'rationale': rationale
})
return plan
def generate_recommendations(confidence_by_segment, total_tracked, min_predictions):
"""Generate actionable recommendations."""
recs = []
if total_tracked < 5:
recs.append("You're in cold-start phase. Post 5+ more times and track results. Calibration will improve prediction accuracy.")
elif total_tracked < min_predictions:
recs.append(f"Good progress! After {min_predictions - total_tracked} more tracked results, medium confidence kicks in.")
else:
recs.append("High confidence across most segments. Predictions are reliable.")
# Identify missing segment/topic combos
zero_segments = [s['segment'] for s in confidence_by_segment if s['predictions_tracked'] == 0]
if zero_segments:
recs.append(f"Build calibration for: {', '.join(zero_segments)}. Post content targeting these segments.")
recs.append("Timeline: 2 posts/week = medium confidence in 4-5 weeks, high confidence in 8+ weeks.")
return recs
npx claudepluginhub stevegustafson32/linkedin-audience-simulator-pluginScores past predictions against actual sprint outcomes, creates calibration claims, computes accuracy scorecards by evidence tier and claim type. Useful for feedback loops after implementations.
Applies Bayesian reasoning to update probability estimates with new evidence, helping make better forecasts, avoid overconfidence, and calibrate judgments under uncertainty.
Guides post-launch AI feature calibration: document production error patterns, review eval performance, decide agency promotion. Uses CC/CD loop with /calibrate shortcuts.