From bc50e8205acef3ecbd9a34f7f4cc1558e08cd97a Mon Sep 17 00:00:00 2001
From: Jared Vititoe <jjvititoe1@gmail.com>
Date: Wed, 22 Apr 2026 21:51:55 -0400
Subject: [PATCH] trivia: strengthen prompt to prevent hallucinated/incoherent
 questions

The previous system prompt was basically empty. Now it explicitly:
- Requires the answer to be unambiguously correct
- Bans vague, ambiguous, or invented facts
- Requires plausible-but-wrong distractors
- Includes a concrete example of a good question
- Tells the model to pick a simpler topic if unsure

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 matrixbot/commands.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/matrixbot/commands.py b/matrixbot/commands.py
index dfca361..94bb74b 100644
--- a/matrixbot/commands.py
+++ b/matrixbot/commands.py
@@ -1019,13 +1019,23 @@ async def _generate_trivia_question(category: str) -> dict | None:
         + "; ".join(f'"{q}"' for q in recent[-10:])
         + "."
     ) if recent else ""
-    prompt = (
+    system_prompt = (
+        "You are a trivia question writer. Respond with ONLY a valid JSON object — no markdown, no explanation.\n"
+        'Format: {"q": "question", "options": ["A answer", "B answer", "C answer", "D answer"], "answer": 0}\n'
+        "where answer is the 0-based index of the correct option.\n\n"
+        "Rules for a good trivia question:\n"
+        "- Ask about a single, specific, verifiable fact. Do not ask vague or ambiguous questions.\n"
+        "- The correct answer must be unambiguously correct. If you are not confident, pick a different topic.\n"
+        "- Wrong options must be plausible but clearly wrong — not trick answers, not obviously absurd.\n"
+        "- The question must be grammatically correct and make sense on its own.\n"
+        "- Do NOT ask questions where the answer depends on interpretation or opinion.\n"
+        "- Do NOT invent facts. If unsure, ask about something simpler and more certain.\n\n"
+        "Example of a good question:\n"
+        '{"q": "What is the chemical symbol for gold?", "options": ["Au", "Ag", "Fe", "Cu"], "answer": 0}'
+    )
+    user_prompt = (
         f"Generate a trivia question about {topic}."
-        + avoid_clause +
-        " Respond with ONLY a JSON object, no markdown, no explanation. "
-        'Format: {"q": "question text", "options": ["A text", "B text", "C text", "D text"], "answer": 0} '
-        "where answer is the 0-based index of the correct option. "
-        "The question should be clear, factual, and have exactly one correct answer."
+        + avoid_clause
     )
     try:
         timeout = aiohttp.ClientTimeout(total=60)
@@ -1036,11 +1046,8 @@ async def _generate_trivia_question(category: str) -> dict | None:
                     "model": ASK_MODEL,
                     "stream": False,
                     "messages": [
-                        {
-                            "role": "system",
-                            "content": "You are a trivia question generator. Respond with only valid JSON, nothing else.",
-                        },
-                        {"role": "user", "content": prompt},
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt},
                     ],
                 },
             ) as response: