FEAT: Chain-of-Thought Rendering for Attack Results

doc/code/output/0_output.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.19.1
+#       jupytext_version: 1.19.5
 # ---
 
 # %% [markdown]
@@ -35,7 +35,6 @@
 # To demonstrate the printers, we'll run a simple attack and use the result.
 
 # %%
-
 from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack
 from pyrit.prompt_target import OpenAIChatTarget
 from pyrit.score import (
@@ -143,6 +142,56 @@
 # print the conversation using the print conversation helper
 await output_conversation_async(messages=conversation)  # type: ignore
 
+# %% [markdown]
+# ## Including Reasoning Summaries
+#
+# Reasoning-summary output is opt in: the conversation and attack-result helpers hide
+# reasoning by default. For OpenAI Responses targets, PyRIT renders
+# provider-generated reasoning summaries exposed by OpenAI, not raw hidden chain-of-thought.
+#
+# - Pretty output labels the summary as **💭 Reasoning** in subdued gray.
+# - Markdown output uses a blockquoted **💭 Reasoning** section.
+# - When a response follows reasoning in the same message, both formats add a
+#   **💬 Response** heading to make the boundary explicit.
+#
+# ```python
+# from pyrit.output import output_attack_async, output_conversation_async
+#
+# # Direct conversation
+# await output_conversation_async(messages=conversation, include_reasoning_trace=True)
+#
+# # Attack result (Pretty or Markdown)
+# await output_attack_async(attack_result, include_reasoning_trace=True)
+# await output_attack_async(attack_result, format="markdown", include_reasoning_trace=True)
+# ```
+
+# %%
+from pyrit.executor.attack import PromptSendingAttack
+from pyrit.output import output_attack_async
+from pyrit.prompt_target import OpenAIResponseTarget
+
+objective_target = OpenAIResponseTarget(reasoning_effort="high", reasoning_summary="detailed")
+
+attack = PromptSendingAttack(objective_target=objective_target)
+prompt = """
+Solve this scheduling problem and return the earliest valid schedule.
+
+Five jobs, A through E, must each occupy one consecutive time slot from 1 to 5.
+
+Constraints:
+- A must occur before D.
+- C must occur immediately after A.
+- E cannot be in slot 1 or slot 5.
+- B must occur after E.
+- D cannot be adjacent to B.
+
+Determine the complete schedule. Verify every constraint in the final answer.
+"""
+
+result = await attack.execute_async(objective=prompt)  # type: ignore
+await output_attack_async(result, include_reasoning_trace=True)
+
+
 # %% [markdown]
 # ## Printing Scores
 #

pyrit/output/attack_result/base.py

@@ -3,7 +3,7 @@
 
 from abc import abstractmethod
 
-from pyrit.models import AttackOutcome, Message, Score
+from pyrit.models import AttackOutcome, AttackResult, Message, Score
 from pyrit.output.base import PrinterBase
 
 
@@ -18,6 +18,31 @@ class AttackResultPrinterBase(PrinterBase):
     Thin-client implementations can fetch data via REST endpoints.
     """
 
+    @abstractmethod
+    async def render_async(
+        self,
+        result: AttackResult,
+        *,
+        include_auxiliary_scores: bool = False,
+        include_pruned_conversations: bool = False,
+        include_adversarial_conversation: bool = False,
+        include_reasoning_trace: bool = False,
+    ) -> str:
+        """
+        Render an attack result.
+
+        Args:
+            result (AttackResult): The attack result to render.
+            include_auxiliary_scores (bool): Whether to include auxiliary scores. Defaults to False.
+            include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False.
+            include_adversarial_conversation (bool): Whether to include the adversarial conversation.
+                Defaults to False.
+            include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False.
+
+        Returns:
+            str: The rendered attack result.
+        """
+
     @abstractmethod
     async def _get_conversation_async(self, conversation_id: str) -> list[Message]:
         """

pyrit/output/attack_result/markdown.py

@@ -67,6 +67,7 @@ async def render_async(
         include_auxiliary_scores: bool = False,
         include_pruned_conversations: bool = False,
         include_adversarial_conversation: bool = False,
+        include_reasoning_trace: bool = False,
     ) -> str:
         """
         Render the complete attack result as markdown and return it as a string.
@@ -77,6 +78,7 @@ async def render_async(
             include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False.
             include_adversarial_conversation (bool): Whether to include the adversarial conversation.
                 Defaults to False.
+            include_reasoning_trace (bool): Whether to include the reasoning trace. Defaults to False.
 
         Returns:
             str: The rendered markdown text.
@@ -93,17 +95,25 @@ async def render_async(
 
         markdown_lines.append("\n## Conversation History\n")
         conversation_lines = await self._get_conversation_markdown_async(
-            result=result, include_scores=include_auxiliary_scores
+            result=result,
+            include_scores=include_auxiliary_scores,
+            include_reasoning_trace=include_reasoning_trace,
         )
         markdown_lines.extend(conversation_lines)
 
         if include_pruned_conversations:
-            pruned_lines = await self._get_pruned_conversations_markdown_async(result)
+            pruned_lines = await self._get_pruned_conversations_markdown_async(
+                result,
+                include_reasoning_trace=include_reasoning_trace,
+            )
             if pruned_lines:
                 markdown_lines.extend(pruned_lines)
 
         if include_adversarial_conversation:
-            adversarial_lines = await self._get_adversarial_conversation_markdown_async(result)
+            adversarial_lines = await self._get_adversarial_conversation_markdown_async(
+                result,
+                include_reasoning_trace=include_reasoning_trace,
+            )
             if adversarial_lines:
                 markdown_lines.extend(adversarial_lines)
 
@@ -123,14 +133,19 @@ async def render_async(
         return "\n".join(markdown_lines)
 
     async def _get_conversation_markdown_async(
-        self, *, result: AttackResult, include_scores: bool = False
+        self,
+        *,
+        result: AttackResult,
+        include_scores: bool = False,
+        include_reasoning_trace: bool = False,
     ) -> list[str]:
         """
         Generate markdown lines for the conversation history.
 
         Args:
             result (AttackResult): The attack result containing the conversation ID.
             include_scores (bool): Whether to include scores. Defaults to False.
+            include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False.
 
         Returns:
             list[str]: Markdown strings for the conversation.
@@ -143,7 +158,11 @@ async def _get_conversation_markdown_async(
         if not messages:
             return [f"*No conversation found for ID: {result.conversation_id}*\n"]
 
-        rendered = await self._conversation_printer.render_async(messages, include_scores=include_scores)
+        rendered = await self._conversation_printer.render_async(
+            messages,
+            include_scores=include_scores,
+            include_reasoning_trace=include_reasoning_trace,
+        )
         return [rendered]
 
     async def _get_summary_markdown_async(self, result: AttackResult) -> list[str]:
@@ -189,12 +208,18 @@ async def _get_summary_markdown_async(self, result: AttackResult) -> list[str]:
 
         return markdown_lines
 
-    async def _get_pruned_conversations_markdown_async(self, result: AttackResult) -> list[str]:
+    async def _get_pruned_conversations_markdown_async(
+        self,
+        result: AttackResult,
+        *,
+        include_reasoning_trace: bool = False,
+    ) -> list[str]:
         """
         Generate markdown lines for pruned conversations.
 
         Args:
             result (AttackResult): The attack result containing related conversations.
+            include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False.
 
         Returns:
             list[str]: Markdown strings for pruned conversations.
@@ -221,11 +246,32 @@ async def _get_pruned_conversations_markdown_async(self, result: AttackResult) -
                 continue
 
             last_message = messages[-1]
+            pieces = self._conversation_printer._get_renderable_pieces(
+                message=last_message,
+                include_reasoning_trace=include_reasoning_trace,
+            )
+            if not pieces:
+                continue
+
             role_label = last_message.api_role.upper()
 
             markdown_lines.append(f"**Last Message ({role_label}):**\n")
 
-            for piece in last_message.message_pieces:
+            reasoning_rendered = False
+            response_heading_rendered = False
+            for piece in pieces:
+                if self._conversation_printer._is_reasoning_piece(piece=piece):
+                    formatted = self._conversation_printer._format_reasoning_summary(
+                        self._conversation_printer._get_reasoning_value(piece=piece)
+                    )
+                    markdown_lines.extend(formatted)
+                    reasoning_rendered = bool(formatted) or reasoning_rendered
+                    continue
+
+                if reasoning_rendered and not response_heading_rendered and last_message.api_role == "assistant":
+                    markdown_lines.extend(self._conversation_printer._format_response_heading())
+                    response_heading_rendered = True
+
                 content = piece.converted_value or ""
                 if "\n" in content:
                     markdown_lines.append("```")
@@ -241,12 +287,18 @@ async def _get_pruned_conversations_markdown_async(self, result: AttackResult) -
 
         return markdown_lines
 
-    async def _get_adversarial_conversation_markdown_async(self, result: AttackResult) -> list[str]:
+    async def _get_adversarial_conversation_markdown_async(
+        self,
+        result: AttackResult,
+        *,
+        include_reasoning_trace: bool = False,
+    ) -> list[str]:
         """
         Generate markdown lines for the adversarial conversation.
 
         Args:
             result (AttackResult): The attack result containing related conversations.
+            include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False.
 
         Returns:
             list[str]: Markdown strings for the adversarial conversation.
@@ -278,6 +330,13 @@ async def _get_adversarial_conversation_markdown_async(self, result: AttackResul
 
             turn_number = 0
             for message in messages:
+                pieces = self._conversation_printer._get_renderable_pieces(
+                    message=message,
+                    include_reasoning_trace=include_reasoning_trace,
+                )
+                if not pieces:
+                    continue
+
                 if message.api_role == "user":
                     turn_number += 1
                     markdown_lines.append(f"\n#### Turn {turn_number} - USER\n")
@@ -286,7 +345,21 @@ async def _get_adversarial_conversation_markdown_async(self, result: AttackResul
                 else:
                     markdown_lines.append(f"\n#### {message.api_role.upper()}\n")
 
-                for piece in message.message_pieces:
+                reasoning_rendered = False
+                response_heading_rendered = False
+                for piece in pieces:
+                    if self._conversation_printer._is_reasoning_piece(piece=piece):
+                        formatted = self._conversation_printer._format_reasoning_summary(
+                            self._conversation_printer._get_reasoning_value(piece=piece)
+                        )
+                        markdown_lines.extend(formatted)
+                        reasoning_rendered = bool(formatted) or reasoning_rendered
+                        continue
+
+                    if reasoning_rendered and not response_heading_rendered and message.api_role == "assistant":
+                        markdown_lines.extend(self._conversation_printer._format_response_heading())
+                        response_heading_rendered = True
+
                     content = piece.converted_value or ""
                     if len(content) > 200 or "\n" in content:
                         markdown_lines.append("```")
@@ -355,6 +428,7 @@ async def render_async(
         include_auxiliary_scores: bool = False,
         include_pruned_conversations: bool = False,
         include_adversarial_conversation: bool = False,
+        include_reasoning_trace: bool = False,
     ) -> str:
         """
         Render the complete attack result as markdown and return it as a string.
@@ -365,6 +439,7 @@ async def render_async(
             include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False.
             include_adversarial_conversation (bool): Whether to include the adversarial conversation.
                 Defaults to False.
+            include_reasoning_trace (bool): Whether to include the reasoning trace. Defaults to False.
 
         Returns:
             str: The rendered markdown text.
@@ -374,6 +449,7 @@ async def render_async(
             include_auxiliary_scores=include_auxiliary_scores,
             include_pruned_conversations=include_pruned_conversations,
             include_adversarial_conversation=include_adversarial_conversation,
+            include_reasoning_trace=include_reasoning_trace,
         )
 
     async def _get_conversation_async(self, conversation_id: str) -> list[Message]: