Reasoning

Reasoning dataclass

Source code in flexeval/core/reasoning_parser/base.py

@dataclass
class Reasoning:
    text: str | None
    reasoning_text: str | None

text instance-attribute

text: str | None

reasoning_text instance-attribute

reasoning_text: str | None

__init__

__init__(
    text: str | None, reasoning_text: str | None
) -> None

ReasoningParser

Base class for parsing raw LLM output text into reasoning and content parts.

Subclasses must implement __call__, which receives the raw text and returns an instance of Reasoning class.

Source code in flexeval/core/reasoning_parser/base.py

class ReasoningParser(ABC):
    """Base class for parsing raw LLM output text into reasoning and content parts.

    Subclasses must implement `__call__`, which receives the raw text and returns
    an instance of `Reasoning` class.
    """

    @abstractmethod
    def __call__(self, raw_text: str) -> Reasoning:
        """Parse raw LLM output and return the reasoning and content parts.

        Args:
            raw_text: Raw output string produced by the language model.

        Returns:
            An instance of `Reasoning` class.
        """
        raise NotImplementedError

__call__ abstractmethod

__call__(raw_text: str) -> Reasoning

Parse raw LLM output and return the reasoning and content parts.

Parameters:

• raw_text (str) –

  Raw output string produced by the language model.

Returns:

• Reasoning –

  An instance of Reasoning class.

Source code in flexeval/core/reasoning_parser/base.py

@abstractmethod
def __call__(self, raw_text: str) -> Reasoning:
    """Parse raw LLM output and return the reasoning and content parts.

    Args:
        raw_text: Raw output string produced by the language model.

    Returns:
        An instance of `Reasoning` class.
    """
    raise NotImplementedError

SeparatedRegexReasoningParser

Extracts reasoning and content using two independent regex patterns.

Each pattern is applied separately. If a pattern has a capture group, the last group is used; otherwise the full match is used.

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

class SeparatedRegexReasoningParser(ReasoningParser):
    """Extracts reasoning and content using two independent regex patterns.

    Each pattern is applied separately.  If a pattern has a capture group,
    the last group is used; otherwise the full match is used.
    """

    def __init__(
        self,
        pattern_for_content: str,
        pattern_for_reasoning_content: str,
    ) -> None:
        super().__init__()
        self.pattern_for_content = re.compile(pattern_for_content, re.DOTALL)
        self.pattern_for_reasoning_content = re.compile(pattern_for_reasoning_content, re.DOTALL)

    def __call__(self, raw_text: str) -> Reasoning:
        content = None
        reasoning_text = None

        found_content = self.pattern_for_content.findall(raw_text)
        if found_content:
            content = found_content[-1]

        found_reasoning = self.pattern_for_reasoning_content.findall(raw_text)
        if found_reasoning:
            reasoning_text = found_reasoning[-1]

        return Reasoning(text=content, reasoning_text=reasoning_text)

pattern_for_content instance-attribute

pattern_for_content = compile(pattern_for_content, DOTALL)

pattern_for_reasoning_content instance-attribute

pattern_for_reasoning_content = compile(
    pattern_for_reasoning_content, DOTALL
)

__init__

__init__(
    pattern_for_content: str,
    pattern_for_reasoning_content: str,
) -> None

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

def __init__(
    self,
    pattern_for_content: str,
    pattern_for_reasoning_content: str,
) -> None:
    super().__init__()
    self.pattern_for_content = re.compile(pattern_for_content, re.DOTALL)
    self.pattern_for_reasoning_content = re.compile(pattern_for_reasoning_content, re.DOTALL)

__call__

__call__(raw_text: str) -> Reasoning

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

def __call__(self, raw_text: str) -> Reasoning:
    content = None
    reasoning_text = None

    found_content = self.pattern_for_content.findall(raw_text)
    if found_content:
        content = found_content[-1]

    found_reasoning = self.pattern_for_reasoning_content.findall(raw_text)
    if found_reasoning:
        reasoning_text = found_reasoning[-1]

    return Reasoning(text=content, reasoning_text=reasoning_text)

UnifiedRegexReasoningParser

Extracts reasoning and content using a single regex with named groups.

The pattern must contain named groups (?P<reasoning_content>...) and/or (?P<content>...) to populate the corresponding fields of :class:Reasoning.

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

class UnifiedRegexReasoningParser(ReasoningParser):
    """Extracts reasoning and content using a single regex with named groups.

    The pattern must contain named groups ``(?P<reasoning_content>...)`` and/or
    ``(?P<content>...)`` to populate the corresponding fields of :class:`Reasoning`.
    """

    def __init__(self, pattern: str) -> None:
        super().__init__()
        self.pattern = pattern

    def __call__(self, raw_text: str) -> Reasoning:
        m = re.search(self.pattern, raw_text, re.DOTALL)
        if m is None:
            return Reasoning(text=None, reasoning_text=None)
        gd = m.groupdict()
        return Reasoning(
            text=gd.get("content"),
            reasoning_text=gd.get("reasoning_content"),
        )

pattern instance-attribute

pattern = pattern

__init__

__init__(pattern: str) -> None

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

def __init__(self, pattern: str) -> None:
    super().__init__()
    self.pattern = pattern

__call__

__call__(raw_text: str) -> Reasoning

Source code in flexeval/core/reasoning_parser/regex_reasoning_parser.py

def __call__(self, raw_text: str) -> Reasoning:
    m = re.search(self.pattern, raw_text, re.DOTALL)
    if m is None:
        return Reasoning(text=None, reasoning_text=None)
    gd = m.groupdict()
    return Reasoning(
        text=gd.get("content"),
        reasoning_text=gd.get("reasoning_content"),
    )