Skip to content

HFMultipleChoiceDataset

MultipleChoiceDataset

Source code in flexeval/core/multiple_choice_dataset/base.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
class MultipleChoiceDataset(Sequence[MultipleChoiceInstance], ABC):
    @abstractmethod
    def __len__(self) -> int:
        """
        Returns the number of instances in the dataset.
        """
        raise NotImplementedError

    @abstractmethod
    def __getitem__(self, i: int) -> MultipleChoiceInstance:
        """
        Returns the i-th instance.
        """
        raise NotImplementedError

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}(num_instances={len(self)})"

__len__ abstractmethod 

__len__() -> int

Returns the number of instances in the dataset.

Source code in flexeval/core/multiple_choice_dataset/base.py 
31
32
33
34
35
36
@abstractmethod
def __len__(self) -> int:
    """
    Returns the number of instances in the dataset.
    """
    raise NotImplementedError

__getitem__ abstractmethod 

__getitem__(i: int) -> MultipleChoiceInstance

Returns the i-th instance.

Source code in flexeval/core/multiple_choice_dataset/base.py 
38
39
40
41
42
43
@abstractmethod
def __getitem__(self, i: int) -> MultipleChoiceInstance:
    """
    Returns the i-th instance.
    """
    raise NotImplementedError

__repr__ 

__repr__() -> str
Source code in flexeval/core/multiple_choice_dataset/base.py 
45
46
def __repr__(self) -> str:
    return f"{self.__class__.__name__}(num_instances={len(self)})"

MultipleChoiceInstance dataclass

A dataclass representing a single input-output pair of a multiple-choice task.

Source code in flexeval/core/multiple_choice_dataset/base.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
@dataclass
class MultipleChoiceInstance:
    """
    A dataclass representing a single input-output pair of a multiple-choice task.
    """

    inputs: dict[str, str]
    """
    Inputs of the multiple-choice task.
    This will be embedded into the prompt for the language model in `PromptTemplate`.
    """
    choices: list[str]
    """
    Choices for the multiple-choice task.
    `LanguageModel` will choose the answer based on the log-probabilities of these choices.
    """
    answer_index: int
    """
    Index of the correct answer in `choices`.
    """

inputs instance-attribute 

inputs: dict[str, str]

Inputs of the multiple-choice task. This will be embedded into the prompt for the language model in PromptTemplate.

choices instance-attribute 

choices: list[str]

Choices for the multiple-choice task. LanguageModel will choose the answer based on the log-probabilities of these choices.

answer_index instance-attribute 

answer_index: int

Index of the correct answer in choices.

__init__ 

__init__(
    inputs: dict[str, str],
    choices: list[str],
    answer_index: int,
) -> None

HFMultipleChoiceDataset

Load MultipleChoiceInstance from a huggingface dataset.

Parameters:

  • path (str) –

    The name or path of the huggingface dataset.

  • split (str) –

    The split of the dataset to use.

  • subset (str | None, default: None ) –

    The subset of the dataset to use.

  • dataset_kwargs (dict[str, Any] | None, default: None ) –

    The keyword arguments for loading the dataset.

Source code in flexeval/core/multiple_choice_dataset/template_based.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
class HFMultipleChoiceDataset(TemplateMultipleChoiceDataset):
    """
    Load MultipleChoiceInstance from a huggingface dataset.

    Args:
        path: The name or path of the huggingface dataset.
        split: The split of the dataset to use.
        subset: The subset of the dataset to use.
        dataset_kwargs: The keyword arguments for loading the dataset.
    """

    def __init__(
        self,
        path: str,
        split: str,
        choices_templates: list[str],
        answer_index_template: str,
        input_templates: dict[str, str] | None = None,
        subset: str | None = None,
        dataset_kwargs: dict[str, Any] | None = None,
        whitespace_before_choices: bool = False,
        data_range: tuple[int, int] | None = None,
        keep_conditions: dict[str, str] | None = None,
        remove_conditions: dict[str, str] | None = None,
    ) -> None:
        dataset_kwargs = dataset_kwargs or {}
        items = datasets.load_dataset(path, split=split, name=subset, **dataset_kwargs)
        items = [dict(item) for item in items]

        super().__init__(
            items=items,
            choices_templates=choices_templates,
            answer_index_template=answer_index_template,
            input_templates=input_templates,
            whitespace_before_choices=whitespace_before_choices,
            data_range=data_range,
            keep_conditions=keep_conditions,
            remove_conditions=remove_conditions,
        )

__init__ 

__init__(
    path: str,
    split: str,
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    subset: str | None = None,
    dataset_kwargs: dict[str, Any] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None
Source code in flexeval/core/multiple_choice_dataset/template_based.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def __init__(
    self,
    path: str,
    split: str,
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    subset: str | None = None,
    dataset_kwargs: dict[str, Any] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None:
    dataset_kwargs = dataset_kwargs or {}
    items = datasets.load_dataset(path, split=split, name=subset, **dataset_kwargs)
    items = [dict(item) for item in items]

    super().__init__(
        items=items,
        choices_templates=choices_templates,
        answer_index_template=answer_index_template,
        input_templates=input_templates,
        whitespace_before_choices=whitespace_before_choices,
        data_range=data_range,
        keep_conditions=keep_conditions,
        remove_conditions=remove_conditions,
    )

JsonlMultipleChoiceDataset

Load MultipleChoiceInstance from a JSONL file.

Source code in flexeval/core/multiple_choice_dataset/template_based.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
class JsonlMultipleChoiceDataset(TemplateMultipleChoiceDataset):
    """
    Load MultipleChoiceInstance from a JSONL file.
    """

    def __init__(
        self,
        path: str,
        choices_templates: list[str],
        answer_index_template: str,
        input_templates: dict[str, str] | None = None,
        whitespace_before_choices: bool = False,
        data_range: tuple[int, int] | None = None,
        keep_conditions: dict[str, str] | None = None,
        remove_conditions: dict[str, str] | None = None,
    ) -> None:
        with open(path) as f:
            items = [json.loads(line) for line in f]

        super().__init__(
            items=items,
            choices_templates=choices_templates,
            answer_index_template=answer_index_template,
            input_templates=input_templates,
            whitespace_before_choices=whitespace_before_choices,
            data_range=data_range,
            keep_conditions=keep_conditions,
            remove_conditions=remove_conditions,
        )

__init__ 

__init__(
    path: str,
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None
Source code in flexeval/core/multiple_choice_dataset/template_based.py 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def __init__(
    self,
    path: str,
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None:
    with open(path) as f:
        items = [json.loads(line) for line in f]

    super().__init__(
        items=items,
        choices_templates=choices_templates,
        answer_index_template=answer_index_template,
        input_templates=input_templates,
        whitespace_before_choices=whitespace_before_choices,
        data_range=data_range,
        keep_conditions=keep_conditions,
        remove_conditions=remove_conditions,
    )

TemplateMultipleChoiceDataset

An abstract dataset class for multiple-choice tasks. This class generates multiple-choice instances from a dict item and Jinja2 templates.

Parameters:

  • items (list[dict[str, Any]]) –

    A list of dict items.

  • choices_templates (list[str]) –

    A list of Jinja2 templates for the choices.

  • answer_index_template (str) –

    A Jinja2 template for the index of the correct answer.

  • input_templates (dict[str, str] | None, default: None ) –

    A dictionary of Jinja2 templates for the inputs.

  • whitespace_before_choices (bool, default: False ) –

    Whether to add a whitespace before each choice. Maybe necessary for language with whitespaces.

  • data_range (tuple[int, int] | None, default: None ) –

    The range of data to use.

  • keep_conditions (dict[str, str] | None, default: None ) –

    A dictionary to indicate the condition to filter certain items. The key is a Jinja2 template string to embed the item into a string, and the value is the value to keep.

  • remove_conditions (dict[str, str] | None, default: None ) –

    A dictionary to indicate the condition to remove certain items. The key is a Jinja2 template string to embed the item into a string, and the value is the value to remove.

Source code in flexeval/core/multiple_choice_dataset/template_based.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
class TemplateMultipleChoiceDataset(MultipleChoiceDataset):
    """
    An abstract dataset class for multiple-choice tasks.
    This class generates multiple-choice instances from a dict item and Jinja2 templates.

    Args:
        items: A list of dict items.
        choices_templates: A list of Jinja2 templates for the choices.
        answer_index_template: A Jinja2 template for the index of the correct answer.
        input_templates: A dictionary of Jinja2 templates for the inputs.
        whitespace_before_choices: Whether to add a whitespace before each choice.
            Maybe necessary for language with whitespaces.
        data_range: The range of data to use.
        keep_conditions: A dictionary to indicate the condition to filter certain items.
            The key is a Jinja2 template string to embed the item into a string, and the value is the value to keep.
        remove_conditions: A dictionary to indicate the condition to remove certain items.
            The key is a Jinja2 template string to embed the item into a string, and the value is the value to remove.
    """

    def __init__(
        self,
        items: list[dict[str, Any]],
        choices_templates: list[str],
        answer_index_template: str,
        input_templates: dict[str, str] | None = None,
        whitespace_before_choices: bool = False,
        data_range: tuple[int, int] | None = None,
        keep_conditions: dict[str, str] | None = None,
        remove_conditions: dict[str, str] | None = None,
    ) -> None:
        if data_range:
            start, end = data_range
            items = items[start:end]

        keep_conditions = keep_conditions or {}
        for template_str, value_to_keep in keep_conditions.items():
            key_template = JINJA2_ENV.from_string(template_str)
            items = [item for item in items if key_template.render(**item) == value_to_keep]
        remove_conditions = remove_conditions or {}
        for template_str, value_to_remove in remove_conditions.items():
            key_template = JINJA2_ENV.from_string(template_str)
            items = [item for item in items if key_template.render(**item) != value_to_remove]
        self.items = items

        input_templates = input_templates or {}
        self.input_templates: dict[str, Template] = {k: JINJA2_ENV.from_string(v) for k, v in input_templates.items()}
        self.choices_templates = [JINJA2_ENV.from_string(t) for t in choices_templates]
        self.answer_index_template = JINJA2_ENV.from_string(
            answer_index_template,
        )
        self.whitespace_before_choices = whitespace_before_choices

    def __len__(self) -> int:
        return len(self.items)

    def __getitem__(self, i: int) -> MultipleChoiceInstance:
        item = self.items[i]
        inputs = dict(item.items())
        inputs.update({k: v.render(**item) for k, v in self.input_templates.items()})

        choices = [t.render(**item) for t in self.choices_templates]
        choices = list(filter(lambda x: len(x) > 0, choices))
        if self.whitespace_before_choices:
            choices = [" " + c for c in choices]

        answer_index = int(self.answer_index_template.render(**item))
        if not (answer_index >= 0 and answer_index < len(choices)):
            msg = f"at least {answer_index+1} choices required, but got {choices}"
            raise ValueError(msg)

        return MultipleChoiceInstance(
            inputs=inputs,
            choices=choices,
            answer_index=answer_index,
        )

items instance-attribute 

items = items

input_templates instance-attribute 

input_templates: dict[str, Template] = {
    k: from_string(v) for (k, v) in items()
}

choices_templates instance-attribute 

choices_templates = [
    from_string(t) for t in choices_templates
]

answer_index_template instance-attribute 

answer_index_template = from_string(answer_index_template)

whitespace_before_choices instance-attribute 

whitespace_before_choices = whitespace_before_choices

__init__ 

__init__(
    items: list[dict[str, Any]],
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None
Source code in flexeval/core/multiple_choice_dataset/template_based.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def __init__(
    self,
    items: list[dict[str, Any]],
    choices_templates: list[str],
    answer_index_template: str,
    input_templates: dict[str, str] | None = None,
    whitespace_before_choices: bool = False,
    data_range: tuple[int, int] | None = None,
    keep_conditions: dict[str, str] | None = None,
    remove_conditions: dict[str, str] | None = None,
) -> None:
    if data_range:
        start, end = data_range
        items = items[start:end]

    keep_conditions = keep_conditions or {}
    for template_str, value_to_keep in keep_conditions.items():
        key_template = JINJA2_ENV.from_string(template_str)
        items = [item for item in items if key_template.render(**item) == value_to_keep]
    remove_conditions = remove_conditions or {}
    for template_str, value_to_remove in remove_conditions.items():
        key_template = JINJA2_ENV.from_string(template_str)
        items = [item for item in items if key_template.render(**item) != value_to_remove]
    self.items = items

    input_templates = input_templates or {}
    self.input_templates: dict[str, Template] = {k: JINJA2_ENV.from_string(v) for k, v in input_templates.items()}
    self.choices_templates = [JINJA2_ENV.from_string(t) for t in choices_templates]
    self.answer_index_template = JINJA2_ENV.from_string(
        answer_index_template,
    )
    self.whitespace_before_choices = whitespace_before_choices

__len__ 

__len__() -> int
Source code in flexeval/core/multiple_choice_dataset/template_based.py 
66
67
def __len__(self) -> int:
    return len(self.items)

__getitem__ 

__getitem__(i: int) -> MultipleChoiceInstance
Source code in flexeval/core/multiple_choice_dataset/template_based.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def __getitem__(self, i: int) -> MultipleChoiceInstance:
    item = self.items[i]
    inputs = dict(item.items())
    inputs.update({k: v.render(**item) for k, v in self.input_templates.items()})

    choices = [t.render(**item) for t in self.choices_templates]
    choices = list(filter(lambda x: len(x) > 0, choices))
    if self.whitespace_before_choices:
        choices = [" " + c for c in choices]

    answer_index = int(self.answer_index_template.render(**item))
    if not (answer_index >= 0 and answer_index < len(choices)):
        msg = f"at least {answer_index+1} choices required, but got {choices}"
        raise ValueError(msg)

    return MultipleChoiceInstance(
        inputs=inputs,
        choices=choices,
        answer_index=answer_index,
    )