Skip to content

MatchMaker

MatchMaker

Generate matches between items from different models.

The output is instances of the Match class.

Source code in flexeval/core/pairwise_comparison/match_maker/base.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
class MatchMaker(ABC):
    """Generate matches between items from different models.

    The output is instances of the `Match` class.
    """

    @abstractmethod
    def generate_matches(
        self,
        model_items: dict[str, list[T]],
        cached_matches: list[Match] | None = None,
    ) -> Iterable[Match]:
        pass

generate_matches abstractmethod 

generate_matches(
    model_items: dict[str, list[T]],
    cached_matches: list[Match] | None = None,
) -> Iterable[Match]
Source code in flexeval/core/pairwise_comparison/match_maker/base.py 
17
18
19
20
21
22
23
@abstractmethod
def generate_matches(
    self,
    model_items: dict[str, list[T]],
    cached_matches: list[Match] | None = None,
) -> Iterable[Match]:
    pass

AllCombinations

Source code in flexeval/core/pairwise_comparison/match_maker/all_combinations.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
class AllCombinations(MatchMaker):
    def __init__(self, include_reversed: bool = True) -> None:
        self.include_reversed = include_reversed

    def generate_matches(self, model_items: dict[str, list[T]], cached_matches: list[Match] | None) -> Iterable[Match]:
        model_names = sorted(model_items.keys())
        all_combinations = list(itertools.combinations(model_names, 2))

        cached_matches = cached_matches or []
        cache_dict = {match.get_key_for_cache(): match for match in cached_matches}

        if self.include_reversed:
            all_combinations += [(m2, m1) for m1, m2 in all_combinations]

        for m1, m2 in all_combinations:
            for item1, item2 in zip(model_items[m1], model_items[m2]):
                match = Match(m1, item1, m2, item2)
                if cached_match := cache_dict.get(match.get_key_for_cache()):
                    yield cached_match
                else:
                    yield match

include_reversed instance-attribute 

include_reversed = include_reversed

__init__ 

__init__(include_reversed: bool = True) -> None
Source code in flexeval/core/pairwise_comparison/match_maker/all_combinations.py 
12
13
def __init__(self, include_reversed: bool = True) -> None:
    self.include_reversed = include_reversed

generate_matches 

generate_matches(
    model_items: dict[str, list[T]],
    cached_matches: list[Match] | None,
) -> Iterable[Match]
Source code in flexeval/core/pairwise_comparison/match_maker/all_combinations.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
def generate_matches(self, model_items: dict[str, list[T]], cached_matches: list[Match] | None) -> Iterable[Match]:
    model_names = sorted(model_items.keys())
    all_combinations = list(itertools.combinations(model_names, 2))

    cached_matches = cached_matches or []
    cache_dict = {match.get_key_for_cache(): match for match in cached_matches}

    if self.include_reversed:
        all_combinations += [(m2, m1) for m1, m2 in all_combinations]

    for m1, m2 in all_combinations:
        for item1, item2 in zip(model_items[m1], model_items[m2]):
            match = Match(m1, item1, m2, item2)
            if cached_match := cache_dict.get(match.get_key_for_cache()):
                yield cached_match
            else:
                yield match

RandomCombinations

Source code in flexeval/core/pairwise_comparison/match_maker/random_combinations.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
class RandomCombinations(MatchMaker):
    def __init__(self, n: int = 100, incremental: bool = False, seed: int = 42) -> None:
        self.n = n
        self.incremental = incremental
        self.seed = seed

    def generate_matches(
        self,
        model_items: dict[str, list[T]],
        cached_matches: list[Match] | None = None,
    ) -> Iterable[Match]:
        model_names = sorted(model_items.keys())
        all_permutations = list(itertools.permutations(model_names, 2))

        cached_matches = cached_matches or []
        cache_dict = {match.get_key_for_cache(): match for match in cached_matches}
        model_match_counter: dict[str, int] = {name: 0 for name in model_names}
        possible_new_matches: list[Match] = []
        matches: list[Match] = []
        for m1, m2 in all_permutations:
            for item1, item2 in zip(model_items[m1], model_items[m2]):
                match = Match(m1, item1, m2, item2)
                if cached_match := cache_dict.get(match.get_key_for_cache()):
                    matches.append(cached_match)
                    model_match_counter[m1] += 1
                    model_match_counter[m2] += 1
                else:
                    possible_new_matches.append(Match(m1, item1, m2, item2))

        # If `self.incremental` is `True`, add n more matches in addition to the cached data.
        max_matches = self.n + len(matches) if self.incremental else self.n

        random.seed(self.seed)

        # For each iteration, assign the model with the fewest matches to a new match.
        while (len(matches) < max_matches) and (len(possible_new_matches) > 0):
            target_model = min(model_match_counter, key=model_match_counter.get)
            candidate_matches = [
                (i, match)
                for i, match in enumerate(possible_new_matches)
                if target_model in (match.model1, match.model2)
            ]
            index, selected_match = random.choice(candidate_matches)
            matches.append(selected_match)
            del possible_new_matches[index]
            model_match_counter[selected_match.model1] += 1
            model_match_counter[selected_match.model2] += 1

        for match in matches:
            yield match

n instance-attribute 

n = n

incremental instance-attribute 

incremental = incremental

seed instance-attribute 

seed = seed

__init__ 

__init__(
    n: int = 100, incremental: bool = False, seed: int = 42
) -> None
Source code in flexeval/core/pairwise_comparison/match_maker/random_combinations.py 
13
14
15
16
def __init__(self, n: int = 100, incremental: bool = False, seed: int = 42) -> None:
    self.n = n
    self.incremental = incremental
    self.seed = seed

generate_matches 

generate_matches(
    model_items: dict[str, list[T]],
    cached_matches: list[Match] | None = None,
) -> Iterable[Match]
Source code in flexeval/core/pairwise_comparison/match_maker/random_combinations.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def generate_matches(
    self,
    model_items: dict[str, list[T]],
    cached_matches: list[Match] | None = None,
) -> Iterable[Match]:
    model_names = sorted(model_items.keys())
    all_permutations = list(itertools.permutations(model_names, 2))

    cached_matches = cached_matches or []
    cache_dict = {match.get_key_for_cache(): match for match in cached_matches}
    model_match_counter: dict[str, int] = {name: 0 for name in model_names}
    possible_new_matches: list[Match] = []
    matches: list[Match] = []
    for m1, m2 in all_permutations:
        for item1, item2 in zip(model_items[m1], model_items[m2]):
            match = Match(m1, item1, m2, item2)
            if cached_match := cache_dict.get(match.get_key_for_cache()):
                matches.append(cached_match)
                model_match_counter[m1] += 1
                model_match_counter[m2] += 1
            else:
                possible_new_matches.append(Match(m1, item1, m2, item2))

    # If `self.incremental` is `True`, add n more matches in addition to the cached data.
    max_matches = self.n + len(matches) if self.incremental else self.n

    random.seed(self.seed)

    # For each iteration, assign the model with the fewest matches to a new match.
    while (len(matches) < max_matches) and (len(possible_new_matches) > 0):
        target_model = min(model_match_counter, key=model_match_counter.get)
        candidate_matches = [
            (i, match)
            for i, match in enumerate(possible_new_matches)
            if target_model in (match.model1, match.model2)
        ]
        index, selected_match = random.choice(candidate_matches)
        matches.append(selected_match)
        del possible_new_matches[index]
        model_match_counter[selected_match.model1] += 1
        model_match_counter[selected_match.model2] += 1

    for match in matches:
        yield match