debeir.datasets.types

 1import string
 2from collections import defaultdict
 3from enum import Enum
 4from typing import List, Union
 5
 6
 7class InputExample:
 8    """
 9    Copied from Sentence Transformer Library
10    Structure for one input example with texts, the label and a unique id
11    """
12
13    def __init__(self, guid: str = '', texts: List[str] = None, label: Union[int, float] = 0):
14        """
15        Creates one InputExample with the given texts, guid and label
16
17        :param guid
18            id for the example
19        :param texts
20            the texts for the example. Note, str.strip() is called on the texts
21        :param label
22            the label for the example
23        """
24        self.guid = guid
25        self.texts = [text.strip() for text in texts]
26        self.label = label
27
28    def __str__(self):
29        return "<InputExample> label: {}, texts: {}".format(str(self.label), "; ".join(self.texts))
30
31    def get_label(self):
32        return self.label
33
34    # def __getattr__(self, key):
35    #    if key == "label":
36    #        return self.get_label()
37
38    #    if key == "texts":
39    #        return self.texts
40
41    #    if key in ["guid", "id"]:
42    #        return self.guid
43
44    #    raise KeyError()
45
46    @classmethod
47    def to_dict(cls, data: List['InputExample']):
48        text_len = len(data[0].texts)
49        processed_data = defaultdict(lambda: [])
50
51        for datum in data:
52            # string.ascii_lowercase
53
54            processed_data["id"].append(datum.guid)
55            processed_data["label"].append(datum.get_label())
56
57            for i in range(text_len):
58                letter = string.ascii_lowercase[i]  # abcdefghi
59                # processed_data[text_a] = ...
60                processed_data[f"text_{letter}"].append(datum.texts[i])
61
62        return processed_data
63
64    @classmethod
65    def from_parser_output(cls, data):
66        pass
67
68
69class RelevanceExample(InputExample):
70    """
71    Converts Relevance Labels to 0 - 1
72    """
73
74    def __init__(self, max_score=2, *args, **kwargs):
75        super().__init__(*args, **kwargs)
76        self.max_score = max_score
77
78    def get_label(self):
79        return self.relevance()
80
81    def relevance(self):
82        """
83        :return:
84            Returns a normalised score for relevance between 0 - 1
85        """
86        return self.label / self.max_score
87
88
89class DatasetTypes(Enum):
90    """
91    A collection of common dataset types that is usable in the library.
92    """
93    List: "List"
94    ListInputExample: "ListInputExample"
95    ListDict: "ListDict"
96    HuggingfaceDataset: "HuggingfaceDataset"