API Reference

class HBaseOps(object):
    """
    demo
    key = 'test'
    db = HBaseHelper(host=hbase_host)
    data = db.query_single_line(table='table', row_key=key)
    print(data)
    """

    def __init__(self, config=global_db_config["hbase"]):
        self.host = config["DEFAULT_HOST"]
        self.port = config["DEFAULT_PORT"]
        self.compat = config["DEFAULT_COMPAT"]
        self.table_prefix = None  # namespace
        self.transport = config["DEFAULT_TRANSPORT"]
        self.protocol = config["DEFAULT_PROTOCOL"]
        self.conn = self.connect()

    def connect(self):
        conn = happybase.Connection(host=self.host, port=self.port, timeout=None, autoconnect=True,
                                    table_prefix=self.table_prefix, compat=self.compat,
                                    transport=self.transport, protocol=self.protocol)
        return conn

    def create_hb_table(self, table_name, **families):
        self.conn.create_table(table_name, families)

    def single_put(self, table_name, row_key, column, data):
        hb = happybase.Table(table_name, self.conn)
        hb.put(row_key,
               data={'{column}:{k}'.format(column=column, k=k): str(v).encode("utf-8") for k, v in data.items()})

    def batch_put(self, table, row_key_name, column, datas, batch_size=1):
        hb = happybase.Table(table, self.conn)
        datas_new = [datas[i:i + batch_size] for i in range(0, len(datas), batch_size)]
        for x in datas_new:
            with hb.batch(batch_size=batch_size) as batch:
                for da in x:
                    da_nw = {'{column}:{k}'.format(column=column, k=k): v for k, v in da.items()}
                    row_key = da_nw.pop('{column}:{k}'.format(column=column, k=row_key_name))
                    batch.put(row_key, da_nw)
        return batch

    def single_put_self(self, table_name, row_keys, datas):
        hb = happybase.Table(table_name, self.conn)
        for row_key, (_, val) in zip(row_keys, datas.items()):
            hb.put(row_key, {'maybe_table_name:maybe_column_name': "%s" % val[0],
                             'maybe_table_name:maybe_column_name2': "%s" % val[1]})

    def scan_table(self, table, row_start=None, row_stop=None, include_timestamp=False, limit=None, timestamps=None,
                   filter=None):
        hb = happybase.Table(table, self.conn)
        scan = hb.scan(row_start=row_start, row_stop=row_stop, limit=limit, timestamp=timestamps, filter=filter)
        hb_dict = dict(scan)
        if hb_dict:
            return {str(k1).decode('utf-8'): {str(k2).decode('utf-8'): str(v2).decode('utf-8') for k2, v2 in v1.items()}
                    for k1, v1 in
                    hb_dict.items()}
        else:
            return {}

    def query_single_line(self, table, row_key):
        conn = self.connect()
        hb = happybase.Table(table, conn)
        hb_dict = hb.row(row_key)
        if hb_dict:
            return {k.decode('utf-8'): v.decode('utf-8') for k, v in hb_dict.items()}
        else:
            return {}

    def query_multi_lines(self, table, row_keys):
        hb = happybase.Table(table, self.conn)
        hb_dict = dict(hb.rows(row_keys))
        if hb_dict:
            return {k1.decode('utf-8'): {k2.decode('utf-8'): v2.decode('utf-8') for k2, v2 in v1.items()} for k1, v1 in
                    hb_dict.items()}
        else:
            return {}

    def single_delete(self, table, row_key):
        hb = happybase.Table(table, self.conn)
        hb.delete(row_key)

    def test_scan(self, table):
        hb = happybase.Table(table, self.conn)
        filter = "SingleColumnValueFilter ('maybe_column_name', 'lang', =, 'regexstring:[regex_string]')"
        scan = hb.scan(limit=1000, filter=filter)

        hb_dict = dict(scan)
        if hb_dict:
            return {str(k1).decode('utf-8'): {str(k2).decode('utf-8'): str(v2).decode('utf-8') for k2, v2 in v1.items()}
                    for k1, v1 in
                    hb_dict.items()}
        else:
            return {}

    def close(self):
        self.conn.close()

class MongoOps(object):
    def __init__(self, config=global_db_config["mongo"]):
        mongo_client = MongoClient(config["uri"])
        db = mongo_client[config["db"]]
        self.collection = db[config["col"]]

    def fetch_all(self):
        """
        读取所有数据
        :return:
        """
        ans = []
        print('提取所有数据.')
        for record in self.collection.find({}):
            record['_id'] = str(record['_id'])
            ans.append(record)
        return ans

    def load_from_mongo(self, special_value):
        """
        读取mongodb该special_value下所有值为special_value的数据
        :param
        :return:
        """
        record = self.collection.find({"{}".format(special_value): special_value})
        record = list(record)
        if not record:
            return None
        else:
            record = sorted(record, key=lambda x: len(x.get("another_value", [])))[0]
            return record

    def delete_all(self):
        query = {}
        deleted = self.collection.delete_many(query)
        return deleted

    def delete_by_time(self, time):
        query = {"name": {"$regex": "^F"}}
        deleted = self.collection.delete_many(query)

    def fetch_by_time(self, year=2022, month=7, day=7, hour=7, minute=7, second=7):
        query = {"query_time": {"$gte": datetime.datetime(year, month, day, hour, minute, second)}}
        sort_sql = [("query_time", -1)]
        ans = []
        print('提取所有数据.')
        for record in self.collection.find(query).sort(sort_sql):
            record['_id'] = str(record['_id'])
            ans.append(record)
        return ans

    def save_to_mongo(self, special_value, each_item):
        """
        数据存入mongo
        :param special_value:
        :param each_item:
        :return:
        """
        query = self.collection.find({"{}".format(special_value): special_value})
        if list(query):
            self.collection.update_one({"{}".format(special_value): special_value},
                                       {"$push": {'each_item': each_item}})
        else:
            insert_item = {
                "special_value": special_value,
                "each_item": [each_item]
            }
            self.collection.insert_one(insert_item)
        print("update success")

    def insert_one(self, data):
        self.collection.insert_one(data)

    def update_to_mongo(self, condition_term, condition_value, new_value):
        """
        根据提供的字段和值，查询出对应的数据，更新数据存入mongo
        类似 updata
        :param condition_term: 条件字段term
        :param condition_value: 条件字段值
        :param new_value: 新的值。最好是dict，不是dict的话不知道行不行
        :return:
        """
        query = self.collection.find({condition_term: condition_value})
        if list(query):
            self.collection.update_one({condition_term: condition_value},
                                       {"$push": new_value})
        else:
            insert_item = {
                condition_term: condition_value,
                "processed_data": new_value
            }
            self.collection.insert_one(insert_item)
        print("update success")

class CopyFunc():
    # from https://github.com/lemon234071/clean-dialog
    def is_chinese_char(cp):
        """Checks whether CP is the codepoint of a CJK character."""
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        return (
                (cp >= 0x4E00 and cp <= 0x9FFF)
                or (cp >= 0x3400 and cp <= 0x4DBF)  #
                or (cp >= 0x20000 and cp <= 0x2A6DF)  #
                or (cp >= 0x2A700 and cp <= 0x2B73F)  #
                or (cp >= 0x2B740 and cp <= 0x2B81F)  #
                or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
                or (cp >= 0xF900 and cp <= 0xFAFF)
                or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
        )

    def contains_Chinese(seq):
        for char in seq:
            cp = ord(char)
            if is_chinese_char(cp):
                return True
        return False

class Pattern:
    """
    >>> pattern_special_char = re.compile("[{}{}]".format(pattern_special_char_x[1:-1], pattern_special_char_u[1:-1]))
        a = "\U000d8be6asdasdas \x00v啊实打实\x00\x00v阿松大\x00"
        res = re.sub(pattern_special_char, "$",a)
    """

    # some from data-prepare

    # emoji
    """
    # 这也是emoji的取法，不知道pattern全不全
    import emoji  # Use version emoji==1.6.1, otherwise it won't have UNICODE_EMOJI
    emoji = list(emoji.UNICODE_EMOJI["en"].keys())
    """
    emoji_pattern = "[\U00010000-\U0010ffff\\uD800-\\uDBFF\\uDC00-\\uDFFF]"

    # 特殊的乱码或不可见字符
    # \x 09:\t 0a:\n 0d:\r
    special_char_x_pattern = "[\x00-\x08\x0b\x0c\x0e\x0f\x10-\x19\x1a-\x1f]"
    # 统计大规模语料出来的非正常字符
    special_char_u_pattern = (
        "[\u3000\U000d8be6\U000e0062\U000e0063\U000e0067\U000e0073\U000e0074\U000e007f]"
    )
    special_char_pattern = "{}{}".format(
        special_char_x_pattern[1:-1], special_char_u_pattern[1:-1]
    )
    non_printing_characters_pattern = (
        f"[{''.join(map(chr, list(range(0, 32)) + list(range(127, 160))))}]"
    )

    # 必须从头匹配，否则无意义的
    # 中文人名
    chinese_name_pattern = "(?:[\u4e00-\u9fa5·]{2,3})"
    # 英文人名
    english_name_pattern = "(^[a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]$)"
    # 纯数字
    pure_num_pattern = "\d+"
    # xxxx图/表 之类的表述
    pic_table_descript_pattern = ".{1,15}图"

    # 无需从头匹配的。
    # hlink
    hlink_pattern = (
        r"(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"
    )
    http_pattern = "(http|https):\/\/([\w.]+\/?)\S*/\S*"
    # 邮箱
    email_pattern = "[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+"
    # html 可能过于严格了
    html_pattern = "<[\s\S]*?>"
    # 重复 “asdasdasdasd”
    repeat_pattern = "(.)\1+"
    # 日期
    day_time_pattern = "\d{1,4}(-)(1[0-2]|0?[1-9])\1(0?[1-9]|[1-2]\d|30|31)"
    # 小时
    hour_time_pattern = "(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d"
    # 股票
    stock_pattern = (
        "(s[hz]|S[HZ])(000[\d]{3}|002[\d]{3}|300[\d]{3}|600[\d]{3}|60[\d]{4})"
    )

    # 一般是需要替换的
    # 多余空格 => " "
    redundancy_space_pattern = " +"
    # 一般用不到 多余换行符号 => " "
    linebreak_pattern = "[\r\n\t]+"

    # 微博视频等
    weibo_pattern = r"([\s]\w+(的微博视频)|#|【|】|转发微博)"
    # @
    at_pattern = "@\w+"

    # from https://github.com/bigscience-workshop/data-preparation pii
    year_patterns = [
        r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([1-2][0-9]{3}[\p{Pd}/][1-2][0-9]{3})(?:$|[\s@,?!;:\'\"(.\p{Han}])",
        # yyyy-yyyy or yyyy/yyyy
        r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([1-2][0-9]{3}[\p{Pd}/.][0-3][0-9][\p{Pd}/.][0-3][0-9])(?:$|[\s@,?!;:\'\"(.\p{Han}])",
        # yyyy-mm-dd or yyyy-dd-mm or yyyy/mm/dd or yyyy/dd/mm or yyyy.mm.dd or yyyy.dd.mm
        r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([0-3][0-9][\p{Pd}/.][0-3][0-9][\p{Pd}/.](?:[0-9]{2}|[1-2][0-9]{3}))(?:$|[\s@,?!;:\'\"(.\p{Han}])",
        # mm-dd-yyyy or dd-mm-yyyy or mm/dd/yyyy or dd/mm/yyyy or mm.dd.yyyy or dd.mm.yyyy or the same but with yy instead of yyyy
        r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([0-3][0-9][\p{Pd}/](?:[0-9]{2}|[1-2][0-9]{3}))(?:$|[\s@,?!;:\'\"(.\p{Han}])",
        # mm-yyyy or mm/yyyy or the same but with yy
        r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([1-2][0-9]{3}-[0-3][0-9])(?:$|[\s@,?!;:\'\"(.\p{Han}])",
        # yyyy-mm or yyyy/mm
    ]

    # Patterns for high-risk character strings
    id_pattern = r'(?:^|[\b\s@?,!;:\'\")(.\p{Han}])([A-Za-z]*(?:[\p{Pd}]*\p{Nd}){6,})(?:$|[\b\s@?,!;:\'\")(.\p{Han}])'
    # https://regex101.com/r/JQkmh8/2
    # key_pattern = r'(?:^|[\b\s@?,!;:\'\")(.\p{Han}])((?:(?:[A-Za-z]+[\p{Nd}\p{Pd}\/\+\=:]+|[\p{Nd}\p{Pd}\/\+\=:]+[A-Za-z]+)){4,}|(?:(?:\p{Nd}{3,}|[A-Z]+\p{Nd}+[A-Z]*|\p{Nd}+[A-Z]+\p{Nd}*)[\s\p{Pd}]?){4,})(?:$|[\b\s\p{Han}@?,!;:\'\"])'
    # https://regex101.com/r/JQkmh8/5
    key_pattern = r'(?:^|[\b\s@?,!:;\'\")(.\p{Han}])((?:(?:[A-Za-z]+[\p{Nd}\p{Pd}\/\+\=:_]+|[\p{Nd}\p{Pd}\/\+\=:]+[A-Za-z]+)){4,}|(?:(?:\p{Nd}{3,}|[A-Z]+\p{Nd}+[A-Z]*|\p{Nd}+[A-Z]+\p{Nd}*)[ \p{Pd}]?){3,})(?:$|[\b\s\p{Han}@?,!;:\'\")(.])'
    ipv4_pattern = r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}'
    ipv6_pattern = r'(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
    ip_pattern = r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])(" + r"|".join(
        [ipv4_pattern, ipv6_pattern]) + ")(?:$|[\s@,?!;:\'\"(.\p{Han}])"

    # https://regex101.com/r/EpA5B7/1
    email_line_pattern = r'''
        (?<= ^ | [\b\s@,?!;:)('".\p{Han}<] )
        (
        [^\b\s@?!;,:)('"<]+
        @
        [^\b\s@!?;,/]*
        [^\b\s@?!;,/:)('">.]
        \.
        \p{L} \w{1,}
        )
        (?= $ | [\b\s@,?!;:)('".\p{Han}>] )
    '''

    # https://regex101.com/r/mOqi1s/3
    # user_pattern = r'(?:^|[\s@,?!;:\'\")(\p{Han}])(@[^\s@,?!;:\'\")(]{3,})'
    user_pattern = r'''
    (?<= ^ | [)(\s@,?!;:'"\p{Han}] )
    (@
        [^)(\s@,?!;:'"]{3,}
    )
    '''

class TextProcess(object):
    """
    数据处理类
    这是基类，如果是定制化的语言处理，请继承该类
    """

    def __init__(
            self,
            patterns_filter: List = None,
            patterns_replace: List[List] = None,
            words_filter: List = []
    ):
        """
        pattern_list:
        """
        self.patterns_filter, self.patterns_replace = self._pre_complie_pattern(
            patterns_filter, patterns_replace
        )
        self.words_filter = words_filter

    @staticmethod
    def _pre_complie_pattern(patterns_filter, patterns_replace):
        complied_patterns_replace, complied_patterns_filter = [], []
        for i in patterns_filter:
            complied_patterns_filter.append(re.compile(i))
        for i in patterns_replace:
            complied_patterns_replace.append((re.compile(i[0]), i[1]))
        return complied_patterns_filter, complied_patterns_replace

    def process(self, text):
        # 进来的数据都要做的标准化
        text = self.full2half(text)
        # text = self.filter_http(text)
        text = self.filter_html(text)
        text = self.filter_html_special(text)
        # 根据类型与语言分别处理
        text = self.filter_exclusive(text)
        # text = self.trandition2simple(text)
        # text = self.remove_stopwords(text)
        return text

    def filter_words(self, text):
        # 根据词典，命中返回True，需要过滤掉

        for word in self.words_filter:
            if word in text:
                return True
        return False

    def filter_whitelist(self, text):
        whitelist = re.compile(
            "[^\u4e00-\u9fa5^0-9a-zA-Z^-^《^》^<^>^【^】^（^）^{^}^–^…^”^“^,^.^;^?^:^‘^~^`^，^。^？^；^！^：^、^·^!^@^#^$^%^&^(^)^|]"
        )
        text = whitelist.sub("", text)
        return text

    def text_split(self, text, language):
        if language == "en":
            text = text[:256]
        elif language == "zh":
            text = text[:510]
        return text

    def trandition2simple(self, text):
        # 仅对中文
        """
        https://juejin.cn/post/7234554420163100728
        """
        text = zhconv.convert("我幹什麼不干你事。", "zh-cn")
        return text

    def remove_stopwords(self, text):
        new_tokens = []
        if self.language == "en":
            tokens = text.split(" ")
        else:
            tokens = jieba.lcut(text)

        for i in tokens:
            if i in self.stopwords:
                pass
            else:
                new_tokens.append(i)

        return new_tokens

    @staticmethod
    def split_sentence(sentence, language="chinese"):
        """
        分句，英文有nltk，中文怎么能没有好的分句工具呢
        :param sentence:
        :param language:
        :return:
        """
        # sentences->Str
        # example '12“345。”“6789”'
        assert language in ["chinese", "english"], "unsupportable for other language"
        if language == "chinese":
            split_signs = list("。！？…\t")
            other_sign = "”"
        elif language == "english":
            split_signs = list(".!?")
            other_sign = '"'
        else:
            split_signs = list(".!?")
            other_sign = '"'
        sentences = []
        start_idx = 0
        for idx, char in enumerate(sentence):
            if idx == len(sentence) - 1:
                if char in split_signs:
                    sentences.append(sentence[start_idx: idx + 1].strip())
                    start_idx = idx + 1
                else:
                    sentences.append(sentence[start_idx:].strip())
            else:
                if char in split_signs:
                    if sentence[idx + 1] == other_sign:
                        if idx < len(sentence) - 2:
                            # 处理。”。
                            if sentence[idx + 2] not in split_signs:
                                sentences.append(sentence[start_idx: idx + 2].strip())
                                start_idx = idx + 2
                    elif sentence[idx + 1] not in split_signs:
                        sentences.append(sentence[start_idx: idx + 1].strip())
                        start_idx = idx + 1
        sentences = [i.strip() for i in sentences if i.strip()]
        return sentences

    def cut_word(self, text, language):
        if language == "en":
            tokens = text.split(" ")
        else:
            tokens = jieba.lcut(text)
        return tokens

    def full2half(self, text):
        """
        全角转化为半角
        :param text:
        :return:
        """
        ret_str = ""
        for i in text:
            if ord(i) >= 33 + 65248 and ord(i) <= 126 + 65248:
                ret_str += chr(ord(i) - 65248)
            else:
                ret_str += i
        return ret_str

    def filter_html(self, text):
        # 这个比较严格
        """
        过滤html标签
        :param text:
        :return:
        """
        patterns = [
            re.compile("//<![CDATA[[^>]*//]]>", re.I),  # 匹配CDATA
            re.compile("<s*script[^>]*>[^<]*<s*/s*scripts*>", re.I),  # Script
            re.compile("<s*style[^>]*>[^<]*<s*/s*styles*>", re.I),  # style
            re.compile("<brs*?/?>"),  # 处理换行
            re.compile("</?w+[^>]*>"),  # HTML标签
            re.compile("<!--[^>]*-->"),  # HTML注释
        ]
        for pattern in patterns:
            text = pattern.sub("", text)
        return text

    def filter_html_special(self, text):
        """
        替换所有html转义字符
        这个好像只有新闻有？
        :param text:
        :return:
        """
        # TODO html标签应该是 &nbsp 这种，\xa0也是吗
        CHAR_ENTITIES = {
            "&nbsp": " ",
            "160": " ",
            "lt": "<",
            "60": "<",
            "gt": ">",
            "62": ">",
            "amp": "&",
            "38": "&",
            "quot": '"',
            "34": '"',
            "ldquo": '"',
            "rdquo": '"',
            "mdash": "",
            "\xa0": "",
        }

        re_charEntity = re.compile(r"&#?(?P<name>\w+);", re.S)
        sz = re.search(re_charEntity, text)
        while sz:
            entity = sz.group()  # entity全称，如>
            key = sz.group("name")  # 去除&;后entity,如>为gt
            try:
                htmlstr = re_charEntity.sub(CHAR_ENTITIES[key], text, 1)
                text = htmlstr
                sz = re.search(re_charEntity, htmlstr)
            except KeyError:
                # 以空串代替
                htmlstr = re_charEntity.sub("", text, 1)
                text = htmlstr
                sz = re_charEntity.search(htmlstr)
        return text

    def filter_exclusive(self, text):
        """
        去除 @、 #、 表情等twitter、微博“特有”的情况
        :return:
        """
        pattern = r"([\s]\w+(的微博视频)|#|【|】|转发微博)"
        p = re.compile(pattern, re.S)
        text = p.sub("", text)

        dr = re.compile("@\w+", re.S)
        text = dr.sub("", text)

        return text

    def filter_html_tag(self, text):
        # res_tr = r'<a (.*?)></a>'
        # m_tr = re.findall(res_tr,text,re.S|re.M)
        res = re.sub(r"<a.*?>", "", text)
        res = re.sub(r"</a>", "", res)
        res = re.sub(r"<span.*?>", "", res)
        res = re.sub(r"</span>", "", res)
        res = re.sub(r"<img.*?>", "", res)
        res = re.sub(r"<br.*?>", "", res)
        res = re.sub(r"//", "", res)
        res = re.sub(r"@", "", res)
        res = re.sub(r"</", "", res)
        # res = re.sub(r',', '', res)
        # res = re.sub(r'&nbsp;', '', res)
        return res

    @staticmethod
    def uniform_whitespace(
            document,
            whitespace=[
                " ",
                " ",
                " ",
                " ",
                " ",
                "　",
                " ",
                " ",
                " ",
                " ",
                "",
                "„",
            ],
    ):
        # from https://github.com/bigscience-workshop/data-preparation
        """There are different whitespace characters."""
        whitespace = set(whitespace)
        document = "".join(
            [char if char not in whitespace else " " for char in document]
        )
        return document

    def filter_pattern(self, text):
        """
        返回True表示命中规则，需要过滤
        """
        for pattern in self.patterns_filter:
            if re.match(pattern, text):
                return True
        return False

    def replace_pattern(self, text):
        for pattern, replace in self.patterns_replace:
            text = re.sub(pattern, replace, text)
        return text

class DataAnalysis:
    @staticmethod
    def draw_pic(df, save_path):
        """
        画直方图，对比两个不同类别差异
        :param df: pd.DataFrame
        :param save_path: str
        :return: 
        """
        sns.distplot(df[df["label"] == 1]["feature"], label="label1")
        sns.distplot(df[df["label"] == 0]["feature"], label="label2")
        plt.legend()
        plt.savefig(save_path)

class STEM(object):

    def __init__(self, IPT_MODEL_PATH):
        self.ltp = LTP(IPT_MODEL_PATH)

    def start_by_dep(self, sentence):
        seg, hidden = self.ltp.seg([sentence])
        dep = self.ltp.dep(hidden)  # , graph=False)
        seg, dep = seg[0], dep[0]
        for i in dep:
            # 主谓宾
            if 'SBV' == i[2]:
                subject = seg[i[0]]
                verb = seg[i[1]]
            if 'VOB' in i[2]:
                if seg[i[1]] == verb:
                    object = seg[i[0]]

                return subject

        return None

    def start_by_srl(self, sentence):
        """
        用语义角色标注工具
        :param sentence: "他叫汤姆去拿外衣。"
        :return:  events: [['他', '叫', '汤姆', '去', '拿', '外衣'], ['汤姆', '拿', '外衣']]
        """
        # 语义角色标注方法
        seg, hidden = self.ltp.seg([sentence])
        srl = self.ltp.srl(hidden)
        seg, srl = seg[0], srl[0]
        events = []
        for wdx, each_srl in enumerate(srl):
            if each_srl:
                args = []
                for arg in each_srl:
                    args.extend(seg[arg[1]:arg[2] + 1])
                # 添加上谓词
                args.insert(each_srl[0][2] - each_srl[0][1] + 1, seg[wdx])
                events.append(args)
        # print(events)
        return events