NLP-文本处理：数据批次化【DataSet、DataLoader构建】

╰半夏微凉° 2023-10-05 18:58 12阅读 0赞

## 一、自定义分词（tokenization） ##

# =======================================进行自定义文本分词【可以用第三方分词工具】：开始=======================================
    def tokenlize(sentence):
        fileters = ['!', '"', '#', '$', '%', '&', '$', '$', '\*', '\+', ',', '-', '\.', '/', ':', ';', '<', '=', '>', '\?', '@', '\[', '\\', '\]', '^', '_', '`', '\{', '\|', '\}', '~', '\t', '\n', '\x97', '\x96', '”', '“', ]
        sentence = sentence.lower()  # 把大写转化为小写
        sentence = re.sub("<br />", " ", sentence)
        # sentence = re.sub("I'm","I am",sentence)  # 当语料量足够多时，可以学习到I'm的含义。
        # sentence = re.sub("isn't","is not",sentence)
        sentence = re.sub("|".join(fileters), " ", sentence)
        result = [i for i in sentence.split(" ") if len(i) > 0]
    
        return result
    # =======================================进行自定义文本分词【可以用第三方分词工具】：结束=======================================

第三方常用分词工具

*  Jieba分词： https://github.com/fxsjy/jieba
 *  SnowNLP： https://github.com/isnowfy/snownlp
 *  LTP： http://www.ltp-cloud.com/
 *  HanNLP： https://github.com/hankcs/HanLP/
 *  PKUseg
 *  清华大学的分词工具THULAC：https://github.com/thunlp/THULAC-Python

中英文分词的方法

*  把句子转化为词语
    
     *  比如：`我爱深度学习` 可以分为`[我，爱， 深度学习]`
 *  把句子转化为单个字
    
     *  比如：`我爱深度学习`的token是`[我，爱，深，度，学，习]`

## 二、构建并保存文本序列化对象 ##

import numpy as np
    
    # =======================================文本序列化：开始=======================================
    class WordSequence:
    	UNK_TAG = "<UNK>"  # 表示未在词典库里出现的未知词汇
        PAD_TAG = "<PAD>"  # 句子长度不够时的填充符
        SOS_TAG = "<SOS>"  # 表示一句文本的开始
        EOS_TAG = "<EOS>"  # 表示一句文本的结束
        UNK = 0
        PAD = 1
        SOS = 2
        EOS = 3
        
        def __init__(self):
            self.word_index_dict = {
        
    		            self.UNK_TAG: self.UNK,
    		            self.PAD_TAG: self.PAD,
    		            self.SOS_TAG: self.SOS,
    		            self.EOS_TAG: self.EOS}  # 初始化词语-数字映射字典
            self.index_word_dict = {
        }  # 初始化数字-词语映射字典
            self.word_count_dict = {
        }  # 初始化词语-词频统计字典
            self.fited = False
    
        def __len__(self):
            return len(self.word_index_dict)
    
    
        # 接受句子，统计词频得到
        def fit(self,sentence,min_count=5,max_count=None,max_features=None):    # 【min_count:最小词频; max_count: 最大词频; max_features: 最大词语数(词典容量大小)】
            """
            :param sentence:[word1,word2,word3]
            :param min_count: 最小出现的次数
            :param max_count: 最大出现的次数
            :param max_feature: 总词语的最大数量
            :return:
            """
            for word in sentence:
                self.word_count_dict[word] = self.word_count_dict.get(word,0)  + 1  #所有的句子fit之后，self.word_count_dict就有了所有词语的词频
            if min_count is not None:   # 根据条件统计词频
                self.word_count_dict = {
        word:count for word,count in self.word_count_dict.items() if count >= min_count}
            if max_count is not None:#   根据条件统计词频
                self.word_count_dict = {
        word:count for word,count in self.word_count_dict.items() if count <= max_count}    # 根据条件构造词典
            if max_features is not None:    # 根据条件保留高词频词语
                self.word_count_dict = dict(sorted(self.word_count_dict.items(),key=lambda x:x[-1],reverse=True)[:max_features])    # 保留词频排名靠前的词汇【self.word_count_dict.items()为待排序的对象，key表示排序指标，reverse=True表示降序排列】
            for word in self.word_count_dict:   # 根据word_count_dict字典构造词语-数字映射字典
                if word not in self.word_index_dict.keys(): # 如果当前词语word还没有添加到word_index_dict字典，则添加
                    self.word_index_dict[word]  = len(self.word_index_dict)  # 每次word对应一个数字【使用self.word_index_dict添加当前word前已有词汇的数量作为其value】
            self.fited = True
            self.index_word_dict = dict(zip(self.word_index_dict.values(),self.word_index_dict.keys()))  #把word_index_dict进行翻转【准备一个index->word的字典】
    
        # word -> index
        def to_index(self,word):
            assert self.fited == True,"必须先进行fit操作"
            return self.word_index_dict.get(word,self.UNK)
    
        # 把句子转化为数字数组(向量)【输入：[str,str,str]；输出：[int,int,int]】
        def transform(self,sentence,max_len=None,add_eos=False):
            if len(sentence) > max_len: # 句子过长，截取句子
            	if add_eos:	# 如果每句文本需要添加<EOS>结束标记
            		sentence = sentence[:max_len-1] + [self.EOS]
            	else:
                	sentence = sentence[:max_len]
            else:   # 句子过短，填充句子
            	if add_eos:	# 如果每句文本需要添加<EOS>结束标记
                	sentence = sentence + [self.EOS] + [self.PAD_TAG] *(max_len - len(sentence) - 1)
                else:
                	sentence = sentence +  [self.PAD_TAG] *(max_len - len(sentence))
            index_sequence = [self.to_index(word) for word in sentence]
            return index_sequence
    
        # index -> word
        def to_word(self,index):
            assert self.fited , "必须先进行fit操作"
            if index in self.inversed_dict:
                return self.inversed_dict[index]
            return self.UNK_TAG
    
        # 把数字数组(向量)转化为句子【输入：[int,int,int]；输出：[str,str,str]】
        def inverse_transform(self,indexes):
            sentence = [self.index_word_dict.get(index,"<UNK>") for index in indexes]
            return sentence
    
    # =======================================文本序列化：结束=======================================
    
    if __name__ == '__main__':
        sentences = [["今天","天气","很","好"],["今天","去","吃","什么"]]
        ws = WordSequence()
        for sentence in sentences:
          ws.fit(sentence)
        print("ws.word_index_dict = {0}".format(ws.word_index_dict))
        print("ws.fited = {0}".format(ws.fited))
        index_sequence = ws.transform(["今天","很","热"],max_len=10)
        print("index_sequence = {0}".format(index_sequence))
    
        # 构建语料的序列化对象
        word_sequence = WordSequence()
        input_path = "../corpus/input.txt"
        target_path = "../corpus/output.txt"
        for line in tqdm(open(input_path).readlines()):
        	tokenlized_line = tokenlize(line.strip())	# 对句子进行分词 >>> ["今天","天气","很","好"]
            word_sequence.fit(tokenlized_line )
        for line in tqdm(open(target_path).readlines()):
        	tokenlized_line = tokenlize(line.strip())	# 对句子进行分词 >>> ["今天","天气","很","好"]
            word_sequence.fit(tokenlized_line )
    	print("word_sequence文本序列化对象里的词汇总数量 = ".format(len(word_sequence)))
        pickle.dump(word_sequence,open("./pkl/ws.pkl","wb"))	# 保存文本序列化对象

输出结果：

ws.word_index_dict = {
        '<UNK>': 1, '<PAD>': 0, '今天': 2, '天气': 3, '很': 4, '好': 5, '去': 6, '吃': 7, '什么': 8}
    ws.fited = True
    index_sequence = [2, 4, 1, 0, 0, 0, 0, 0, 0, 0]

## 三、数据批次化【批次化过程中进行文本序列化】 ##

import numpy as np
    import pickle
    from torch.utils.data import DataLoader, Dataset
    import torch
    import os
    import re
    import pickle
    from tqdm import tqdm
    
    BATCH_SIZE_TRAIN = 2
    BATCH_SIZE_TEST = 2
    MAX_LEN = 100
    
    
    # =======================================进行自定义文本分词【可以用第三方分词工具】：开始=======================================
    def tokenlize(sentence):
        fileters = ['!', '"', '#', '$', '%', '&', '$', '$', '\*', '\+', ',', '-', '\.', '/', ':', ';', '<', '=', '>', '\?', '@', '\[', '\\', '\]', '^', '_', '`', '\{', '\|', '\}', '~', '\t', '\n', '\x97', '\x96', '”', '“', ]
        sentence = sentence.lower()  # 把大写转化为小写
        sentence = re.sub("<br />", " ", sentence)
        # sentence = re.sub("I'm","I am",sentence)  # 当语料量足够多时，可以学习到I'm的含义。
        # sentence = re.sub("isn't","is not",sentence)
        sentence = re.sub("|".join(fileters), " ", sentence)
        result = [i for i in sentence.split(" ") if len(i) > 0]
    
        return result
    # =======================================进行自定义文本分词【可以用第三方分词工具】：结束=======================================
    
    # =======================================Dataset数据准备：开始=======================================
    class ImdbDataset(Dataset):
        def __init__(self, wordSequence=None, train=True):
            super(ImdbDataset,self).__init__()
            self.wordSequence = wordSequence
            data_path = r"./data/aclImdb"
            data_path += r"/train" if train else r"/test" # 文件名拼接【等价于os.path.join()】
            self.total_path = []  # 保存所有的文件路径
            for temp_path in [r"/pos", r"/neg"]:
                cur_path = data_path + temp_path
                self.total_path += [os.path.join(cur_path, i) for i in os.listdir(cur_path) if i.endswith(".txt")]  # 将所有文件路径加入到total_path列表中
    
        def __getitem__(self, idx):
            file = self.total_path[idx]
            review = tokenlize(open(file, encoding="utf-8").read())  # 读取文件内容, 对句子进行分词 >>> ["今天","天气","很","好"]
            label = int(file.split("_")[-1].split(".")[0])
            label = 0 if label < 5 else 1
            if self.wordSequence is not None:
            	# *****使用“词语-数字”映射器进行文本序列化*****
                review = self.wordSequence.transform(review, max_len=MAX_LEN)  #  将字符串通过已经保存的“词语-数字”映射器转为数字
            return review, label
    
        def __len__(self):
            return len(self.total_path)
    # =======================================Dataset数据准备：结束=======================================
    
    # =======================================DataLoader数据数据批次化：开始=======================================
    # 自定义collate_fn方式，对batch数据进行处理【batch是list，其中是一个一个元组，每个元组是dataset中__getitem__的结果】
    def collate_fn(batch):
        reviews, labels = zip(*batch)
        lengths = [len(review) if len(review) < MAX_LEN else MAX_LEN for review in reviews]
        reviews, labels = torch.LongTensor(np.array(list(reviews))), torch.LongTensor(np.array(list(labels)))  # 将tuple类型转为Tensor类型
        return reviews, labels, lengths
    
    
    def get_dataloader(dataset, train=True):
        batch_size = BATCH_SIZE_TRAIN if train else BATCH_SIZE_TEST
        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
        return dataloader
    # =======================================DataLoader数据批次化：结束=======================================
    
    if __name__ == '__main__':
        wordSequence = pickle.load(open("./models/ws.pkl", "rb"))
        dataset = ImdbDataset(wordSequence=wordSequence)	# 在dataset中使用已保存的“词语-数字”映射器
        dataLoader = get_dataloader(dataset=dataset, train=True)
        for batch_index,(reviews,labels,lengths) in enumerate(dataLoader):
            print("batch_index = {0}".format(batch_index))
            print("reviews in this batch = {0}".format(reviews))
            print("labels in this batch = {0}".format(labels))
            print("lengths in this batch = {0}".format(lengths))
            break

输出如下

batch_index = 0
    reviews in this batch = (
    	tensor([   58,   103,    98,   739,   607,     4,   456,     6,     2,     8,
             2667,    58,   278,   259,  5242,   464,     2,   763,   290,   382,
              459,    98,   215,  5808,    21,  7498,   100,    58,   175,   224,
                2,   167,    15,  3353,   152,  2394,  9842,    58,   215,  7449,
               14,  1456,    22,    27,  4335,   280,   182, 10739,    98,    16,
             7419,   100,    21,     4,   804,  2772,     6,    82,  1609,    73,
             2649,   208,     2,  2667,    21,  1643, 10740,   108,   896,    21,
             3177,  3746,  8328,    13,  3845,  1238,   140,     2,    54,    55,
             1862,    13,   791,    51,   202,  8995,    53,    13,    58,  2682,
             3326,  3387,   248,   165,     2,  2305,    58,   909,    26,    64]), 
    	tensor([    2,  5991,   671,    27,     4,   209,   106,   152,  6751,     4,
             3762,  1626,   168,   208,    24,    19,   262, 12221,  6467,    21,
              540,    53,  1026,     6,     4,   671,   230,     9,   445,  5991,
              465,  3612,    87,     4,  1169,   572,     2,   168,    61,   338,
              459,  4215,    15,    98,     9,   109,   165,  6751,    98,     9,
                2,   133,  1312,    54,    55,   106,   959, 10310,  5693, 10606,
               54,    55,  2347,   182,  7884,   651,  7838,    64,   176,  9010,
              726,   491,    54,    55,   244,   316,    64,  1776,   214,   316,
               64,   176,   896,  1108,     2,    54,    55,   165,   399,   671,
                2,  2745,     6,     2, 11479, 11480,   290,    64,   229,    47]))
    
    labels in this batch = (tensor(1, dtype=torch.int32), tensor(0, dtype=torch.int32))
    
    lengths in this batch = [100, 100]