阿里nlp2018初赛算法，Python代码

女爷i 2022-05-19 04:24 137阅读 0赞

主要是用lightmbg库：

# coding: utf-8
    
    # In[1]:
    
    
    # -*- coding: utf-8 -*-
    import sys
    stdi,stdo,stde=sys.stdin,sys.stdout,sys.stderr 
    reload(sys)
    sys.stdin,sys.stdout,sys.stderr=stdi,stdo,stde 
    # 合并两个csv文件到
    
    
    def combine(combine_file, filename1, filename2):
        f = open("true_value.txt",'w')
        len_merge_sum = 0
        with open(combine_file, 'w') as fout:
            with open(filename1, 'r') as f1:
                for eachLine in f1:
                    lineno, sen1, sen2, label = eachLine.strip().split('\t')
                    fout.write(lineno + '\t' + sen1 + '\t' + sen2 + '\t' + label + '\n')
                    if int(label) == 1:
                        f.write(sen1 + '\t' + sen2 + '\t' + label + '\n')
                    len_merge_sum += 1
            with open(filename2, 'r') as f1:
                for eachLine in f1:
                    lineno, sen1, sen2, label = eachLine.strip().split('\t')
                    fout.write(lineno + '\t' + sen1 + '\t' + sen2 + '\t' + label + '\n')
                    if int(label) == 1:
                        f.write(sen1 + '\t' + sen2 + '\t' + label + '\n')
                    len_merge_sum += 1
        fout.close()
        f.close()
        return combine_file, len_merge_sum
    
    
    # In[2]:
    
    
    # -*- coding: utf-8 -*-
    from gensim.models import word2vec
    import pandas as pd
    import numpy as np
    import sys
    import time
    import re
    import jieba
    import io
    stdi,stdo,stde=sys.stdin,sys.stdout,sys.stderr 
    reload(sys)
    sys.stdin,sys.stdout,sys.stderr=stdi,stdo,stde 
    sys.setdefaultencoding('utf-8')
    
    
    def process_simi_stop(simiwords, stopwords, line):
        for word, subword in simiwords.iteritems():
            if word in line:
                # print line
                #line = re.sub(word, subword, line)
                line = line.replace(word,subword)
                # print subword
        words1 = [w for w in jieba.cut(line) if w.strip()]
        word1 = []
        for i in words1:
            if i not in stopwords:
                word1.append(i)
        return word1,line
    
    
    def splitSentence(inputFile, inpath, segment, submit):
        print u'分词开始！', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间
        start = time.clock()
        jieba.load_userdict("jieba_dict.txt")
        corpus = []
    
        simiwords = {}
        with io.open("simiwords.txt", encoding='utf-8') as fr:
            for line in fr:
                words = re.split(",", line.strip())
                simiwords[words[0]] = words[1]
    
        stopwords = []  # 停用词
        fstop = open('chinese_stopwords.txt', 'r')
        for eachWord in fstop:
            stopwords.append(eachWord.strip())
            # print eachWord.strip()
    
        fin = open(inputFile, 'r')  # 以读的方式打开文件 inputFile
        fin1 = open('sentences_1.txt','w')
        for eachLine in fin:
            # line = eachLine.strip()  # 去除每行首尾可能出现的空格
            # line = re.sub("[0-9\s+\.\!\/_,$%^*()?;；:-【】+\"\']+|[+——！，;:。？、~@#￥%……&*（）]+", "", eachLine)
            eachLine = re.sub("\*", " ", eachLine)
            # jieba.del_word('年')
            lineno, sen1, sen2, label = eachLine.strip().split('\t')
            word1,sen_1 = process_simi_stop(simiwords, stopwords, sen1)
            word2,sen_2 = process_simi_stop(simiwords, stopwords, sen2)
            fin1.write(sen_1)
            fin1.write("\n")
            fin1.write(sen_2)
            fin1.write("\n")
    #         sen_11 = ' '.join(sen_1.decode('utf8'))
    #         sen_12 = sen_11.split(" ")
    #         for s in sen_12:
    #             if s != sen_12[-1]:
    #                 fin1.write(s+" ")
    #             else:
    #                 fin1.write(s)
    #         fin1.write('\n')
    #         sen_21 = ' '.join(sen_2.decode('utf8'))
    #         sen_22 = sen_21.split(" ") 
    #         for s in sen_22:
    #             if s != sen_22[-1]:
    #                 fin1.write(s+" ")
    #             else:
    #                 fin1.write(s)
    #         fin1.write('\n')
            corpus.append(word1)
            corpus.append(word2)
        print len(corpus)
        with open(inpath, 'r') as fin2:  # inpath
            for eachLine in fin2:
                eachLine = re.sub("\*", " ", eachLine)
                if submit:
                    lineno, sen1, sen2 = eachLine.strip().split('\t')
                    #print "ceshijieshisha:", sen1, sen2
                else:
                    lineno, sen1, sen2, label = eachLine.strip().split('\t')   # 无label
                    #print "ceshijieshisha:", sen1, sen2
                word1,sen_1 = process_simi_stop(simiwords, stopwords, sen1)
                word2,sen_2 = process_simi_stop(simiwords, stopwords, sen2)
                fin1.write(sen_1)
                fin1.write("\n")
                fin1.write(sen_2)
                fin1.write("\n")
    #             sen_11 = ' '.join(sen_1.decode('utf8'))
    #             sen_12 = sen_11.split(" ")
    #             for s in sen_12:
    #                 if s != sen_12[-1]:
    #                     fin1.write(s+" ")
    #                 else:
    #                     fin1.write(s)
    #             fin1.write('\n')
    #             sen_21 = ' '.join(sen_2.decode('utf8'))
    #             sen_22 = sen_21.split(" ") 
    #             for s in sen_22:
    #                 if s != sen_22[-1]:
    #                     fin1.write(s+" ")
    #                 else:
    #                     fin1.write(s)
    #             fin1.write('\n')
                corpus.append(word1)
                corpus.append(word2)
        print len(corpus)  # 204954
        fin1.close()
        with open(segment, 'w') as fs:
            for word in corpus:
                # print type(word)
                for w in word:
                    # print w
                    fs.write(w)  # 将分词好的结果写入到输出文件
                    fs.writelines(' ')
                fs.write('\n')
        end = time.clock()
        print u'分词实际用时：', end - start
        return corpus
    
    
    def filter_word_in_model(model, filename):
        a = []
        with open(filename, 'r') as file_to_read:
            for line in file_to_read:
                if True:
                    if not line:
                        break
                    a.append(line)
        sentences = []  # 读sentences 里面的词
        for i in range(len(a)):
            b = a[i].strip().split()
            sentences.append(b)
        print 'sentences length:', len(sentences)
        new_sentences = []  # 完成获取模型训练，剩余含有词向量序列
        for i in range(len(sentences)):
            new_sentence = []
            for j in range(len(sentences[i])):
                if sentences[i][j].decode('utf8') in model:
                    new_sentence.append(sentences[i][j])
            new_sentences.append(new_sentence)
        print 'new_sentences length: ', len(new_sentences)
        # print(np.array(new_sentences).shape)
        print u'new_sentences,用时', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间
        with open('new_sentences.txt', 'w') as fs:  # 写入new_sentences
            for word in new_sentences:
                for w in word:
                    fs.write(w)  # 将分词好的结果写入到输出文件
                    fs.writelines(' ')
                fs.write('\n')
        return new_sentences
    
    
    def eval_file(label1, pre):
        tp, tn, fp, fn = 0.0000001, 0.0000001, 0.0000001, 0.00000001
        for la, pr in zip(label1, pre):
            if la == 1 and pr == 1:
                tp += 1
            elif la == 1 and pr == 0:
                fn += 1
            elif la == 0 and pr == 0:
                tn += 1
            elif la == 0 and pr == 1:
                fp += 1
        recall = float(tp)/float(tp+fn)
        precision = float(tp)/float(tp+fp)
        f11 = 2*recall*precision/(recall+precision)
        return f11
    
    
    def cos_Vector(x, y):  # 用cos求夹角
        if len(x) != len(y):
            print u'error input,x and y is not in the same space'
            return
        x = np.array(x)
        y = np.array(y)
        num = (x * y.T)
        num = float(num.sum())
        if num == 0:
            return 0
        denom = np.linalg.norm(x) * np.linalg.norm(y)
        if denom == 0:
            return 0
        cos = num / denom  # 余弦值
        sim = 0.5 + 0.5 * cos  # 归一化
        return sim
    
    
    def vec_minus(x, y):  # 相减
        if len(x) != len(y):
            print u'error input,x and y is not in the same space'
            return
        x = np.array(x)
        y = np.array(y)
        sim = abs(x-y)
        return sim
    
    
    def vec_multi(x, y):  # 相乘
        if len(x) != len(y):
            print u'error input,x and y is not in the same space'
            return
        x = np.array(x)
        y = np.array(y)
        sim1 = x * y
        return sim1
    
    
    def calEuclideanDistance(x, y):
        if len(x) != len(y):
            print u'error input,x and y is not in the same space'
            return
        dist = np.sqrt(np.sum(np.square(x - y)))
        return dist
    
    #相同数组中相同元素的数量统计
    def cal_jaccard(list1, list2):
        set1 = set(list1)
        set2 = set(list2)
        avg_len = (len(set1) + len(set2)) / 2
        min_len = min(len(set1), len(set2))
        # return len(set1 & set2) * 1.0 / (len(set1) + len(set2) - len(set1 & set2))
        if min_len == 0:
            return 0
        else:
            return len(set1 & set2) * 1.0 / min_len
    
    
    def zishu(X,useStatus):  
        if useStatus:  
            return 1.0 / (1 + np.exp(-(X)));  
        else:  
            return (X);  
    
    
    # In[3]:
    
    
    def fenge(input_file,out_file,out_file1):
        f = open(input_file,'r')
        f_1 = open(out_file,'w')
        f_2 = open(out_file1,'w')
        lines = f.readlines()
        Row = len(lines)
        D = int(Row*0.85)
        for i in range(Row):
            if i < D:
                lineno, sen1, sen2, label = lines[i].strip().split('\t')
                f_1.write(lineno + '\t' + sen1 + '\t' + sen2 + '\t' + label + '\n')
            else:
                lineno, sen1, sen2, label = lines[i].strip().split('\t')
                f_2.write(lineno + '\t' + sen1 + '\t' + sen2 + '\t' + label + '\n')
        f.close()
        f_1.close()
        f_2.close()
        return D
    def feature_extraction(new_sentences,model,size):
        vec_titles = []  # 获取句子的向量
        for val in range(len(new_sentences)):
            vec = np.zeros(shape=(1, size))
            for i in range(len(new_sentences[val])):
                vec += model[new_sentences[val][i].decode('utf8')]
            if len(new_sentences[val]):
                vec = vec/len(new_sentences[val])
            vec_titles.append(vec)
        return vec_titles
                
    
    
    # In[4]:
    
    
    '''
    import threading
    import time
    gl_num = 0
    vec_titles = []
    lock = threading.RLock()
    def feature_extraction(new_sentences,model,size,val_start):
        lock.acquire()
        global vec_titles
        for val in range(val_start,len(new_sentences),4):
            vec = np.zeros(shape=(1, size))
            for i in range(len(new_sentences[val])):
                vec += model[new_sentences[val][i].decode('utf8')]
            if len(new_sentences[val]):
                vec = vec/len(new_sentences[val])
            vec_titles.append(vec)
        time.sleep(1)
        print len(vec_titles)
        lock.release()
    thread_list = [] 
    for i in range(4):
        t = threading.Thread(target=feature_extraction,args = (new_sentences,model,size,i))
        thread_list.append(t)
    for t in thread_list:
        t.start()
    '''
    
    
    # In[5]:
    
    
    # import jieba.analyse
    # def get_keyword(model, num_keywords, new_sentences):
    #     # 获取关键词
    #     content = open('new_sentences.txt', 'rb').read()
    #     jieba.analyse.set_stop_words('chinese_stopwords.txt')
    #     keywords = jieba.analyse.extract_tags(content, topK=num_keywords, withWeight=False, allowPOS=())
    #     print u'keywords长度：', len(keywords)
    
    #     # 获取在模型中的关键词
    #     keywords_in_model = []
    #     for i in range(len(keywords)):
    #         if keywords[i].decode('utf8') in model:
    #             keywords_in_model.append(keywords[i])
    #     print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    
    #     # 计算每一个keywords中的一个词，与句子中所有词语最大的值
    #     keywords_indexes = []  # 20w * 2k
    #     for i in range(len(new_sentences)):
    #         keywords_million_value = []
    #         for val in range(num_keywords):  # key_words 2k
    #             similar_values = []
    #             for j in range(len(new_sentences[i])):  # 每一个title里的词
    #                 try:
    #                     value = model.similarity(new_sentences[i][j].decode('utf-8'), keywords_in_model[val].decode('utf-8'))
    #                     similar_values.append(max(value, 0))
    #                 except:
    #                     print new_sentences[i][j]
    #             try:
    #                 keywords_one_value = max(similar_values)  # 得到第一个句子与第一词的相似度最大值
    #             except:
    #                 keywords_one_value = 0
    #                 print i
    #             keywords_million_value.append(keywords_one_value)  # 1w个
    #         keywords_indexes.append(keywords_million_value)
    #     print np.array(keywords_indexes).shape
    #     # 每个标题 1w维向量！
    #     np.save("train_data_similar_vec.npy", keywords_indexes)
    #     print u'词袋生成完毕:', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    #     return keywords_indexes
    # keywords_indexes = get_keyword(model,80,new_sentences)
    
    
    # In[6]:
    
    
    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.feature_extraction.text import TfidfTransformer  
    def get_tiidf_vec(filename):
        # 把new_sentences 写成 corpus要求的类型
        corpus = [' '.join(a) for a in filename]
        stopword = [u' ']
        #vectorizer = CountVectorizer(min_df=0,stop_words=stopword,token_pattern='(?u)\\b\\w+\\b')  # 词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
        #vectorizer = CountVectorizer(token_pattern=r"(?u)\b\w+\b")
        vectorizer = CountVectorizer(min_df=0,token_pattern=r"(?u)\W{1}|(?u)\b\w+\b")
        result = vectorizer.fit_transform(corpus)  # 文本转为词频矩阵
        transformer = TfidfTransformer()  # 统计每个词语的tf-idf权值
        tfidf = transformer.fit_transform(result)  # fit_transform是计算tf-idf
        vecs = []  # 每一个值的tfidf值
    #     hz = result.toarray()
        weight=tfidf.toarray()
        word=vectorizer.get_feature_names()#获取词袋模型中的所有词语
        print len(word)
    #     print hz.shape
        print weight.shape 
       # return word,weight
    #     for i in range(len(weight)):#打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重  
    #         print u"-------这里输出第",i,u"类文本的词语tf-idf权重------"  
    #         for j in range(len(word)):  
    #             print word[j],weight[i][j] 
        tfidf_cos = []
        hz_cos = []
        jk_cos = []
        for i in range(weight.shape[0]/2):
    #         numerator = np.sum(np.min(hz[2*i:2*i+2,:], axis=0))
    #         denominator = np.sum(np.max(hz[2*i:2*i+2,:], axis=0))
            value = np.dot(weight[2*i], weight[2*i+1])
    #         value_hz = np.dot(hz[2*i], hz[2*i+1]) / (norm(hz[2*i]) * norm(hz[2*i+1]))
            value = 0.5 + 0.5 * value
    #         value_hz = 0.5 + 0.5 * value_hz
            tfidf_cos.append([value])
    #         hz_cos.append([value_hz])
    #         jk_cos.append([1.0 * numerator / denominator])
        return tfidf_cos,hz_cos,jk_cos
    # word_list,tfidf = get_tiidf_vec(new_sentences)
    
    
    # In[5]:
    
    
    # # -*- coding: utf-8 -*-
    # import numpy as np
    # import sys
    # import time
    # from gensim.models import word2vec
    # import lightgbm as lgb
    # from sklearn.model_selection import train_test_split   # 随机分割
    # from scipy.linalg import norm
    # # import bm25
    # stdi,stdo,stde=sys.stdin,sys.stdout,sys.stderr 
    # reload(sys)
    # sys.stdin,sys.stdout,sys.stderr=stdi,stdo,stde 
    # sys.setdefaultencoding('utf-8')
    # # 合并两个csv文件到
    # filename1 = 'atec_nlp_sim_train.csv'
    # filename2 = 'atec_nlp_sim_train_add.csv'
    # combine_file, len_merge_sum = combine('merge_sum.csv', filename1, filename2)
    # if __name__ == '__main__':
    #     SUBMIT = False
    #     if SUBMIT:
    #         inpath, outpath = sys.argv[1], sys.argv[2]
    #         testpath = combine_file
    #         test_num = len_merge_sum
    #     else:
    #         num = fenge("merge_sum.csv","merge_train.csv","merge_test.csv")
    #         inpath, outpath = 'merge_test.csv', 'output.csv'
    #         testpath = 'merge_train.csv'
    #         #test_num = 92228
    #         test_num = 87105
    #         # inpath, outpath = 'empty.csv', 'output.csv'
    #         #         # testpath = 'merge_sum.csv'
    #         #         # test_num = 92228
    #     filename = 'sentences.txt'
    #     corpus = splitSentence(testpath, inpath, filename, SUBMIT)  # jieba 分词
    
    #     print u'语料corpus生成完毕:', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    
    # # # 训练词向量模型
    
    #     sentences = word2vec.Text8Corpus('sentences.txt')
    #     model = word2vec.Word2Vec(sentences, sg=1, size=100, window=5, min_count=5, negative=3, sample=0.001, hs=1, workers=4)
    #     model.save('result')  # save
    #     size = 100  # model_train size
    #     print u'词向量训练完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间
    #     # 导入model
    #     model = word2vec.Word2Vec.load('result')
    #     new_sentences = filter_word_in_model(model, filename)
        
        
    #     print "开始计算特征向量"
    #     tfidf_cos,hz_cos,jk_cos = get_tiidf_vec(new_sentences)
    # #     feature_2 = np.hstack((tfidf_cos,hz_cos,jk_cos))
    #     vec_titles = []  # 获取句子的向量
    #     max_titles = []
    
    #     for val in range(len(new_sentences)):
    #         vec = np.zeros(shape=(1, size))
    #         mat = np.zeros(shape=(30, size))
    #         for i in range(len(new_sentences[val])):
    #             print len(new_sentences[val])
    #             if i < 30:
    #                 vec += model[new_sentences[val][i].decode('utf8')]
    #                 mat[i] = model[new_sentences[val][i].decode('utf8')]
    #         if len(new_sentences[val]):
    #             vec = vec/len(new_sentences[val])
    #         vec_titles.append(vec)
    #         max_titles.append(mat)
    #     print "计算特征向量完毕"
    
    #     #vec_titles = feature_extraction(new_sentences,model,size)
    #     vec_titles = list(map(lambda x: x[0], vec_titles))  # 去掉外部的[], 获得title 的向量形式
    #     print(np.array(max_titles).shape)
    #     np.save("train_data_title_vec.npy", vec_titles)
    #     np.save("train_data_title_max.npy", max_titles)
    #     print u'生成train_data_title_vec完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出时间
    #     trains = np.load('train_data_title_max.npy')
    
    
    # In[7]:
    
    
    # # -*- coding: utf-8 -*-
    # import numpy as np
    # import sys
    # import time
    # from gensim.models import word2vec
    # import lightgbm as lgb
    # from sklearn.model_selection import train_test_split   # 随机分割
    # from scipy.linalg import norm
    
    # # from keras.datasets import mnist  
    # # from keras.models import Sequential  
    # # from keras.layers import Dense, Dropout, Activation, Flatten  
    # # from keras.layers import Convolution2D, MaxPooling2D  
    # # from keras.utils import np_utils  
    # # from keras import backend as K  
    # # import bm25
    # stdi,stdo,stde=sys.stdin,sys.stdout,sys.stderr 
    # reload(sys)
    # sys.stdin,sys.stdout,sys.stderr=stdi,stdo,stde 
    # sys.setdefaultencoding('utf-8')
    # # 合并两个csv文件到
    # filename1 = 'atec_nlp_sim_train.csv'
    # filename2 = 'atec_nlp_sim_train_add.csv'
    # combine_file, len_merge_sum = combine('merge_sum.csv', filename1, filename2)
    # if __name__ == '__main__':
    #     SUBMIT = False
    #     if SUBMIT:
    #         inpath, outpath = sys.argv[1], sys.argv[2]
    #         testpath = combine_file
    #         test_num = len_merge_sum
    #     else:
    #         num = fenge("merge_sum.csv","merge_train.csv","merge_test.csv")
    #         inpath, outpath = 'merge_test.csv', 'output.csv'
    #         testpath = 'merge_train.csv'
    #         #test_num = 92228
    #         test_num = 87105
    #         # inpath, outpath = 'empty.csv', 'output.csv'
    #         #         # testpath = 'merge_sum.csv'
    #         #         # test_num = 92228
    #     filename = 'sentences.txt'
    #     corpus = splitSentence(testpath, inpath, filename, SUBMIT)  # jieba 分词
    
    #     print u'语料corpus生成完毕:', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    
    # # # 训练词向量模型
    
    #     sentences = word2vec.Text8Corpus('sentences.txt')
    #     model = word2vec.Word2Vec(sentences, sg=1, size=100, window=5, min_count=5, negative=3, sample=0.001, hs=1, workers=4)
    #     model.save('result')  # save
    #     size = 100  # model_train size
    #     print u'词向量训练完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间
    #     # 导入model
    #     model = word2vec.Word2Vec.load('result')
    #     new_sentences = filter_word_in_model(model, filename)
        
        
    #     print "开始计算特征向量"
    # #     tfidf_cos,hz_cos,jk_cos = get_tiidf_vec(new_sentences)
    # #     feature_2 = np.hstack((tfidf_cos,hz_cos,jk_cos))
    #     vec_titles = []  # 获取句子的向量
    #     max_titles = []
    
    #     for val in range(len(new_sentences)/2):
    #         mat = np.zeros(shape=(50, size))
    #         for i in range(len(new_sentences[2*val])):
    #             if i < 25:
    #                 mat[i] = model[new_sentences[2*val][i].decode('utf8')]
    #         for i in range(len(new_sentences[2*val+1])):
    #             if i < 25:
    #                 mat[i+25] = model[new_sentences[2*val+1][i].decode('utf8')]        
    #         max_titles.append(mat)
    #     print "计算特征向量完毕"
    
    #     #vec_titles = feature_extraction(new_sentences,model,size)
    # #     vec_titles = list(map(lambda x: x[0], vec_titles))  # 去掉外部的[], 获得title 的向量形式
    #     print (np.array(max_titles).shape)
    #     np.save("train_data_title_max.npy", max_titles)
    #     print u'生成train_data_title_vec完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出时间
    #     trains = np.load('train_data_title_max.npy')    
    # #     nb_filters = 28  
    # #     # size of pooling area for max pooling  
    # #     pool_size = (2, 2)  
    # #     # convolution kernel size  
    # #     kernel_size = (3, 100)  
    # #     input_shape = (img_rows, img_cols, 1)  
    # #     model = Sequential()  
    # # """ 
    # # model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1], 
    # #                         border_mode='same', 
    # #                         input_shape=input_shape)) 
    # # """  
    # #     model.add(Convolution2D(nb_filters, (kernel_size[0], kernel_size[1]),  
    # #                         padding='same',  
    # #                         input_shape=input_shape)) # 卷积层1  
    # #     model.add(Activation('relu')) #激活层  
    # #     model.add(Convolution2D(nb_filters, (kernel_size[0], kernel_size[1]))) #卷积层2  
    # #     model.add(Activation('relu')) #激活层  
    # #     model.add(MaxPooling2D(pool_size=pool_size)) #池化层  
    # #     model.add(Dropout(0.25)) #神经元随机失活  
    # #     model.add(Flatten()) #拉成一维数据  
    # #     model.add(Dense(128)) #全连接层1  
    # #     model.add(Activation('relu')) #激活层  
    # #     model.add(Dropout(0.5)) #随机失活  
    # #     model.add(Dense(nb_classes)) #全连接层2  
    # #     model.add(Activation('softmax')) #Softmax评分  
      
    # #     #编译模型  
    # #     model.compile(loss='categorical_crossentropy',  
    # #               optimizer='adadelta',  
    # #               metrics=['accuracy'])  
    # #     #训练模型  
    # #     model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs,  
    # #           verbose=1, validation_data=(X_test, Y_test))  
    
    
    # In[8]:
    
    
    # print trains[0]
    
    
    # In[28]:
    
    
    # -*- coding: utf-8 -*-
    import numpy as np
    import sys
    import time
    from gensim.models import word2vec
    import lightgbm as lgb
    from sklearn.model_selection import train_test_split   # 随机分割
    from scipy.linalg import norm
    # import bm25
    stdi,stdo,stde=sys.stdin,sys.stdout,sys.stderr 
    reload(sys)
    sys.stdin,sys.stdout,sys.stderr=stdi,stdo,stde 
    sys.setdefaultencoding('utf-8')
    # 合并两个csv文件到
    filename1 = 'atec_nlp_sim_train.csv'
    filename2 = 'atec_nlp_sim_train_add.csv'
    combine_file, len_merge_sum = combine('merge_sum.csv', filename1, filename2)
    if __name__ == '__main__':
        SUBMIT = True
        if SUBMIT:
            inpath, outpath = sys.argv[1], sys.argv[2]
            testpath = combine_file
            test_num = len_merge_sum
        else:
            num = fenge("merge_sum.csv","merge_train.csv","merge_test.csv")
            inpath, outpath = 'merge_test.csv', 'output.csv'
            testpath = 'merge_train.csv'
            #test_num = 92228
            test_num = 87105
            # inpath, outpath = 'empty.csv', 'output.csv'
            #         # testpath = 'merge_sum.csv'
            #         # test_num = 92228
        filename = 'sentences.txt'
        corpus = splitSentence(testpath, inpath, filename, SUBMIT)  # jieba 分词
    
        print u'语料corpus生成完毕:', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    
    # # 训练词向量模型
    
        sentences = word2vec.Text8Corpus('sentences.txt')
        model = word2vec.Word2Vec(sentences, sg=1, size=120, window=5, min_count=5, negative=3, sample=0.001, hs=1, workers=4)
        model.save('result')  # save
        size = 120  # model_train size
        print u'词向量训练完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间
        # 导入model
        model = word2vec.Word2Vec.load('result')
        new_sentences = filter_word_in_model(model, filename)
        
        
        print "开始计算特征向量"
        tfidf_cos,hz_cos,jk_cos = get_tiidf_vec(new_sentences)
    #     feature_2 = np.hstack((tfidf_cos,hz_cos,jk_cos))
        vec_titles = []  # 获取句子的向量
        value_eig = []
        v_max = []
        v_v = [0,0,0]
        for val in range(len(new_sentences)):
            vec = np.zeros(shape=(1, size))
    #         mat = np.zeros(shape=(len(new_sentences[val]), size))
            for i in range(len(new_sentences[val])):
                vec += model[new_sentences[val][i].decode('utf8')]
    #             mat[i] = model[new_sentences[val][i].decode('utf8')]
            if len(new_sentences[val]):
                vec = vec/len(new_sentences[val])
    #             a,b,c = np.linalg.svd(mat)
    #             b_l = list(b)
    #             b_s = sorted(b_l)
    #             b_1 = np.max(b)
    #         v_max.append(b_s[-1])
    #         if len(b_s) < 2:
    #             v_max.append(0)
    #         else:
    #             v_max.append(b_s[-2])
    #         if len(b_s) < 3:
    #             v_max.append(0)
    #         else:
    #             v_max.append(b_s[-3])
            vec_titles.append(vec)
    #         if len(v_max) == 6:
    #             v_v[0] = abs(v_max[3]-v_max[0])
    #             v_v[1] = abs(v_max[4]-v_max[1])
    #             v_v[2] = abs(v_max[5]-v_max[2])
    #             value_eig.append(v_v)
    #             v_max = []
        print "计算特征向量完毕"
    
    #     vec_titles = []
    #     for val in range(len(new_sentences)):
    #         vec = np.zeros(shape=(len(new_sentences[val]), size))
    #         for i in range(len(new_sentences[val])):
    #             vec[i] = model[new_sentences[val][i].decode('utf8')]
    #         if len(new_sentences[val]):
    #             V = np.dot(vec.transpose(),vec)
    #             a,b = np.linalg.eig(V)
    #         vec1 = np.zeros(shape = (size,1))
    #         for k in range(a.shape[0]):
    #             if a[k] > 0.1:
    #                 vec1 = vec1+(a[k]*b[:,k]).reshape(size,1)
    #         print len(vec_titles)
    #         vec_titles.append(abs(vec1).reshape(1,size))
    
    
        #vec_titles = feature_extraction(new_sentences,model,size)
        vec_titles = list(map(lambda x: x[0], vec_titles))  # 去掉外部的[], 获得title 的向量形式
        print(np.array(vec_titles).shape)
        np.save("train_data_title_vec.npy", vec_titles)
        print u'生成train_data_title_vec完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出时间
        trains = np.load('train_data_title_vec.npy')
        # bm_score = np.load('bm_score.npy')
        # size = 100
        new_sentences = []
        with open('new_sentences.txt', 'r') as f:
            for eachLine in f:
                word = eachLine.decode('utf8').strip().split(' ')
                new_sentences.append(word)
    # 一维特征
        # 6  杰拉德距离
            J_dist = []
            for val in range(len(new_sentences) / 2):
                j = cal_jaccard(new_sentences[2 * val], new_sentences[2 * val + 1])
                J_dist.append(j)
        #  1向量夹角，2向量距离(最小) 3 bm_score
    
        juzi = []
        f1 = open('sentences_1.txt','r')
        for eachLine in f1:
            word = eachLine.decode('utf8').strip()
            juzi.append(list(word))
        distance = []
        for i in range(len(juzi)/2):
            j = cal_jaccard(juzi[2 * i], juzi[2 * i + 1])
            distance.append([j])
        f1.close()    
        
        cos_val = []
        E_Dist = []
        print 'train', len(trains)
        for i in range(len(trains) / 2):
            score1 = cos_Vector(trains[2 * i], trains[2 * i + 1])
            cos_val += [score1]
            score2 = calEuclideanDistance(trains[2 * i], trains[2 * i + 1])
            E_Dist += [score2]
        #添加一个二维特征
        combine_feature1 = np.vstack((cos_val, J_dist)).transpose()  # 2个
        np.save("cos_val.npy", cos_val), np.save('E_Dist.npy', E_Dist)
        # 4,5 各自的长度(2维)，6 长度差异(删)
        len_ = []
        dif_length = []
        for val in range(len(new_sentences) / 2):
            a = [0.1*len(new_sentences[2 * val]), 0.1*len(new_sentences[2 * val + 1])]
            #长度特征
            len_.append(a)
            b = abs(len(new_sentences[2 * val]) - len(new_sentences[2 * val + 1]))
            #长度差特征
            dif_length.append(b)
        # combine_feature2 = np.vstack((np.transpose(len_), J_dist)).transpose()  # 3个
        # combine_feature = np.hstack((combine_feature1, combine_feature2))  # 5个
        # combine_feature2 = len_ #二维特征
        combine_feature3 = tfidf_cos
        combine_feature4 = distance
        # combine_feature = np.vstack((np.transpose(combine_feature), J_dist)).transpose()  # 6个
        combine_feature = np.hstack((combine_feature1, combine_feature3, combine_feature4))  # 4个
        print combine_feature.shape
        # 编辑距离
        dim_1_num = 4  # 一维特征列数
        # dim_1_num = 6  # 一维特征列数
        print 'combine_feature length:', len(combine_feature)
    # 产生新特征(size*num 维度)
        feature1_val = []
        feature2_val = []
        feature3_val = []
        feature4_val = []
        feature5_val = []
        feature6_val = []
        print 'train', len(trains)
        #train就是句子向量对应的矩阵
        for i in range(len(trains)/2):
            vec1 = trains[2*i]
            vec2 = trains[2*i+1]
            feature1_val.append(vec1)
            feature2_val.append(vec2)
        for i in range(len(trains)/2):
            vec3 = vec_minus(trains[2*i], trains[2*i+1])
            vec4 = vec_multi(trains[2*i], trains[2*i+1])
            feature3_val.append(vec3)
            feature4_val.append(vec4)
        print 'feature1_val length:', len(feature3_val)
        # feature_val = np.hstack((feature1_val, feature2_val, feature3_val, feature4_val, combine_feature))  # 400列///406
        #合成总特征feature_val
        feature_val = np.hstack((feature1_val, feature2_val,combine_feature))  # 300列///406
        #feature_val = combine_feature
        np.save("feature_val.npy", feature_val)
        # feature_num = 4
        feature_num = 2 #几个feature1_val向量组成的特征
        print u'特征生成完毕'
    
        y_true = []
        with open(testpath, 'r') as f, open(inpath, 'r') as fin:
            for line in f.readlines():
                pair_id, sen1, sen2, label = line.strip().split('\t')
                label = int(label)
                y_true += [label]
        np.save('y_true.npy', y_true)
        print 'y_true length', len(y_true)
    
        # 读取数据
        y_true = np.load('y_true.npy')
        # bm_score = np.load('bm_score.npy')
        # bm_score_train = bm_score[:test_num]
        # print 'bm_score_train length', len(bm_score_train)
        # bm_score_test = bm_score[test_num:]
        # bm_score = pd.Series([bm_score], index=['bm_score'])
        cos_val = np.load('cos_val.npy')
        cos_val_train = cos_val[:test_num]
        cos_val_test = cos_val[test_num:]
        feature_val = np.load('feature_val.npy')
        feature_val_train = feature_val[:test_num]
        feature_val_test = feature_val[test_num:]
        #每一行对应一个训练集，最后一列是标号
        trains = np.vstack((np.transpose(feature_val_train), y_true)).transpose()
        print np.array(trains).shape
        print u"数据拆分"
        train, val = train_test_split(trains, test_size=0.2, random_state=21)
        print 'train length', len(train)
        print 'val length', len(val)
        print u"训练集"
        y = [train[i][feature_num*size+dim_1_num] for i in range(len(train))]  # 训练集标签
        X = [train[i][:feature_num*size+dim_1_num] for i in range(len(train))]  # 训练集特征矩阵
        print u"验证集"
        val_y = [val[i][feature_num*size+dim_1_num] for i in range(len(val))]  # 验证集标签
        val_X = [val[i][:feature_num*size+dim_1_num] for i in range(len(val))]  # 验证集特征矩阵
        print u"测试集"
        tests = feature_val_test
        # 数据转换
        lgb_train = lgb.Dataset(X, y, free_raw_data=False)
        lgb_eval = lgb.Dataset(val_X, val_y, reference=lgb_train, free_raw_data=False)
        # 开始训练
        print u'设置参数'
        params = {
            'num_threads' : '4',
            'boosting_type': 'gbdt',
            'boosting': 'gbdt',
            'objective': 'binary',
            'metric': 'binary_logloss',
    
            'learning_rate': 0.1,
            'num_leaves': 25,
            'max_depth': 3,
    
            'max_bin': 10,
            'min_data_in_leaf': 8,
    
            'feature_fraction': 1,
            'bagging_fraction': 0.7,
            'bagging_freq': 5,
    
            'lambda_l1': 0,
            'lambda_l2': 0,
            'min_split_gain': 0
            
        }
        print u"开始训练"
        gbm = lgb.train(params,  # 参数字典
                        lgb_train,  # 训练集
                        num_boost_round=3000,  # 迭代次数
                        valid_sets=lgb_eval,  # 验证集
                        early_stopping_rounds=30)  # 早停系数
        # 保存模型
        from sklearn.externals import joblib
        joblib.dump(gbm, 'gbm.pkl')
        print u"预测,并输出在 outpath"
        preds_offline = gbm.predict(tests, num_iteration=gbm.best_iteration)  # 输出概率
        np.save('preds.npy', preds_offline)
    
        if not SUBMIT:
            N = 200
            score_best = 0
            with open('merge_test.csv', 'r') as f1:
                y_true_10 = []
                for eachLine in f1:
                    lineno, sen1, sen2, label = eachLine.strip().split('\t')
                    a = int(label)
                    y_true_10.append(a)
    
            for thred in range(1,N):  # 阈值的选取，如何找到最好的阈值
                thred = thred * (np.max(preds_offline) - np.min(preds_offline)) / N + np.min(preds_offline)
                pred = []
                for i in range(len(preds_offline)):
                    if preds_offline[i] > thred:
                        pred.append(1)
                    else:
                        pred.append(0)
                score = eval_file(y_true_10, pred)
                if score > score_best:
                    score_best = score
                    thred_best = thred
            print u'最优阈值：', thred_best
    
            for i in range(len(preds_offline)):
                if preds_offline[i] > thred_best:
                    preds_offline[i] = 1
                else:
                    preds_offline[i] = 0
            print len(preds_offline)
            f1_score = eval_file(y_true_10, preds_offline)
            print 'F1 score is :' + str(f1_score)
            fout = open(outpath,'w')
            for t in preds_offline:
                fout.write(str(t))
                fout.write('\n')
            fout.write('F1 score is :' + str(f1_score))
            fout.close()
        else:
            with open(inpath, 'r') as fin, open(outpath, 'w') as fout:
                line_id = []
                for line in fin:
                    lineno, sen1, sen2 = line.strip().split('\t')
                    line_id.append(lineno)
                for i in range(len(line_id)):
                    if preds_offline[i] >= 0.246:
                        fout.write(line_id[i] + '\t1\n')
                    else:
                        fout.write(line_id[i] + '\t0\n')
        print u'运行完毕', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))  # 输出当前时间