# 采用TF-IDF方法提取文本關鍵詞
# http://scikit-learn.org/stable/modules/feature_extraction.html#tfidf-term-weighting
import sys,codecs
import pandas as pd
import numpy as np
import jieba.posseg
import jieba.analyse
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
"""
       TF-IDF權重：
           1、CountVectorizer 構建詞頻矩陣
           2、TfidfTransformer 構建tfidf權值計算
           3、文本的關鍵字
           4、對應的tfidf矩陣
"""
# 數(shù)據(jù)預處理操作：分詞，去停用詞，詞性篩選
def dataPrepos(text, stopkey):
    l = []
    pos = ['n', 'nz', 'v', 'vd', 'vn', 'l', 'a', 'd']  # 定義選取的詞性
    seg = jieba.posseg.cut(text)  # 分詞
    for i in seg:
        if i.word not in stopkey and i.flag in pos:  # 去停用詞 + 詞性篩選
            l.append(i.word)
    return l

# tf-idf獲取文本top10關鍵詞
#@profile
def getKeywords_tfidf(data,stopkey,topK):
    idList, titleList, abstractList = data['id'], data['title'], data['abstract']
    corpus = [] # 將所有文檔輸出到一個list中，一行就是一個文檔
    for index in range(len(idList)):
        text = '%s。%s' % (titleList[index], abstractList[index]) # 拼接標題和摘要
        text = dataPrepos(text,stopkey) # 文本預處理
        text = " ".join(text) # 連接成字符串，空格分隔
        corpus.append(text)
    # 1、構建詞頻矩陣，將文本中的詞語轉換成詞頻矩陣
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus) # 詞頻矩陣,a[i][j]:表示j詞在第i個文本中的詞頻
    # 2、統(tǒng)計每個詞的tf-idf權值
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(X)
    # 3、獲取詞袋模型中的關鍵詞
    word = vectorizer.get_feature_names_out()
    # 4、獲取tf-idf矩陣，a[i][j]表示j詞在i篇文本中的tf-idf權重
    weight = tfidf.toarray()
    # 5、打印詞語權重
    ids, titles, keys = [], [], []
    for i in range(len(weight)):
        print("-------這里輸出第", i+1 , "篇文本的詞語tf-idf------")
        ids.append(idList[i])
        titles.append(titleList[i])
        df_word,df_weight = [],[] # 當前文章的所有詞匯列表、詞匯對應權重列表
        for j in range(len(word)):
            print (word[j],weight[i][j])
            df_word.append(word[j])
            df_weight.append(weight[i][j])
        df_word = pd.DataFrame(df_word,columns=['word'])
        df_weight = pd.DataFrame(df_weight,columns=['weight'])
        word_weight = pd.concat([df_word, df_weight], axis=1) # 拼接詞匯列表和權重列表
        word_weight = word_weight.sort_values(by="weight",ascending = False) # 按照權重值降序排列
        keyword = np.array(word_weight['word']) # 選擇詞匯列并轉成數(shù)組格式
        word_split = [keyword[x] for x in range(0,topK)] # 抽取前topK個詞匯作為關鍵詞
        word_split = " ".join(word_split)
        keys.append(word_split)
    result = pd.DataFrame({"id": ids, "title": titles, "key": keys},columns=['id','title','key'])
    return result

def main():
    # 讀取數(shù)據(jù)集
    dataFile = 'data/sample_data.csv'
    data = pd.read_csv(dataFile)
    # 停用詞表
    stopkey = [w.strip() for w in codecs.open('data/stopWord.txt', 'r', encoding='utf-8').readlines()]
    # tf-idf關鍵詞抽取
    result = getKeywords_tfidf(data,stopkey,10)
    result.to_csv("result/keys_TFIDF.csv",index=False)

if __name__ == '__main__':
    main()

# 采用TextRank方法提取文本關鍵詞
import sys
import pandas as pd
import jieba.analyse
"""
       TextRank權重：

            1、將待抽取關鍵詞的文本進行分詞、去停用詞、篩選詞性
            2、以固定窗口大小(默認為5，通過span屬性調整)，詞之間的共現(xiàn)關系，構建圖
            3、計算圖中節(jié)點的PageRank，注意是無向帶權圖
"""

# 處理標題和摘要，提取關鍵詞
def getKeywords_textrank(data,topK):
    idList,titleList,abstractList = data['id'],data['title'],data['abstract']
    ids, titles, keys = [], [], []
    for index in range(len(idList)):
        text = '%s。%s' % (titleList[index], abstractList[index]) # 拼接標題和摘要
        jieba.analyse.set_stop_words("data/stopWord.txt") # 加載自定義停用詞表
        print("\"",titleList[index],"\" ,  10 Keywords - TextRank :")
        keywords = jieba.analyse.textrank(text, topK=topK, allowPOS=('n','nz','v','vd','vn','l','a','d'))  # TextRank關鍵詞提取，詞性篩選
        word_split = " ".join(keywords)
        print(word_split)
        keys.append(word_split)
        ids.append(idList[index])
        titles.append(titleList[index])

    result = pd.DataFrame({"id": ids, "title": titles, "key": keys}, columns=['id', 'title', 'key'])
    return result

def main():
    dataFile = 'data/sample_data.csv'
    data = pd.read_csv(dataFile,encoding='utf-8')
    result = getKeywords_textrank(data,10)
    result.to_csv("result/keys_TextRank.csv",index=False,encoding='utf-8')

if __name__ == '__main__':
    main()

# 采用Word2Vec詞聚類方法抽取關鍵詞1——獲取文本詞向量表示
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')  # 忽略警告
import sys, codecs
import pandas as pd
import numpy as np
import jieba
import jieba.posseg
import gensim

# 返回特征詞向量
def getWordVecs(wordList, model):
    name = []
    vecs = []
    for word in wordList:
        word = word.replace('\n', '')
        try:
            if word in model:  # 模型中存在該詞的向量表示
                name.append(word)
                vecs.append(model[word])
        except KeyError:
            continue
    a = pd.DataFrame(name, columns=['word'])
    b = pd.DataFrame(np.array(vecs, dtype='float'))
    return pd.concat([a, b], axis=1)

# 數(shù)據(jù)預處理操作：分詞，去停用詞，詞性篩選
def dataPrepos(text, stopkey):
    l = []
    pos = ['n', 'nz', 'v', 'vd', 'vn', 'l', 'a', 'd']  # 定義選取的詞性
    seg = jieba.posseg.cut(text)  # 分詞
    for i in seg:
        if i.word not in l and i.word not in stopkey and i.flag in pos:  # 去重 + 去停用詞 + 詞性篩選
            # print i.word
            l.append(i.word)
    return l

# 根據(jù)數(shù)據(jù)獲取候選關鍵詞詞向量
def buildAllWordsVecs(data, stopkey, model):
    idList, titleList, abstractList = data['id'], data['title'], data['abstract']
    for index in range(len(idList)):
        id = idList[index]
        title = titleList[index]
        abstract = abstractList[index]
        l_ti = dataPrepos(title, stopkey)  # 處理標題
        l_ab = dataPrepos(abstract, stopkey)  # 處理摘要
        # 獲取候選關鍵詞的詞向量
        words = np.append(l_ti, l_ab)  # 拼接數(shù)組元素
        words = list(set(words))  # 數(shù)組元素去重,得到候選關鍵詞列表
        wordvecs = getWordVecs(words, model)  # 獲取候選關鍵詞的詞向量表示
        # 詞向量寫入csv文件，每個詞400維
        data_vecs = pd.DataFrame(wordvecs)
        data_vecs.to_csv('result/vecs/wordvecs_' + str(id) + '.csv', index=False)
        print("document ", id, " well done.")

def main():
    # 讀取數(shù)據(jù)集
    dataFile = 'data/sample_data.csv'
    data = pd.read_csv(dataFile)
    # 停用詞表
    stopkey = [w.strip() for w in codecs.open('data/stopWord.txt', 'r').readlines()]
    # 詞向量模型
    inp = 'wiki.zh.text.vector'
    model = gensim.models.KeyedVectors.load_word2vec_format(inp, binary=False)
    buildAllWordsVecs(data, stopkey, model)

if __name__ == '__main__':
    main()

# 采用Word2Vec詞聚類方法抽取關鍵詞2——根據(jù)候選關鍵詞的詞向量進行聚類分析
import sys,os
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
import os

# 對詞向量采用K-means聚類抽取TopK關鍵詞
def getkeywords_kmeans(data,topK):
    words = data["word"] # 詞匯
    vecs = data.iloc[:,1:] # 向量表示

    kmeans = KMeans(n_clusters=1,n_init=10,random_state=10).fit(vecs)
    labels = kmeans.labels_ #類別結果標簽
    labels = pd.DataFrame(labels,columns=['label'])
    new_df = pd.concat([labels,vecs],axis=1)
    df_count_type = new_df.groupby('label').size() #各類別統(tǒng)計個數(shù)
    # print df_count_type
    vec_center = kmeans.cluster_centers_ #聚類中心

    # 計算距離（相似性） 采用歐幾里得距離（歐式距離）
    distances = []
    vec_words = np.array(vecs) # 候選關鍵詞向量，dataFrame轉array
    vec_center = vec_center[0] # 第一個類別聚類中心,本例只有一個類別
    length = len(vec_center) # 向量維度
    for index in range(len(vec_words)): # 候選關鍵詞個數(shù)
        cur_wordvec = vec_words[index] # 當前詞語的詞向量
        dis = 0 # 向量距離
        for index2 in range(length):
            dis += (vec_center[index2]-cur_wordvec[index2])*(vec_center[index2]-cur_wordvec[index2])
        dis = math.sqrt(dis)
        distances.append(dis)
    distances = pd.DataFrame(distances,columns=['dis'])

    result = pd.concat([words, labels ,distances], axis=1) # 拼接詞語與其對應中心點的距離
    result = result.sort_values(by="dis",ascending = True) # 按照距離大小進行升序排序
    """
    # 將用于聚類的數(shù)據(jù)的特征維度降到2維
    pca = PCA(n_components=2)
    new_pca = pd.DataFrame(pca.fit_transform(new_df))
    print(new_pca)
    # 可視化
    d = new_pca[new_df['label'] == 0]
    plt.plot(d[0],d[1],'r.')
    d = new_pca[new_df['label'] == 1]
    plt.plot(d[0], d[1], 'go')
    d = new_pca[new_df['label'] == 2]
    plt.plot(d[0], d[1], 'b*')
    plt.gcf().savefig('kmeans.png')
    plt.show()
    """
    # 抽取排名前topK個詞語作為文本關鍵詞
    wordlist = np.array(result['word']) # 選擇詞匯列并轉成數(shù)組格式
    word_split = [wordlist[x] for x in range(0,topK)] # 抽取前topK個詞匯
    word_split = " ".join(word_split)
    return word_split

def main():
    # 讀取數(shù)據(jù)集
    dataFile = 'data/sample_data.csv'
    articleData = pd.read_csv(dataFile,encoding='utf-8')
    
    ids, titles, keys = [], [], []

    rootdir = "result/vecs" # 詞向量文件根目錄
    fileList = os.listdir(rootdir) #列出文件夾下所有的目錄與文件
    # 遍歷文件
    for i in range(len(fileList)):
        filename = fileList[i]
        path = os.path.join(rootdir,filename)
        if os.path.isfile(path):
            data = pd.read_csv(path, encoding='utf-8') # 讀取詞向量文件數(shù)據(jù)
            artile_keys = getkeywords_kmeans(data,10) # 聚類算法得到當前文件的關鍵詞
            
            # 根據(jù)文件名獲得文章id以及標題
            (shortname, extension) = os.path.splitext(filename) # 得到文件名和文件擴展名
            t = shortname.split("_")
            article_id = int(t[len(t)-1]) # 獲得文章id
            artile_tit = articleData[articleData.id==article_id]['title'] # 獲得文章標題
            artile_tit = list(artile_tit)[0] # series轉成字符串
            ids.append(article_id)
            titles.append(artile_tit)
            keys.append(artile_keys)
    # 所有結果寫入文件
    result = pd.DataFrame({"id": ids, "title": titles, "key": keys}, columns=['id', 'title', 'key'])
    result = result.sort_values(by="id",ascending=True) # 排序
    result.to_csv("result/keys_word2vec.csv", index=False,encoding='utf-8')

if __name__ == '__main__':
    main()