> pip install paddlepaddle numpy elasticsearch

> pip install requests bs4

PUT http://127.0.0.1/page
{
    "settings": {
        "number_of_shards": "5",
        "number_of_replicas": "0"
    },
    "mappings": {
        "properties": {
            "title": {
                "type": "text",
                "analyzer": "ik_max_word"
            },
            "weight": {
                "type": "double"
            },
            "content" : {
                "type" : "text",
                "analyzer": "ik_max_word"
            },
            "content_type": {
                "type": "text"
            },
            "url": {
                "type": "text",
                "analyzer": "ik_max_word"
            },
            "update_date": {
                "type": "date",
                "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
            }
        }
    }
}

> cd WebApp
> java -jar *.jar

> cd DataCrawler
> python crawler.py

> cd DataProcess
> python PageRank.py

> cd DataProcess/Text_Classification
> python Classify.py

import os
import sys
import json
import numpy as np
import time
import codecs
 
dir_path = os.path.split(os.path.realpath(sys.argv[0]))[0] + '/../RawData'
 
print(dir_path)
Vexname = list(os.listdir(dir_path))
Vexnum = len(Vexname)
epoch = 50
 
# 初始化，去掉所有未在url中出現(xiàn)的link以及錯誤文件
def init():
    global Vexnum
    falsefiles={}
    idx=0
    start = time.perf_counter()
    for file in Vexname:
        if idx % 100 == 0:
            a = '=' * int(idx / Vexnum * 100)
            b = ' ' * (100 - int(idx / Vexnum * 100))
            c = int(idx / Vexnum * 100)
            dur = time.perf_counter() - start
            sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
            sys.stdout.flush()
        with codecs.open(os.path.join(dir_path, file), 'r', encoding='utf-8') as load_f:
            try:
                text = json.load(load_f)
            except:
                falsefiles[file]=Vexname.index(file)-len(falsefiles)
                continue
            try:
                links = []
                for link in text['link']:
                    if link+'.json' in Vexname:
                        links.append(link)
                text['link'] = links.copy()
            except:
                pass
            finally:
                if 'link' in text:
                    text['link'].clear()
                else:
                    text['link'] = []
        with codecs.open(os.path.join(dir_path, file), 'w', encoding='utf-8') as dump_f:
            json.dump(text, dump_f, ensure_ascii=False,indent=4)
        idx += 1
    print('正在刪除錯誤文件及鏈接...')
    Vexnum -= len(falsefiles)
    checknum=0
    checkfalse=0
    for file in list(falsefiles.keys()):
        os.remove(os.path.join(dir_path,file))
        Vexname.remove(file)
        for i in range(checknum,falsefiles[file]):
            with codecs.open(os.path.join(dir_path, Vexname[i]), 'r', encoding='utf-8') as load_f:
                text = json.load(load_f)
                try:
                    for falsefile in list(falsefiles.keys())[checkfalse:]:
                        if falsefile in text['link']:
                            text['link'].remove(falsefile)
                except:
                    text['link'].clear()
            with codecs.open(os.path.join(dir_path, Vexname[i]), 'w', encoding='utf-8') as dump_f:
                json.dump(text, dump_f, ensure_ascii=False,indent=4)
        checknum += falsefiles[file]
        checkfalse += 1
 
# 計算PageRank
def Rank(Value, start):
    NewValue=np.zeros(Vexnum,dtype=np.double)
    for iter in range(1,epoch):
        a = '=' * int(iter / epoch * 100)
        b = ' ' * (100 - int(iter / epoch * 100))
        c = int(iter / epoch * 100)
        dur = time.perf_counter() - start
        sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
        sys.stdout.flush()
        for i in range(Vexnum):
            with open(os.path.join(dir_path, Vexname[i]), 'r', encoding='utf-8') as load_f:
                text = json.load(load_f)
 
                count = len(text['link'])
 
                if count == 0:
                    NewValue[i] = Value[i]
                    continue
                for link in text['link']:
                    link += '.json'
                    NewValue[Vexname.index(link)] += Value[i] / count
        for i in range(Vexnum):
            NewValue[i] = NewValue[i] / (iter + 1) + Value[i] * (iter / (iter + 1))
        Value=NewValue.copy()
    return Value
 
 
def run():
    print('開始計算PageRank...')
    print('數(shù)據(jù)初始化...')
    init()
    Value = np.ones(len(Vexname),dtype=np.double)*(1000.0/Vexnum)
    print('錯誤文件刪除完畢！')
    print('正在計算PageRank(迭代次數(shù){})...'.format(epoch))
    start = time.perf_counter()
    Value = Rank(Value, start)
    a = '=' * 100
    b = ' ' * 0
    c = 100
    dur = time.perf_counter() - start
    sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
    sys.stdout.flush()
    print('\nPageRank計算完畢，正在往JSON中寫入數(shù)據(jù)...')
    max = {}
    for file in Vexname:  # 將PageRank寫入JSON
        with open(os.path.join(dir_path, file), 'r', encoding='utf-8') as load_f:
            text = json.load(load_f)
        with open(os.path.join(dir_path, file), 'w', encoding='utf-8') as dump_f:
            text['weight'] = Value[Vexname.index(file)]
            max[file] = text['weight']
            json.dump(text, dump_f, ensure_ascii=False,indent=4)
    print('數(shù)據(jù)寫入完畢...')
 
 
if __name__ == '__main__':
    run()

# 導入必要的包
import json
import os
import sys
import time
import math
import gc
 
import elasticsearch
import numpy as np
import paddle.fluid as fluid
 
dir_path = os.path.dirname(os.path.realpath(__file__))
# 用訓練好的模型進行預測并輸出預測結果
# 創(chuàng)建執(zhí)行器
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
 
save_path = os.path.join(dir_path, 'infer_model/')
 
# 從模型中獲取預測程序、輸入數(shù)據(jù)名稱列表、分類器
[infer_program, feeded_var_names, target_var] = fluid.io.load_inference_model(dirname=save_path, executor=exe)
 
# 主機
host = "py7hon.com:9200"
 
# 建立 elasticsearch 連接
try:
    es = elasticsearch.Elasticsearch(hosts=host)
except Exception as e:
    print(e)
    exit()
 
 
# 獲取數(shù)據(jù)
def get_data(sentence):
    # 讀取數(shù)據(jù)字典
    with open(os.path.join(dir_path, 'dict_txt.txt'), 'r', encoding='utf-8') as f_data:
        dict_txt = eval(f_data.readlines()[0])
    dict_txt = dict(dict_txt)
    # 把字符串數(shù)據(jù)轉換成列表數(shù)據(jù)
    keys = dict_txt.keys()
    data = []
    for s in sentence:
        # 判斷是否存在未知字符
        if not s in keys:
            s = '<unk>'
        data.append((np.int64)(dict_txt[s]))
    return data
 
def batch_reader(Json_list,json_path):
    datas = []
    gc.collect()
    json_files = []
    falsefiles = []
    datas.clear()
    falsefiles.clear()
    json_files.clear()
    start = time.perf_counter()
    i=0
    scale = 100
    for file in Json_list:
        if i % 100 == 0:
            a = '=' * int(i / len(Json_list) * 100)
            b = ' ' * (scale - int(i / len(Json_list) * 100))
            c = int(i / len(Json_list) * 100)
            dur = time.perf_counter() - start
            sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
            sys.stdout.flush()
        i+=1
        with open(os.path.join(json_path, file), "r", encoding='utf-8') as f:
            try:
                text = json.load(f)
            except:
                falsefiles.append(file)
                continue
            json_files.append(os.path.join(json_path, file))
            json_text = text['content']
            data = get_data(json_text)
            datas.append(data)
    for file in falsefiles:
        os.remove(os.path.join(dir_path, file))
    file_count = len(Json_list) - len(falsefiles)
    a = '=' * 100
    b = ' ' * 0
    c = 100
    dur = time.perf_counter() - start
    sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
    sys.stdout.flush()
    print('\n文本數(shù)據(jù)獲取完畢，共計{0}條文本數(shù)據(jù)，有效數(shù)據(jù){2}條，無效數(shù)據(jù){1}條（已刪除）！'.format(len(Json_list),len(falsefiles),file_count))
    print('AI正在加載分類模型...')
    # 獲取每句話的單詞數(shù)量
    base_shape = [[len(c) for c in datas]]
 
    # 生成預測數(shù)據(jù)
    tensor_words = fluid.create_lod_tensor(datas, base_shape, place)
 
    # 執(zhí)行預測
    result = exe.run(program=infer_program,
                     feed={feeded_var_names[0]: tensor_words},
                     fetch_list=target_var)
    print('模型加載完畢！')
    # 分類名稱
    names = ['文化', '娛樂', '體育', '財經', '房產', '汽車', '教育', '科技', '國際', '證券']
    count = np.zeros(10)
    print('AI正在對文本數(shù)據(jù)進行分類并上傳至ES：')
    # 獲取結果概率最大的label
    start = time.perf_counter()
    for i in range(file_count):
        if i % 100 == 0:
            a = '=' * int(i / file_count * 100)
            b = ' ' * (scale - int(i / file_count * 100))
            c = int(i / file_count * 100)
            dur = time.perf_counter() - start
            sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
            sys.stdout.flush()
        lab = np.argsort(result)[0][i][-1]
        # print('預測結果標簽為：%d，  名稱為：%s， 概率為：%f' % (lab, names[lab], result[0][i][lab]))
        count[lab] += 1
        with open(json_files[i], 'r', encoding='utf-8') as load_f:
            try:
                text = json.load(load_f)
            except:
                continue
        text['content_type'] = names[lab]
 
        id = json_files[i].split('\\')[-1].split('.')[0]
        #try:
        del text['link']
        response = es.index(index='page', doc_type='_doc', id=id, body=text)
        #except Exception:
        # print("\n" + "數(shù)據(jù) " + id + " 插入失敗，錯誤信息：" + response)
 
        # with open(os.path.join(json_path,json_files[i].split('\\')[-1]),'w') as dump_f:
        #     json.dump(text,dump_f)
    a = '=' * 100
    b = ' ' * 0
    c = 100
    dur = time.perf_counter() - start
    sys.stdout.write("\r{:^3.0f}%[{}=>{}]{:.2f}s".format(c, a, b, dur))
    sys.stdout.flush()
    print("\n" + "%d條文本數(shù)據(jù)分類結束！已全部上傳至ES" % (file_count))
 
 
def run():
    # 獲取圖片數(shù)據(jù)
    print('AI正在獲取文本數(shù)據(jù)...')
    json_path = os.path.realpath(__file__) + '/../../../RawData'
    Json_list = os.listdir(json_path)
    batch_size=500
    if len(Json_list)>batch_size:
        Json_batch=0
        print('當前文本數(shù)量為{0}條，正在分批處理...'.format(len(Json_list)))
        for batch_id in range(math.ceil(len(Json_list)/batch_size)):
            a=(batch_size if batch_size<(len(Json_list)-Json_batch) else len(Json_list)-Json_batch)
            print('正在處理第{0}批，數(shù)量為{1}...'.format(batch_id+1,a))
            batch_reader(Json_list[Json_batch:Json_batch+a],json_path)
            Json_batch += a
    else:
        batch_reader(Json_list,json_path)
 
 
if __name__ == '__main__':
    run()