import pandas as pd
import requests
from bs4 import BeautifulSoup
import time
import re
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import lxml
import openpyxl
import codecs
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"
}
url1 = "https://data.stats.gov.cn/easyquery.htm?cn=E0105"
url2 = "http://data.stats.gov.cn"
'''爬取近五年主要城市數(shù)據(jù)'''
def driver1(url1):
    driver = webdriver.Chrome()
    driver.get(url1)
    time.sleep(3)
    xRdata = '//*[@id="details-button"]'  # 點(diǎn)擊高級(jí)
    ActionChains(driver).click(driver.find_element_by_xpath(xRdata)).perform()
    xRdata = '//*[@id="proceed-link"]'  # 點(diǎn)擊
    ActionChains(driver).click(driver.find_element_by_xpath(xRdata)).perform()
    time.sleep(3)
    time.sleep(4)  # 加載頁(yè)面，停留時(shí)間長(zhǎng)一些
    xTable = '//*[@class="table-container"]/div/div[3]/table/tbody'  # 主要城市年度數(shù)據(jù)表
    table = driver.find_element_by_xpath(xTable).get_attribute('innerHTML')
    soup = BeautifulSoup(table, 'html.parser')  # 解析表單
    tb = soup.find_all('tr')  # 查找表內(nèi)tr標(biāo)簽
    tb_p = []  # 各城市名字
    tb_ = []
    tb_1 = [[], [], [], [], []]
    for row in tb:
        row_text = row.text
        # 查找第一個(gè)數(shù)字的索引
        ret1 = re.search('\d', row_text).start()
        # 將各個(gè)城市寫(xiě)入列表中
        tb_p.append(row_text[:ret1])
        # 截?cái)嘀恋谝粋€(gè)數(shù)字
        row_text = row_text[ret1:]
        for i in range(5):
            try:
                # 順序查找點(diǎn)
                ret = row_text.find('.')
                # 將第一年年數(shù)據(jù)保存到列表中
                tb_.append(row_text[:ret + 3])
                # 刪去第一年的數(shù)據(jù)
                row_text = row_text[ret + 3:]
            except:
                break
    for i in range(5):
        for j in range(i, len(tb_), 5):
            try:
                tb_1[i].append(tb_[j])
            except:
                break
    data = {'地區(qū)': tb_p,
            '2019年': tb_1[0],
            '2018年': tb_1[1],
            '2017年': tb_1[2],
            '2016年': tb_1[3],
            '2015年': tb_1[4],
            }
    dataframe = pd.DataFrame(data)
    dataframe.to_csv('City_data.csv', index=False, sep=',', encoding='utf-8-sig')
    driver.close()
'''爬取2019年各省GDP'''
def driver2(url2):
    driver = webdriver.Chrome()
    driver.get(url2)
    time.sleep(3)
    xRdata = '//*[@id="details-button"]'  # 點(diǎn)擊高級(jí)
    ActionChains(driver).click(driver.find_element_by_xpath(xRdata)).perform()
    xRdata = '//*[@id="proceed-link"]'  # 點(diǎn)擊
    ActionChains(driver).click(driver.find_element_by_xpath(xRdata)).perform()
    time.sleep(3)
    xRdata = '//*[@id="nav"]/ul/li[6]/a'  # 地區(qū)數(shù)據(jù)
    ActionChains(driver).click(driver.find_element_by_xpath(xRdata)).perform()
    time.sleep(3)
    xPdata = '//*[@id="menuE01"]/li[3]/a'  # 分省年度數(shù)據(jù)
    ActionChains(driver).click(driver.find_element_by_xpath(xPdata)).perform()
    time.sleep(4)  # 加載頁(yè)面，停留時(shí)間長(zhǎng)一些
    xTable = '//*[@class="table-container"]/div/div[3]/table/tbody'  # 數(shù)據(jù)表
    table = driver.find_element_by_xpath(xTable).get_attribute('innerHTML')
    soup = BeautifulSoup(table, 'html.parser')  # 解析表單
    tb = soup.find_all('tr')  # 查找表內(nèi)tr標(biāo)簽
    tb_p = []  # 各省市名字
    tb_ = []
    tb_1 = [[], [], [], [], [], [], [], [], [], []]
    for row in tb:
        row_text = row.text
        # 查找第一個(gè)數(shù)字的索引
        ret1 = re.search('\d', row_text).start()
        # 將各個(gè)省份寫(xiě)入列表中
        tb_p.append(row_text[:ret1])
        # 截?cái)嘀恋谝粋€(gè)數(shù)字
        row_text = row_text[ret1:]
        for i in range(1):
            try:
                # 順序查找點(diǎn)
                ret = row_text.find('.')
                # 將第一年年數(shù)據(jù)保存到列表中
                tb_.append(row_text[:ret + 3])
                # 刪去第一年的數(shù)據(jù)
                row_text = row_text[ret + 3:]
            except:
                break
    for i in range(1):
        for j in range(i, len(tb_), 1):
            try:
                tb_1[i].append(tb_[j])
            except:
                break
    data = {'Province': tb_p,
            'GDP': tb_1[0],
            }
    dataframe = pd.DataFrame(data)
    dataframe.to_csv('2019_Province_GDP.csv', index=False, sep=',', encoding='utf-8-sig')
    driver.close()
'''爬取豆瓣電影Top250'''
wb = openpyxl.Workbook()    # 創(chuàng)建工作簿對(duì)象
sheet = wb.active           # 獲取工作簿的活動(dòng)表
sheet.title = "movie"       # 工作簿重命名
sheet.append(["排名", "電影名", "導(dǎo)演和主演", "上映時(shí)間", "上映地區(qū)", "電影類(lèi)型", "評(píng)分", "評(píng)價(jià)人數(shù)", "引言"])
def get_movie(urls):
    rank = 1
    for url in urls:
        res = requests.get(url, headers=headers).text
        html = lxml.etree.HTML(res)
        # 先xpath定位提取到每個(gè)頁(yè)面的所有l(wèi)i標(biāo)簽
        lis = html.xpath('//*[@id="content"]/div/div[1]/ol/li')
        # 每個(gè)li標(biāo)簽里有每部電影的基本信息
        for li in lis:
            name = li.xpath('.//div[@class="hd"]/a/span[1]/text()')[0]
            director_actor = li.xpath('.//div[@class="bd"]/p/text()')[0].strip()
            info = li.xpath('.//div[@class="bd"]/p/text()')[1].strip()
            # 按"/"切割成列表
            _info = info.split("/")
            # 得到 上映時(shí)間  上映地區(qū)  電影類(lèi)型信息   去除兩端多余空格
            time, area, genres = _info[0].strip(), _info[1].strip(), _info[2].strip()
            # print(time, area, genres)
            rating_score = li.xpath('.//div[@class="star"]/span[2]/text()')[0]
            rating_num = li.xpath('.//div[@class="star"]/span[4]/text()')[0]
            quote = li.xpath('.//p[@class="quote"]/span/text()')
            # 最后一頁(yè)有部電影  九品芝麻官  沒(méi)有一句話引言  加條件判斷  防止報(bào)錯(cuò)
            if len(quote) == 0:
                quote = None
            else:
                quote = quote[0]
            sheet.append([rank, name, director_actor, time, area, genres, rating_score, rating_num, quote])
            rank += 1
    # 保存到文件
    wb.save("movie.xlsx")
'''豆瓣電影Top250'''
index_url = 'https://movie.douban.com/top250'
def get_html(url):
    html = requests.get(url, headers=headers).text
    return html
def create_list(html):
    soup = BeautifulSoup(html, 'lxml')
    movie_names = []
    movie_info = []
    for t in soup.find_all('div', 'hd'):
        name = t.find('span', 'title').get_text()
        movie_names.append(name)
    for t in soup.find_all('div', 'info'):
        info = t.find('p').get_text().replace(' ','')
        movie_info.append(info)
        next_page = soup.find('span', 'next').find('a')
    if next_page:
         return movie_names, movie_info, index_url + next_page['href']
    else:
         return movie_names, movie_info, None
def main():
    order = 1
    url = index_url
    with codecs.open('top250.txt', 'wb', encoding='utf-8') as f:
        while url:
            html = get_html(url)
            names, info, url = create_list(html)
            for n in range(25):
                f.write('Top ' + str(order) + ' ' + names[n] + '\r\n')
                f.write(info[n] + '\r\n')
                order = order+1
if __name__ == "__main__":
    # 列表推導(dǎo)式得到url列表  10頁(yè)的電影信息  Top250
    url_list = ["https://movie.douban.com/top250?start={}&filter=".format(i * 25) for i in range(10)]
    driver1(url1)
    driver2(url2)
    get_movie(url_list)
    main()

import pandas as pd
import re
import matplotlib.pyplot as plt
import collections
import seaborn as sns
from stylecloud import gen_stylecloud
import jieba
import squarify
from pyecharts import Pie
sns.set() #恢復(fù)seaborn的默認(rèn)主題
# 中文亂碼和坐標(biāo)軸負(fù)號(hào)的處理
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False
'''條形圖'''
GDP = pd.read_csv('2019_Province_GDP.csv')
# 設(shè)置繪圖風(fēng)格
plt.style.use('ggplot')
# 繪制條形圖
plt.bar(x= range(GDP.shape[0]), # 指定條形圖x軸的刻度值
        height = GDP.GDP, # 指定條形圖y軸的數(shù)值
        tick_label = GDP.Province, # 指定條形圖x軸的刻度標(biāo)簽
        color = 'steelblue', # 指定條形圖的填充色
       )
# 添加y軸的標(biāo)簽
plt.ylabel('GDP(萬(wàn)億)')
# 添加條形圖的標(biāo)題
plt.title('2019年各省份GDP分布')
# 為每個(gè)條形圖添加數(shù)值標(biāo)簽
for x,y in enumerate(GDP.GDP):
    plt.text(x,y+0.1,'%s' %round(y,1),ha='center')
# 顯示圖形
plt.savefig('test1.png')
plt.show()
'''餅圖'''
# 構(gòu)造數(shù)據(jù)
edu = [0.2515,0.3724,0.3336,0.0368,0.0057]
labels = ['中專(zhuān)','大專(zhuān)','本科','碩士','其他']
explode = [0,0.1,0,0,0]  # 生成數(shù)據(jù)，用于突出顯示大專(zhuān)學(xué)歷人群
colors=['#9999ff','#ff9999','#7777aa','#2442aa','#dd5555']  # 自定義顏色
# 將橫、縱坐標(biāo)軸標(biāo)準(zhǔn)化處理，確保餅圖是一個(gè)正圓，否則為橢圓
plt.axes(aspect='equal')
# 繪制餅圖
plt.pie(x = edu, # 繪圖數(shù)據(jù)
        explode=explode, # 突出顯示大專(zhuān)人群
        labels=labels, # 添加教育水平標(biāo)簽
        colors=colors, # 設(shè)置餅圖的自定義填充色
        autopct='%.1f%%', # 設(shè)置百分比的格式，這里保留一位小數(shù)
        pctdistance=0.8,  # 設(shè)置百分比標(biāo)簽與圓心的距離
        labeldistance = 1.1, # 設(shè)置教育水平標(biāo)簽與圓心的距離
        startangle = 180, # 設(shè)置餅圖的初始角度
        radius = 1.2, # 設(shè)置餅圖的半徑
        counterclock = False, # 是否逆時(shí)針，這里設(shè)置為順時(shí)針?lè)较?
        wedgeprops = {'linewidth': 1.5, 'edgecolor':'green'},# 設(shè)置餅圖內(nèi)外邊界的屬性值
        textprops = {'fontsize':10, 'color':'black'}, # 設(shè)置文本標(biāo)簽的屬性值
        )
# 添加圖標(biāo)題
plt.title('失信用戶(hù)的受教育水平分布')
# 顯示圖形
plt.savefig('test2.png')
plt.show()
'''箱線圖'''
data=pd.read_csv('City_data.csv')
#箱線圖
data.boxplot()
plt.savefig('test3.png')
plt.show()
'''樹(shù)地圖'''
# 創(chuàng)建數(shù)據(jù)
name = ['國(guó)內(nèi)增值稅', '國(guó)內(nèi)消費(fèi)稅', '企業(yè)所得稅', '個(gè)人所得稅',
        '進(jìn)口增值稅、消費(fèi)稅', '出口退稅', '城市維護(hù)建設(shè)稅',
        '車(chē)輛購(gòu)置稅', '印花稅', '資源稅', '土地和房稅', '車(chē)船稅煙葉稅等']
income = [3908, 856, 801, 868, 1361, 1042, 320, 291, 175, 111, 414, 63]
# 繪圖
colors = ['steelblue', '#9999ff', 'red', 'indianred',
          'green', 'yellow', 'orange']
plot = squarify.plot(sizes=income,  # 指定繪圖數(shù)據(jù)
                     label=name,  # 指定標(biāo)簽
                     color=colors,  # 指定自定義顏色
                     alpha=0.6,  # 指定透明度
                     value=income,  # 添加數(shù)值標(biāo)簽
                     edgecolor='white',  # 設(shè)置邊界框?yàn)榘咨?
                     linewidth=3  # 設(shè)置邊框?qū)挾葹?
                     )
# 設(shè)置標(biāo)簽大小
plt.rc('font', size=8)
# 設(shè)置標(biāo)題大小
plot.set_title('2020年12月中央財(cái)政收支情況', fontdict={'fontsize': 15})
# 去除坐標(biāo)軸
plt.axis('off')
# 去除上邊框和右邊框刻度
plt.tick_params(top='off', right='off')
# 顯示圖形
plt.savefig('test4.png')
plt.show()
'''玫瑰圖'''
attr =["碳酸飲料", "其他", "綠茶", "礦泉水", "果汁"]
v1 =[6, 2, 7, 6,1]
v2 =[9, 6, 4, 4,5]
pie =Pie("玫瑰圖", title_pos='center', width=800)
pie.add("男", attr, v1, center=[25, 50], is_random=True, radius=[20, 50], rosetype='radius')
pie.add("女", attr, v2, center=[75, 50], is_random=True, radius=[20, 50], rosetype='area',
        is_legend_show=False, is_label_show=True)
pie.show_config()
pie.render('玫瑰圖.html')
# 讀取數(shù)據(jù)
df = pd.read_excel("movie.xlsx",engine='openpyxl')
# print(type(df))    # <class 'pandas.core.frame.DataFrame'>
'''上映高分電影數(shù)量最多的年份Top10'''
show_time = list(df["上映時(shí)間"])
# 有上映時(shí)間數(shù)據(jù)是1961(中國(guó)大陸)這樣的  處理一下  字符串切片
show_time = [s[:4] for s in show_time]
show_time_count = collections.Counter(show_time)
# 取數(shù)量最多的前10  得到一個(gè)列表  里面每個(gè)元素是元組
# (年份, 數(shù)量)
show_time_count = show_time_count.most_common(10)
# 字典推導(dǎo)式
show_time_dic = {k: v for k, v in show_time_count}
# 按年份排序
show_time = sorted(show_time_dic)
# 年份對(duì)應(yīng)高分電影數(shù)量
counts = [show_time_dic[k] for k in show_time]
plt.figure(figsize=(9, 6), dpi=100)
# 繪制條形圖
plt.bar(show_time, counts, width=0.5, color="cyan")
# y軸刻度重新設(shè)置一下
plt.yticks(range(0, 16, 2))
# 添加描述信息
plt.xlabel("年份")
plt.ylabel("高分電影數(shù)量")
plt.title("上映高分電影數(shù)量最多的年份Top10", fontsize=15)
# 添加網(wǎng)格  網(wǎng)格的透明度  線條樣式
plt.grid(alpha=0.2, linestyle=":")
plt.savefig('test5.png')
plt.show()
'''國(guó)家或地區(qū)上榜電影數(shù)量最多的Top10'''
area = list(df['上映地區(qū)'])
sum_area = []
for x in area:
    x = x.split(" ")
    for i in x:
        sum_area.append(i)
area_count = collections.Counter(sum_area)
area_dic = dict(area_count)
area_count = [(k, v) for k, v in list(area_dic.items())]
# 按國(guó)家或地區(qū)上榜電影數(shù)量排序
area_count.sort(key=lambda k: k[1])
# 取國(guó)家或地區(qū)上榜電影數(shù)量最多的前十
area = [m[0] for m in area_count[-10:]]
nums = [m[1] for m in area_count[-10:]]
plt.figure(figsize=(9, 6), dpi=100)
# 繪制橫著的條形圖
plt.barh(area, nums, color='red')
# 添加描述信息
plt.xlabel('電影數(shù)量')
plt.title('國(guó)家或地區(qū)上榜電影數(shù)量最多的Top10')
plt.savefig('test6.png')
plt.show()
'''豆瓣電影Top250-評(píng)價(jià)人數(shù)Top10'''
name = list(df['電影名'])
ranting_num = list(df['評(píng)價(jià)人數(shù)'])
# (電影名, 評(píng)價(jià)人數(shù))
info = [(m, int(n.split('人')[0])) for m, n in list(zip(name, ranting_num))]
# 按評(píng)價(jià)人數(shù)排序
info.sort(key=lambda x: x[1])
# print(info)
name = [x[0] for x in info[-10:]]
ranting_num = [x[1] for x in info[-10:]]
plt.figure(figsize=(12, 6), dpi=100)
# 繪制橫著的條形圖
plt.barh(name, ranting_num, color='cyan', height=0.4)
# 添加描述信息
plt.xlabel('評(píng)價(jià)人數(shù)')
plt.title('豆瓣電影Top250-評(píng)價(jià)人數(shù)Top10')
plt.savefig('test7.png')
plt.show()
'''豆瓣電影Top250評(píng)分-排名的散點(diǎn)分布'''
# 豆瓣電影Top250  排名  評(píng)分  散點(diǎn)圖   描述關(guān)系
rating = list(df["排名"])
rating_score = list(df["評(píng)分"])
plt.figure(figsize=(9, 6), dpi=100)
# 繪制散點(diǎn)圖  設(shè)置點(diǎn)的顏色
plt.scatter(rating_score, rating, c='r')
# 添加描述信息  設(shè)置字體大小
plt.xlabel("評(píng)分", fontsize=12)
plt.ylabel("排名", fontsize=12)
plt.title("豆瓣電影Top250評(píng)分-排名的散點(diǎn)分布", fontsize=15)
# 添加網(wǎng)格  網(wǎng)格的透明度  線條樣式
plt.grid(alpha=0.5, linestyle=":")
plt.savefig('test8.png')
plt.show()
'''豆瓣電影Top250詞云'''
# 讀取數(shù)據(jù)
with open('top250.txt', encoding='utf-8') as f:
    data = f.read()
# 文本預(yù)處理  去除一些無(wú)用的字符   只提取出中文出來(lái)
new_data = re.findall('[\u4e00-\u9fa5]+', data, re.S)
new_data = " ".join(new_data)
# 文本分詞
seg_list_exact = jieba.cut(new_data, cut_all=False)
result_list = []
with open('top250.txt', encoding='utf-8') as f:
    con = f.readlines()
    stop_words = set()
    for i in con:
        i = i.replace("\n", "")   # 去掉讀取每一行數(shù)據(jù)的\n
        stop_words.add(i)
for word in seg_list_exact:
    # 設(shè)置停用詞并去除單個(gè)詞
    if word not in stop_words and len(word) > 1:
        result_list.append(word)
print(result_list)
gen_stylecloud(
    text=' '.join(result_list),
    size=500,
    collocations=False,
    font_path=r'C:\Windows\Fonts\msyh.ttc',
    output_name='test9.png',
    icon_name='fas fa-video',
    palette='colorbrewer.qualitative.Dark2_7'
)