pip install requests beautifulsoup4 pandas

import requests
from bs4 import BeautifulSoup
 
# 目標(biāo)網(wǎng)頁URL
url = 'https://example.com/news'  # 替換為實(shí)際的URL
 
# 發(fā)送HTTP請(qǐng)求獲取網(wǎng)頁內(nèi)容
response = requests.get(url)
response.raise_for_status()  # 檢查請(qǐng)求是否成功
 
# 使用Beautiful Soup解析HTML
soup = BeautifulSoup(response.text, 'html.parser')

# 提取新聞標(biāo)題、發(fā)布時(shí)間和正文內(nèi)容
articles = []
for article in soup.select('.news-article'):  # 假設(shè)新聞文章都有class="news-article"
    title = article.select_one('h2.title').text.strip()
    publish_time = article.select_one('.publish-time').text.strip()
    content = article.select_one('.content').text.strip()
    articles.append({
        'title': title,
        'publish_time': publish_time,
        'content': content
    })

import pandas as pd
 
# 將數(shù)據(jù)轉(zhuǎn)換為DataFrame
df = pd.DataFrame(articles)
 
# 打印前幾行數(shù)據(jù)查看
print(df.head())
 
# 數(shù)據(jù)清洗步驟
# 1. 去除字符串前后的空格（已在提取時(shí)處理）
# 2. 替換特殊字符（例如換行符為空格）
df['content'] = df['content'].str.replace('\n', ' ')
 
# 3. 刪除缺失值或無效數(shù)據(jù)（假設(shè)空標(biāo)題或空內(nèi)容的數(shù)據(jù)無效）
df = df.dropna(subset=['title', 'content'])
 
# 4. 統(tǒng)一時(shí)間格式（假設(shè)發(fā)布時(shí)間為"YYYY-MM-DD HH:MM:SS"格式）
# 這里我們假設(shè)發(fā)布時(shí)間已經(jīng)是字符串格式，且格式統(tǒng)一，如果需要轉(zhuǎn)換格式，可以使用pd.to_datetime()
# df['publish_time'] = pd.to_datetime(df['publish_time'], format='%Y-%m-%d %H:%M:%S')
 
# 打印清洗后的數(shù)據(jù)查看
print(df.head())

# 數(shù)據(jù)處理步驟
# 1. 提取發(fā)布日期的日期部分（如果需要）
# df['publish_date'] = df['publish_time'].dt.date
 
# 2. 統(tǒng)計(jì)每個(gè)發(fā)布日期的新聞數(shù)量（如果需要）
# daily_counts = df['publish_date'].value_counts().reset_index()
# daily_counts.columns = ['publish_date', 'count']
# print(daily_counts)
 
# 3. 根據(jù)關(guān)鍵詞過濾新聞（例如只保留包含"疫情"關(guān)鍵詞的新聞）
keyword = '疫情'
filtered_df = df[df['content'].str.contains(keyword, na=False, case=False)]
 
# 打印過濾后的數(shù)據(jù)查看
print(filtered_df.head())

# 保存數(shù)據(jù)為CSV文件
csv_file_path = 'cleaned_news_data.csv'
df.to_csv(csv_file_path, index=False, encoding='utf-8-sig')
 
# 保存數(shù)據(jù)為Excel文件
excel_file_path = 'cleaned_news_data.xlsx'
df.to_excel(excel_file_path, index=False, engine='openpyxl')

import requests
from bs4 import BeautifulSoup
import pandas as pd
 
# 目標(biāo)網(wǎng)頁URL（請(qǐng)?zhí)鎿Q為實(shí)際的URL）
url = 'https://example.com/news'
 
# 發(fā)送HTTP請(qǐng)求獲取網(wǎng)頁內(nèi)容
response = requests.get(url)
response.raise_for_status()
 
# 使用Beautiful Soup解析HTML
soup = BeautifulSoup(response.text, 'html.parser')
 
# 提取新聞標(biāo)題、發(fā)布時(shí)間和正文內(nèi)容
articles = []
for article in soup.select('.news-article'):  # 假設(shè)新聞文章都有class="news-article"
    title = article.select_one('h2.title').text.strip()
    publish_time = article.select_one('.publish-time').text.strip()
    content = article.select_one('.content').text.strip()
    articles.append({
        'title': title,
        'publish_time': publish_time,
        'content': content
    })
 
# 將數(shù)據(jù)轉(zhuǎn)換為DataFrame
df = pd.DataFrame(articles)
 
# 數(shù)據(jù)清洗步驟
df['content'] = df['content'].str.replace('\n', ' ')
df = df.dropna(subset=['title', 'content'])
 
# 數(shù)據(jù)處理步驟（示例：根據(jù)關(guān)鍵詞過濾新聞）
keyword = '疫情'
filtered_df = df[df['content'].str.contains(keyword, na=False, case=False)]
 
# 保存數(shù)據(jù)為CSV文件和Excel文件
csv_file_path = 'cleaned_news_data.csv'
excel_file_path = 'cleaned_news_data.xlsx'
df.to_csv(csv_file_path, index=False, encoding='utf-8-sig')
df.to_excel(excel_file_path, index=False, engine='openpyxl')
 
# 打印過濾后的數(shù)據(jù)查看
print(filtered_df.head())