pip install requests beautifulsoup4 selenium

import requests
from bs4 import BeautifulSoup

url = 'https://example.com'
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')

# 查找所有的div元素
divs = soup.find_all('div')
for div in divs:
    print(div.text)

pip install selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 初始化WebDriver
driver = webdriver.Chrome()

# 打開目標(biāo)網(wǎng)頁(yè)
url = 'https://example.com'
driver.get(url)

# 等待頁(yè)面加載完成
try:
    # 等待特定的元素出現(xiàn)
    element = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, 'comments'))
    )
finally:
    # 獲取頁(yè)面源代碼
    page_source = driver.page_source
    driver.quit()

# 解析頁(yè)面源代碼
soup = BeautifulSoup(page_source, 'html.parser')

# 查找所有的評(píng)論div
comment_divs = soup.find_all('div', class_='comment')
for comment in comment_divs:
    print(comment.text)

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 初始化WebDriver
driver = webdriver.Chrome()

# 打開目標(biāo)網(wǎng)頁(yè)
url = 'https://example.com'
driver.get(url)

# 等待按鈕出現(xiàn)
button = WebDriverWait(driver, 10).until(
    EC.element_to_be_clickable((By.ID, 'show-comments-button'))
)

# 點(diǎn)擊按鈕
button.click()

# 等待評(píng)論內(nèi)容出現(xiàn)
try:
    # 等待特定的元素出現(xiàn)
    element = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, 'comments'))
    )
finally:
    # 獲取頁(yè)面源代碼
    page_source = driver.page_source
    driver.quit()

# 解析頁(yè)面源代碼
soup = BeautifulSoup(page_source, 'html.parser')

# 查找所有的評(píng)論div
comment_divs = soup.find_all('div', class_='comment')
for comment in comment_divs:
    print(comment.text)

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

# 設(shè)置Chrome選項(xiàng)
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

# 初始化WebDriver
driver = webdriver.Chrome(options=chrome_options)

# 打開目標(biāo)網(wǎng)頁(yè)
url = 'https://example.com'
driver.get(url)

# ... 其他代碼 ...

import concurrent.futures
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup

def fetch_comments(url):
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    page_source = driver.page_source
    driver.quit()
    soup = BeautifulSoup(page_source, 'html.parser')
    comment_divs = soup.find_all('div', class_='comment')
    return [comment.text for comment in comment_divs]

urls = ['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3']

with concurrent.futures.ThreadPoolExecutor() as executor:
    results = list(executor.map(fetch_comments, urls))

for result in results:
    for comment in result:
        print(comment)

import pandas as pd

# 假設(shè)我們已經(jīng)抓取到了評(píng)論數(shù)據(jù)
comments = [
    {'text': 'Great product!', 'date': '2023-01-01'},
    {'text': 'Not so good.', 'date': '2023-01-02'},
    {'text': 'Excellent service!', 'date': '2023-01-03'}
]

# 將數(shù)據(jù)轉(zhuǎn)換為DataFrame
df = pd.DataFrame(comments)

# 清洗數(shù)據(jù)
df['date'] = pd.to_datetime(df['date'])
df['text'] = df['text'].str.strip()

print(df)

df.to_csv('comments.csv', index=False)

import sqlite3

conn = sqlite3.connect('comments.db')
df.to_sql('comments', conn, if_exists='replace', index=False)
conn.close()