import threading
import requests
from queue import Queue

class Spider:
    def __init__(self):
        self.urls = Queue()  # 待爬取的鏈接隊(duì)列
        self.results = []  # 存儲(chǔ)爬取結(jié)果的列表
        self.lock = threading.Lock()  # 線程鎖
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
    # 獲取鏈接列表
    def get_urls(self):
        # 這里可以從文件、數(shù)據(jù)庫(kù)、網(wǎng)頁(yè)等方式獲取待爬取的鏈接
        # 這里以一個(gè)示例鏈接列表作為例子
        urls = ['<https://www.example.com/page1>', '<https://www.example.com/page2>', '<https://www.example.com/page3>']
        for url in urls:
            self.urls.put(url)
 
    # 爬取頁(yè)面并處理結(jié)果
    def crawl(self):
        while not self.urls.empty():
            url = self.urls.get()
            try:
                response = requests.get(url, headers=self.headers)
                # 這里可以對(duì)response進(jìn)行解析，獲取需要的信息
                # 這里以抓取頁(yè)面title作為例子
                title = response.text.split('<title>')[1].split('</title>')[0]
                self.results.append(title)
            except Exception as e:
                print(e)
            finally:
                self.urls.task_done()
 
    # 啟動(dòng)多線程爬蟲(chóng)
    def run(self, thread_num=10):
        self.get_urls()
        for i in range(thread_num):
            t = threading.Thread(target=self.crawl)
            t.start()
        self.urls.join()
 
        # 將結(jié)果寫(xiě)入文件或者數(shù)據(jù)庫(kù)
        with self.lock:
            with open('result.txt', 'a') as f:
                for result in self.results:
                    f.write(result + '\\n')