🐍 Python 爬虫全栈开发：从入门到反爬攻防

import requests

url = 'http://www.example.com'
res = requests.get(url)

# 常用属性
print(res.status_code) # 状态码
print(res.text)        # 文本数据 (HTML/JSON)
print(res.content)     # 字节数据 (图片/音频/视频)
print(res.encoding)    # 字符编码

# 编码修正示例
# res.encoding = 'utf-8' 

# 保存图片示例
img_data = res.content
with open('image.jpg', 'wb') as f:
    f.write(img_data)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)...'
}
res = requests.get(url, headers=headers)

# 带参数的搜索请求 (GET)
params = {'wd': 'Python爬虫'}
res = requests.get('https://www.baidu.com/s', params=params)

# 带数据的提交请求 (POST)
data = {'username': 'admin', 'password': '123'}
res = requests.post(url, data=data)

import re

text = '我叫麒麟，手机号是19120412166，座机是6013-8214006'

# 1. 匹配手机号 (1开头，11位数字)
print(re.findall(r'1\d{10}', text))

# 2. 贪婪与非贪婪 (重要)
html = '<p>多喝热水</p><p>好好学习</p>'
# (.*?) 是非贪婪匹配，只匹配标签内的内容
print(re.findall(r'<p>(.*?)</p>', html)) 

import requests
import re
import pymysql

class DoubanSpider:
    def __init__(self):
        self.url = 'https://movie.douban.com/top250?start={}'
        self.headers = {'User-Agent': 'Mozilla/5.0...'}
        # 数据库连接
        self.db = pymysql.connect(host='localhost', user='root', password='...', database='chairman')
        self.cursor = self.db.cursor()

    def get_data(self, url):
        res = requests.get(url, headers=self.headers)
        return res.text

    def parse_data(self, html):
        # 使用正则提取：标题、导演、主演
        # re.S 让 . 能匹配换行符
        pattern = r'<span class="title">(.*?)</span>.*?<p class="">(.*?)</p>'
        data = re.findall(pattern, html, re.S)
        self.save_data(data)

    def save_data(self, datas):
        for item in datas:
            title = item[0]
            content = item[1]
            # 进一步清洗数据...
            print(f"保存电影：{title}")
            # sql = "INSERT INTO..."
            # self.cursor.execute(sql)

    def run(self):
        for i in range(0, 250, 25):
            self.get_data(self.url.format(i))

if __name__ == '__main__':
    spider = DoubanSpider()
    spider.run()

from lxml import etree

# 1. 解析 HTML 字符串
html = etree.HTML(text)
# 2. 解析 HTML 文件
# html = etree.parse('index.html')

# 3. 提取数据
# 获取所有 li 标签下的 href 属性
link_list = html.xpath('//li/a/@href')

# 获取特定 class 的 div 下的文本
content = html.xpath('//div[@class="box"]/text()')

import openpyxl

# 创建工作簿
wb = openpyxl.Workbook()
sheet = wb.active
sheet.title = "爬虫数据"

# 添加一行数据
sheet.append(['标题', '链接', '价格'])

# 保存
wb.save('data.xlsx')

# 读取
wb2 = openpyxl.load_workbook('data.xlsx')
print(wb2.active['A1'].value)

from multiprocessing import Process

class MyProcess(Process):
    def __init__(self, name):
        super().__init__()
        self.name = name

    def run(self):
        print(f"{self.name} 开始工作...")

if __name__ == '__main__':
    p = MyProcess('爬虫1号')
    p.start()

import tkinter

root = tkinter.Tk()
root.geometry('500x400')
tkinter.Label(root, text="Hello Spider").pack()
root.mainloop()

from multiprocessing import Queue
q = Queue()
q.put("任务1")
data = q.get()

from concurrent.futures import ProcessPoolExecutor

def task(url):
    print(f"下载: {url}")

if __name__ == '__main__':
    with ProcessPoolExecutor(max_workers=4) as pool:
        urls = ['url1', 'url2', 'url3']
        pool.map(task, urls)

from selenium import webdriver
from selenium.webdriver.common.by import By

# 1. 启动浏览器
driver = webdriver.Chrome()

# 2. 访问页面
driver.get('http://www.baidu.com')

# 3. 定位元素 (新版写法)
el = driver.find_element(By.ID, 'kw')

# 4. 交互操作
el.send_keys('Python') # 输入
driver.find_element(By.ID, 'su').click() # 点击

# 5. 无头模式 (不显示界面)
options = webdriver.ChromeOptions()
options.add_argument('--headless')

# 6. JS 执行 (滚动页面/懒加载)
driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')

driver.quit()

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'content'))
)