从零到接单 10：动态网页爬取——Selenium 与 Playwright
pythontutorial爬虫自动化接单SeleniumPlaywright
系列目录：本文是「从零到接单：Python 自动化与爬虫实战」系列的第 10 篇。上一篇我们用 BeautifulSoup 爬静态网页，但很多网站用 JavaScript 动态渲染内容——BeautifulSoup 拿到的 HTML 里根本没有数据。这篇我们用 Selenium 和 Playwright 控制真实浏览器，所见即所得。
一、静态 vs 动态网页
# 静态网页：HTML 里直接有数据
# requests.get(url) → HTML 里有 → BeautifulSoup 直接提取 ✅
# 动态网页：JavaScript 运行后才生成数据
# requests.get(url) → HTML 里只有 <div id="app"></div>
# 数据是 JS 渲染的，BS4 看不到！❌
判断一个网站是否动态：右键 → 查看网页源代码，看看数据在不在 HTML 里。不在的话就需要 Selenium 或 Playwright。
二、Selenium：老牌浏览器自动化
安装
pip install selenium webdriver-manager
webdriver-manager 帮你自动下载和管理浏览器驱动，不需要手动配置。
第一个 Selenium 脚本
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 自动下载并配置 ChromeDriver
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
# 打开网页
driver.get("https://httpbin.org/html")
# 获取页面源码（此时 JS 已执行完毕）
html = driver.page_source
print(html[:500])
# 获取标题
print(driver.title)
# 查找元素
element = driver.find_element(By.TAG_NAME, "h1")
print(element.text)
# 截图
driver.save_screenshot("screenshot.png")
# 关闭浏览器
driver.quit()
常用元素查找方式
from selenium.webdriver.common.by import By
# 8 种定位策略
driver.find_element(By.ID, "username")
driver.find_element(By.CLASS_NAME, "submit-btn")
driver.find_element(By.NAME, "email")
driver.find_element(By.TAG_NAME, "input")
driver.find_element(By.CSS_SELECTOR, ".container > .item")
driver.find_element(By.XPATH, "//div[@class='article']/h2")
driver.find_element(By.LINK_TEXT, "下一页")
driver.find_element(By.PARTIAL_LINK_TEXT, "下一")
# 找多个元素
elements = driver.find_elements(By.CLASS_NAME, "article")
for el in elements:
print(el.text)
页面交互
from selenium.webdriver.common.keys import Keys
# 输入文字
search_box = driver.find_element(By.NAME, "q")
search_box.send_keys("Python 爬虫")
# 点击按钮
search_button = driver.find_element(By.NAME, "btnK")
search_button.click()
# 清空输入框
search_box.clear()
# 键盘操作
search_box.send_keys(Keys.RETURN) # 回车
search_box.send_keys(Keys.ESCAPE) # ESC
# 下拉选择框
from selenium.webdriver.support.ui import Select
select = Select(driver.find_element(By.ID, "category"))
select.select_by_value("python")
select.select_by_visible_text("Python")
select.select_by_index(1)
等待元素加载（最重要！）
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# ❌ 不等待——元素可能还没加载出来
element = driver.find_element(By.ID, "content")
# ✅ 显式等待——等元素出现再操作
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "content"))
)
# 其他等待条件
EC.visibility_of_element_located(...) # 元素可见
EC.element_to_be_clickable(...) # 可点击
EC.text_to_be_present_in_element(...) # 文本出现
EC.alert_is_present() # 弹窗出现
EC.invisibility_of_element_located(...) # 元素消失
三、Selenium 实战：登录并爬取数据
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
import time
import csv
class SeleniumCrawler:
def __init__(self, headless=False):
"""初始化浏览器"""
options = webdriver.ChromeOptions()
if headless:
options.add_argument("--headless") # 无头模式（不显示浏览器窗口）
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
self.driver = webdriver.Chrome(
service=Service(ChromeDriverManager().install()),
options=options
)
self.wait = WebDriverWait(self.driver, 10)
def login(self, url, username, password):
"""登录网站"""
self.driver.get(url)
# 等待登录表单加载
username_input = self.wait.until(
EC.presence_of_element_located((By.NAME, "username"))
)
password_input = self.driver.find_element(By.NAME, "password")
submit_btn = self.driver.find_element(By.CSS_SELECTOR, "button[type='submit']")
# 填写并提交
username_input.send_keys(username)
password_input.send_keys(password)
submit_btn.click()
# 等待登录成功
time.sleep(2)
print(f"登录完成，当前 URL：{self.driver.current_url}")
def scroll_to_bottom(self):
"""滚动到页面底部（加载更多内容）"""
last_height = self.driver.execute_script("return document.body.scrollHeight")
while True:
# 滚动到底部
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2)
# 检查是否有新内容加载
new_height = self.driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
print("已滚动到底部")
def extract_items(self, item_selector, fields):
"""提取列表项"""
items = []
elements = self.driver.find_elements(By.CSS_SELECTOR, item_selector)
for el in elements:
item = {}
for field_name, css_selector in fields.items():
try:
item[field_name] = el.find_element(By.CSS_SELECTOR, css_selector).text
except:
item[field_name] = ""
items.append(item)
return items
def save_csv(self, data, filename="data.csv"):
"""保存为 CSV"""
if not data:
print("无数据")
return
with open(filename, "w", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=data[0].keys())
writer.writeheader()
writer.writerows(data)
print(f"已保存 {len(data)} 条数据到 {filename}")
def close(self):
self.driver.quit()
# 使用示例
if __name__ == "__main__":
crawler = SeleniumCrawler(headless=False)
try:
crawler.driver.get("https://httpbin.org/html")
print(f"页面标题：{crawler.driver.title}")
crawler.driver.save_screenshot("page.png")
print("截图已保存")
finally:
crawler.close()
四、Playwright：新一代浏览器自动化
Playwright 是微软出品的现代化方案，比 Selenium 更快、更稳定。
安装
pip install playwright
playwright install chromium # 安装 Chromium 浏览器
第一个 Playwright 脚本
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
# 启动浏览器
browser = p.chromium.launch(headless=False)
page = browser.new_page()
# 打开网页
page.goto("https://httpbin.org/html")
# 等待元素
page.wait_for_selector("h1")
# 获取内容
print(page.title())
print(page.content()[:500])
# 截图
page.screenshot(path="screenshot.png", full_page=True)
browser.close()
Playwright 常用操作
# ── 查找元素 ──
page.locator(".article") # CSS 选择器
page.locator("text=Python 爬虫") # 文本内容
page.locator("button:has-text('提交')") # 包含文本
page.locator("input[name='username']") # 属性选择
page.get_by_role("button", name="提交") # 按角色和名称
# ── 获取元素列表 ──
articles = page.locator(".article").all()
for article in articles:
title = article.locator("h2").inner_text()
print(title)
# ── 交互 ──
page.fill("input[name='q']", "Python") # 输入
page.click("button[type='submit']") # 点击
page.select_option("select#category", "python") # 选择下拉
# ── 等待 ──
page.wait_for_selector(".result", timeout=10000) # 等待元素出现
page.wait_for_load_state("networkidle") # 等待网络空闲
page.wait_for_timeout(2000) # 等待 2 秒（不推荐过度使用）
# ── 提取数据 ──
title = page.locator("h1").inner_text()
html = page.content()
url = page.url
# ── 反反爬 ──
page.set_extra_http_headers({
"User-Agent": "Mozilla/5.0 ...",
})
五、Playwright 实战：滚动加载爬虫
from playwright.sync_api import sync_playwright
import csv
def crawl_infinite_scroll(url, max_items=100):
"""爬取无限滚动页面"""
results = []
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto(url, wait_until="networkidle")
while len(results) < max_items:
# 获取当前页面的所有项目
items = page.locator(".item-card").all()
for item in items[len(results):]: # 只处理新增的
results.append({
"title": item.locator(".title").inner_text(),
"desc": item.locator(".desc").inner_text(),
"price": item.locator(".price").inner_text(),
})
print(f"已收集 {len(results)} 条数据...")
if len(results) >= max_items:
break
# 滚动到底部
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(2000) # 等待新内容加载
# 检查是否到底了（没有新内容加载）
before = len(results)
page.wait_for_timeout(1000)
after = page.evaluate("""() => {
return document.querySelectorAll('.item-card').length;
}""")
if after <= before:
print("已到页面底部")
break
browser.close()
return results
# 使用（实际使用时替换 URL 和选择器）
if __name__ == "__main__":
# data = crawl_infinite_scroll("https://example.com/products")
# for item in data:
# print(f"{item['title']} - {item['price']}")
print("请替换 URL 和选择器后使用")
六、Selenium vs Playwright：怎么选？
| 维度 | Selenium | Playwright | |------|----------|------------| | 浏览器支持 | Chrome/Firefox/Edge/Safari | Chromium/Firefox/WebKit | | 速度 | 较慢 | 更快 | | API 设计 | 传统 | 现代化，更简洁 | | 自动等待 | 需手动设置 | 内置自动等待 | | 社区生态 | 非常成熟 | 快速增长 | | 屏幕录制 | ❌ | ✅ 内置 trace viewer | | 对爬虫友好度 | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
建议：新项目优先选 Playwright，维护老项目或需要 Safari 时用 Selenium。
七、动态爬虫最佳实践
# 1. 设置合理的视窗大小（像真人）
page.set_viewport_size({"width": 1920, "height": 1080})
# 2. 随机延时
import random
import time
time.sleep(random.uniform(1, 3))
# 3. 随机鼠标移动（Playwright）
page.mouse.move(x, y)
# 4. 循环翻页而非暴力抓取
while True:
# ... 抓取当前页 ...
next_btn = page.locator("a.next-page")
if not next_btn.is_visible():
break
next_btn.click()
page.wait_for_load_state("networkidle")
time.sleep(random.uniform(1, 3))
总结
| 工具 | 适用场景 | 安装 |
|------|----------|------|
| requests + BS4 | 静态 HTML | pip install requests beautifulsoup4 |
| Selenium | 动态网页、兼容老项目 | pip install selenium webdriver-manager |
| Playwright | 动态网页、新项目首选 | pip install playwright |
下一篇，我们学习数据存储——抓回来的数据怎么存到 CSV、Excel、数据库里，形成可交付的成果。
练习：用 Playwright 打开一个带搜索功能的网站（如百度），输入搜索关键词，等待搜索结果出现，提取前 10 条结果的标题和链接。截图保存整个页面。
Comments
Sign in to leave a comment.