Scrapling深度测评：这个10.6k星的Python库如何让爬虫“抗揍”又聪明？

# 步骤1：安装核心库（建议用虚拟环境）
pip install "scrapling[fetchers]"

# 步骤2：安装浏览器和指纹库（必须！）
scrapling install

# 1. 安装Python 3.8+（官网下载）
# 2. 打开PowerShell（管理员）
python -m venv scrapling_env
.\scrapling_env\Scripts\activate

# 3. 安装Microsoft C++ Build Tools（某些依赖需要）
# 下载：https://visualstudio.microsoft.com/downloads/#build-tools

# 4. 安装Scrapling
pip install --upgrade pip
pip install "scrapling[all]"

# 5. 安装浏览器（如果权限不够，用管理员PowerShell）
scrapling install

# 1. 安装Homebrew（如果没有）
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"

# 2. 安装Python和依赖
brew install python3
pip3 install --upgrade pip

# 3. 创建虚拟环境
python3 -m venv ~/scrapling_env
source ~/scrapling_env/bin/activate

# 4. 安装Scrapling
pip install "scrapling[all]"

# 5. 安装浏览器（Apple Silicon可能需要Rosetta2）
scrapling install

# 如果浏览器安装失败，尝试：
softwareupdate --install-rosetta  # 仅M1/M2/M3芯片需要

# 1. 更新系统
sudo apt update && sudo apt upgrade -y

# 2. 安装Python和编译工具
sudo apt install -y python3 python3-pip python3-venv build-essential libssl-dev libffi-dev

# 3. 安装Playwright系统依赖
pip install playwright
playwright install-deps  # 这步很重要，装一堆.so库

# 4. 创建虚拟环境
python3 -m venv ~/scrapling_env
source ~/scrapling_env/bin/activate

# 5. 安装Scrapling
pip install --upgrade pip
pip install "scrapling[all]"

# 6. 安装浏览器
scrapling install

# 拉取镜像
docker pull pyd4vinci/scrapling:latest

# 运行交互式容器
docker run -it --rm pyd4vinci/scrapling bash

# 或者挂载本地代码
docker run -it --rm -v $(pwd):/app pyd4vinci/scrapling python /app/your_scraper.py

from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher

# 测试HTTP fetcher
resp = Fetcher.get('https://httpbin.org/get')
print(f"HTTP fetcher: {resp.status}")

# 测试Stealthy fetcher（如果能过Cloudflare就说明装好了）
stealth = StealthyFetcher.fetch('https://nowsecure.nl', solve_cloudflare=True)
print(f"Stealth bypass: {stealth.status}")

pip install "scrapling[ai]"

which mcp-server-scrapling
# 输出类似：/usr/local/bin/mcp-server-scrapling

{
  "mcpServers": {
    "scrapling": {
      "command": "/usr/local/bin/mcp-server-scrapling",
      "args": [],
      "env": {}
    }
  }
}

import json
from scrapling.fetchers import Fetcher
from scrapling.adaptivity import AdaptiveSelector

def scrape_products(url):
    """
    爬取商品列表，启用自适应模式
    """
    # 1. 发送请求（使用HTTP fetcher，速度快）
    print(f"正在获取页面: {url}")
    page = Fetcher.get(url, impersonate="chrome")
    
    if page.status != 200:
        print(f"页面获取失败: {page.status}")
        return []
    
    # 2. 使用自适应选择器定位商品容器
    # 注意adaptive=True：第一次爬会建立指纹，以后即使结构变了也能匹配
    product_containers = page.css(
        ".product-item, .goods-card, [class*=product]", 
        adaptive=True  # 开启自适应模式
    )
    
    print(f"找到 {len(product_containers)} 个商品容器")
    
    products = []
    
    # 3. 遍历每个商品，提取字段
    for idx, container in enumerate(product_containers):
        # 同样是自适应选择
        title_elem = container.css_first(".title, .name, h2", adaptive=True)
        price_elem = container.css_first(".price, .sale-price, .current-price", adaptive=True)
        img_elem = container.css_first("img", adaptive=True)
        
        # 提取文本和属性
        title = title_elem.get_all_text(strip=True) if title_elem else "无标题"
        
        # 价格处理：去除货币符号，保留数字
        price_text = price_elem.get_all_text(strip=True) if price_elem else "0"
        # 简单提取数字（实际场景可能需要正则）
        import re
        price_match = re.search(r'[\d,]+\.?\d*', price_text.replace(',', ''))
        price = price_match.group() if price_match else "0"
        
        # 图片链接
        img_url = img_elem.attrib.get('src', '') if img_elem else ''
        if img_url and not img_url.startswith('http'):
            img_url = 'https:' + img_url  # 补全协议
        
        product = {
            'index': idx + 1,
            'title': title,
            'price': float(price) if price != "0" else 0.0,
            'image_url': img_url
        }
        products.append(product)
    
    return products

def save_results(products, filename='products.json'):
    """
    保存结果到JSON文件
    """
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(products, f, ensure_ascii=False, indent=2)
    print(f"已保存 {len(products)} 条记录到 {filename}")

# 主程序
if __name__ == "__main__":
    # 测试用的目标网站（使用Quotes to Scrape的变体，模拟电商）
    # 实际使用时换成你要爬的电商URL
    test_url = "https://books.toscrape.com/"  # 图书网站，当作电商示例
    
    results = scrape_products(test_url)
    
    # 打印前3个结果
    for i, p in enumerate(results[:3]):
        print(f"{i+1}. {p['title']} - ￥{p['price']}")
        print(f"   图片: {p['image_url']}")
    
    save_results(results)

正在获取页面: https://books.toscrape.com/
找到 20 个商品容器
1. A Light in the Attic - ￥51.77
   图片: https://books.toscrape.com/media/cache/fe/72/fe72f0532301ec28892ae79a629a293c.jpg
2. Tipping the Velvet - ￥53.74
   图片: https://books.toscrape.com/media/cache/08/e9/08e94b8a2d1979c6ad3c9a07994acce2.jpg
3. Soumission - ￥50.10
   图片: https://books.toscrape.com/media/cache/ee/cf/eecfe9981bc183bb6c6e4c32f192aa3a.jpg
已保存 20 条记录到 products.json