Firecrawl MCP Server测评：让AI成为你的网页抓取专家

npm install -g firecrawl-mcp

env FIRECRAWL_API_KEY=fc-YOUR_API_KEY npx -y firecrawl-mcp

{
  "mcpServers": {
    "firecrawl-mcp": {
      "command": "npx",
      "args": ["-y", "firecrawl-mcp"],
      "env": {
        "FIRECRAWL_API_KEY": "YOUR_API_KEY"
      }
    }
  }
}

cmd /c "set FIRECRAWL_API_KEY=your-api-key && npx -y firecrawl-mcp"

# 为不同类型页面创建JSON结构模板
product_schema = {
    "product_name": "string",
    "description": "string", 
    "price": "string",
    "update_date": "string",
    "key_features": ["list"]
}

blog_schema = {
    "title": "string",
    "publish_date": "string",
    "author": "string", 
    "content_summary": "string",
    "tags": ["list"]
}

job_schema = {
    "job_title": "string",
    "department": "string",
    "location": "string",
    "requirements": ["list"]
}

import requests
import json
import os

# MCP服务器配置
MCP_SERVER_URL = "https://your-mcp-server-endpoint"  # 或使用本地MCP服务器
API_KEY = os.getenv("FIRECRAWL_API_KEY")

# 竞品URL列表
target_urls = [
    {"url": "https://competitor-a.com/product", "schema": product_schema},
    {"url": "https://competitor-b.com/blog", "schema": blog_schema},
    {"url": "https://competitor-c.com/careers", "schema": job_schema},
    # ... 更多URL
]

headers = {
    "Authorization": f"Bearer {API_KEY}",
    "Content-Type": "application/json"
}

def scrape_competitor_data():
    results = []
    
    for target in target_urls:
        payload = {
            "url": target["url"],
            "schema": target["schema"]
        }
        
        try:
            response = requests.post(MCP_SERVER_URL, json=payload, headers=headers)
            if response.status_code == 200:
                structured_data = response.json()
                results.append({
                    "url": target["url"],
                    "data": structured_data,
                    "status": "success"
                })
                print(f"✅ 成功抓取 {target['url']}")
            else:
                results.append({
                    "url": target["url"], 
                    "error": response.text,
                    "status": "failed"
                })
                print(f"❌ 抓取失败: {response.status_code}, {response.text}")
        except Exception as e:
            results.append({
                "url": target["url"],
                "error": str(e),
                "status": "error"
            })
            print(f"❌ 请求异常: {str(e)}")
    
    return results

# 执行抓取
if __name__ == "__main__":
    data = scrape_competitor_data()
    
    # 这里可以添加数据存储逻辑
    # save_to_database(data)
    
    print(f"抓取完成！成功: {len([d for d in data if d['status'] == 'success'])}个, 失败: {len([d for d in data if d['status'] != 'success'])}个")

# 从不同结构的电商页面提取统一格式的产品信息
extraction_schema = {
    "type": "object",
    "properties": {
        "product_name": {"type": "string"},
        "price": {"type": "string"},
        "description": {"type": "string"},
        "key_features": {"type": "array", "items": {"type": "string"}},
        "availability": {"type": "string"}
    },
    "required": ["product_name", "price"]
}

extraction_payload = {
    "url": "https://example-ecommerce.com/product/123",
    "schema": extraction_schema
}

# 调用firecrawl_extract功能
response = requests.post(
    f"{MCP_SERVER_URL}/extract", 
    json=extraction_payload, 
    headers=headers
)

batch_payload = {
    "urls": [
        "https://competitor-a.com/products",
        "https://competitor-b.com/services", 
        "https://competitor-c.com/solutions"
    ],
    "options": {
        "onlyMainContent": True,
        "format": "markdown"
    }
}

# 调用firecrawl_batch_scrape功能
response = requests.post(
    f"{MCP_SERVER_URL}/batch-scrape",
    json=batch_payload,
    headers=headers
)