AI Scraping & Shadow AI | Is Your Data Training Rivals?

Typical bot requests: 500-1000 req/min per bot
Bandwidth cost: $0.08 per GB transferred
CPU overhead: 15-25% additional processing
Result: $2000-5000/month extra in infrastructure

# Traditional robots.txt - INEFFECTIVE
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Example of evasive bot
headers_pool = [
    {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"},
    {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"},
    {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}
]

# Automatic identity rotation
def scrape_with_rotation(urls):
    for url in urls:
        headers = random.choice(headers_pool)
        proxy = get_residential_proxy()
        response = requests.get(url, headers=headers, proxies=proxy)

graph TD
    A[Employee] --> B[Copies sensitive data]
    B --> C[Pastes into ChatGPT/Claude]
    C --> D[AI processes and memorizes]
    D --> E[Data appears in future responses]
    E --> F[Competitors access information]

graph LR
    A[Bot Request] --> B[Azion Edge]
    B --> C[Behavioral Analysis]
    C --> D{Bot Score}
    D -->|High| E[Block/Challenge]
    D -->|Low| F[Forward to Origin]

    G[Origin Server] -.-> H[Zero bot traffic]
    H -.-> I[Reduced costs]

// Behavioral analysis at the Edge
export default async function botDetection(request) {
    const userAgent = request.headers.get('user-agent') || '';
    const clientIP = request.headers.get('cf-connecting-ip');
    const acceptLanguage = request.headers.get('accept-language');

    // Detect known AI bots
    const aiBotsPattern = /(GPTBot|ClaudeBot|CCBot|ChatGPT-User|Bard|Bing.*Bot)/i;

    if (aiBotsPattern.test(userAgent)) {
        return new Response('Access Denied - AI Scraping Not Allowed', {
            status: 403,
            headers: { 'content-type': 'text/plain' }
        });
    }

    // Continue to origin if not suspicious bot
    return fetch(request);
}

Traditional Firewall:
IP 192.168.1.1 + Port 80 = Allow/Block

Advanced AI Scraper:
Rotating IP + Human headers + Natural timing = Total bypass

# Basic configuration for ethical bots
User-agent: GPTBot
Disallow: api/
Disallow: admin/
Disallow: private/

User-agent: ClaudeBot
Disallow: /

# Honeypot to detect violations
User-agent: *
Disallow: trap/

{
  "name": "Block AI Scrapers",
  "criteria": [
    [
      {
        "variable": "${http_user_agent}",
        "operator": "matches",
        "conditional": "if",
        "input_value": "(GPTBot|ClaudeBot|CCBot|ChatGPT-User|Bard|Bing.*Bot)"
      }
    ]
  ],
  "behaviors": [
    {
      "name": "deny",
      "target": {
        "status_code": 403,
        "content_type": "text/plain",
        "content_body": "Access Denied - AI Scraping Not Allowed"
      }
    }
  ]
}

graph TD
    A[Employee] --> B[Request AI]
    B --> C[Internal AI Gateway]
    C --> D{Data Classification}
    D -->|Public| E[Allow ChatGPT]
    D -->|Sensitive| F[Internal LLM]
    D -->|Confidential| G[Block + Alert]

// Internal AI gateway on Azion Edge Functions
export default async function aiGateway(request) {
    try {
        const body = await request.json();
        const { prompt, classification } = body;

        // Check sensitive data using patterns
        const sensitivePatterns = [
            /\b\d{3}-\d{2}-\d{4}\b/, // SSN
            /\b\d{2}-\d{7}\b/, // Tax ID
            /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/ // Email
        ];

        const hasSensitiveData = sensitivePatterns.some(pattern =>
            pattern.test(prompt)
        );

        if (hasSensitiveData) {
            return new Response(JSON.stringify({
                error: "Sensitive data detected",
                suggestion: "Use internal model or remove personal information"
            }), {
                status: 403,
                headers: { 'content-type': 'application/json' }
            });
        }

        // Routing based on classification
        if (classification === 'public') {
            // Allow external AI use
            return new Response(JSON.stringify({
                status: "allowed",
                message: "Request approved for external AI"
            }), {
                headers: { 'content-type': 'application/json' }
            });
        } else {
            // Redirect to internal model
            return new Response(JSON.stringify({
                status: "redirect",
                message: "Use company's internal model"
            }), {
                headers: { 'content-type': 'application/json' }
            });
        }
    } catch (error) {
        return new Response('Invalid request', { status: 400 });
    }
}

// Advanced Bot Detection on Azion Edge
export default async function advancedBotDetection(request) {
    const userAgent = request.headers.get('user-agent') || '';
    const clientIP = request.headers.get('cf-connecting-ip');
    const referer = request.headers.get('referer') || '';
    const acceptHeader = request.headers.get('accept') || '';

    // Score based on multiple factors
    let suspicionScore = 0;

    // Check suspicious User-Agent
    const botPatterns = [
        new RegExp('GPTBot|ClaudeBot|CCBot|ChatGPT-User', 'i'),
        new RegExp('python-requests|curl|wget', 'i'),
        new RegExp('bot|crawler|spider|scraper', 'i')
    ];

    if (botPatterns.some(pattern => pattern.test(userAgent))) {
        suspicionScore += 0.4;
    }

    // Check absence of common browser headers
    if (!acceptHeader.includes('text/html')) {
        suspicionScore += 0.3;
    }

    // Check navigation patterns
    if (!referer && request.method === 'GET') {
        suspicionScore += 0.2;
    }

    // Action based on score
    if (suspicionScore >= 0.7) {
        // Log suspicious event
        console.log(JSON.stringify({
            timestamp: new Date().toISOString(),
            type: 'ai_scraper_blocked',
            ip: clientIP,
            userAgent: userAgent,
            score: suspicionScore,
            url: request.url
        }));

        return new Response('Access Denied - Automated Access Detected', {
            status: 403,
            headers: {
                'content-type': 'text/plain',
                'x-blocked-reason': 'ai-scraper-detection'
            }
        });
    }

    // Allow legitimate request
    return fetch(request);
}

export default {
  build: {
    entry: 'functions/bot-detection.js',
    preset: {
      name: 'javascript'
    }
  },
  rules: {
    request: [
      {
        name: 'Bot Detection',
        match: '.*',
        behavior: {
          runFunction: {
            path: './functions/bot-detection.js'
          }
        }
      }
    ]
  }
};

{
  "security_metrics": {
    "ai_threats_blocked": 15420,
    "shadow_ai_prevented": 89,
    "cost_savings": "$8,450/month",
    "false_positive_rate": "0.02%"
  }
}