import subprocess
import sys


print("📦 Installing system dependencies...")
subprocess.run(['apt-get', 'update', '-qq'], capture_output=True)
subprocess.run(['apt-get', 'install', '-y', '-qq',
               'libnss3', 'libnspr4', 'libatk1.0-0', 'libatk-bridge2.0-0',
               'libcups2', 'libdrm2', 'libxkbcommon0', 'libxcomposite1',
               'libxdamage1', 'libxfixes3', 'libxrandr2', 'libgbm1',
               'libasound2', 'libpango-1.0-0', 'libcairo2'], capture_output=True)
print("✅ System dependencies installed!")


print("\n📦 Installing Python packages...")
subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', 'crawl4ai', 'nest_asyncio', 'pydantic', '-q'])
print("✅ Python packages installed!")


print("\n📦 Installing Playwright browsers (this may take a minute)...")
subprocess.run([sys.executable, '-m', 'playwright', 'install', 'chromium'], capture_output=True)
subprocess.run([sys.executable, '-m', 'playwright', 'install-deps', 'chromium'], capture_output=True)
print("✅ Playwright browsers installed!")


import nest_asyncio
nest_asyncio.apply()


import asyncio
import json
from typing import List, Optional
from pydantic import BaseModel, Field


print("\n" + "="*60)
print("✅ INSTALLATION COMPLETE! Ready to crawl!")
print("="*60)


print("\n" + "="*60)
print("📖 PART 2: BASIC CRAWLING")
print("="*60)


from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode


async def basic_crawl():
   """The simplest possible crawl - fetch a webpage and get markdown."""
   print("\n🔍 Running basic crawl on example.com...")
  
   async with AsyncWebCrawler() as crawler:
       result = await crawler.arun(url="https://example.com")
      
       print(f"\n✅ Crawl successful: {result.success}")
       print(f"📄 Title: {result.metadata.get('title', 'N/A')}")
       print(f"📝 Markdown length: {len(result.markdown.raw_markdown)} characters")
       print(f"\n--- First 500 chars of markdown ---")
       print(result.markdown.raw_markdown[:500])
      
   return result


result = asyncio.run(basic_crawl())


print("\n" + "="*60)
print("⚙️ PART 3: CONFIGURED CRAWLING")
print("="*60)


async def configured_crawl():
   """Crawling with custom browser and crawler configurations."""
   print("\n🔧 Running configured crawl with custom settings...")
  
   browser_config = BrowserConfig(
       headless=True,
       verbose=True,
       viewport_width=1920,
       viewport_height=1080,
       user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
   )
  
   run_config = CrawlerRunConfig(
       cache_mode=CacheMode.BYPASS,
       word_count_threshold=10,
       page_timeout=30000,
       wait_until="networkidle",
       verbose=True
   )
  
   async with AsyncWebCrawler(config=browser_config) as crawler:
       result = await crawler.arun(
           url="https://httpbin.org/html",
           config=run_config
       )
      
       print(f"\n✅ Success: {result.success}")
       print(f"📊 Status code: {result.status_code}")
       print(f"\n--- Content Preview ---")
       print(result.markdown.raw_markdown[:400])
      
   return result


result = asyncio.run(configured_crawl())


print("\n" + "="*60)
print("📝 PART 4: MARKDOWN GENERATION")
print("="*60)


from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator


async def markdown_generation_demo():
   """Demonstrates raw vs fit markdown with content filtering."""
   print("\n🎯 Demonstrating markdown generation strategies...")
  
   browser_config = BrowserConfig(headless=True, verbose=False)
  
   run_config = CrawlerRunConfig(
       cache_mode=CacheMode.BYPASS,
       markdown_generator=DefaultMarkdownGenerator(
           content_filter=PruningContentFilter(
               threshold=0.4,
               threshold_type="fixed",
               min_word_threshold=20
           )
       )
   )
  
   async with AsyncWebCrawler(config=browser_config) as crawler:
       result = await crawler.arun(
           url="https://en.wikipedia.org/wiki/Web_scraping",
           config=run_config
       )
      
       raw_len = len(result.markdown.raw_markdown)
       fit_len = len(result.markdown.fit_markdown) if result.markdown.fit_markdown else 0
      
       print(f"\n📊 Markdown Comparison:")
       print(f"   Raw Markdown:  {raw_len:,} characters")
       print(f"   Fit Markdown:  {fit_len:,} characters")
       print(f"   Reduction:     {((raw_len - fit_len) / raw_len * 100):.1f}%")
      
       print(f"\n--- Fit Markdown Preview (first 600 chars) ---")
       print(result.markdown.fit_markdown[:600] if result.markdown.fit_markdown else "N/A")
      
   return result


result = asyncio.run(markdown_generation_demo())