requirements.txt

# Core dependencies
requests>=2.31.0
beautifulsoup4>=4.12.0
aiohttp>=3.9.0
lxml==4.9.2
html5lib==1.1
pydantic>=2.0,<3.0
pymongo>=4.6.0
redis>=5.0.0
boto3==1.26.123
docopt==0.6.2

# URL and DNS handling
dnspython==2.3.0
tldextract>=5.1.1
validators==0.20.0
robotexclusionrulesparser==1.7.1
urllib3==1.26.15

# Monitoring and metrics
prometheus-client>=0.19.0

# HTML processing
html2text==2020.1.16

# Async and concurrency
anyio==3.6.2
asyncio==3.4.3

# Utilities
python-dateutil>=2.8.2
pytz==2023.3
retry==0.9.2
cryptography==40.0.1
cachetools==5.3.0

# Added from the code block
openai>=1.12.0
gradio>=4.16.0
chardet==5.2.0 

# Dotenv
python-dotenv>=1.0.0

# New dependencies
mmh3>=4.0.0
httpx>=0.26.0