[badges.maintenance]
status = "as-is"
[dependencies.ahash]
default-features = false
features = ["std"]
version = "0.8"
[dependencies.aho-corasick]
optional = true
version = "1"
[dependencies.async-openai]
optional = true
version = "0.26"
[dependencies.async-trait]
optional = true
version = "0.1"
[dependencies.async_job]
optional = true
version = "0.1"
[dependencies.auto_encoder]
version = "0.1"
[dependencies.base64]
optional = true
version = "0.22"
[dependencies.bytes]
features = ["serde"]
version = "1"
[dependencies.case_insensitive_string]
features = ["compact", "serde"]
version = "0.2"
[dependencies.chrono]
optional = true
version = "0.4"
[dependencies.const_format]
optional = true
version = "0.2"
[dependencies.cron]
optional = true
version = "0.12"
[dependencies.cssparser]
version = "0.31.2"
[dependencies.ego-tree]
version = "0.10"
[dependencies.fast_html5ever]
version = "0.26.6"
[dependencies.fastrand]
optional = true
version = "2"
[dependencies.flexbuffers]
optional = true
version = "2"
[dependencies.hashbrown]
default-features = true
version = "0.15"
[dependencies.http]
optional = true
version = "1"
[dependencies.http-cache]
default-features = false
optional = true
version = "0.20"
[dependencies.http-cache-reqwest]
default-features = false
optional = true
version = "0.15"
[dependencies.http-cache-semantics]
optional = true
version = "2"
[dependencies.httpdate]
optional = true
version = "1"
[dependencies.itertools]
optional = true
version = "0.13"
[dependencies.lazy_static]
version = "1"
[dependencies.log]
version = "0.4"
[dependencies.lol_html]
version = "2"
[dependencies.moka]
features = ["future"]
optional = true
version = "0.12"
[dependencies.num_cpus]
version = "1"
[dependencies.percent-encoding]
version = "2"
[dependencies.phf]
version = "0.11"
[dependencies.phf_codegen]
version = "0.11"
[dependencies.quick-xml]
features = ["serde", "serialize", "async-tokio"]
version = "0.37"
[dependencies.rand]
optional = true
version = "0.8"
[dependencies.regex]
version = "1"
[dependencies.reqwest-middleware]
default-features = false
optional = true
version = "0.4"
[dependencies.selectors]
version = "0.25.0"
[dependencies.serde]
features = ["derive"]
optional = true
version = "1"
[dependencies.serde_json]
optional = true
version = "1"
[dependencies.serde_regex]
optional = true
version = "1"
[dependencies.sitemap]
optional = true
version = "0.4"
[dependencies.smallvec]
version = "1"
[dependencies.spider_chrome]
features = ["bytes", "stream"]
optional = true
version = "2"
[dependencies.sqlx]
features = ["runtime-tokio", "sqlite"]
optional = true
version = "0.8"
[dependencies.statrs]
optional = true
version = "0.17"
[dependencies.string-interner]
default-features = false
features = ["std", "inline-more", "backends"]
optional = true
version = "0.18"
[dependencies.string_concat]
version = "0.0.1"
[dependencies.strum]
features = ["derive"]
version = "0.26"
[dependencies.sysinfo]
default-features = false
features = ["system"]
optional = true
version = "0.33"
[dependencies.tendril]
version = "0.4.3"
[dependencies.tiktoken-rs]
optional = true
version = "0.5"
[dependencies.tokio-stream]
version = "0.1"
[dependencies.tracing]
default-features = false
features = ["std"]
optional = true
version = "0.1"
[dependencies.ua_generator]
optional = true
version = "^0.5"
[dependencies.url]
version = "2"
[features]
adblock = ["chrome", "spider_chrome/adblock"]
balance = ["dep:sysinfo"]
cache = ["cache_request", "http-cache-reqwest/manager-cacache"]
cache_chrome_hybrid = ["cache_request", "chrome", "dep:http-cache-semantics", "dep:http-cache", "dep:http"]
cache_mem = ["cache_request", "http-cache-reqwest/manager-moka"]
cache_openai = ["dep:moka"]
cache_request = ["dep:reqwest-middleware", "dep:http-cache-reqwest"]
chrome = ["dep:spider_chrome", "dep:base64"]
chrome_cpu = ["chrome"]
chrome_headed = ["chrome"]
chrome_headless_new = ["chrome"]
chrome_intercept = ["chrome"]
chrome_screenshot = ["chrome"]
chrome_stealth = ["chrome"]
chrome_store_page = ["chrome", "serde"]
control = []
cookies = ["reqwest/cookies"]
cowboy = []
cron = ["dep:async_job", "dep:chrono", "dep:cron", "dep:async-trait"]
decentralized = ["serde", "flexbuffers"]
decentralized_headers = ["dep:const_format", "dep:itertools"]
default = ["sync", "reqwest_native_tls_native_roots", "disk_native_tls", "cookies", "ua_generator", "encoding", "string_interner_buffer_backend", "balance"]
disk = ["dep:sqlx", "dep:aho-corasick"]
disk_aws = ["disk", "sqlx/tls-rustls-aws-lc-rs"]
disk_native_tls = ["disk", "sqlx/runtime-tokio-native-tls"]
encoding = []
flexbuffers = ["dep:flexbuffers"]
fs = ["tokio/fs"]
full_resources = []
glob = ["dep:itertools"]
headers = ["dep:httpdate"]
inline-more = []
jemalloc = ["tikv-jemallocator"]
openai = ["chrome", "serde", "chrome_intercept", "dep:async-openai", "dep:tiktoken-rs", "dep:serde_json"]
openai_slim_fit = []
real_browser = ["dep:statrs", "dep:rand"]
regex = []
remote_addr = []
reqwest_hickory_dns = ["reqwest/hickory-dns"]
reqwest_json = ["reqwest/json"]
reqwest_multipart = ["reqwest/multipart"]
reqwest_native_tls = ["reqwest/native-tls"]
reqwest_native_tls_alpn = ["reqwest/native-tls-alpn"]
reqwest_native_tls_manual_roots = ["reqwest/rustls-tls-manual-roots"]
reqwest_native_tls_native_roots = ["reqwest/rustls-tls-native-roots"]
reqwest_native_tls_vendored = ["reqwest/native-tls-vendored"]
reqwest_native_tls_webpki_roots = ["reqwest/rustls-tls-webpki-roots"]
reqwest_rustls_tls = ["reqwest/rustls-tls"]
serde = ["dep:serde", "hashbrown/serde", "string-interner/serde", "dep:serde_regex", "smallvec/serde"]
sitemap = ["dep:sitemap"]
smart = ["chrome", "dep:rand", "chrome_intercept", "dep:aho-corasick"]
socks = ["reqwest/socks"]
spoof = ["dep:fastrand"]
string_interner_bucket_backend = ["dep:string-interner"]
string_interner_buffer_backend = ["dep:string-interner"]
string_interner_string_backend = ["dep:string-interner"]
sync = ["tokio/sync"]
time = []
tokio_io_std = ["tokio/io-std"]
tracing = ["tokio/tracing", "dep:tracing"]
ua_generator = ["dep:ua_generator"]
[lib]
name = "spider"
path = "src/lib.rs"
[package]
authors = ["j-mendez <jeff@spider.cloud>"]
autobenches = false
autobins = false
autoexamples = false
autotests = false
build = false
categories = ["web-programming", "command-line-utilities"]
description = "The fastest web crawler written in Rust."
documentation = "https://docs.rs/spider"
edition = "2018"
keywords = ["crawler", "spider"]
license = "MIT"
name = "spider"
readme = "README.md"
repository = "https://github.com/spider-rs/spider"
resolver = "2"
version = "2.22.8"
[package.metadata.docs.rs]
cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]
[target.'cfg(all(not(windows), not(target_os = "android"), not(target_env = "musl")))'.dependencies.tikv-jemallocator]
optional = true
version = "0.6"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.fastrand]
optional = true
version = "2"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.reqwest]
features = ["brotli", "gzip", "deflate", "zstd", "stream"]
version = "0.12"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.tokio]
default-features = false
features = ["macros", "time", "rt-multi-thread"]
version = "1"
[target.'cfg(target_arch = "wasm32")'.dependencies.fastrand]
features = ["js"]
optional = true
version = "2"
[target.'cfg(target_arch = "wasm32")'.dependencies.reqwest]
features = ["brotli", "gzip", "deflate", "stream"]
version = "0.12"
[target.'cfg(target_arch = "wasm32")'.dependencies.tokio]
default-features = false
features = ["macros", "time", "rt"]
version = "1"