spider 2.22.8

The fastest web crawler written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

[package]
name = "spider"
version = "2.22.8"
authors = [
    "j-mendez <jeff@spider.cloud>"
]
description = "The fastest web crawler written in Rust."
repository = "https://github.com/spider-rs/spider"
readme = "README.md"
keywords = ["crawler", "spider"]
categories = ["web-programming", "command-line-utilities"]
license = "MIT"
documentation = "https://docs.rs/spider"
edition = "2018"

[badges]
maintenance = { status = "as-is" }

[dependencies]
url = "2"
tokio-stream = "0.1"
hashbrown = { version = "0.15", default-features = true }
log = "0.4"
percent-encoding = "2"
regex = { version = "1" }
ua_generator = { version = "^0.5", optional = true }
string_concat = "0.0.1"
lazy_static = "1"
fast_html5ever = "0.26.6"
selectors = "0.25.0"
tendril = "0.4.3"
ahash = { version = "0.8", default-features = false, features = ["std"] }
cssparser = "0.31.2"
smallvec = "1"
ego-tree = "0.10"
num_cpus = "1"
bytes = { version = "1", features = ["serde"] }
serde = { version = "1", optional = true, features = ["derive"] }
flexbuffers = { version = "2", optional = true }
itertools = { version = "0.13", optional = true }
case_insensitive_string = { version = "0.2", features = ["compact", "serde"] }
sitemap = { version = "0.4", optional = true }
chrono = { version = "0.4", optional = true }
cron = { version = "0.12", optional = true }
async-trait = { version = "0.1", optional = true }
strum = { version = "0.26", features = ["derive"] }
async_job = { version = "0.1", optional = true }
reqwest-middleware = { version = "0.4", optional = true,  default-features = false }
http-cache-reqwest = { version = "0.15", optional = true, default-features = false }
const_format = { version = "0.2", optional = true }
async-openai = { version = "0.26", optional = true }
tiktoken-rs = { version = "0.5", optional = true }
lol_html = { version = "2" }
serde_json = { version = "1", optional = true }
quick-xml = { version = "0.37", features = ["serde", "serialize", "async-tokio"]}
moka = { version = "0.12", features = ["future"], optional = true }
fastrand = { version = "2", optional = true }
http-cache-semantics = { version = "2", optional = true }
http-cache = { version = "0.20", optional = true, default-features = false }
http = { version = "1", optional = true }
phf = "0.11"
phf_codegen = "0.11"
auto_encoder = { version = "0.1" }
base64 = { version = "0.22", optional = true }
string-interner = { version = "0.18", default-features = false, features = ["std", "inline-more", "backends"], optional = true }
httpdate = { version = "1", optional = true }
rand = { version = "0.8", optional = true }
serde_regex = { version = "1", optional = true }
statrs = { version = "0.17", optional = true }
aho-corasick = { version = "1", optional = true }
tracing = { version = "0.1", default-features = false, features = ["std"], optional = true }
sysinfo = { version = "0.33", default-features = false, features = ["system"], optional = true }
sqlx = { version = "0.8", features = [ "runtime-tokio", "sqlite" ], optional = true }

[dependencies.spider_chrome]
version = "2"
path = "../spider_chrome"
optional = true
features = [
    "bytes",
    "stream"
]

[target.'cfg(all(not(windows), not(target_os = "android"), not(target_env = "musl")))'.dependencies]
tikv-jemallocator = { version = "0.6", optional = true }

[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
tokio = { version = "1", default-features = false, features = [
    "macros",
    "time",
    "rt-multi-thread"
] }
fastrand = { version = "2", optional = true }
reqwest = { version = "0.12", features = [
    "brotli",
    "gzip",
    "deflate",
    "zstd",
    "stream",
] }

[target.'cfg(target_arch = "wasm32")'.dependencies]
tokio = { version = "1", default-features = false, features = [
    "macros",
    "time",
    "rt"
] }
fastrand = { version = "2", optional = true, features = ["js"] }
reqwest = { version = "0.12", features = [
    "brotli",
    "gzip",
    "deflate",
    "stream",
] }

[features]
default = ["sync", "reqwest_native_tls_native_roots", "disk_native_tls", "cookies", "ua_generator", "encoding", "string_interner_buffer_backend", "balance"]
disk = ["dep:sqlx", "dep:aho-corasick"]
disk_native_tls = ["disk", "sqlx/runtime-tokio-native-tls"]
disk_aws = ["disk", "sqlx/tls-rustls-aws-lc-rs"]
adblock = ["chrome", "spider_chrome/adblock"]
balance = ["dep:sysinfo"]
regex = []
glob = [ "dep:itertools"]
ua_generator = ["dep:ua_generator"]
jemalloc = ["tikv-jemallocator"]
decentralized = ["serde", "flexbuffers"]
control = []
time = []
sync = ["tokio/sync"]
flexbuffers = ["dep:flexbuffers"]
serde = ["dep:serde", "hashbrown/serde", "string-interner/serde", "dep:serde_regex", "smallvec/serde"]
fs = ["tokio/fs"]
full_resources = []
socks = ["reqwest/socks"]
reqwest_json = ["reqwest/json"]
sitemap = ["dep:sitemap"]
cache_request = ["dep:reqwest-middleware", "dep:http-cache-reqwest"]
cache = ["cache_request", "http-cache-reqwest/manager-cacache"]
cache_mem = ["cache_request", "http-cache-reqwest/manager-moka"]
cache_openai = ["dep:moka"]
cache_chrome_hybrid = ["cache_request", "chrome", "dep:http-cache-semantics", "dep:http-cache", "dep:http"]
chrome = ["dep:spider_chrome", "dep:base64"]
chrome_headed = ["chrome"]
chrome_cpu = ["chrome"]
chrome_stealth = ["chrome"]
chrome_screenshot = ["chrome"]
chrome_store_page = ["chrome", "serde"]
chrome_intercept = ["chrome"]
chrome_headless_new = ["chrome"]
cookies = ["reqwest/cookies"]
cron = ["dep:async_job", "dep:chrono", "dep:cron", "dep:async-trait"]
smart = ["chrome", "dep:rand", "chrome_intercept", "dep:aho-corasick"]
encoding = []
headers = ["dep:httpdate"]
remote_addr = []
real_browser = ["dep:statrs", "dep:rand"]
openai = ["chrome", "serde", "chrome_intercept", "dep:async-openai", "dep:tiktoken-rs", "dep:serde_json"]
openai_slim_fit = []
decentralized_headers = ["dep:const_format", "dep:itertools"]
spoof = ["dep:fastrand"]
reqwest_rustls_tls = ["reqwest/rustls-tls"]
reqwest_native_tls = ["reqwest/native-tls"]
reqwest_native_tls_alpn = ["reqwest/native-tls-alpn"]
reqwest_native_tls_vendored = ["reqwest/native-tls-vendored"]
reqwest_native_tls_manual_roots = ["reqwest/rustls-tls-manual-roots"]
reqwest_native_tls_webpki_roots = ["reqwest/rustls-tls-webpki-roots"]
reqwest_native_tls_native_roots = ["reqwest/rustls-tls-native-roots"]
reqwest_hickory_dns = ["reqwest/hickory-dns"]
reqwest_multipart = ["reqwest/multipart"]
tokio_io_std = ["tokio/io-std"]
tracing = ["tokio/tracing", "dep:tracing"]
string_interner_buffer_backend = ["dep:string-interner"]
string_interner_string_backend = ["dep:string-interner"]
string_interner_bucket_backend = ["dep:string-interner"]

# Allows full concurrency.
cowboy = []

# Enables usage of `#[inline]` on far more functions than by default in this
# crate. This may lead to a performance increase but often comes at a compile
# time cost.
inline-more = []

[package.metadata.docs.rs]
cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]