tokenizers 0.21.0

Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.
Documentation
Build #1565914 2024-11-27T13:31:10.733719+00:00
# rustc version
rustc 1.85.0-nightly (dff3e7ccd 2024-11-26)# docs.rs version
docsrs 0.6.0 (36c721fb 2024-11-06)# build log
[INFO] running `Command { std: "docker" "create" "-v" "/home/cratesfyi/workspace/builds/tokenizers-0.21.0/target:/opt/rustwide/target:rw,Z" "-v" "/home/cratesfyi/workspace/builds/tokenizers-0.21.0/source:/opt/rustwide/workdir:ro,Z" "-v" "/home/cratesfyi/workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/home/cratesfyi/workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "DOCS_RS=1" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "6442450944" "--cpus" "6" "--user" "1001:1001" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4a844ea9eb2546a2d2c7022eacef16ef2e8229c7fbb2c7d4d55a9ceca922f72d" "/opt/rustwide/cargo-home/bin/cargo" "+nightly" "rustdoc" "--lib" "-Zrustdoc-map" "--config" "build.rustdocflags=[\"--cfg\", \"docsrs\", \"-Z\", \"unstable-options\", \"--emit=invocation-specific\", \"--resource-suffix\", \"-20241126-1.85.0-nightly-dff3e7ccd\", \"--static-root-path\", \"/-/rustdoc.static/\", \"--cap-lints\", \"warn\", \"--extern-html-root-takes-precedence\"]" "--offline" "-Zunstable-options" "--config=doc.extern-map.registries.crates-io=\"https://docs.rs/{pkg_name}/{version}/i686-pc-windows-msvc\"" "-Zrustdoc-scrape-examples" "-j6" "--target" "i686-pc-windows-msvc", kill_on_drop: false }`
[INFO] [stderr] WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap.
[INFO] [stdout] ae26fc0c5d611e1e867b742f068d225d6b18c2cf01624b570e25d2cbe6e78e88
[INFO] running `Command { std: "docker" "start" "-a" "ae26fc0c5d611e1e867b742f068d225d6b18c2cf01624b570e25d2cbe6e78e88", kill_on_drop: false }`
[INFO] [stderr] warning: Rustdoc did not scrape the following examples because they require dev-dependencies: encode_batch, serialization
[INFO] [stderr]     If you want Rustdoc to scrape these examples, then add `doc-scrape-examples = true`
[INFO] [stderr]     to the [[example]] target configuration of at least one example.
[INFO] [stderr] warning: target filter specified, but no targets matched; this is a no-op
[INFO] [stderr]     Checking crossbeam-epoch v0.9.18
[INFO] [stderr]    Compiling onig_sys v69.8.1
[INFO] [stderr]     Checking rand_core v0.6.4
[INFO] [stderr]     Checking ppv-lite86 v0.2.20
[INFO] [stderr]     Checking aho-corasick v1.1.3
[INFO] [stderr]     Checking minimal-lexical v0.2.1
[INFO] [stderr] warning: onig_sys@69.8.1: GNU compiler is not supported for this target
[INFO] [stderr] warning: onig_sys@69.8.1: In file included from oniguruma/src/regenc.h:36,
[INFO] [stderr] warning: onig_sys@69.8.1:                  from oniguruma/src/regint.h:103,
[INFO] [stderr] warning: onig_sys@69.8.1:                  from oniguruma/src/regexec.c:36:
[INFO] [stderr] warning: onig_sys@69.8.1: /opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out/config.h:33:15: error: two or more data types in declaration specifiers
[INFO] [stderr] warning: onig_sys@69.8.1:    33 | #define gid_t int
[INFO] [stderr] warning: onig_sys@69.8.1:       |               ^~~
[INFO] [stderr] warning: onig_sys@69.8.1: /opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out/config.h:32:15: error: two or more data types in declaration specifiers
[INFO] [stderr] warning: onig_sys@69.8.1:    32 | #define uid_t int
[INFO] [stderr] warning: onig_sys@69.8.1:       |               ^~~
[INFO] [stderr] warning: onig_sys@69.8.1: In file included from oniguruma/src/regexec.c:36:
[INFO] [stderr] warning: onig_sys@69.8.1: oniguruma/src/regint.h:422:9: error: unknown type name 'uint32_t'
[INFO] [stderr] warning: onig_sys@69.8.1:   422 | typedef uint32_t  Bits;
[INFO] [stderr] warning: onig_sys@69.8.1:       |         ^~~~~~~~
[INFO] [stderr] error: failed to run custom build command for `onig_sys v69.8.1`
[INFO] [stderr] 
[INFO] [stderr] Caused by:
[INFO] [stderr]   process didn't exit successfully: `/opt/rustwide/target/debug/build/onig_sys-d3d2e1d4c5ac8ea8/build-script-build` (exit status: 1)
[INFO] [stderr]   --- stdout
[INFO] [stderr]   cargo:rerun-if-env-changed=RUSTONIG_DYNAMIC_LIBONIG
[INFO] [stderr]   cargo:rerun-if-env-changed=RUSTONIG_STATIC_LIBONIG
[INFO] [stderr]   cargo:rerun-if-env-changed=RUSTONIG_SYSTEM_LIBONIG
[INFO] [stderr]   OUT_DIR = Some(/opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out)
[INFO] [stderr]   OPT_LEVEL = Some(0)
[INFO] [stderr]   TARGET = Some(i686-pc-windows-msvc)
[INFO] [stderr]   cargo:rerun-if-env-changed=VCINSTALLDIR
[INFO] [stderr]   VCINSTALLDIR = None
[INFO] [stderr]   HOST = Some(x86_64-unknown-linux-gnu)
[INFO] [stderr]   cargo:rerun-if-env-changed=CC_i686-pc-windows-msvc
[INFO] [stderr]   CC_i686-pc-windows-msvc = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CC_i686_pc_windows_msvc
[INFO] [stderr]   CC_i686_pc_windows_msvc = None
[INFO] [stderr]   cargo:rerun-if-env-changed=TARGET_CC
[INFO] [stderr]   TARGET_CC = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CC
[INFO] [stderr]   CC = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CROSS_COMPILE
[INFO] [stderr]   CROSS_COMPILE = None
[INFO] [stderr]   RUSTC_LINKER = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CC_ENABLE_DEBUG_OUTPUT
[INFO] [stderr]   RUSTC_WRAPPER = None
[INFO] [stderr]   cargo:warning=GNU compiler is not supported for this target
[INFO] [stderr]   cargo:rerun-if-env-changed=CRATE_CC_NO_DEFAULTS
[INFO] [stderr]   CRATE_CC_NO_DEFAULTS = None
[INFO] [stderr]   DEBUG = Some(true)
[INFO] [stderr]   CARGO_CFG_TARGET_FEATURE = Some(fxsr,sse,sse2)
[INFO] [stderr]   cargo:rerun-if-env-changed=CFLAGS_i686-pc-windows-msvc
[INFO] [stderr]   CFLAGS_i686-pc-windows-msvc = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CFLAGS_i686_pc_windows_msvc
[INFO] [stderr]   CFLAGS_i686_pc_windows_msvc = None
[INFO] [stderr]   cargo:rerun-if-env-changed=TARGET_CFLAGS
[INFO] [stderr]   TARGET_CFLAGS = None
[INFO] [stderr]   cargo:rerun-if-env-changed=CFLAGS
[INFO] [stderr]   CFLAGS = None
[INFO] [stderr]   cargo:warning=In file included from oniguruma/src/regenc.h:36,
[INFO] [stderr]   cargo:warning=                 from oniguruma/src/regint.h:103,
[INFO] [stderr]   cargo:warning=                 from oniguruma/src/regexec.c:36:
[INFO] [stderr]   cargo:warning=/opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out/config.h:33:15: error: two or more data types in declaration specifiers
[INFO] [stderr]   cargo:warning=   33 | #define gid_t int
[INFO] [stderr]   cargo:warning=      |               ^~~
[INFO] [stderr]   cargo:warning=/opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out/config.h:32:15: error: two or more data types in declaration specifiers
[INFO] [stderr]   cargo:warning=   32 | #define uid_t int
[INFO] [stderr]   cargo:warning=      |               ^~~
[INFO] [stderr]   cargo:warning=In file included from oniguruma/src/regexec.c:36:
[INFO] [stderr]   cargo:warning=oniguruma/src/regint.h:422:9: error: unknown type name 'uint32_t'
[INFO] [stderr]   cargo:warning=  422 | typedef uint32_t  Bits;
[INFO] [stderr]   cargo:warning=      |         ^~~~~~~~
[INFO] [stderr] 
[INFO] [stderr]   --- stderr
[INFO] [stderr] 
[INFO] [stderr] 
[INFO] [stderr]   error occurred: Command "cc" "-O0" "-ffunction-sections" "-fdata-sections" "-g" "-fno-omit-frame-pointer" "-m32" "-I" "/opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out" "-I" "oniguruma/src" "-o" "/opt/rustwide/target/i686-pc-windows-msvc/debug/build/onig_sys-0a76f71cd5958e7f/out/c77b18e714869709-regexec.o" "-c" "oniguruma/src/regexec.c" with args cc did not execute successfully (status code exit status: 1).
[INFO] [stderr] 
[INFO] [stderr] 
[INFO] [stderr] warning: build failed, waiting for other jobs to finish...
[INFO] running `Command { std: "docker" "inspect" "ae26fc0c5d611e1e867b742f068d225d6b18c2cf01624b570e25d2cbe6e78e88", kill_on_drop: false }`
[INFO] running `Command { std: "docker" "rm" "-f" "ae26fc0c5d611e1e867b742f068d225d6b18c2cf01624b570e25d2cbe6e78e88", kill_on_drop: false }`
[INFO] [stdout] ae26fc0c5d611e1e867b742f068d225d6b18c2cf01624b570e25d2cbe6e78e88