use super::Worker;
use anyhow::{anyhow, bail, Context, Result};
use directories_next::ProjectDirs;
use log::{trace, warn};
use serde::{
de::{self, Deserializer},
Deserialize,
};
use std::fmt::Debug;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
use std::sync::Arc;
use std::time::Duration;
#[derive(serde_derive::Deserialize, Debug)]
#[serde(deny_unknown_fields)]
struct Config {
cache: CacheConfig,
}
#[derive(serde_derive::Deserialize, Debug, Clone)]
#[serde(deny_unknown_fields)]
pub struct CacheConfig {
enabled: bool,
directory: Option<PathBuf>,
#[serde(
default,
rename = "worker-event-queue-size",
deserialize_with = "deserialize_si_prefix"
)]
worker_event_queue_size: Option<u64>,
#[serde(rename = "baseline-compression-level")]
baseline_compression_level: Option<i32>,
#[serde(rename = "optimized-compression-level")]
optimized_compression_level: Option<i32>,
#[serde(
default,
rename = "optimized-compression-usage-counter-threshold",
deserialize_with = "deserialize_si_prefix"
)]
optimized_compression_usage_counter_threshold: Option<u64>,
#[serde(
default,
rename = "cleanup-interval",
deserialize_with = "deserialize_duration"
)]
cleanup_interval: Option<Duration>,
#[serde(
default,
rename = "optimizing-compression-task-timeout",
deserialize_with = "deserialize_duration"
)]
optimizing_compression_task_timeout: Option<Duration>,
#[serde(
default,
rename = "allowed-clock-drift-for-files-from-future",
deserialize_with = "deserialize_duration"
)]
allowed_clock_drift_for_files_from_future: Option<Duration>,
#[serde(
default,
rename = "file-count-soft-limit",
deserialize_with = "deserialize_si_prefix"
)]
file_count_soft_limit: Option<u64>,
#[serde(
default,
rename = "files-total-size-soft-limit",
deserialize_with = "deserialize_disk_space"
)]
files_total_size_soft_limit: Option<u64>,
#[serde(
default,
rename = "file-count-limit-percent-if-deleting",
deserialize_with = "deserialize_percent"
)]
file_count_limit_percent_if_deleting: Option<u8>,
#[serde(
default,
rename = "files-total-size-limit-percent-if-deleting",
deserialize_with = "deserialize_percent"
)]
files_total_size_limit_percent_if_deleting: Option<u8>,
#[serde(skip)]
worker: Option<Worker>,
#[serde(skip)]
state: Arc<CacheState>,
}
#[derive(Default, Debug)]
struct CacheState {
hits: AtomicUsize,
misses: AtomicUsize,
}
pub fn create_new_config<P: AsRef<Path> + Debug>(config_file: Option<P>) -> Result<PathBuf> {
trace!("Creating new config file, path: {:?}", config_file);
let config_file = match config_file {
Some(path) => path.as_ref().to_path_buf(),
None => default_config_path()?,
};
if config_file.exists() {
bail!(
"Configuration file '{}' already exists.",
config_file.display()
);
}
let parent_dir = config_file
.parent()
.ok_or_else(|| anyhow!("Invalid cache config path: {}", config_file.display()))?;
fs::create_dir_all(parent_dir).with_context(|| {
format!(
"Failed to create config directory, config path: {}",
config_file.display(),
)
})?;
let content = "\
# Comment out certain settings to use default values.
# For more settings, please refer to the documentation:
# https://bytecodealliance.github.io/wasmtime/cli-cache.html
[cache]
enabled = true
";
fs::write(&config_file, content).with_context(|| {
format!(
"Failed to flush config to the disk, path: {}",
config_file.display(),
)
})?;
Ok(config_file.to_path_buf())
}
const ZSTD_COMPRESSION_LEVELS: std::ops::RangeInclusive<i32> = 0..=21;
const DEFAULT_WORKER_EVENT_QUEUE_SIZE: u64 = 0x10;
const WORKER_EVENT_QUEUE_SIZE_WARNING_TRESHOLD: u64 = 3;
const DEFAULT_BASELINE_COMPRESSION_LEVEL: i32 = zstd::DEFAULT_COMPRESSION_LEVEL;
const DEFAULT_OPTIMIZED_COMPRESSION_LEVEL: i32 = 20;
const DEFAULT_OPTIMIZED_COMPRESSION_USAGE_COUNTER_THRESHOLD: u64 = 0x100;
const DEFAULT_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
const DEFAULT_OPTIMIZING_COMPRESSION_TASK_TIMEOUT: Duration = Duration::from_secs(30 * 60);
const DEFAULT_ALLOWED_CLOCK_DRIFT_FOR_FILES_FROM_FUTURE: Duration =
Duration::from_secs(60 * 60 * 24);
const DEFAULT_FILE_COUNT_SOFT_LIMIT: u64 = 0x10_000;
const DEFAULT_FILES_TOTAL_SIZE_SOFT_LIMIT: u64 = 1024 * 1024 * 512;
const DEFAULT_FILE_COUNT_LIMIT_PERCENT_IF_DELETING: u8 = 70;
const DEFAULT_FILES_TOTAL_SIZE_LIMIT_PERCENT_IF_DELETING: u8 = 70;
fn project_dirs() -> Option<ProjectDirs> {
ProjectDirs::from("", "BytecodeAlliance", "wasmtime")
}
fn default_config_path() -> Result<PathBuf> {
match project_dirs() {
Some(dirs) => Ok(dirs.config_dir().join("config.toml")),
None => bail!("config file not specified and failed to get the default"),
}
}
macro_rules! generate_deserializer {
($name:ident($numname:ident: $numty:ty, $unitname:ident: &str) -> $retty:ty {$body:expr}) => {
fn $name<'de, D>(deserializer: D) -> Result<$retty, D::Error>
where
D: Deserializer<'de>,
{
let text = Option::<String>::deserialize(deserializer)?;
let text = match text {
None => return Ok(None),
Some(text) => text,
};
let text = text.trim();
let split_point = text.find(|c: char| !c.is_numeric());
let (num, unit) = split_point.map_or_else(|| (text, ""), |p| text.split_at(p));
let deserialized = (|| {
let $numname = num.parse::<$numty>().ok()?;
let $unitname = unit.trim();
$body
})();
if deserialized.is_some() {
Ok(deserialized)
} else {
Err(de::Error::custom(
"Invalid value, please refer to the documentation",
))
}
}
};
}
generate_deserializer!(deserialize_duration(num: u64, unit: &str) -> Option<Duration> {
match unit {
"s" => Some(Duration::from_secs(num)),
"m" => Some(Duration::from_secs(num * 60)),
"h" => Some(Duration::from_secs(num * 60 * 60)),
"d" => Some(Duration::from_secs(num * 60 * 60 * 24)),
_ => None,
}
});
generate_deserializer!(deserialize_si_prefix(num: u64, unit: &str) -> Option<u64> {
match unit {
"" => Some(num),
"K" => num.checked_mul(1_000),
"M" => num.checked_mul(1_000_000),
"G" => num.checked_mul(1_000_000_000),
"T" => num.checked_mul(1_000_000_000_000),
"P" => num.checked_mul(1_000_000_000_000_000),
_ => None,
}
});
generate_deserializer!(deserialize_disk_space(num: u64, unit: &str) -> Option<u64> {
match unit {
"" => Some(num),
"K" => num.checked_mul(1_000),
"Ki" => num.checked_mul(1u64 << 10),
"M" => num.checked_mul(1_000_000),
"Mi" => num.checked_mul(1u64 << 20),
"G" => num.checked_mul(1_000_000_000),
"Gi" => num.checked_mul(1u64 << 30),
"T" => num.checked_mul(1_000_000_000_000),
"Ti" => num.checked_mul(1u64 << 40),
"P" => num.checked_mul(1_000_000_000_000_000),
"Pi" => num.checked_mul(1u64 << 50),
_ => None,
}
});
generate_deserializer!(deserialize_percent(num: u8, unit: &str) -> Option<u8> {
match unit {
"%" => Some(num),
_ => None,
}
});
static CACHE_IMPROPER_CONFIG_ERROR_MSG: &str =
"Cache system should be enabled and all settings must be validated or defaulted";
macro_rules! generate_setting_getter {
($setting:ident: $setting_type:ty) => {
pub fn $setting(&self) -> $setting_type {
self.$setting.expect(CACHE_IMPROPER_CONFIG_ERROR_MSG)
}
};
}
impl CacheConfig {
generate_setting_getter!(worker_event_queue_size: u64);
generate_setting_getter!(baseline_compression_level: i32);
generate_setting_getter!(optimized_compression_level: i32);
generate_setting_getter!(optimized_compression_usage_counter_threshold: u64);
generate_setting_getter!(cleanup_interval: Duration);
generate_setting_getter!(optimizing_compression_task_timeout: Duration);
generate_setting_getter!(allowed_clock_drift_for_files_from_future: Duration);
generate_setting_getter!(file_count_soft_limit: u64);
generate_setting_getter!(files_total_size_soft_limit: u64);
generate_setting_getter!(file_count_limit_percent_if_deleting: u8);
generate_setting_getter!(files_total_size_limit_percent_if_deleting: u8);
pub fn enabled(&self) -> bool {
self.enabled
}
pub fn directory(&self) -> &PathBuf {
self.directory
.as_ref()
.expect(CACHE_IMPROPER_CONFIG_ERROR_MSG)
}
pub fn new_cache_disabled() -> Self {
Self {
enabled: false,
directory: None,
worker_event_queue_size: None,
baseline_compression_level: None,
optimized_compression_level: None,
optimized_compression_usage_counter_threshold: None,
cleanup_interval: None,
optimizing_compression_task_timeout: None,
allowed_clock_drift_for_files_from_future: None,
file_count_soft_limit: None,
files_total_size_soft_limit: None,
file_count_limit_percent_if_deleting: None,
files_total_size_limit_percent_if_deleting: None,
worker: None,
state: Arc::new(CacheState::default()),
}
}
fn new_cache_enabled_template() -> Self {
let mut conf = Self::new_cache_disabled();
conf.enabled = true;
conf
}
pub fn from_file(config_file: Option<&Path>) -> Result<Self> {
let mut config = Self::load_and_parse_file(config_file)?;
config.validate_directory_or_default()?;
config.validate_worker_event_queue_size_or_default();
config.validate_baseline_compression_level_or_default()?;
config.validate_optimized_compression_level_or_default()?;
config.validate_optimized_compression_usage_counter_threshold_or_default();
config.validate_cleanup_interval_or_default();
config.validate_optimizing_compression_task_timeout_or_default();
config.validate_allowed_clock_drift_for_files_from_future_or_default();
config.validate_file_count_soft_limit_or_default();
config.validate_files_total_size_soft_limit_or_default();
config.validate_file_count_limit_percent_if_deleting_or_default()?;
config.validate_files_total_size_limit_percent_if_deleting_or_default()?;
config.spawn_worker();
Ok(config)
}
fn spawn_worker(&mut self) {
if self.enabled {
self.worker = Some(Worker::start_new(self));
}
}
pub(super) fn worker(&self) -> &Worker {
assert!(self.enabled);
self.worker.as_ref().unwrap()
}
pub fn cache_hits(&self) -> usize {
self.state.hits.load(SeqCst)
}
pub fn cache_misses(&self) -> usize {
self.state.misses.load(SeqCst)
}
pub(crate) fn on_cache_get_async(&self, path: impl AsRef<Path>) {
self.state.hits.fetch_add(1, SeqCst);
self.worker().on_cache_get_async(path)
}
pub(crate) fn on_cache_update_async(&self, path: impl AsRef<Path>) {
self.state.misses.fetch_add(1, SeqCst);
self.worker().on_cache_update_async(path)
}
fn load_and_parse_file(config_file: Option<&Path>) -> Result<Self> {
let (config_file, user_custom_file) = match config_file {
Some(path) => (path.to_path_buf(), true),
None => (default_config_path()?, false),
};
let entity_exists = config_file.exists();
match (entity_exists, user_custom_file) {
(false, false) => Ok(Self::new_cache_enabled_template()),
_ => {
let bytes = fs::read(&config_file).context(format!(
"failed to read config file: {}",
config_file.display()
))?;
let config = toml::from_slice::<Config>(&bytes[..]).context(format!(
"failed to parse config file: {}",
config_file.display()
))?;
Ok(config.cache)
}
}
}
fn validate_directory_or_default(&mut self) -> Result<()> {
if self.directory.is_none() {
match project_dirs() {
Some(proj_dirs) => self.directory = Some(proj_dirs.cache_dir().to_path_buf()),
None => {
bail!("Cache directory not specified and failed to get the default");
}
}
}
let cache_dir = self.directory.as_ref().unwrap();
if !cache_dir.is_absolute() {
bail!(
"Cache directory path has to be absolute, path: {}",
cache_dir.display(),
);
}
fs::create_dir_all(cache_dir).context(format!(
"failed to create cache directory: {}",
cache_dir.display()
))?;
let canonical = fs::canonicalize(cache_dir).context(format!(
"failed to canonicalize cache directory: {}",
cache_dir.display()
))?;
self.directory = Some(canonical);
Ok(())
}
fn validate_worker_event_queue_size_or_default(&mut self) {
if self.worker_event_queue_size.is_none() {
self.worker_event_queue_size = Some(DEFAULT_WORKER_EVENT_QUEUE_SIZE);
}
if self.worker_event_queue_size.unwrap() < WORKER_EVENT_QUEUE_SIZE_WARNING_TRESHOLD {
warn!("Detected small worker event queue size. Some messages might be lost.");
}
}
fn validate_baseline_compression_level_or_default(&mut self) -> Result<()> {
if self.baseline_compression_level.is_none() {
self.baseline_compression_level = Some(DEFAULT_BASELINE_COMPRESSION_LEVEL);
}
if !ZSTD_COMPRESSION_LEVELS.contains(&self.baseline_compression_level.unwrap()) {
bail!(
"Invalid baseline compression level: {} not in {:#?}",
self.baseline_compression_level.unwrap(),
ZSTD_COMPRESSION_LEVELS
);
}
Ok(())
}
fn validate_optimized_compression_level_or_default(&mut self) -> Result<()> {
if self.optimized_compression_level.is_none() {
self.optimized_compression_level = Some(DEFAULT_OPTIMIZED_COMPRESSION_LEVEL);
}
let opt_lvl = self.optimized_compression_level.unwrap();
let base_lvl = self.baseline_compression_level.unwrap();
if !ZSTD_COMPRESSION_LEVELS.contains(&opt_lvl) {
bail!(
"Invalid optimized compression level: {} not in {:#?}",
opt_lvl,
ZSTD_COMPRESSION_LEVELS
);
}
if opt_lvl < base_lvl {
bail!(
"Invalid optimized compression level is lower than baseline: {} < {}",
opt_lvl,
base_lvl
);
}
Ok(())
}
fn validate_optimized_compression_usage_counter_threshold_or_default(&mut self) {
if self.optimized_compression_usage_counter_threshold.is_none() {
self.optimized_compression_usage_counter_threshold =
Some(DEFAULT_OPTIMIZED_COMPRESSION_USAGE_COUNTER_THRESHOLD);
}
}
fn validate_cleanup_interval_or_default(&mut self) {
if self.cleanup_interval.is_none() {
self.cleanup_interval = Some(DEFAULT_CLEANUP_INTERVAL);
}
}
fn validate_optimizing_compression_task_timeout_or_default(&mut self) {
if self.optimizing_compression_task_timeout.is_none() {
self.optimizing_compression_task_timeout =
Some(DEFAULT_OPTIMIZING_COMPRESSION_TASK_TIMEOUT);
}
}
fn validate_allowed_clock_drift_for_files_from_future_or_default(&mut self) {
if self.allowed_clock_drift_for_files_from_future.is_none() {
self.allowed_clock_drift_for_files_from_future =
Some(DEFAULT_ALLOWED_CLOCK_DRIFT_FOR_FILES_FROM_FUTURE);
}
}
fn validate_file_count_soft_limit_or_default(&mut self) {
if self.file_count_soft_limit.is_none() {
self.file_count_soft_limit = Some(DEFAULT_FILE_COUNT_SOFT_LIMIT);
}
}
fn validate_files_total_size_soft_limit_or_default(&mut self) {
if self.files_total_size_soft_limit.is_none() {
self.files_total_size_soft_limit = Some(DEFAULT_FILES_TOTAL_SIZE_SOFT_LIMIT);
}
}
fn validate_file_count_limit_percent_if_deleting_or_default(&mut self) -> Result<()> {
if self.file_count_limit_percent_if_deleting.is_none() {
self.file_count_limit_percent_if_deleting =
Some(DEFAULT_FILE_COUNT_LIMIT_PERCENT_IF_DELETING);
}
let percent = self.file_count_limit_percent_if_deleting.unwrap();
if percent > 100 {
bail!(
"Invalid files count limit percent if deleting: {} not in range 0-100%",
percent
);
}
Ok(())
}
fn validate_files_total_size_limit_percent_if_deleting_or_default(&mut self) -> Result<()> {
if self.files_total_size_limit_percent_if_deleting.is_none() {
self.files_total_size_limit_percent_if_deleting =
Some(DEFAULT_FILES_TOTAL_SIZE_LIMIT_PERCENT_IF_DELETING);
}
let percent = self.files_total_size_limit_percent_if_deleting.unwrap();
if percent > 100 {
bail!(
"Invalid files total size limit percent if deleting: {} not in range 0-100%",
percent
);
}
Ok(())
}
}
#[cfg(test)]
#[macro_use]
pub mod tests;