#![no_std]
extern crate no_std_compat as std;
use std::prelude::v1::*;
use std::collections::BTreeMap;
use product_os_net::SocketAddr;
use std::str::FromStr;
use serde::{Deserialize, Deserializer, Serialize};
use url::Url;
use core::default::Default;
use product_os_connector::Definition;
#[cfg(feature = "config_file")]
use std::fs;
use crate::RelationalKind::Postgres;
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Configuration {
pub environment: String,
pub root_path: String,
pub network: Network,
pub logging: Logging,
pub certificate: Option<Certificate>,
pub compression: Option<Compression>,
pub command_control: Option<CommandControl>,
pub security: Option<Security>,
pub oidc: Option<OIDC>,
pub store: Option<Stores>,
pub authentication: Option<Authentication>,
pub content: Option<Content>,
pub proxy: Option<NetworkProxy>,
pub browser: Option<Browser>,
pub browser_automation: Option<BrowserAutomation>,
pub crawler: Option<Crawler>,
pub vpn: Option<VPN>,
pub connectors: Option<BTreeMap<String, Definition>>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ProductOSFileFormat {
csv,
json
}
impl Default for ProductOSFileFormat {
fn default() -> Self {
ProductOSFileFormat::csv
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Network {
pub protocol: String,
pub secure: bool,
pub host: String,
pub port: u16,
pub listen_all_interfaces: bool,
pub allow_insecure: bool,
pub insecure_port: u16,
pub insecure_use_different_port: bool,
pub insecure_force_secure: bool
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Certificate {
pub files: Option<CertificateFiles>,
pub attributes: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CertificateFiles {
pub cert_file: String,
pub key_file: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Compression {
pub enable: bool,
pub gzip: bool,
pub deflate: bool,
pub brotli: bool
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Logging {
pub level: LogLevel
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CommandControl {
pub enable: bool,
pub max_servers: u8,
pub max_failures: u8,
pub pulse_check: bool,
pub pulse_check_cron: String,
pub monitor: bool,
pub monitor_cron: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Security {
pub enable: bool,
pub csrf: bool,
pub csp: Option<CSPConfig>
}
#[derive(Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "kebab-case")]
pub struct CSPConfig {
pub base_uri: Option<Vec<String>>,
pub block_all_mixed_content: Option<Vec<String>>,
pub child_src: Option<Vec<String>>,
pub connect_src: Option<Vec<String>>,
pub default_src: Option<Vec<String>>,
pub font_src: Option<Vec<String>>,
pub form_action: Option<Vec<String>>,
pub frame_ancestors: Option<Vec<String>>,
pub frame_src: Option<Vec<String>>,
pub img_src: Option<Vec<String>>,
pub manifest_src: Option<Vec<String>>,
pub media_src: Option<Vec<String>>,
pub navigate_to: Option<Vec<String>>,
pub object_src: Option<Vec<String>>,
pub plugin_types: Option<Vec<Vec<String>>>,
pub prefetch_src: Option<Vec<String>>,
pub report_to: Option<Vec<String>>,
pub report_uri: Option<Vec<String>>,
pub require_sri_for: Option<Vec<String>>,
pub sandbox_allow: Option<Vec<String>>,
pub script_src: Option<Vec<String>>,
pub script_src_attr: Option<Vec<String>>,
pub script_src_elem: Option<Vec<String>>,
pub style_src: Option<Vec<String>>,
pub style_src_attr: Option<Vec<String>>,
pub style_src_elem: Option<Vec<String>>,
pub trusted_types: Option<Vec<String>>,
pub upgrade_insecure_requests: Option<Vec<String>>,
pub worker_src: Option<Vec<String>>,
}
impl CSPConfig {
pub fn new() -> Self {
Self {
base_uri: Some(vec!("self".to_string())),
block_all_mixed_content: Some(vec!()),
child_src: Some(vec!("self".to_string())),
connect_src: Some(vec!("self".to_string())),
default_src: Some(vec!("self".to_string())),
font_src: Some(vec!("self".to_string())),
form_action: Some(vec!("self".to_string())),
frame_ancestors: Some(vec!("self".to_string())),
frame_src: Some(vec!("self".to_string())),
img_src: Some(vec!("self".to_string())),
manifest_src: Some(vec!("self".to_string())),
media_src: Some(vec!("self".to_string())),
navigate_to: Some(vec!("self".to_string())),
object_src: Some(vec!("self".to_string())),
plugin_types: Some(vec!()),
prefetch_src: Some(vec!("self".to_string())),
report_to: Some(vec!()),
report_uri: Some(vec!()),
require_sri_for: Some(vec!()),
sandbox_allow: Some(vec!()),
script_src: Some(vec!("self".to_string())),
script_src_attr: Some(vec!("self".to_string())),
script_src_elem: Some(vec!("self".to_string())),
style_src: Some(vec!("self".to_string())),
style_src_attr: Some(vec!("self".to_string())),
style_src_elem: Some(vec!("self".to_string())),
trusted_types: Some(vec!()),
upgrade_insecure_requests: Some(vec!()),
worker_src: Some(vec!("self".to_string())),
}
}
}
impl Clone for CSPConfig {
fn clone(&self) -> Self {
Self {
base_uri: self.base_uri.clone(),
block_all_mixed_content: self.block_all_mixed_content.clone(),
child_src: self.child_src.clone(),
connect_src: self.connect_src.clone(),
default_src: self.default_src.clone(),
font_src: self.font_src.clone(),
form_action: self.form_action.clone(),
frame_ancestors: self.frame_ancestors.clone(),
frame_src: self.frame_src.clone(),
img_src: self.img_src.clone(),
manifest_src: self.manifest_src.clone(),
media_src: self.media_src.clone(),
navigate_to: self.navigate_to.clone(),
object_src: self.object_src.clone(),
plugin_types: self.plugin_types.clone(),
prefetch_src: self.prefetch_src.clone(),
report_to: self.report_to.clone(),
report_uri: self.report_uri.clone(),
require_sri_for: self.require_sri_for.clone(),
sandbox_allow: self.sandbox_allow.clone(),
script_src: self.script_src.clone(),
script_src_attr: self.script_src_attr.clone(),
script_src_elem: self.script_src_elem.clone(),
style_src: self.style_src.clone(),
style_src_attr: self.style_src_attr.clone(),
style_src_elem: self.style_src_elem.clone(),
trusted_types: self.trusted_types.clone(),
upgrade_insecure_requests: self.upgrade_insecure_requests.clone(),
worker_src: self.worker_src.clone()
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OIDC {
pub authorization_url: Url,
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Stores {
pub key_value: Option<KeyValueStore>,
pub queue: Option<QueueStore>,
pub relational: Option<RelationalStore>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum KeyValueKind {
Redis,
Memory,
File,
Sink
}
impl Default for KeyValueKind {
fn default() -> Self {
KeyValueKind::Sink
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct KeyValueStore {
pub enabled: bool,
pub kind: KeyValueKind,
pub host: String,
pub port: u16,
pub secure: bool,
pub db_number: u8,
pub db_name: Option<String>,
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub password: Option<String>,
pub pool_size: u8,
pub default_limit: u32,
pub default_offset: u32,
pub prefix: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum TunnelType {
Tor,
Vpn
}
impl Default for TunnelType {
fn default() -> Self {
TunnelType::Vpn
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum QueueKind {
Redis,
Memory,
File,
Sink
}
impl Default for QueueKind {
fn default() -> Self {
QueueKind::Sink
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct QueueStore {
pub enabled: bool,
pub kind: QueueKind,
pub host: String,
pub port: u16,
pub secure: bool,
pub db_number: u8,
pub db_name: Option<String>,
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub password: Option<String>,
pub pool_size: u8,
pub default_limit: u32,
pub default_offset: u32,
pub prefix: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum RelationalKind {
Postgres,
MSSql,
MySql,
Sqlite,
Memory,
Sink
}
impl Default for RelationalKind {
fn default() -> Self {
RelationalKind::Sink
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct RelationalStore {
pub enabled: bool,
pub kind: RelationalKind,
pub host: String,
pub port: u16,
pub secure: bool,
pub db_name: String,
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub password: Option<String>,
pub pool_size: u8,
pub default_limit: u32,
pub default_offset: u32,
pub prefix: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub enum LogLevel {
ERROR,
WARN,
INFO,
DEBUG,
TRACE,
OFF
}
impl Default for LogLevel {
fn default() -> Self {
LogLevel::OFF
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Authentication {
pub app_id: String,
pub client_id: String,
pub client_secret: String,
pub client_key: String,
pub default_scope: String,
pub audience: String,
pub token_lifetime: u32,
pub scopes: BTreeMap<String, String>,
pub oidc: AuthenticationOIDC
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AuthenticationOIDC {
pub providers: Vec<AuthenticationOIDCProvider>,
pub redirect_uri: Url
}
#[derive(Debug, Clone, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct AuthenticationOIDCProvider {
pub provider: String,
pub enabled: bool,
pub client: String,
pub secret: String,
pub scope: Vec<String>,
pub issuer_endpoint: String,
pub auth_endpoint: Option<String>,
pub token_endpoint: Option<String>,
pub revoke_endpoint: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Content {
pub setup: ContentSetup
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct ContentSetup {
pub base_path: String,
pub add_templates: bool,
pub add_javascript: bool,
pub add_stylesheet: bool,
pub add_fonts: bool,
pub add_images: bool,
pub add_files: bool,
pub add_error: bool,
pub add_views: bool,
pub add_applets: bool,
pub add_cms: bool
}
impl ContentSetup {
pub fn new() -> Self {
Self {
base_path: "".to_string(),
add_templates: false,
add_javascript: false,
add_stylesheet: false,
add_fonts: false,
add_images: false,
add_files: false,
add_error: false,
add_views: false,
add_applets: false,
add_cms: false,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Browser {
pub mode: BrowserMode,
pub debug: bool,
pub head_check: bool,
pub document_matcher: String,
pub max_success_clicks: i32,
pub max_click_attempts: usize,
pub on_click_matcher: String,
pub button_x_path_search: String,
pub max_scrolls: i32,
pub busy_wait: usize,
pub busy_retry: usize,
pub identifiers: BrowserIdentifiers,
#[serde(alias = "type")]
pub kind: String,
pub windows: usize,
pub multi_window: bool,
pub instances: usize,
pub multi_instance: bool,
pub args: Vec<String>,
pub browser_logging: BrowserLogging,
pub selenium: bool,
pub binary_path: String,
pub profile_path: String,
pub server_path: String,
pub driver_name: String,
pub driver_path: String,
pub extensions_path: String,
pub default_preferences_path: String,
pub headless: bool,
pub display: i32,
pub display_command: String,
pub browser_name: String,
pub wait_for_actions: bool,
pub action_wait_timeout: i32,
pub action_wait_internal_timeout: i32,
pub block_ads: bool,
pub block_images: bool,
pub block_video: bool,
pub pipeline: bool,
pub max_requests: i32,
pub max_connections: i32,
pub max_persistent_connections: i32,
pub cache: bool,
pub cache_size: i32,
pub history_size: i32,
pub page_load_timeout: u64,
pub script_load_timeout: u64,
pub implicit_element_find_timeout: u64,
pub conditional_timeout: u64,
pub conditional_polling: u64,
pub javascript: bool,
pub render_mode: BrowserRenderMode,
pub ssl_trust_all: bool,
pub reload_browser: bool,
pub close_overlays: bool,
pub reload_every_x_pages: i32,
pub enable_dev_tools: bool,
pub enable_default_extensions: bool,
pub enable_faster_extension: bool,
pub enable_plugins: bool,
pub network: BrowserNetwork,
pub setup_driver_max_attempts: i32,
pub socket_host: Option<String>,
pub socket_endpoint: Option<String>,
pub socket_port: Option<u16>,
pub socket_identifier: Option<String>,
pub browser_options: Option<BTreeMap<String, serde_json::Value>>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum BrowserMode {
Socket,
Custom
}
impl Default for BrowserMode {
fn default() -> Self {
BrowserMode::Socket
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum BrowserRenderMode {
Normal,
None,
Eager,
}
impl Default for BrowserRenderMode {
fn default() -> Self {
BrowserRenderMode::Normal
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct BrowserAutomation {
pub browser_agent_list_path: String,
pub browser_agent_list_type: ProductOSFileFormat,
pub use_browser_agent_list: bool,
pub suppress_automation_markers: bool,
pub mask_bot_events: bool,
pub monitor_objects: bool,
pub monitor_objects_log_functions: bool,
pub monitor_objects_log_objects: bool,
pub monitor_objects_list: Vec<String>
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct BrowserLogging {
pub level: LogLevel,
pub path: String,
pub filename: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct BrowserNetwork {
pub secure: bool,
pub host: String,
pub port_min: usize,
pub port_max: usize
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Crawler {
pub link_chain_depth: usize,
pub focus_depth: usize,
pub deep_depth: usize,
pub broad_deep_breadth: usize,
pub broad_deep_depth: usize,
pub crawl_delay: u32,
pub deep_crawl_delay: u32,
pub broad_deep_crawl_delay: u32,
pub max_concurrent_crawls: usize,
pub revisit_delay: u32,
pub revisit_score: u32,
pub domain_chunk_size: u32,
pub resolver_timeout: u32,
pub seeder: CrawlerSeeder,
pub method: String,
pub randomiser: i32,
pub do_processing: bool,
pub do_acting: bool,
pub do_scoring: bool,
pub do_following: bool,
pub do_scrolls: bool,
pub do_clicks: bool,
pub idle_check_frequency: i32,
pub idle_timeout: i32,
pub pause: bool,
pub kill_list: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CrawlerSeeder {
pub instances: usize,
pub multi_instance: bool,
pub queue_wait_min: usize,
pub queue_wait_max: usize,
pub throttling: CrawlerThrottling,
pub seed_frequency: u32
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CrawlerThrottling {
pub enable: bool,
pub default_frequency: u32,
pub default_rate: u32,
pub default_wait: u32,
pub frequency_limit: u32,
pub rate_limit: u32,
pub wait_limit: u32,
pub change_tight_threshold: u32,
pub change_loose_threshold: u32,
pub change_tight_range: u32,
pub change_loose_range: u32,
pub multiplier: u32,
pub auto_adjust: bool
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct BrowserIdentifiers {
pub string_type: String,
pub string_formatter: String,
pub anchor_reference: String,
pub on_click_identifier: String,
pub window_identifier: String,
pub query_identifier: String,
pub anchor_element: String,
pub link_attribute: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxy {
pub enable: bool,
pub network: NetworkProxyNetwork,
pub enable_tunnel: bool,
pub tunnel_settings: NetworkProxyTunnel,
pub certificate_authority: Option<NetworkProxyCertificateAuthority>,
pub custom_requester: bool,
pub compression: NetworkProxyCompression,
pub enable_content_type_matchers: bool,
pub enable_content_type_filters: bool,
pub enable_domain_filter: bool,
pub enable_cache: bool,
pub trust_all_certificates: bool,
pub trust_any_certificate_for_hostname: bool,
pub cache_size: i32,
pub block_extensions: bool,
pub block_urls: bool,
pub content_matcher: String,
pub content_filter: String,
pub cache_matcher: String,
pub extension_filter: String,
pub resource_matcher: String,
pub content_replacers: BTreeMap<String, NetworkProxyContentReplacers>,
pub content_injectors: BTreeMap<String, NetworkProxyContentInjectors>,
pub content_security_policy_manipulators: BTreeMap<String, NetworkProxyCSPManipulators>,
pub request_header_manipulators: BTreeMap<String, NetworkProxyHeaderManipulators>,
pub response_header_manipulators: BTreeMap<String, NetworkProxyHeaderManipulators>,
pub url_filter: String,
pub connect_timeout: u64,
pub request_timeout: u64,
pub use_user_agent_list: bool,
pub user_agent_list_path: String,
pub user_agent_list_type: ProductOSFileFormat,
pub rotate_user_agent: bool,
pub rotate_user_agent_frequency: u32,
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyNetwork {
pub secure: bool,
pub host: String,
pub port: u16,
pub listen_all_interfaces: bool,
pub allow_insecure: bool,
pub insecure_port: u16
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum NetworkProxyCompression {
None,
Gzip,
Brotli
}
impl Default for NetworkProxyCompression {
fn default() -> Self {
NetworkProxyCompression::Gzip
}
}
impl NetworkProxyCompression {
fn to_string(self) -> String {
match self {
NetworkProxyCompression::None => String::from("none"),
NetworkProxyCompression::Gzip => String::from("gzip"),
NetworkProxyCompression::Brotli => String::from("br")
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyTunnel {
pub tunnel_type: TunnelType,
pub pipe_count: u16
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyCertificateAuthority {
pub files: Option<NetworkProxyCertificateAuthorityFiles>,
pub attributes: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyCertificateAuthorityFiles {
pub cert_file: String,
pub key_file: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyContentInjectors {
pub source: String,
pub base_relative: bool,
pub find: String,
pub inject: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyContentReplacers {
pub content_matcher: String,
pub find: String,
pub replace: String
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyHeaderManipulators {
pub header_name_matcher: Option<String>,
pub header_value_matcher: Option<String>,
pub action: NetworkProxyManipulatorAction,
pub name: Option<String>,
pub value: Option<String>,
pub value_match: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct NetworkProxyCSPManipulators {
pub directive_name_matcher: Option<String>,
pub directive_value_matcher: Option<String>,
pub action: NetworkProxyManipulatorAction,
pub name: Option<String>,
pub value: Option<String>,
pub value_match: Option<String>
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum NetworkProxyManipulatorAction {
Insert,
Update,
Upsert,
Remove
}
impl Default for NetworkProxyManipulatorAction {
fn default() -> Self {
NetworkProxyManipulatorAction::Upsert
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct VPN {
pub base_relative: bool,
pub config_path: String,
pub credentials_path: String,
pub protocol: VPNProtocol,
pub provider: VPNProvider,
pub bind_host: Option<String>,
pub identifier: String,
pub key: String,
pub config_server_archive: Option<String>,
pub config_server_page: Option<String>,
pub network_protocol: NetworkProtocol
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum NetworkProtocol {
TCP,
UDP,
ANY
}
impl Default for NetworkProtocol {
fn default() -> Self {
NetworkProtocol::TCP
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum VPNProvider {
NordVPN
}
impl Default for VPNProvider {
fn default() -> Self {
VPNProvider::NordVPN
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum VPNProtocol {
OpenVPN
}
impl Default for VPNProtocol {
fn default() -> Self {
VPNProtocol::OpenVPN
}
}
impl Configuration {
#[cfg(feature = "config_file")]
pub fn from_file(path: &str) -> Self {
let config_file = match fs::read_to_string(path) {
Ok(data) => data,
Err(e) => {
panic!("Error parsing configuration file {}", e);
}
};
match serde_json::from_str(config_file.as_str()) {
Ok(configuration) => configuration,
Err(e) => {
panic!("Error parsing configuration file {}", e);
}
}
}
pub fn new() -> Self {
tracing::subscriber::with_default(
tracing_subscriber::FmtSubscriber::builder().with_max_level(tracing::Level::INFO).finish(),
|| {
Self {
environment: String::from("development"),
root_path: String::from("/"),
network: Network {
protocol: "".to_string(),
secure: false,
host: "".to_string(),
port: 0,
listen_all_interfaces: false,
allow_insecure: false,
insecure_port: 0,
insecure_use_different_port: false,
insecure_force_secure: false
},
certificate: None,
compression: Some(Compression {
enable: false,
gzip: true,
deflate: false,
brotli: true
}),
logging: Logging {
level: LogLevel::INFO
},
command_control: Some(CommandControl {
enable: false,
max_servers: 50,
max_failures: 1,
pulse_check: true,
pulse_check_cron: String::from("*/30 * * * * *"),
monitor: true,
monitor_cron: String::from("0 */1 * * * *")
}),
oidc: Some(OIDC {
authorization_url: Url::parse("http://127.0.0.1:8080/oidc/request/").unwrap()
}),
store: Some(Stores {
key_value: Some(KeyValueStore {
enabled: true,
kind: KeyValueKind::Redis,
host: "localhost".to_string(),
port: 6379,
secure: false,
db_number: 0,
db_name: None,
username: None,
password: None,
pool_size: 0,
default_limit: 0,
default_offset: 0,
prefix: None
}),
queue: Some(QueueStore {
enabled: true,
kind: QueueKind::Redis,
host: "localhost".to_string(),
port: 6379,
secure: false,
db_number: 0,
db_name: None,
username: None,
password: None,
pool_size: 0,
default_limit: 0,
default_offset: 0,
prefix: None
}),
relational: Some(RelationalStore {
enabled: true,
kind: Postgres,
host: "localhost".to_string(),
port: 5432,
secure: false,
db_name: "test".to_string(),
username: None,
password: None,
pool_size: 0,
default_limit: 0,
default_offset: 0,
prefix: None
})
}),
authentication: Some(Authentication {
app_id: "".to_string(),
client_id: "".to_string(),
client_secret: "".to_string(),
client_key: "".to_string(),
default_scope: "".to_string(),
audience: "".to_string(),
token_lifetime: 0,
scopes: Default::default(),
oidc: AuthenticationOIDC {
providers: vec![],
redirect_uri: Url::parse("http://127.0.0.1:8080/auth/redirect/").unwrap()
}
}),
content: Some(Content {
setup: ContentSetup {
base_path: "".to_string(),
add_templates: false,
add_javascript: false,
add_stylesheet: false,
add_fonts: false,
add_images: false,
add_files: false,
add_error: false,
add_views: false,
add_applets: false,
add_cms: false
}
}),
proxy: Some(NetworkProxy {
enable: false,
network: NetworkProxyNetwork {
secure: false,
host: "".to_string(),
port: 0,
listen_all_interfaces: false,
allow_insecure: false,
insecure_port: 0
},
enable_tunnel: false,
tunnel_settings: NetworkProxyTunnel {
tunnel_type: TunnelType::Tor,
pipe_count: 0
},
certificate_authority: None,
custom_requester: false,
compression: NetworkProxyCompression::None,
enable_content_type_matchers: false,
enable_content_type_filters: false,
enable_domain_filter: false,
enable_cache: false,
trust_all_certificates: false,
trust_any_certificate_for_hostname: false,
cache_size: 0,
block_extensions: false,
block_urls: false,
content_matcher: "".to_string(),
content_filter: "".to_string(),
cache_matcher: "".to_string(),
extension_filter: "".to_string(),
resource_matcher: "".to_string(),
content_replacers: Default::default(),
content_injectors: Default::default(),
content_security_policy_manipulators: Default::default(),
request_header_manipulators: Default::default(),
response_header_manipulators: Default::default(),
url_filter: "".to_string(),
connect_timeout: 0,
request_timeout: 0,
use_user_agent_list: false,
user_agent_list_path: "".to_string(),
user_agent_list_type: ProductOSFileFormat::csv,
rotate_user_agent: false,
rotate_user_agent_frequency: 0,
}),
browser: Some(Browser {
mode: BrowserMode::Socket,
debug: false,
head_check: false,
document_matcher: "".to_string(),
max_success_clicks: 0,
max_click_attempts: 0,
on_click_matcher: "".to_string(),
button_x_path_search: "".to_string(),
max_scrolls: 0,
busy_wait: 0,
kind: "".to_string(),
windows: 0,
multi_window: false,
instances: 0,
multi_instance: false,
args: vec![],
browser_logging: BrowserLogging {
level: LogLevel::OFF,
path: "".to_string(),
filename: "".to_string()
},
selenium: false,
binary_path: "".to_string(),
profile_path: "".to_string(),
server_path: "".to_string(),
driver_name: "".to_string(),
driver_path: "".to_string(),
extensions_path: "".to_string(),
default_preferences_path: "".to_string(),
headless: false,
display: 0,
display_command: "".to_string(),
browser_name: "".to_string(),
wait_for_actions: false,
action_wait_timeout: 0,
action_wait_internal_timeout: 0,
block_ads: false,
block_images: false,
block_video: false,
pipeline: false,
max_requests: 0,
max_connections: 0,
max_persistent_connections: 0,
cache: false,
cache_size: 0,
history_size: 0,
page_load_timeout: 0,
script_load_timeout: 0,
implicit_element_find_timeout: 0,
conditional_timeout: 0,
conditional_polling: 0,
javascript: false,
render_mode: BrowserRenderMode::Eager,
ssl_trust_all: false,
reload_browser: false,
close_overlays: false,
reload_every_x_pages: 0,
enable_dev_tools: false,
enable_default_extensions: false,
enable_faster_extension: false,
enable_plugins: false,
network: BrowserNetwork {
secure: false,
host: "".to_string(),
port_min: 0,
port_max: 0
},
setup_driver_max_attempts: 0,
identifiers: BrowserIdentifiers {
string_type: "".to_string(),
string_formatter: "".to_string(),
anchor_reference: "".to_string(),
on_click_identifier: "".to_string(),
window_identifier: "".to_string(),
query_identifier: "".to_string(),
anchor_element: "".to_string(),
link_attribute: "".to_string()
},
busy_retry: 0,
socket_host: None,
socket_endpoint: None,
socket_port: None,
socket_identifier: None,
browser_options: None,
}),
browser_automation: Some(BrowserAutomation {
browser_agent_list_path: "".to_string(),
browser_agent_list_type: ProductOSFileFormat::csv,
use_browser_agent_list: false,
suppress_automation_markers: false,
mask_bot_events: false,
monitor_objects: false,
monitor_objects_log_functions: false,
monitor_objects_log_objects: false,
monitor_objects_list: vec![],
}),
crawler: Some(Crawler {
link_chain_depth: 0,
focus_depth: 0,
deep_depth: 0,
broad_deep_breadth: 0,
broad_deep_depth: 0,
crawl_delay: 0,
deep_crawl_delay: 0,
broad_deep_crawl_delay: 0,
max_concurrent_crawls: 0,
revisit_delay: 0,
revisit_score: 0,
domain_chunk_size: 0,
resolver_timeout: 0,
seeder: CrawlerSeeder {
instances: 0,
multi_instance: false,
queue_wait_min: 0,
queue_wait_max: 0,
throttling: CrawlerThrottling {
enable: false,
default_frequency: 0,
default_rate: 0,
default_wait: 0,
frequency_limit: 0,
rate_limit: 0,
wait_limit: 0,
change_tight_threshold: 0,
change_loose_threshold: 0,
change_tight_range: 0,
change_loose_range: 0,
multiplier: 0,
auto_adjust: false
},
seed_frequency: 0
},
method: "".to_string(),
randomiser: 0,
idle_check_frequency: 0,
idle_timeout: 0,
do_processing: false,
do_acting: false,
do_scoring: false,
do_following: false,
do_scrolls: false,
do_clicks: false,
pause: false,
kill_list: "".to_string()
}),
security: Some(Security {
enable: false,
csrf: false,
csp: Some(CSPConfig::new())
}),
vpn: Some(VPN {
base_relative: false,
config_path: "".to_string(),
credentials_path: "".to_string(),
protocol: VPNProtocol::OpenVPN,
provider: VPNProvider::NordVPN,
bind_host: None,
identifier: "".to_string(),
key: "".to_string(),
config_server_archive: None,
config_server_page: None,
network_protocol: NetworkProtocol::ANY
}),
connectors: None,
}
}
)
}
pub fn environment(self) -> String {
self.environment
}
pub fn log_level(&self) -> tracing::Level {
match &self.logging.level {
LogLevel::ERROR => tracing::Level::ERROR,
LogLevel::WARN => tracing::Level::WARN,
LogLevel::INFO => tracing::Level::INFO,
LogLevel::DEBUG => tracing::Level::DEBUG,
LogLevel::TRACE => tracing::Level::TRACE,
LogLevel::OFF => tracing::Level::ERROR
}
}
pub fn get_host(&self) -> String {
self.network.host.to_owned()
}
pub fn url_address(&self) -> Url {
let protocol = self.network.protocol.to_owned();
let host = self.network.host.to_owned();
let port = self.network.port.to_owned();
let path = self.root_path.strip_suffix("/").unwrap();
let mut url_string = protocol;
url_string.push_str("://");
url_string.push_str(host.as_str());
url_string.push_str(":");
url_string.push_str(port.to_string().as_str());
url_string.push_str(path);
Url::parse(url_string.as_str()).unwrap()
}
pub fn socket_address(&self, port: Option<u16>, default_all: bool) -> SocketAddr {
let socket_port = match port {
None => self.network.port.to_owned(),
Some(p) => p
};
let host = self.network.host.to_owned();
Configuration::get_socket_address(host, socket_port, default_all)
}
pub fn get_socket_address(host: String, port: u16, default_all: bool) -> SocketAddr {
product_os_utilities::ProductOSUtilities::get_socket_address(host, port, default_all)
}
pub fn is_secure(&self) -> bool {
self.network.secure.to_owned()
}
pub fn all_insecure(&self) -> bool {
self.network.allow_insecure.to_owned()
}
pub fn insecure_port(&self) -> u16 {
self.network.insecure_port.to_owned()
}
pub fn insecure_force_secure(&self) -> bool {
self.network.insecure_force_secure.to_owned()
}
pub fn is_compression_gzip(&self) -> bool {
match &self.compression {
None => false,
Some(c) => c.gzip.to_owned()
}
}
pub fn is_compression_deflate(&self) -> bool {
match &self.compression {
None => false,
Some(c) => c.deflate.to_owned()
}
}
pub fn is_compression_brotli(&self) -> bool {
match &self.compression {
None => false,
Some(c) => c.brotli.to_owned()
}
}
pub fn get_cc_max_servers(&self) -> u8 {
match &self.command_control {
None => 1,
Some(c) => c.max_servers.to_owned()
}
}
pub fn get_cc_max_failures(&self) -> u8 {
match &self.command_control {
None => 1,
Some(c) => c.max_failures.to_owned()
}
}
pub fn is_monitor_enabled(&self) -> bool {
match &self.command_control {
None => false,
Some(c) => c.monitor.to_owned()
}
}
pub fn is_pulse_check_enabled(&self) -> bool {
match &self.command_control {
None => false,
Some(c) => c.pulse_check.to_owned()
}
}
pub fn get_monitor_cron(&self) -> String {
match &self.command_control {
None => String::new(),
Some(c) => c.monitor_cron.to_owned()
}
}
pub fn get_hearbeat_cron(&self) -> String {
match &self.command_control {
None => String::new(),
Some(c) => c.pulse_check_cron.to_owned()
}
}
pub fn get_csp(&self) -> CSPConfig {
match &self.security {
None => CSPConfig::new(),
Some(s) => match &s.csp {
None => CSPConfig::new(),
Some(c) => c.to_owned()
}
}
}
pub fn get_oidc_authorization_url(&self) -> Url {
match &self.oidc {
None => url::Url::from_str("https://none").unwrap(),
Some(o) => o.authorization_url.to_owned()
}
}
pub fn get_store_key_value_enabled(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.key_value {
None => false,
Some(kv) => kv.enabled.to_owned()
}
}
}
pub fn get_store_key_value_kind(&self) -> KeyValueKind {
match &self.store {
None => KeyValueKind::Sink,
Some(s) => match &s.key_value {
None => KeyValueKind::Sink,
Some(kv) => kv.kind.to_owned()
}
}
}
pub fn get_store_key_value_host(&self) -> String {
match &self.store {
None => String::new(),
Some(s) => match &s.key_value {
None => String::new(),
Some(kv) => kv.host.to_owned()
}
}
}
pub fn get_store_key_value_port(&self) -> u16 {
match &self.store {
None => 6379,
Some(s) => match &s.key_value {
None => 6379,
Some(kv) => kv.port.to_owned()
}
}
}
pub fn get_store_key_value_pool_size(&self) -> u8 {
match &self.store {
None => 8,
Some(s) => match &s.key_value {
None => 8,
Some(kv) => kv.pool_size.to_owned()
}
}
}
pub fn get_store_key_value_username(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.key_value {
None => None,
Some(kv) => kv.username.to_owned()
}
}
}
pub fn get_store_key_value_password(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.key_value {
None => None,
Some(kv) => kv.password.to_owned()
}
}
}
pub fn get_store_key_value_is_secure(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.key_value {
None => false,
Some(kv) => kv.secure.to_owned()
}
}
}
pub fn get_store_key_value_db_identifier(&self) -> u8 {
match &self.store {
None => 0,
Some(s) => match &s.key_value {
None => 0,
Some(kv) => kv.db_number.to_owned()
}
}
}
pub fn get_store_key_value_default_limit(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.key_value {
None => 0,
Some(kv) => kv.default_limit.to_owned()
}
}
}
pub fn get_store_key_value_default_offset(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.key_value {
None => 0,
Some(kv) => kv.default_offset.to_owned()
}
}
}
pub fn get_store_queue_enabled(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.queue {
None => false,
Some(q) => q.enabled.to_owned()
}
}
}
pub fn get_store_queue_kind(&self) -> QueueKind {
match &self.store {
None => QueueKind::Sink,
Some(s) => match &s.queue {
None => QueueKind::Sink,
Some(q) => q.kind.to_owned()
}
}
}
pub fn get_store_queue_host(&self) -> String {
match &self.store {
None => String::new(),
Some(s) => match &s.queue {
None => String::new(),
Some(q) => q.host.to_owned()
}
}
}
pub fn get_store_queue_port(&self) -> u16 {
match &self.store {
None => 6379,
Some(s) => match &s.queue {
None => 6379,
Some(q) => q.port.to_owned()
}
}
}
pub fn get_store_queue_pool_size(&self) -> u8 {
match &self.store {
None => 8,
Some(s) => match &s.queue {
None => 8,
Some(q) => q.pool_size.to_owned()
}
}
}
pub fn get_store_queue_username(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.queue {
None => None,
Some(q) => q.username.to_owned()
}
}
}
pub fn get_store_queue_password(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.queue {
None => None,
Some(q) => q.password.to_owned()
}
}
}
pub fn get_store_queue_is_secure(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.queue {
None => false,
Some(q) => q.secure.to_owned()
}
}
}
pub fn get_store_queue_db_identifier(&self) -> u8 {
match &self.store {
None => 0,
Some(s) => match &s.queue {
None => 0,
Some(q) => q.db_number.to_owned()
}
}
}
pub fn get_store_queue_default_limit(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.queue {
None => 0,
Some(q) => q.default_limit.to_owned()
}
}
}
pub fn get_store_queue_default_offset(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.queue {
None => 0,
Some(q) => q.default_offset.to_owned()
}
}
}
pub fn get_store_relational_enabled(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.relational {
None => false,
Some(r) => r.enabled.to_owned()
}
}
}
pub fn get_store_relational_kind(&self) -> RelationalKind {
match &self.store {
None => RelationalKind::Postgres,
Some(s) => match &s.relational {
None => RelationalKind::Postgres,
Some(r) => r.kind.to_owned()
}
}
}
pub fn get_store_relational_host(&self) -> String {
match &self.store {
None => String::new(),
Some(s) => match &s.relational {
None => String::new(),
Some(r) => r.host.to_owned()
}
}
}
pub fn get_store_relational_port(&self) -> u16 {
match &self.store {
None => 6379,
Some(s) => match &s.relational {
None => 6379,
Some(r) => r.port.to_owned()
}
}
}
pub fn get_store_relational_pool_size(&self) -> u8 {
match &self.store {
None => 8,
Some(s) => match &s.relational {
None => 8,
Some(r) => r.pool_size.to_owned()
}
}
}
pub fn get_store_relational_username(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.relational {
None => None,
Some(r) => r.username.to_owned()
}
}
}
pub fn get_store_relational_password(&self) -> Option<String> {
match &self.store {
None => None,
Some(s) => match &s.relational {
None => None,
Some(r) => r.password.to_owned()
}
}
}
pub fn get_store_relational_is_secure(&self) -> bool {
match &self.store {
None => false,
Some(s) => match &s.relational {
None => false,
Some(r) => r.secure.to_owned()
}
}
}
pub fn get_store_relational_db_identifier(&self) -> String {
match &self.store {
None => String::new(),
Some(s) => match &s.relational {
None => String::new(),
Some(r) => r.db_name.to_owned()
}
}
}
pub fn get_store_relational_default_limit(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.relational {
None => 0,
Some(r) => r.default_limit.to_owned()
}
}
}
pub fn get_store_relational_default_offset(&self) -> u32 {
match &self.store {
None => 0,
Some(s) => match &s.relational {
None => 0,
Some(r) => r.default_offset.to_owned()
}
}
}
pub fn get_authentication_app_id(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.app_id.to_owned()
}
}
pub fn get_authentication_client_id(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.client_id.to_owned()
}
}
pub fn get_authentication_client_secret(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.client_secret.to_owned()
}
}
pub fn get_authentication_client_key(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.client_key.to_owned()
}
}
pub fn get_authentication_default_scope(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.default_scope.to_owned()
}
}
pub fn get_authentication_audience(&self) -> String {
match &self.authentication {
None => String::new(),
Some(a) => a.audience.to_owned()
}
}
pub fn get_authentication_token_lifetime(&self) -> u32 {
match &self.authentication {
None => 86400,
Some(a) => a.token_lifetime.to_owned()
}
}
pub fn get_authentication_scopes(&self) -> BTreeMap<String, String> {
match &self.authentication {
None => BTreeMap::new(),
Some(a) => a.scopes.to_owned()
}
}
pub fn get_authentication_oidc_providers(&self) -> Vec<AuthenticationOIDCProvider> {
match &self.authentication {
None => Vec::new(),
Some(a) => a.oidc.providers.to_owned()
}
}
pub fn get_authentication_oidc_redirect_uri(&self) -> Url {
match &self.authentication {
None => Url::from_str("http://none").unwrap(),
Some(a) => a.oidc.redirect_uri.to_owned()
}
}
pub fn get_content_setup(&self) -> ContentSetup {
match &self.content {
None => ContentSetup::new(),
Some(c) => c.setup.to_owned()
}
}
}