magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove hardcoded 300kb limit on spider size with config option of 1MB default

Browse source 
modern websites are sadly massive, 300kb is pretty low. 1MB should be enough.

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-02-10 13:29:12 -05:00 • committed by June
parent 2ea895199a
commit 48e4b71dd1
4 changed files with 19 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/config/mod.rs
View file

@ -136,12 +136,12 @@ pub struct Config {
#[serde(default = "Vec::new")]
pub url_preview_domain_contains_allowlist: Vec<String>,
#[serde(default = "Vec::new")]
pub url_preview_domain_explicit_allowlist: Vec<String>,
#[serde(default = "Vec::new")]
pub url_preview_url_contains_allowlist: Vec<String>,
#[serde(default = "default_url_preview_max_spider_size")]
pub url_preview_max_spider_size: usize,
#[serde(default = "RegexSet::empty")]
#[serde(with = "serde_regex")]
@ -370,6 +370,10 @@ impl fmt::Display for Config {
"URL preview URL contains allowlist",
&self.url_preview_url_contains_allowlist.join(", "),
),
(
"URL preview maximum spider size",
&self.url_preview_max_spider_size.to_string(),
),
];
let mut msg: String = "Active config values:\n\n".to_owned();
@ -495,3 +499,7 @@ fn default_ip_range_denylist() -> Vec<String> {
"fec0::/10".to_owned(),
]
}
fn default_url_preview_max_spider_size() -> usize {
1_000_000 // 1MB
}