remove hardcoded 300kb limit on spider size with config option of 1MB default

modern websites are sadly massive, 300kb is pretty low. 1MB should be enough.

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-02-10 13:29:12 -05:00 committed by June
parent 2ea895199a
commit 48e4b71dd1
4 changed files with 19 additions and 5 deletions

View file

@ -180,6 +180,9 @@ url_preview_domain_explicit_allowlist = []
# Setting this to "*" will allow all URL previews. Please note that this opens up significant attack surface to your server, you are expected to be aware of the risks by doing so.
url_preview_url_contains_allowlist = []
# Maximum amount of bytes allowed in a URL preview body size when spidering. Defaults to 1MB (1_000_000 bytes)
url_preview_max_spider_size = 1_000_000
### Misc