refactor: drop MAX_FILE_CHARS cap on --doc file size
All checks were successful
CI / test (pull_request) Successful in 1m23s

It didn't actually guard what it claimed to: std::fs::read_to_string
loads the whole file into memory before truncation ever ran, so it
never prevented the OOM risk it was documented as protecting against
— it only capped how much of the already-fully-read text got chunked
afterward. Directory scanning is non-recursive and --doc is an
explicit opt-in, so an oversized file is on the caller; chunking
already handles arbitrarily long documents correctly. Can add a
pre-read fs::metadata size check back later if real usage shows a
need for it.
This commit is contained in:
Austin Schaefer 2026-08-19 15:23:52 +02:00
parent 19c92ccd52
commit 5e55861a3f

View file

@ -2,12 +2,6 @@ use rig::embeddings::{EmbedError, TextEmbedder};
use std::path::{Path, PathBuf};
use text_splitter::{Characters, TextSplitter};
/// Safety valve against accidentally pointing `--doc` at a huge binary or
/// log file — not a content limit. Real documents are chunked in full (see
/// `CHUNK_CHARS` below), so nothing meaningful gets silently dropped short
/// of this.
const MAX_FILE_CHARS: usize = 2_000_000;
/// Target chunk size handed to the embedding model: small enough that a
/// handful of retrieved chunks stays well within a local model's context
/// window, large enough to keep a paragraph or two of context in each one.
@ -61,13 +55,7 @@ pub(crate) fn collect_documents(paths: &[PathBuf]) -> anyhow::Result<Vec<Documen
fn push_document(path: &Path, splitter: &TextSplitter<Characters>, documents: &mut Vec<Document>) {
match std::fs::read_to_string(path) {
Ok(mut text) => {
// Cheap upper bound on char count, so the vast majority of
// documents (well under the limit) skip the char-by-char walk
// entirely.
if text.len() > MAX_FILE_CHARS && text.chars().count() > MAX_FILE_CHARS {
text = text.chars().take(MAX_FILE_CHARS).collect();
}
Ok(text) => {
let chunks: Vec<&str> = splitter.chunks(&text).collect();
let total = chunks.len();
let source = |i: usize| {