From 5e55861a3f33068209ad43aed7631b15975418e4 Mon Sep 17 00:00:00 2001 From: Austin Schaefer Date: Wed, 19 Aug 2026 15:23:52 +0200 Subject: [PATCH] refactor: drop MAX_FILE_CHARS cap on --doc file size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It didn't actually guard what it claimed to: std::fs::read_to_string loads the whole file into memory before truncation ever ran, so it never prevented the OOM risk it was documented as protecting against — it only capped how much of the already-fully-read text got chunked afterward. Directory scanning is non-recursive and --doc is an explicit opt-in, so an oversized file is on the caller; chunking already handles arbitrarily long documents correctly. Can add a pre-read fs::metadata size check back later if real usage shows a need for it. --- src/documents.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/documents.rs b/src/documents.rs index e3fe5f5..a167330 100644 --- a/src/documents.rs +++ b/src/documents.rs @@ -2,12 +2,6 @@ use rig::embeddings::{EmbedError, TextEmbedder}; use std::path::{Path, PathBuf}; use text_splitter::{Characters, TextSplitter}; -/// Safety valve against accidentally pointing `--doc` at a huge binary or -/// log file — not a content limit. Real documents are chunked in full (see -/// `CHUNK_CHARS` below), so nothing meaningful gets silently dropped short -/// of this. -const MAX_FILE_CHARS: usize = 2_000_000; - /// Target chunk size handed to the embedding model: small enough that a /// handful of retrieved chunks stays well within a local model's context /// window, large enough to keep a paragraph or two of context in each one. @@ -61,13 +55,7 @@ pub(crate) fn collect_documents(paths: &[PathBuf]) -> anyhow::Result, documents: &mut Vec) { match std::fs::read_to_string(path) { - Ok(mut text) => { - // Cheap upper bound on char count, so the vast majority of - // documents (well under the limit) skip the char-by-char walk - // entirely. - if text.len() > MAX_FILE_CHARS && text.chars().count() > MAX_FILE_CHARS { - text = text.chars().take(MAX_FILE_CHARS).collect(); - } + Ok(text) => { let chunks: Vec<&str> = splitter.chunks(&text).collect(); let total = chunks.len(); let source = |i: usize| {