mirror of
https://github.com/instructkr/claw-code.git
synced 2026-11-19 00:00:00 +08:00
Revert "Merge pull request #3318 from ultraworkers/pr-3292"
This reverts commit2d920ce915, reversing changes made to80bfe94b77.
This commit is contained in:
1 parent
2d920ce915
commit
f021597942
18 files changed
+464
-1940
No files matched your search
@@ -11,48 +11,8 @@ use serde::{Deserialize, Serialize};
|
||||
use walkdir::{DirEntry, WalkDir};
|
||||
|
||||
/// Maximum file size that can be read (10 MB).
|
||||
///
|
||||
/// This is a *process* guard — it stops us slurping a huge or binary blob into memory. It is
|
||||
/// not a context guard: 10 MB is on the order of 2.5 M tokens, so it never fires on the files
|
||||
/// that actually blow a model's context window. `DEFAULT_READ_LINES` / `MAX_READ_CHARS` below
|
||||
/// are the context guard.
|
||||
const MAX_READ_SIZE: u64 = 10 * 1024 * 1024;
|
||||
|
||||
/// Lines returned by `read_file` when the caller does not pass an explicit `limit`.
|
||||
///
|
||||
/// This is a *paging* size, not a refusal. An unbounded read is the easiest way for a model to
|
||||
/// destroy its own context: one 100 KB source file is ~25 k tokens, which does not fit alongside
|
||||
/// a system prompt in a 32 k window. When that happens the backend rejects the whole request and
|
||||
/// the session dies — so reading everything at once is what makes it stop. Returning a page plus
|
||||
/// `nextOffset` means the read always succeeds and the model can keep going.
|
||||
const DEFAULT_READ_LINES: usize = 2000;
|
||||
|
||||
/// Hard ceiling on the characters `read_file` returns, applied even when `limit` is explicit.
|
||||
///
|
||||
/// `DEFAULT_READ_LINES` alone does not bound the payload: minified bundles, JSON blobs and
|
||||
/// generated sources routinely put megabytes on a handful of lines. ~64 KB is ~16 k tokens.
|
||||
///
|
||||
/// That default assumes a large (200 k) context window. Against a small self-hosted backend it
|
||||
/// is far too generous: two reads at this size put a request over 47 k tokens, which a 32 k
|
||||
/// window rejects outright — and compaction cannot recover it, because the preserved recent
|
||||
/// messages ARE the oversized tool results. Override with [`MAX_READ_CHARS_ENV_VAR`].
|
||||
const DEFAULT_MAX_READ_CHARS: usize = 64 * 1024;
|
||||
|
||||
/// Environment override for [`DEFAULT_MAX_READ_CHARS`], in characters.
|
||||
const MAX_READ_CHARS_ENV_VAR: &str = "CLAW_MAX_READ_CHARS";
|
||||
|
||||
/// Effective character ceiling for one `read_file` page.
|
||||
///
|
||||
/// Read per call rather than cached: a long-lived process should pick up a changed budget, and
|
||||
/// this runs once per read, not per line.
|
||||
fn max_read_chars() -> usize {
|
||||
std::env::var(MAX_READ_CHARS_ENV_VAR)
|
||||
.ok()
|
||||
.and_then(|raw| raw.trim().parse::<usize>().ok())
|
||||
.filter(|chars| *chars > 0)
|
||||
.unwrap_or(DEFAULT_MAX_READ_CHARS)
|
||||
}
|
||||
|
||||
/// Maximum file size that can be written (10 MB).
|
||||
const MAX_WRITE_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
@@ -120,17 +80,6 @@ pub struct TextFilePayload {
|
||||
pub start_line: usize,
|
||||
#[serde(rename = "totalLines")]
|
||||
pub total_lines: usize,
|
||||
/// True when the file has more lines past this page. Signals the caller that it has NOT seen
|
||||
/// the whole file, so it does not silently reason over a partial view.
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
/// Line index to pass back as `offset` to continue reading. Absent once the file is exhausted.
|
||||
#[serde(
|
||||
rename = "nextOffset",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub next_offset: Option<usize>,
|
||||
}
|
||||
|
||||
/// Output envelope for the `read_file` tool.
|
||||
@@ -279,28 +228,10 @@ pub fn read_file(
|
||||
let content = fs::read_to_string(&absolute_path)?;
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
let start_index = offset.unwrap_or(0).min(lines.len());
|
||||
// An absent `limit` pages rather than reading to EOF: see DEFAULT_READ_LINES.
|
||||
let requested_end = start_index
|
||||
.saturating_add(limit.unwrap_or(DEFAULT_READ_LINES))
|
||||
.min(lines.len());
|
||||
|
||||
// Apply the character ceiling on top of the line window, so a file with very long lines
|
||||
// cannot blow the budget the line cap was meant to enforce. Always yield at least one line,
|
||||
// otherwise an oversized single line would return nothing and the caller could not advance.
|
||||
let mut end_index = start_index;
|
||||
let mut chars = 0usize;
|
||||
let char_ceiling = max_read_chars();
|
||||
for line in &lines[start_index..requested_end] {
|
||||
let next = chars + line.chars().count() + 1;
|
||||
if next > char_ceiling && end_index > start_index {
|
||||
break;
|
||||
}
|
||||
chars = next;
|
||||
end_index += 1;
|
||||
}
|
||||
|
||||
let end_index = limit.map_or(lines.len(), |limit| {
|
||||
start_index.saturating_add(limit).min(lines.len())
|
||||
});
|
||||
let selected = lines[start_index..end_index].join("\n");
|
||||
let truncated = end_index < lines.len();
|
||||
|
||||
Ok(ReadFileOutput {
|
||||
kind: String::from("text"),
|
||||
@@ -310,8 +241,6 @@ pub fn read_file(
|
||||
num_lines: end_index.saturating_sub(start_index),
|
||||
start_line: start_index.saturating_add(1),
|
||||
total_lines: lines.len(),
|
||||
truncated,
|
||||
next_offset: truncated.then_some(end_index),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -883,18 +812,11 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
component_contains_glob, derive_glob_walk_root, edit_file, expand_braces, glob_search,
|
||||
grep_search, is_symlink_escape, max_read_chars, normalize_for_comparison, read_file,
|
||||
grep_search, is_symlink_escape, normalize_for_comparison, read_file,
|
||||
read_file_in_workspace, write_file, write_file_in_workspace, GrepSearchInput,
|
||||
DEFAULT_MAX_READ_CHARS, DEFAULT_READ_LINES, MAX_READ_CHARS_ENV_VAR, MAX_WRITE_SIZE,
|
||||
MAX_WRITE_SIZE,
|
||||
};
|
||||
|
||||
/// Serializes the tests that mutate `CLAW_MAX_READ_CHARS`; env vars are process-global.
|
||||
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
LOCK.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn temp_path(name: &str) -> std::path::PathBuf {
|
||||
let unique = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
@@ -915,132 +837,6 @@ mod tests {
|
||||
assert_eq!(read_output.file.content, "two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unbounded_read_of_a_large_file_pages_instead_of_returning_everything() {
|
||||
// Regression: an unbounded read used to return the whole file, which on a small-context
|
||||
// backend overflows the window and the request is rejected outright — the session dies.
|
||||
let path = temp_path("paged-read.txt");
|
||||
let total = DEFAULT_READ_LINES + 500;
|
||||
let body = (0..total)
|
||||
.map(|i| format!("line {i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
write_file(path.to_string_lossy().as_ref(), &body).expect("write should succeed");
|
||||
|
||||
let first = read_file(path.to_string_lossy().as_ref(), None, None).expect("first page");
|
||||
assert_eq!(first.file.num_lines, DEFAULT_READ_LINES);
|
||||
assert_eq!(first.file.total_lines, total);
|
||||
assert!(first.file.truncated, "caller must be told it is partial");
|
||||
assert_eq!(first.file.next_offset, Some(DEFAULT_READ_LINES));
|
||||
assert!(first.file.content.starts_with("line 0\n"));
|
||||
|
||||
// The advertised next_offset must actually resume where the last page stopped.
|
||||
let second = read_file(
|
||||
path.to_string_lossy().as_ref(),
|
||||
first.file.next_offset,
|
||||
None,
|
||||
)
|
||||
.expect("second page");
|
||||
assert_eq!(second.file.num_lines, 500);
|
||||
assert!(!second.file.truncated, "the tail is the final page");
|
||||
assert_eq!(second.file.next_offset, None);
|
||||
assert!(second
|
||||
.file
|
||||
.content
|
||||
.starts_with(&format!("line {DEFAULT_READ_LINES}\n")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_ceiling_bounds_files_whose_lines_are_enormous() {
|
||||
// A line cap alone does not bound the payload: minified bundles put megabytes on a few
|
||||
// lines, so the byte ceiling has to apply too — and must still yield forward progress.
|
||||
let _guard = env_lock();
|
||||
std::env::remove_var(MAX_READ_CHARS_ENV_VAR);
|
||||
let path = temp_path("long-lines.txt");
|
||||
let huge = "x".repeat(DEFAULT_MAX_READ_CHARS);
|
||||
let body = format!("{huge}\n{huge}\n{huge}");
|
||||
write_file(path.to_string_lossy().as_ref(), &body).expect("write should succeed");
|
||||
|
||||
let page = read_file(path.to_string_lossy().as_ref(), None, None).expect("first page");
|
||||
assert_eq!(page.file.num_lines, 1, "one oversized line still advances");
|
||||
assert!(page.file.truncated);
|
||||
assert_eq!(page.file.next_offset, Some(1));
|
||||
|
||||
// An explicit limit must not be able to defeat the ceiling.
|
||||
let greedy = read_file(path.to_string_lossy().as_ref(), None, Some(3)).expect("greedy");
|
||||
assert_eq!(greedy.file.num_lines, 1);
|
||||
assert!(greedy.file.truncated);
|
||||
}
|
||||
|
||||
/// The 64 KB default is a 200 k-context number. A small self-hosted backend must be able to
|
||||
/// shrink it, or two reads overflow a 32 k window in a single turn.
|
||||
#[test]
|
||||
fn read_char_ceiling_is_configurable_via_the_environment() {
|
||||
let _guard = env_lock();
|
||||
let path = temp_path("configurable-ceiling.txt");
|
||||
// 40 lines of 100 chars = ~4 KB: far under the default ceiling, over a 1 KB one.
|
||||
let body = (0..40)
|
||||
.map(|_| "y".repeat(100))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
write_file(path.to_string_lossy().as_ref(), &body).expect("write should succeed");
|
||||
|
||||
std::env::remove_var(MAX_READ_CHARS_ENV_VAR);
|
||||
let whole = read_file(path.to_string_lossy().as_ref(), None, None).expect("default read");
|
||||
assert_eq!(
|
||||
whole.file.num_lines, 40,
|
||||
"default ceiling returns the file whole"
|
||||
);
|
||||
assert!(!whole.file.truncated);
|
||||
|
||||
std::env::set_var(MAX_READ_CHARS_ENV_VAR, "1024");
|
||||
let clipped = read_file(path.to_string_lossy().as_ref(), None, None).expect("clipped read");
|
||||
std::env::remove_var(MAX_READ_CHARS_ENV_VAR);
|
||||
assert!(
|
||||
clipped.file.num_lines < 40,
|
||||
"a smaller ceiling must actually clip, got {} lines",
|
||||
clipped.file.num_lines
|
||||
);
|
||||
assert!(
|
||||
clipped.file.truncated,
|
||||
"a clipped page must report truncated"
|
||||
);
|
||||
assert_eq!(
|
||||
clipped.file.next_offset,
|
||||
Some(clipped.file.num_lines),
|
||||
"the caller must be told where to resume"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_char_ceiling_ignores_junk_and_zero_values() {
|
||||
let _guard = env_lock();
|
||||
for value in ["", "0", "-5", "not-a-number"] {
|
||||
std::env::set_var(MAX_READ_CHARS_ENV_VAR, value);
|
||||
assert_eq!(
|
||||
max_read_chars(),
|
||||
DEFAULT_MAX_READ_CHARS,
|
||||
"{value:?} must fall back to the default rather than disabling reads"
|
||||
);
|
||||
}
|
||||
std::env::set_var(MAX_READ_CHARS_ENV_VAR, "24000");
|
||||
assert_eq!(max_read_chars(), 24_000);
|
||||
std::env::remove_var(MAX_READ_CHARS_ENV_VAR);
|
||||
assert_eq!(max_read_chars(), DEFAULT_MAX_READ_CHARS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_files_are_returned_whole_and_not_marked_truncated() {
|
||||
let path = temp_path("small-read.txt");
|
||||
write_file(path.to_string_lossy().as_ref(), "one\ntwo\nthree").expect("write");
|
||||
|
||||
let output = read_file(path.to_string_lossy().as_ref(), None, None).expect("read");
|
||||
assert_eq!(output.file.content, "one\ntwo\nthree");
|
||||
assert_eq!(output.file.num_lines, 3);
|
||||
assert!(!output.file.truncated);
|
||||
assert_eq!(output.file.next_offset, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edits_file_contents() {
|
||||
let path = temp_path("edit.txt");
|
||||
|
||||
Reference in new issue
Block a user