Skip to content

Commit fc8054e

Browse files
falsydotdevaeppling
authored andcommitted
feat(rewrite): rewrite each line of multi-line Bash blocks
Coding agents routinely emit multi-line Bash blocks of sequential commands. The rewriter tokenized newlines as plain whitespace, so a block was treated as one command: the rtk prefix landed on the first line and every following line ran raw and unfiltered. Split multi-line input at the newline tokens the quote-aware lexer emits (newlines inside quoted strings are never split points) and rewrite each line through the existing single-line path. Blank lines, comment lines, indentation, and CRLF separators are preserved verbatim. Per-line rewriting only applies when every line is an independent command. The whole block passes through untouched when: - a shell keyword opens control flow (for/if/while/case/...) - a list or pipeline continues across the line break (&&, ||, |, |& at a line edge) - a subshell or group spans lines - the block contains a heredoc (existing gate) Ref #1243
1 parent 8a24ce2 commit fc8054e

2 files changed

Lines changed: 255 additions & 2 deletions

File tree

‎src/discover/lexer.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@ pub fn tokenize(input: &str) -> Vec<ParsedToken> {
2626
tokenize_inner(input, false)
2727
}
2828

29+
/// Like [`tokenize`] but emits a `\n` operator token for each newline that
30+
/// sits outside quotes. Newlines inside quoted strings stay part of their
31+
/// argument, so callers can use the emitted offsets as safe line-split points.
32+
pub fn tokenize_with_newlines(input: &str) -> Vec<ParsedToken> {
33+
tokenize_inner(input, true)
34+
}
35+
2936
fn tokenize_inner(input: &str, emit_newline: bool) -> Vec<ParsedToken> {
3037
let mut tokens = Vec::new();
3138
let mut current = String::new();
@@ -1342,4 +1349,17 @@ mod tests {
13421349
assert!(split_for_permissions("").is_empty());
13431350
assert!(split_for_permissions(" ").is_empty());
13441351
}
1352+
1353+
#[test]
1354+
fn test_tokenize_with_newlines_emits_operator_outside_quotes_only() {
1355+
let newline_ops = |input: &str| {
1356+
tokenize_with_newlines(input)
1357+
.iter()
1358+
.filter(|t| t.kind == TokenKind::Operator && t.value == "\n")
1359+
.count()
1360+
};
1361+
assert_eq!(newline_ops("git status\ngit log"), 1);
1362+
assert_eq!(newline_ops("echo 'line1\nline2'"), 0);
1363+
assert_eq!(newline_ops("git status\r\ngit log"), 2);
1364+
}
13451365
}

‎src/discover/registry.rs‎

Lines changed: 235 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ use regex::{Regex, RegexSet};
55
use std::path::Path;
66
use std::sync::LazyLock;
77

8-
use super::lexer::{shell_split, split_on_operators, tokenize, ParsedToken, PipeKind, TokenKind};
8+
use super::lexer::{
9+
shell_split, split_on_operators, tokenize, tokenize_with_newlines, ParsedToken, PipeKind,
10+
TokenKind,
11+
};
912
use super::rules::{IGNORED_EXACT, IGNORED_PREFIXES, RULES};
1013

1114
const PHP_TOOL_NAMES: [&str; 6] = ["phpunit", "phpstan", "ecs", "pest", "paratest", "pint"];
@@ -583,6 +586,19 @@ pub fn rewrite_command(
583586
let compiled = compile_exclude_patterns(excluded);
584587
let normalized_prefixes = normalize_transparent_prefixes(transparent_prefixes);
585588

589+
if trimmed.contains('\n') {
590+
return rewrite_multiline_block(trimmed, &compiled, &normalized_prefixes);
591+
}
592+
593+
rewrite_single(trimmed, &compiled, &normalized_prefixes)
594+
}
595+
596+
/// Rewrite one logical command line (no unquoted newlines).
597+
fn rewrite_single(
598+
trimmed: &str,
599+
excluded: &[ExcludePattern],
600+
transparent_prefixes: &[String],
601+
) -> Option<String> {
586602
// Simple (non-compound) already-RTK command — return as-is.
587603
// For compound commands that start with "rtk" (e.g. "rtk git add . && cargo test"),
588604
// fall through to rewrite_compound so the remaining segments get rewritten.
@@ -595,7 +611,103 @@ pub fn rewrite_command(
595611
return Some(trimmed.to_string());
596612
}
597613

598-
rewrite_compound(trimmed, &compiled, &normalized_prefixes)
614+
rewrite_compound(trimmed, excluded, transparent_prefixes)
615+
}
616+
617+
/// Shell keywords that open or close a multi-line construct. A line inside a
618+
/// loop, conditional, case arm, function body, or group is not an independent
619+
/// command, so the whole block passes through untouched.
620+
const BLOCK_KEYWORDS: &[&str] = &[
621+
"for", "while", "until", "if", "then", "else", "elif", "fi", "do", "done", "case", "esac",
622+
"select", "function", "coproc", "{", "}", "(", ")",
623+
];
624+
625+
/// A list or pipeline that continues across a line break makes adjacent lines
626+
/// one logical command; grouping chars at a line edge mean a construct spans
627+
/// lines. Either way, per-line rewriting is unsafe.
628+
fn line_breaks_independence(line: &str) -> bool {
629+
let first = line.split_whitespace().next().unwrap_or("");
630+
if BLOCK_KEYWORDS.contains(&first) {
631+
return true;
632+
}
633+
["&&", "||", "|&", "|"]
634+
.iter()
635+
.any(|op| line.ends_with(op) || line.starts_with(op))
636+
|| line.ends_with('(')
637+
|| line.ends_with('{')
638+
|| line.starts_with(')')
639+
|| line.starts_with('}')
640+
}
641+
642+
/// Rewrite each line of a multi-line block independently (issue #1243).
643+
///
644+
/// Split points are the newline tokens the quote-aware lexer emits, so a
645+
/// newline inside a quoted string (e.g. a multi-line commit message) never
646+
/// becomes a boundary. The whole block passes through unchanged unless every
647+
/// line is an independent command (see [`line_breaks_independence`]); blank
648+
/// lines and comment lines are preserved verbatim, as is indentation and the
649+
/// original separator bytes (`\n` vs `\r\n`).
650+
fn rewrite_multiline_block(
651+
cmd: &str,
652+
excluded: &[ExcludePattern],
653+
transparent_prefixes: &[String],
654+
) -> Option<String> {
655+
let newline_offsets: Vec<usize> = tokenize_with_newlines(cmd)
656+
.iter()
657+
.filter(|t| t.kind == TokenKind::Operator && t.value == "\n")
658+
.map(|t| t.offset)
659+
.collect();
660+
661+
// Newlines only inside quotes — a single logical command.
662+
if newline_offsets.is_empty() {
663+
return rewrite_single(cmd, excluded, transparent_prefixes);
664+
}
665+
666+
let mut segments = Vec::with_capacity(newline_offsets.len() + 1);
667+
let mut start = 0;
668+
for &off in &newline_offsets {
669+
segments.push(&cmd[start..off]);
670+
start = off + 1;
671+
}
672+
segments.push(&cmd[start..]);
673+
674+
if segments
675+
.iter()
676+
.map(|seg| seg.trim())
677+
.filter(|line| !line.is_empty() && !line.starts_with('#'))
678+
.any(line_breaks_independence)
679+
{
680+
return None;
681+
}
682+
683+
let mut any_changed = false;
684+
let mut result = String::with_capacity(cmd.len() + 32);
685+
for (i, seg) in segments.iter().enumerate() {
686+
if i > 0 {
687+
let off = newline_offsets[i - 1];
688+
result.push_str(&cmd[off..off + 1]);
689+
}
690+
let line = seg.trim();
691+
if line.is_empty() || line.starts_with('#') {
692+
result.push_str(seg);
693+
continue;
694+
}
695+
match rewrite_single(line, excluded, transparent_prefixes) {
696+
Some(rewritten) if rewritten != line => {
697+
any_changed = true;
698+
let indent = &seg[..seg.len() - seg.trim_start().len()];
699+
result.push_str(indent);
700+
result.push_str(&rewritten);
701+
}
702+
_ => result.push_str(seg),
703+
}
704+
}
705+
706+
if any_changed {
707+
Some(result)
708+
} else {
709+
None
710+
}
599711
}
600712

601713
/// Pipeline boundaries used to rewrite its final stage.
@@ -1159,6 +1271,127 @@ mod tests {
11591271
super::rewrite_command(cmd, excluded, &[])
11601272
}
11611273

1274+
mod multiline_blocks {
1275+
use super::rewrite_command_no_prefixes;
1276+
1277+
#[test]
1278+
fn test_rewrites_each_line() {
1279+
assert_eq!(
1280+
rewrite_command_no_prefixes("git status\ngit log --oneline -3", &[]),
1281+
Some("rtk git status\nrtk git log --oneline -3".into())
1282+
);
1283+
}
1284+
1285+
#[test]
1286+
fn test_preserves_blank_lines_comments_and_indentation() {
1287+
assert_eq!(
1288+
rewrite_command_no_prefixes("git status\n\n# check history\n git log -3", &[]),
1289+
Some("rtk git status\n\n# check history\n rtk git log -3".into())
1290+
);
1291+
}
1292+
1293+
#[test]
1294+
fn test_compound_line_inside_block() {
1295+
assert_eq!(
1296+
rewrite_command_no_prefixes("cd /tmp && git status\ngrep -rn foo src", &[]),
1297+
Some("cd /tmp && rtk git status\nrtk grep -rn foo src".into())
1298+
);
1299+
}
1300+
1301+
#[test]
1302+
fn test_crlf_separators_preserved() {
1303+
assert_eq!(
1304+
rewrite_command_no_prefixes("git status\r\ngit log -3", &[]),
1305+
Some("rtk git status\r\nrtk git log -3".into())
1306+
);
1307+
}
1308+
1309+
#[test]
1310+
fn test_newline_inside_quotes_is_not_a_split_point() {
1311+
// The quoted body must never be treated as a command line of its own.
1312+
let result =
1313+
rewrite_command_no_prefixes("git commit -m \"subject\ngit status in body\"", &[]);
1314+
if let Some(rewritten) = result {
1315+
assert!(!rewritten.contains("rtk git status"));
1316+
}
1317+
}
1318+
1319+
#[test]
1320+
fn test_no_rewritable_line_passes_through() {
1321+
assert_eq!(rewrite_command_no_prefixes("echo one\necho two", &[]), None);
1322+
}
1323+
1324+
#[test]
1325+
fn test_already_rtk_lines_count_as_unchanged() {
1326+
assert_eq!(
1327+
rewrite_command_no_prefixes("rtk git status\necho done", &[]),
1328+
None
1329+
);
1330+
}
1331+
1332+
#[test]
1333+
fn test_mixed_rtk_and_rewritable_line() {
1334+
assert_eq!(
1335+
rewrite_command_no_prefixes("rtk git status\ngit log -3", &[]),
1336+
Some("rtk git status\nrtk git log -3".into())
1337+
);
1338+
}
1339+
1340+
#[test]
1341+
fn test_for_loop_block_passes_through() {
1342+
assert_eq!(
1343+
rewrite_command_no_prefixes("for f in a b; do\n grep -n foo $f\ndone", &[]),
1344+
None
1345+
);
1346+
}
1347+
1348+
#[test]
1349+
fn test_if_block_passes_through() {
1350+
assert_eq!(
1351+
rewrite_command_no_prefixes("if [ -d src ]; then\n git status\nfi", &[]),
1352+
None
1353+
);
1354+
}
1355+
1356+
#[test]
1357+
fn test_cross_line_and_list_passes_through() {
1358+
assert_eq!(
1359+
rewrite_command_no_prefixes("git status &&\ngit log -3", &[]),
1360+
None
1361+
);
1362+
}
1363+
1364+
#[test]
1365+
fn test_cross_line_pipeline_passes_through() {
1366+
assert_eq!(
1367+
rewrite_command_no_prefixes("git log |\ngrep feat", &[]),
1368+
None
1369+
);
1370+
assert_eq!(
1371+
rewrite_command_no_prefixes("cargo test |&\ngrep FAILED", &[]),
1372+
None
1373+
);
1374+
}
1375+
1376+
#[test]
1377+
fn test_subshell_spanning_lines_passes_through() {
1378+
assert_eq!(rewrite_command_no_prefixes("(\n git status\n)", &[]), None);
1379+
}
1380+
1381+
#[test]
1382+
fn test_group_spanning_lines_passes_through() {
1383+
assert_eq!(rewrite_command_no_prefixes("{\n git status\n}", &[]), None);
1384+
}
1385+
1386+
#[test]
1387+
fn test_heredoc_block_passes_through() {
1388+
assert_eq!(
1389+
rewrite_command_no_prefixes("git status\ncat <<EOF\nhello\nEOF", &[]),
1390+
None
1391+
);
1392+
}
1393+
}
1394+
11621395
fn analyze_test_pipeline(cmd: &str) -> PipelineAnalysis {
11631396
let tokens = tokenize(cmd);
11641397
let first_pipe_offset = tokens

0 commit comments

Comments
 (0)