Skip to content

Commit 54a9d5b

Browse files
authored
feat(scan): read @openshell comments in Ruby, Rust, Java and more; report unscanned languages (#21)
## What and why Inline `@openshell` comments in Ruby, Rust, Java, Kotlin, C#, PHP and other languages were silently ignored. They are now read, and files in languages we cannot read are reported instead of skipped silently. - Generic `#` and `//` comment lexers, picked by file extension. Built-in: `rb rake gemspec pl pm r` (hash) and `rs java kt kts cs php swift scala dart c h cc cpp hpp` (slash). - `[scan.comment_leaders]` in `openshell-annotate.toml` adds extensions or switches defaults off (`hash`, `slash`, `none`); a bad value is a config error. - `[languages]` default executables for these languages are keyed by file extension (`rb = "/usr/bin/ruby"`). - Known source files that are still not scanned (for example `.lua`, `.ex`, `.hs`, or an extension set to `none`) produce a `note: N .ext files were not scanned ...` line on stderr. stdout and `--json` are unchanged. - No call-site lint for these languages (directives only); documented in the README. The `.sh` sidecar workaround is documented. A first-class sidecar file is tracked separately in #20. Verified on copies of the real candidates: an inline directive in pocketrb's `web_search.rb` and in anthropic-proxy-rs's `proxy.rs` now scans and generates. Tests: three new CLI tests. `TOOLCHAIN=1.94 scripts/ci-local.sh` and stable clippy pass. Independent of the other PRs (touches `scan.rs`, `lexer.rs`, `config.rs`, README); it can merge in any order. Closes #7 --------- Signed-off-by: Ilan Harel <[email protected]> Co-authored-by: Ilan Harel <[email protected]>
1 parent 3be648d commit 54a9d5b

10 files changed

Lines changed: 264 additions & 14 deletions

File tree

‎README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,10 @@ exclude = [] # "^prefix/" anchors at the repo root; anything else
218218
categories = [] # restrict call-site heuristics to any of "net", "fs", "env", "exec" (empty = all)
219219
[scan]
220220
exclude = [] # same pattern syntax as [lint].exclude
221+
[scan.comment_leaders] # extension -> "hash" (#) | "slash" (// and /* */) | "none"; extends/overrides the built-ins
222+
# rb = "hash" # built in: rb rake gemspec pl pm r (hash); rs java kt kts cs php swift scala dart c h cc cpp hpp (slash)
223+
[languages]
224+
# rb = "/usr/bin/ruby" # default executable for generic languages is keyed by file extension
221225
```
222226

223227
## Limits (be aware)
@@ -227,6 +231,12 @@ exclude = [] # same pattern syntax as [lint].exclude
227231
a write whose path is computed cannot be proven statically, so it is one repo-level info (or one warning per file when no
228232
root is declared at all). Writes through an already-open `FileHandle` are not seen. `fs.mkdir` etc. are only recognised
229233
as fs calls when imported from `fs`, `node:fs`, `fs/promises`, `fs-extra` (or called on an object named like `fs`).
234+
- Languages: JS/TS, Python, Go and shell are first-class (string-aware lexing and lint call-site detection). Ruby, Rust,
235+
Java, Kotlin, C#, PHP, Swift, Scala, Dart, C/C++ and a few more are read for `@openshell` comments only (generic `#` or `//`
236+
lexer, no call-site lint). Map more extensions with `[scan.comment_leaders]`. Files in other known source languages (for
237+
example `.lua`, `.ex`, `.hs`) are counted and reported on stderr as `note: N .ext files were not scanned`. When you
238+
cannot or do not want to edit the source, keep the directives in a sidecar `.sh` file (comments only); it is scanned like any
239+
other shell file. A first-class sidecar annotations file is tracked in #20.
230240
- JS regex literals are not parsed; Python docstrings are strings (directives must be `#` comments).
231241
- Filesystem/process/landlock are startup-only in OpenShell; drift labels them `startup`.
232242
- `diff-logs` only sees network and L7 events: OpenShell 0.1.6 does not log Landlock/filesystem denials (upstream #4270), log

‎src/ast.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ fn urls_in(nodes: &[Node], src: &str) -> Vec<String> {
194194

195195
/// Parse and extract sites. `None` when the grammar cannot be loaded or parsing fails outright.
196196
pub fn sites(lang: Language, rel: &str, text: &str) -> Option<Vec<Site>> {
197+
if matches!(lang, Language::Hash | Language::Slash) {
198+
return None; // no grammar: directives only, no call-site heuristics
199+
}
197200
let mut parser = Parser::new();
198201
parser.set_language(&language_for(lang, rel)).ok()?;
199202
let tree = parser.parse(text, None)?;
@@ -207,6 +210,7 @@ pub fn sites(lang: Language, rel: &str, text: &str) -> Option<Vec<Site>> {
207210
Language::Python => py(n, text, &mut out),
208211
Language::Go => go(n, text, &mut out),
209212
Language::Shell => sh(n, text, &mut out),
213+
Language::Hash | Language::Slash => {}
210214
}
211215
url_literal(n, text, &mut out);
212216
if cur.goto_first_child() {
@@ -233,14 +237,17 @@ fn language_for(lang: Language, rel: &str) -> tree_sitter::Language {
233237
},
234238
Language::Python => tree_sitter_python::LANGUAGE.into(),
235239
Language::Go => tree_sitter_go::LANGUAGE.into(),
236-
Language::Shell => tree_sitter_bash::LANGUAGE.into(),
240+
Language::Shell | Language::Hash | Language::Slash => tree_sitter_bash::LANGUAGE.into(),
237241
}
238242
}
239243

240244
/// Comments from the real syntax tree, one entry per line with the leader removed. The hand-written lexer cannot
241245
/// tell a JS regex literal from a division, so a stray quote inside a regex can turn the rest of a big file into a
242246
/// "string" and hide its directives; the grammar does not have that problem.
243247
pub fn comments(lang: Language, rel: &str, text: &str) -> Option<Vec<crate::lexer::Comment>> {
248+
if matches!(lang, Language::Hash | Language::Slash) {
249+
return None;
250+
}
244251
let mut parser = Parser::new();
245252
parser.set_language(&language_for(lang, rel)).ok()?;
246253
let tree = parser.parse(text, None)?;

‎src/config.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@ pub struct ScanConfig {
121121
/// Path patterns (see `path_matches`) never scanned (in addition to .gitignore, node_modules, dist, build, target).
122122
#[serde(default)]
123123
pub exclude: Vec<String>,
124+
/// File extension (no dot) -> comment style for languages without first-class support:
125+
/// `hash` (`#`), `slash` (`//` and `/* */`) or `none` (do not scan). Built-in defaults cover
126+
/// rb, rs, java, kt, cs, php, swift, scala and a few more; entries here override them.
127+
#[serde(default)]
128+
pub comment_leaders: BTreeMap<String, String>,
124129
}
125130

126131
fn auto() -> String {
@@ -149,6 +154,14 @@ impl Config {
149154
path.display()
150155
);
151156
}
157+
for (ext, kind) in &c.scan.comment_leaders {
158+
if !matches!(kind.as_str(), "hash" | "slash" | "none") {
159+
anyhow::bail!(
160+
"{}: scan.comment_leaders.{ext} must be `hash`, `slash` or `none`, not `{kind}`",
161+
path.display()
162+
);
163+
}
164+
}
152165
Ok(c)
153166
}
154167

‎src/diff_logs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,7 @@ pub fn analyze(events: &[Event], scan: &Scan, cfg: &Config) -> Report {
604604

605605
pub fn leader(lang: Language) -> &'static str {
606606
match lang {
607-
Language::Python | Language::Shell => "#",
607+
Language::Python | Language::Shell | Language::Hash => "#",
608608
_ => "//",
609609
}
610610
}

‎src/generate.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -164,16 +164,19 @@ pub(crate) fn find_bin(
164164
}
165165
}
166166
if let Some(lang) = d.lang {
167-
if let Some(p) = cfg.languages.get(lang.config_key()) {
168-
return Ok((
169-
p.clone(),
170-
format!("[languages].{} default", lang.config_key()),
171-
));
167+
// Generic `#` / `//` languages are keyed by file extension (`[languages] rb = "/usr/bin/ruby"`).
168+
let key = match lang {
169+
crate::lexer::Language::Hash | crate::lexer::Language::Slash => {
170+
d.file.rsplit('.').next().unwrap_or("").to_ascii_lowercase()
171+
}
172+
_ => lang.config_key().to_string(),
173+
};
174+
if let Some(p) = cfg.languages.get(&key) {
175+
return Ok((p.clone(), format!("[languages].{key} default")));
172176
}
173177
return Err(format!(
174-
"{}: no executable known for this egress: add `bin=/real/path` or a `@openshell bin /real/path` line in the file, or set [languages].{} in openshell-annotate.toml (an empty `binaries` list allows nothing)",
178+
"{}: no executable known for this egress: add `bin=/real/path` or a `@openshell bin /real/path` line in the file, or set [languages].{key} in openshell-annotate.toml (an empty `binaries` list allows nothing)",
175179
d.loc(),
176-
lang.config_key()
177180
));
178181
}
179182
Err(format!("{}: cannot determine the executable", d.loc()))

‎src/lexer.rs‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//! Comment lexer for JS/TS, Python, Go and shell.
1+
//! Comment lexer for JS/TS, Python, Go and shell, plus generic `#` and `//` leaders for other languages.
22
//!
33
//! The lexer is string-aware: `//` or `#` inside a string literal, template
44
//! literal, Python triple-quoted string or shell heredoc is not a comment.
@@ -20,9 +20,67 @@ pub enum Language {
2020
Python,
2121
Go,
2222
Shell,
23+
/// Any other language whose line comments start with `#` (Ruby, Perl, R, ...), chosen by extension.
24+
Hash,
25+
/// Any other language whose comments start with `//` or `/* */` (Rust, Java, C#, Kotlin, ...), chosen by extension.
26+
Slash,
2327
}
2428

29+
/// Extensions with a built-in comment leader beyond the four first-class languages.
30+
/// Override or extend with `[scan.comment_leaders]` in `openshell-annotate.toml`.
31+
pub const DEFAULT_LEADERS: &[(&str, &str)] = &[
32+
("rb", "hash"),
33+
("rake", "hash"),
34+
("gemspec", "hash"),
35+
("pl", "hash"),
36+
("pm", "hash"),
37+
("r", "hash"),
38+
("rs", "slash"),
39+
("java", "slash"),
40+
("kt", "slash"),
41+
("kts", "slash"),
42+
("cs", "slash"),
43+
("php", "slash"),
44+
("swift", "slash"),
45+
("scala", "slash"),
46+
("dart", "slash"),
47+
("c", "slash"),
48+
("h", "slash"),
49+
("cc", "slash"),
50+
("cpp", "slash"),
51+
("hpp", "slash"),
52+
];
53+
54+
/// Source-code extensions we do not know a comment leader for; files with these are reported as "not scanned".
55+
pub const KNOWN_UNSCANNED: &[&str] = &[
56+
"lua", "ex", "exs", "erl", "hs", "ml", "clj", "cljs", "jl", "zig", "groovy", "vb", "fs", "nim",
57+
"cr", "v", "sol", "tf",
58+
];
59+
2560
impl Language {
61+
/// Language of a file, honouring `[scan.comment_leaders]` (extension -> `hash` | `slash` | `none`).
62+
/// First-class languages always win; `none` switches a built-in default off.
63+
pub fn detect(
64+
path: &Path,
65+
overrides: &std::collections::BTreeMap<String, String>,
66+
) -> Option<Self> {
67+
if let Some(l) = Self::from_path(path) {
68+
return Some(l);
69+
}
70+
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
71+
let kind = overrides.get(&ext).map(String::as_str).or_else(|| {
72+
DEFAULT_LEADERS
73+
.iter()
74+
.find(|(e, _)| *e == ext)
75+
.map(|(_, k)| *k)
76+
})?;
77+
match kind {
78+
"hash" => Some(Language::Hash),
79+
"slash" => Some(Language::Slash),
80+
_ => None,
81+
}
82+
}
83+
2684
pub fn from_path(path: &Path) -> Option<Self> {
2785
let name = path.file_name()?.to_str()?;
2886
if name == "Dockerfile"
@@ -49,6 +107,8 @@ impl Language {
49107
Language::Python => "python",
50108
Language::Go => "go",
51109
Language::Shell => "shell",
110+
Language::Hash => "hash",
111+
Language::Slash => "slash",
52112
}
53113
}
54114
}
@@ -76,6 +136,8 @@ pub fn lex(lang: Language, src: &str) -> Lexed {
76136
Language::Go => lex_clike(src, true),
77137
Language::Python => lex_hash(src, true, false),
78138
Language::Shell => lex_hash(src, false, true),
139+
Language::Hash => lex_hash(src, false, false),
140+
Language::Slash => lex_clike(src, true),
79141
}
80142
}
81143

‎src/lint.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ impl Hints {
107107
fn new(lang: Language, cfg: &Config) -> Self {
108108
Hints {
109109
leader: match lang {
110-
Language::Python | Language::Shell => "#",
110+
Language::Python | Language::Shell | Language::Hash => "#",
111111
_ => "//",
112112
},
113113
window: cfg.lint.window,
@@ -292,7 +292,10 @@ fn patterns(lang: Language) -> &'static Vec<Pat> {
292292
static PY: OnceLock<Vec<Pat>> = OnceLock::new();
293293
static GO: OnceLock<Vec<Pat>> = OnceLock::new();
294294
static SH: OnceLock<Vec<Pat>> = OnceLock::new();
295+
static NONE: OnceLock<Vec<Pat>> = OnceLock::new();
295296
match lang {
297+
// generic `#` / `//` languages: directives are read, but there are no call-site heuristics
298+
Language::Hash | Language::Slash => NONE.get_or_init(Vec::new),
296299
Language::JavaScript => JS.get_or_init(|| {
297300
vec![
298301
p("net", "fetch()", r"(?:^|[^.\w])fetch\s*\(|globalThis\.fetch\s*\("),
@@ -354,6 +357,7 @@ fn env_name_re(lang: Language) -> Regex {
354357
}
355358
Language::Go => r#"os\.Getenv\(\s*"([A-Z][A-Z0-9_]*)""#,
356359
Language::Shell => r"\$\{?([A-Z][A-Z0-9_]*)",
360+
Language::Hash | Language::Slash => r"\A\z\b", // never matches
357361
})
358362
.unwrap()
359363
}

‎src/main.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,11 @@ fn load_profiles(cfg: &Config, cli_dir: Option<&Path>, root: &Path) -> Result<Ve
195195
}
196196

197197
fn do_scan(root: &Path, paths: &[PathBuf], cfg: &Config) -> Result<Scan> {
198-
scan::scan(root, paths, cfg)
198+
let s = scan::scan(root, paths, cfg)?;
199+
for n in s.unscanned_notes() {
200+
eprintln!("note: {n}");
201+
}
202+
Ok(s)
199203
}
200204

201205
fn style_for(o: &OutputOpts, stream_is_tty: bool) -> Style {

‎src/scan.rs‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,23 @@ pub struct Scan {
3131
pub sources: Vec<Source>,
3232
pub directives: Vec<Directive>,
3333
pub errors: Vec<ParseError>,
34+
/// Source files in languages with no known comment syntax, by extension (see `lexer::KNOWN_UNSCANNED`).
35+
pub unscanned: std::collections::BTreeMap<String, usize>,
36+
}
37+
38+
impl Scan {
39+
/// One human-readable note per extension that was skipped, for stderr.
40+
pub fn unscanned_notes(&self) -> Vec<String> {
41+
self.unscanned
42+
.iter()
43+
.map(|(ext, n)| {
44+
format!(
45+
"{n} .{ext} file{} not scanned (no comment syntax known); if its comments start with `#` or `//`, set `[scan.comment_leaders] {ext} = \"hash\"` (or \"slash\") in openshell-annotate.toml to read @openshell comments there",
46+
if *n == 1 { " was" } else { "s were" }
47+
)
48+
})
49+
.collect()
50+
}
3451
}
3552

3653
pub fn rel_path(root: &Path, p: &Path) -> String {
@@ -67,7 +84,21 @@ pub fn scan(root: &Path, paths: &[PathBuf], cfg: &Config) -> Result<Scan> {
6784
continue;
6885
}
6986
let path = ent.path();
70-
if Language::from_path(path).is_none() {
87+
if Language::detect(path, &cfg.scan.comment_leaders).is_none() {
88+
let rel = rel_path(root, path);
89+
let skipped = cfg
90+
.scan
91+
.exclude
92+
.iter()
93+
.any(|x| crate::config::path_matches(&rel, x));
94+
if let (Some(ext), false) = (path.extension().and_then(|e| e.to_str()), skipped) {
95+
let ext = ext.to_ascii_lowercase();
96+
if crate::lexer::KNOWN_UNSCANNED.contains(&ext.as_str())
97+
|| cfg.scan.comment_leaders.get(&ext).map(String::as_str) == Some("none")
98+
{
99+
*out.unscanned.entry(ext).or_insert(0) += 1;
100+
}
101+
}
71102
continue;
72103
}
73104
if ent.metadata().map(|m| m.len() > MAX_BYTES).unwrap_or(false) {
@@ -91,7 +122,7 @@ pub fn scan(root: &Path, paths: &[PathBuf], cfg: &Config) -> Result<Scan> {
91122
let Ok(src) = std::fs::read_to_string(&path) else {
92123
continue;
93124
};
94-
let lang = Language::from_path(&path).unwrap();
125+
let lang = Language::detect(&path, &cfg.scan.comment_leaders).unwrap();
95126
let mut lexed = lex(lang, &src);
96127
if lang == Language::JavaScript {
97128
// the grammar knows regex literals and JSX; the hand-written lexer does not

0 commit comments

Comments
 (0)