Paper review 2026-08-04: 25 candidates #350

[{"paper": {"id": "2607.28631", "title": "Can AI Evaluate AI Scientists? A Benchmarking Study of Autonomous Research Generation Systems Using Automated Multi-Model Review", "authors": ["Vaibhava Lakshmi Ravideshik", "Mayank Kejriwal"], "venue": "arXiv 2026/04", "category": "", "published": "2026-04-18", "links": {"paper": "https://arxiv.org/abs/2607.28631", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical", "benchmark"], "summary": "Proposes an automated multi-model LLM peer-review protocol to benchmark and compare autonomous AI Scientist research-generation systems (Sakana AI, CycleResearcher, Data-to-Paper) against a commercial baseline. Establishes a quantitative benchmark for evaluating the quality of AI-generated scientific papers.", "reason": "An empirical benchmarking study whose object is autonomous research (AI Scientist) agents themselves, not a new agent performing a code task.", "source": "crawl"}, {"paper": {"id": "2607.28650", "title": "Unanticipated Effects of Generative AI on Expertise Pathways and Performance Perception in System Administration", "authors": ["Rana Abou Khamis", "Hala Assal", "Ashraf Matrawy"], "venue": "arXiv 2026/06", "category": "", "published": "2026-06-02", "links": {"paper": "https://arxiv.org/abs/2607.28650", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "An interview-based study of 14 IT professionals examining how GenAI reshapes system-administration practice, finding compressed expertise-development pathways and shifting productivity-perception norms. Explores socio-technical effects of GenAI-assisted troubleshooting and scripting rather than proposing a new agent.", "reason": "An empirical/behavioral study of human-agent interaction and adoption in sysadmin practice, matching the studies leaf's real-user usage-in-the-wild scope.", "source": "crawl"}, {"paper": {"id": "2607.28802", "title": "Model or Harness? An Interaction-Centric Taxonomy for Localizing Agent Failures", "authors": ["Harsh Raj", "Vipul Gupta", "Anas Mahmoud", "Razvan-Gabriel Dumitru", "Darvin Yi", "Aakash Sabharwal", "Yunzhong He"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2607.28802", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "Proposes an interaction-centric taxonomy of 41 agent failure modes that localizes faults to specific component interactions (model, harness, environment, grader) across coding assistants, personal assistants, and multi-agent systems. Validates reproducibility of the taxonomy using LLM judges against human labels.", "reason": "Cross-cutting taxonomy studying agent failures across many agent types/worlds, not tied to one activity and proposing no task-performing method.", "source": "crawl"}, {"paper": {"id": "2607.28825", "title": "Building a Process-Modeling Tool using Agentic AI: An Experience Report on PM4Py-UCM", "authors": ["Daniel Amyot"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2607.28825", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "An experience report mining the development record of PM4Py-UCM, a process-mining tool built largely by an AI coding agent (Claude Code), characterizing human turns, agent corrections, and code-quality outcomes across 18 sessions and 65 hours. Contributes a reproducible taxonomy of human-agent interaction patterns and lessons for responsibly engineering AI-built tools.", "reason": "An empirical case study of human-agent interaction during AI-assisted software construction, proposing no generalizable method, matching the studies leaf.", "source": "crawl"}, {"paper": {"id": "2507.03156", "title": "The Impact of LLM-Assistants on Software Developer Productivity: A Systematic Review and Mapping Study", "authors": ["Amr Mohamed", "Maram Assi", "Mariam Guizani"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2507.03156", "github": "", "website": ""}}, "category": "studies", "tags": ["survey"], "summary": "Systematic review and mapping of 39 studies examining how LLM-assistants affect software developer productivity across coding, testing, debugging, and documentation tasks. Finds mixed evidence on code quality and highlights gaps in longitudinal and team-based evaluation.", "reason": "Field-wide empirical/survey study whose object is code-agent tool usage and developer productivity, matching the studies leaf.", "source": "crawl"}, {"paper": {"id": "2608.00101", "title": "Agentic Coding in the Wild: Characterizing GitHub Copilot Traces at Production Scale", "authors": ["Banruo Liu", "Haoran Qiu", "Íñigo Goiri", "Rodrigo Fonseca", "Ricardo Bianchini", "Esha Choukse"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2608.00101", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "A production-scale empirical characterization of GitHub Copilot agentic coding traces (3.2M users, 761M LLM calls), analyzing session structure, KV-cache behavior, and idle-time patterns to inform agent-native serving infrastructure. It derives a lightweight idle-time predictor from these workload observations.", "reason": "An empirical/behavioral study whose object is the coding-agent workload itself, not a proposed task-performing agent.", "source": "crawl"}, {"paper": {"id": "2608.00181", "title": "Cross-Benchmark Generalization in Long-Horizon Agents", "authors": ["Sushant Mehta", "Logan Ritchie", "Liudas Panavas", "Edwin Chen"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2608.00181", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "Post-trains a mixture-of-experts model on long-horizon MCP tool-use tasks via SFT+RL and measures transfer to held-out benchmarks (Toolathlon, tau2-Bench, BFCL-V4, SWE-Bench Pro, Terminal-Bench 2), finding consistent gains. A paired-trajectory analysis identifies recurring behavioral changes (goal formation, state tracking, verification) that generalize across domains.", "reason": "Object of study is cross-benchmark behavioral generalization of trained agents, an empirical study spanning both artifact and agency benchmarks rather than one task.", "source": "crawl"}, {"paper": {"id": "2608.00355", "title": "CurveShift: Is Agent Progress Scalar? Separating Level from Shape", "authors": ["Hanwen Xing", "Pengyun Wang", "BingXu Meng", "Kumail Alhamoud", "Xiang Li", "Jicheng Wang", "Xin Yu", "Xinyang Han", "Xiaomin Li", "Philip Torr", "Yuexing Hao"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2608.00355", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical", "benchmark"], "summary": "An empirical analysis disentangling whether LLM/agent capability gains reflect genuine shifts in the difficulty-response curve versus ceiling effects, using LiveCodeBench to isolate a real hard-problem effect since September 2024. Releases a new LiveCodeBench Difficulty Panel and analysis code.", "reason": "The paper's object of study is measuring and explaining agent/model capability trends, not proposing a task-performing agent, matching the empirical studies leaf.", "source": "crawl"}, {"paper": {"id": "2608.00966", "title": "AgenTag: Attribution of AI Coding Agents from Behavioral Fingerprints", "authors": ["Taher A. Ghaleb"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-02", "links": {"paper": "https://arxiv.org/abs/2608.00966", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "AgenTag is a multimodal classifier that attributes pull requests to specific AI coding agents using textual, behavioral, and code-diff signals, evaluated on thousands of real PRs. It finds that agents are distinguished mainly by communication style rather than code content, even after removing self-disclosed markers.", "reason": "Empirically studies behavioral fingerprints of coding agents in the wild rather than proposing an agent that writes or executes code.", "source": "crawl"}, {"paper": {"id": "2608.01347", "title": "Prompt-Induced Waste in Large Reasoning Models: A Preregistered Two-Harness Benchmark of Coding Agents", "authors": ["Sarel Weinberger", "Amir Hozez"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-02", "links": {"paper": "https://arxiv.org/abs/2608.01347", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "A preregistered benchmark measuring how prompt wording and agent harness choice affect reasoning-token cost of coding agents across six LRMs and 24 deterministic coding tasks. Finds certain prompts multiply reasoning cost without improving correctness, and harness design causes large cost variance.", "reason": "An empirical/behavioral study of coding-agent cost and prompting practices, not a method proposing a task-performing agent.", "source": "crawl"}, {"paper": {"id": "2608.01604", "title": "Post-Training on Office Work Improves Software Engineering: A Behavioral Account of Cross-Domain Transfer", "authors": ["Logan Ritchie", "Sushant Mehta", "Liudas Panavas", "Edwin Chen"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2608.01604", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "This paper post-trains an agent on long-horizon office-workflow tasks (no software engineering data) and finds unexpected improvement on SWE-Bench Pro, then behaviorally analyzes the transfer via goal-directed execution behaviors. It offers a behavioral account of cross-domain generalization rather than a new task-specific tool.", "reason": "The paper's object is empirical behavioral analysis of agent transfer across domains, not a proposed task-specific agent or resource, fitting the empirical-study branch.", "source": "crawl"}, {"paper": {"id": "2608.01995", "title": "Long-Horizon Autonomous Architecture Research with a Language-Model Agent: A Behavioural Case Study", "authors": ["Aon Safdar", "Mohamed Saadeldin"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2608.01995", "github": "", "website": ""}}, "category": "studies", "tags": ["empirical"], "summary": "A behavioral case study tracking a single LLM agent autonomously conducting long-horizon neural architecture research over ~100 experiments. It reports productivity phase patterns, saturation and recovery dynamics, and how workflow design shaped the agent's research behavior.", "reason": "Object of study is the agent's behavior/process during research, not a proposed method to perform the task, matching the studies/empirical precedent.", "source": "crawl"}, {"paper": {"id": "2608.02582", "title": "ACEM: A Cost Estimation Model for Agentic Software Engineering", "authors": ["Mohammad El-Ramly"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2608.02582", "github": "", "website": ""}}, "category": "studies", "tags": ["position"], "summary": "ACEM proposes a cost estimation framework for agentic software engineering that decomposes development cost into LLM, human-in-the-loop, and infrastructure dimensions, mapping traditional sizing metrics to token consumption estimates.", "reason": "A position/framework paper whose object is the economics and practice of agentic software engineering, not a task-performing agent, so it fits studies.", "source": "crawl"}, {"paper": {"id": "2607.28815", "title": "Preventing Premature Commitment in Coding Agents with an Evidence-Conditioned Execution Layer", "authors": ["Yisen Xu", "Chenglin Li", "Zehao Wang", "Jinqiu Yang", "Tse-Hsun Chen"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2607.28815", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "ECLoop is an execution layer that enforces evidence-conditioned execution, postponing code edits or patch submissions by coding agents until repository evidence conditions are met. Evaluated on SWE-bench Verified, it raises Pass@1 by 4.8-11.8 points while lowering token consumption.", "reason": "Improves issue-resolution/patch-submission behavior on SWE-bench, so routes to debugging.", "source": "crawl"}, {"paper": {"id": "2607.28871", "title": "Validation Evidence in LLM Repair Agents: How Much of What Passes Actually Tests the Bug?", "authors": ["Xiaonan Xu", "Wenjing Wu"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2607.28871", "github": "", "website": ""}}, "category": "software_debugging", "tags": ["empirical"], "summary": "BSG-VA replays repair-agent test validations against buggy, candidate, and gold-fix code states to measure how often a passing test actually discriminates the reported bug, finding nearly half of positive evidence in LLM repair trajectories is uninformative. A feedback intervention returning buggy-state replay results modestly reduces evidence-inadequate patch closures.", "reason": "Centered on diagnosing and improving evidence quality within program-repair agent pipelines, a single activity (software_debugging).", "source": "crawl"}, {"paper": {"id": "2607.28887", "title": "To Add Is Machine, To Delete Is Human: Measuring and Mitigating Deletion Avoidance in LLM Code Editing", "authors": ["Amir M. Ebrahimi", "Mohammed Mehedi Hasan", "Aaditya Bhatia", "Gopi Krishnan Rajbahadur", "Ahmed E. Hassan"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-30", "links": {"paper": "https://arxiv.org/abs/2607.28887", "github": "", "website": ""}}, "category": "software_debugging", "tags": ["empirical", "benchmark"], "summary": "An empirical study of 'deletion avoidance' in LLM code editing, showing frontier models on SWE-bench Verified frequently fail to delete required code, often wrapping it in guards instead. Introduces CanItDelete, a benchmark of pure-deletion repair tasks, and shows post-training can mitigate the behavior.", "reason": "Measures and benchmarks a failure mode in SWE-bench-style issue-resolution patches, so it routes to software_debugging plus benchmark tag.", "source": "crawl"}, {"paper": {"id": "2607.29658", "title": "Reusing Past Repairs Through Hierarchical Trajectory Abstraction for Coding Agents", "authors": ["Yisen Xu", "Jiayuan Zhou", "Ruiqi Pan", "Tse-Hsun Chen"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2607.29658", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "STAIR converts historical repair trajectories into hierarchical, reusable multi-level plans that guide coding agents on new issues, improving Pass@1 on SWE-bench Verified across multiple agent backbones including a structurally different agent without code changes.", "reason": "Directly serves SWE-bench-style repository issue resolution via reusable repair strategies.", "source": "crawl"}, {"paper": {"id": "2603.29122", "title": "ReLog: Execution-Aware Logging with Runtime Feedback for LLM-Oriented Debugging", "authors": ["Xin Wang", "Yang Feng", "Xiaoqian Jiao", "Yang Zhang", "Zhenhao Li", "Zishuo Ding"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-02", "links": {"paper": "https://arxiv.org/abs/2603.29122", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "ReLog is an LLM-driven framework that iteratively generates, executes, evaluates, and refines logging statements using runtime feedback, evaluated on defect localization and repair via Defects4J.", "reason": "An agentic code-generation/execution loop serving debugging (fault localization and repair), matching software_debuggings dynamic-analysis debugging-assistant scope.", "source": "crawl"}, {"paper": {"id": "2608.00808", "title": "Turning Interaction History into Execution State: A Runtime Layer for Long-Horizon Coding Agents", "authors": ["Zehao Wang", "Yisen Xu", "Chenglin Li", "Chao Peng", "Bram Adams", "Ahmed E. Hassan", "Tse-Hsun", "Chen"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-01", "links": {"paper": "https://arxiv.org/abs/2608.00808", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "Ledger is a deterministic runtime layer that tracks execution state (observed, modified, attempted) for long-horizon coding agents, preventing redundant or stale actions during repository work. Evaluated on SWE-bench Verified, it improves Pass@1 while cutting cost across multiple LLM-based coding agents.", "reason": "Evaluated on SWE-bench issue-resolution tasks, so it serves software_debugging per benchmark routing.", "source": "crawl"}, {"paper": {"id": "2602.06090", "title": "SVRepair: Structured Visual Reasoning for Automated Program Repair", "authors": ["Jincheng Wang", "Liwei Luo", "Xiaoxuan Tang", "Jingxuan Xu", "Sheng Zhou", "Dajun Chen", "Wei Jiang", "Yong Li"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2602.06090", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "SVRepair is a multimodal automated program repair framework that converts visual bug artifacts (screenshots, control-flow graphs) into structured scene graphs to guide a coding agent's fault localization and patch synthesis. It is evaluated on SWE-Bench M and OmniGIRL, improving repair rates over unimodal baselines.", "reason": "An agent that localizes faults and synthesizes patches for reported bugs is issue resolution/APR.", "source": "crawl"}, {"paper": {"id": "2608.01918", "title": "HarnessCompass: Guiding Automatic Harness Evolution toward Generalizable and Effective Agent Harnesses", "authors": ["Luan Zhang", "Ruochen Zhou", "Dandan Song", "Zhengyu Chen", "Yuhang Tian", "Jun Yang", "Huipeng Ma", "Chenhao Li", "Guangyuan Feng", "Xudong Li", "Yizhou Jin", "Yan Xu"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2608.01918", "github": "", "website": ""}}, "category": "software_debugging", "tags": [], "summary": "HarnessCompass automatically evolves LLM agent harnesses using constrained, component-wise optimization guided by proactive agent feedback, improving generalization over prior harness-evolution methods. Evaluated on SWE-bench Verified, it raises GPT-5.4 Pass@1 from 54% to 66% in five iterations.", "reason": "Improves a coding agent's scaffolding and is evaluated on SWE-bench issue resolution, routing to that activity per benchmark-routing rule.", "source": "crawl"}, {"paper": {"id": "2608.02499", "title": "SWE-Touch: Benchmarking Coding Agents When Users Touch the Code", "authors": ["Yuqiao Tan", "Jinxiang Meng", "Fangyu Lei", "Minzheng Wang", "Shizhu He", "Jun Zhao", "Kang Liu"], "venue": "arXiv 2026/08", "category": "", "published": "2026-08-03", "links": {"paper": "https://arxiv.org/abs/2608.02499", "github": "", "website": ""}}, "category": "software_debugging", "tags": ["benchmark"], "summary": "SWE-Touch is a benchmark that stress-tests coding agents on SWE-bench-style issue resolution when users concurrently edit task-relevant code, via injected 'Counter-Edits.' It shows current agents lack sufficient workspace awareness, degrading resolve rates on shared-workspace repair tasks.", "reason": "Evaluates issue-resolution coding agents on repository repair tasks (SWE-bench Verified/Pro), matching software_debugging.", "source": "crawl"}, {"paper": {"id": "2607.28645", "title": "Looks Right, Works Right: A Project-Level Benchmark for Multi-Screen Mobile App Generation", "authors": ["Fan Wu", "Cuiyun Gao", "Yiming Huang", "Yang Xiao", "Yujia Chen", "Qing Liao"], "venue": "arXiv 2026/05", "category": "", "published": "2026-05-29", "links": {"paper": "https://arxiv.org/abs/2607.28645", "github": "", "website": ""}}, "category": "software_code_generation", "tags": ["benchmark"], "summary": "MobileForge is a project-level benchmark for generating complete multi-screen mobile apps from designs, testing buildability, cross-page navigation, visual fidelity, and maintainability. It evaluates six frontier multimodal LLMs and finds navigation and maintainability still lag behind compilable output.", "reason": "Whole-application generation from a specification is software_code_generation per the repo/whole-app generation-from-scratch boundary.", "source": "crawl"}, {"paper": {"id": "2607.29389", "title": "Simulation Code Generation for Fluid Systems using Large Language Models: Benchmarking Models and Prompting Strategies", "authors": ["Jan Marius Stürmer", "Jascha Knack", "Tobias Koch", "Andreas Weinmann"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2607.29389", "github": "", "website": ""}}, "category": "software_code_generation", "tags": ["benchmark"], "summary": "Benchmarks ten LLMs and six prompting strategies for translating a neutral graph representation of fluid system models into executable simulation code for WNTR and Modelica. Evaluates generated code via software-quality metrics and functional fidelity against benchmark scenarios.", "reason": "Task is producing domain-specific simulation code from a specification, matching software_code_generation.", "source": "crawl"}, {"paper": {"id": "2607.29529", "title": "AuditCoder: Responsibility-Preserving Task Graphs for Auditable Code Generation and Bounded Repair", "authors": ["Kangjie Huang", "Chen Lyu"], "venue": "arXiv 2026/07", "category": "", "published": "2026-07-31", "links": {"paper": "https://arxiv.org/abs/2607.29529", "github": "", "website": ""}}, "category": "software_code_generation", "tags": [], "summary": "AuditCoder generates code alongside an auditable construction trace using contract-annotated task graphs that assign stable responsibility identities, enabling bounded repair localized to failing regions. Evaluated on APPS and ClassEval, it achieves competitive pass@1 while providing high decision-code trace coverage.", "reason": "The served task is producing code from specifications on APPS/ClassEval benchmarks, matching software_code_generation.", "source": "crawl"}]