Skip to content

Commit 80c2547

Browse files
feat(cli): send missing features and host-app friction as ratingless feedback (#5140)
* docs(skills): ask agents inside a host app to report its friction as feedback * feat(cli): send missing features and host-app friction as ratingless feedback * fix(cli): keep the rating error, refuse blank reports and document ratingless feedback * docs(skills): leave names and figures out of missing-feature reports * docs(skills): keep anything private out of missing-feature reports
1 parent 9e0d05d commit 80c2547

12 files changed

Lines changed: 169 additions & 30 deletions

File tree

‎docs/guides/feedback.mdx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ npx hyperframes feedback --rating 7 --comment "The render finished, but the capt
2121

2222
The rating is from 0 to 10. The comment is optional, but a specific example is usually more useful than a score alone.
2323

24+
A comment with no rating is a report with no score, for something you needed that HyperFrames cannot do yet. It never counts in the rating:
25+
26+
```bash
27+
npx hyperframes feedback --comment "MISSING FEATURE: trim a clip from the timeline | WORKAROUND: none"
28+
```
29+
2430
## Report a reproducible bug
2531

2632
Use `--file-issue` when the project can be shared publicly:

‎docs/packages/cli.mdx‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,10 +1248,11 @@ the CLI attaches your email (below).
12481248
npx hyperframes feedback --rating 10
12491249
npx hyperframes feedback --rating 7 --comment "render succeeded but GSAP timeline didn't animate"
12501250
npx hyperframes feedback --rating 3 --comment "GSAP timeline froze on seek" --file-issue
1251+
npx hyperframes feedback --comment "MISSING FEATURE: trim a clip from the timeline | WORKAROUND: none"
12511252
```
12521253

1253-
`--rating` is required, 0–10. `--comment` adds free text. `--source person`
1254-
or `--source agent` says who wrote a rating report (a `--search-miss` report
1254+
`--rating` is 0–10. `--comment` adds free text; a `--comment` with no `--rating` is a report with no score (a missing feature, or friction in the app that launched the CLI), kept out of the rating metric. `--source person`
1255+
or `--source agent` says who wrote a rating or comment report (a `--search-miss` report
12551256
carries neither), so the team can tell a person's feedback from an agent's. `--file-issue`
12561257
also opens a GitHub issue, `--dir` picks the project published as its repro
12571258
(default: current directory), and `--yes, -y` skips the consent prompt for

‎packages/cli/src/commands/coreSkillContent.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,15 @@ describe("hyperframes-core contract docs", () => {
4646
expect(renderReference).toContain("WORKAROUND:");
4747
});
4848

49+
it("sends missing features and host-app friction as ratingless comments", () => {
50+
const skill = read("skills", "hyperframes-cli", "SKILL.md");
51+
52+
expect(skill).toContain(
53+
'npx hyperframes feedback --comment "MISSING FEATURE: <what the person asked for, in their words, with names, clients, figures and paths left out> | WORKAROUND:',
54+
);
55+
expect(skill).toMatch(/HYPERFRAMES_CLIENT[^\n]*npx hyperframes feedback --comment "HOST APP:/);
56+
});
57+
4958
it("mandates a composition-structure block for visual-defect feedback", () => {
5059
const skill = read("skills", "hyperframes-cli", "SKILL.md");
5160
const renderReference = read("skills", "hyperframes-cli", "references", "preview-render.md");

‎packages/cli/src/commands/feedback.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ const mocks = vi.hoisted(() => ({
88
}));
99
vi.mock("../telemetry/client.js", () => ({ shouldTrack: () => true, flush: mocks.flush }));
1010
const trackRenderFeedback = vi.hoisted(() => vi.fn());
11+
const trackFeedbackComment = vi.hoisted(() => vi.fn());
1112
vi.mock("../telemetry/events.js", () => ({
1213
trackCatalogSearchMiss: vi.fn(),
14+
trackFeedbackComment,
1315
trackRenderFeedback,
1416
}));
1517
vi.mock("../telemetry/feedback.js", () => ({ getDoctorSummary: async () => "os=test" }));
@@ -58,3 +60,39 @@ it("refuses a source that is neither person nor agent, and sends nothing", async
5860
await expect(feedback.run?.({ args: { rating: "6", source: "bot" } } as never)).rejects.toThrow();
5961
expect(mocks.submitFeedback).not.toHaveBeenCalled();
6062
});
63+
64+
it("sends a comment with no rating as its own report, never as a rating", async () => {
65+
vi.spyOn(console, "log").mockImplementation(() => {});
66+
trackRenderFeedback.mockClear();
67+
const comment = "MISSING FEATURE: trim a clip | WORKAROUND: none";
68+
const { default: feedback } = await import("./feedback.js");
69+
await feedback.run?.({ args: { comment, source: "agent" } } as never);
70+
expect(trackRenderFeedback).not.toHaveBeenCalled();
71+
expect(trackFeedbackComment).toHaveBeenLastCalledWith(expect.objectContaining({ comment }));
72+
expect(mocks.submitFeedback).toHaveBeenLastCalledWith(
73+
expect.objectContaining({ rating: undefined, comment }),
74+
);
75+
});
76+
77+
it.each([
78+
["neither a rating nor a comment", {}],
79+
["a blank comment with no rating", { comment: " " }],
80+
["--file-issue with no rating", { comment: "MISSING FEATURE: x", "file-issue": true }],
81+
])("refuses %s, and sends nothing", async (_, args) => {
82+
vi.spyOn(console, "error").mockImplementation(() => {});
83+
mocks.submitFeedback.mockClear();
84+
const { default: feedback } = await import("./feedback.js");
85+
await expect(feedback.run?.({ args } as never)).rejects.toThrow();
86+
expect(mocks.submitFeedback).not.toHaveBeenCalled();
87+
});
88+
89+
it("keeps the rating error for an unreadable rating, so a retry keeps the rating", async () => {
90+
const errors = vi.spyOn(console, "error").mockImplementation(() => {});
91+
const { default: feedback } = await import("./feedback.js");
92+
await expect(
93+
feedback.run?.({ args: { rating: "8/10", comment: "x" } } as never),
94+
).rejects.toThrow();
95+
expect(String(errors.mock.calls.at(-1)?.[0])).toContain(
96+
"Rating must be an integer between 0 and 10",
97+
);
98+
});

‎packages/cli/src/commands/feedback.ts‎

Lines changed: 62 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ import { defineCommand } from "citty";
55
import * as clack from "@clack/prompts";
66
import open from "open";
77
import type { Example } from "./_examples.js";
8-
import { trackCatalogSearchMiss, trackRenderFeedback } from "../telemetry/events.js";
8+
import {
9+
trackCatalogSearchMiss,
10+
trackFeedbackComment,
11+
trackRenderFeedback,
12+
} from "../telemetry/events.js";
913
import { shouldTrack } from "../telemetry/client.js";
1014
import { getDoctorSummary } from "../telemetry/feedback.js";
1115
import { readConfig, type RecentRenderRecord } from "../telemetry/config.js";
@@ -21,6 +25,10 @@ import { lintFeedbackComment, type FeedbackLintInput } from "../utils/feedbackLi
2125
export const examples: Example[] = [
2226
["Submit render feedback", 'hyperframes feedback --rating 8 --comment "fast but font missing"'],
2327
["Quick rating only", "hyperframes feedback --rating 10"],
28+
[
29+
"Report something you needed that is missing (no rating)",
30+
'hyperframes feedback --comment "MISSING FEATURE: trim a clip from the timeline | WORKAROUND: none"',
31+
],
2432
[
2533
"Report a catalog gap after a fruitless search",
2634
'hyperframes feedback --search-miss "typewriter that deletes" --wanted "text that types then backspaces"',
@@ -38,8 +46,8 @@ function normalizeComment(raw?: string): string | undefined {
3846
/**
3947
* Compact PostHog join keys appended to the environment string that rides
4048
* along with the forwarded report (and therefore lands verbatim in the wild
41-
* feedback channel): `fid` = this submission's PostHog `cli_render_feedback`
42-
* `feedback_id`; `tid` = the install's telemetry distinct_id; `renders` =
49+
* feedback channel): `fid` = the PostHog `feedback_id` (`cli_render_feedback` or
50+
* `cli_feedback_comment`); `tid` = the install's telemetry distinct_id; `renders` =
4351
* recent `render_job_id`s (newest last, `!` suffix = the render failed).
4452
* Together they turn a wild report into an exact telemetry lookup instead of
4553
* a hardware-fingerprint hunt.
@@ -155,18 +163,62 @@ async function fileGithubIssue(opts: {
155163
await openAndPrintIssue(url);
156164
}
157165

166+
// A comment with no rating is its own report (a missing feature, host-app
167+
// friction): it scores nothing, so it must stay out of the rating metric.
168+
function reportRating(
169+
raw: string | undefined,
170+
comment: string | undefined,
171+
fileIssue: boolean,
172+
): number | undefined {
173+
const rating = raw === undefined ? undefined : parseFeedbackRating(raw);
174+
if (rating === null) {
175+
console.error(c.error("Rating must be an integer between 0 and 10"));
176+
failCommand();
177+
}
178+
if (rating === undefined && !comment?.trim()) {
179+
console.error(c.error("Give a --rating (an integer from 0 to 10), a --comment, or both"));
180+
failCommand();
181+
}
182+
if (rating === undefined && fileIssue) {
183+
console.error(c.error("--file-issue needs a --rating"));
184+
failCommand();
185+
}
186+
return rating;
187+
}
188+
189+
function trackFeedback(report: {
190+
rating: number | undefined;
191+
comment: string | undefined;
192+
doctorSummary: string;
193+
feedbackId: string;
194+
recentRenderIds: string[] | undefined;
195+
}): void {
196+
const { rating, comment, ...keys } = report;
197+
if (rating === undefined) {
198+
if (comment) trackFeedbackComment({ comment, ...keys });
199+
return;
200+
}
201+
// Soft-warn (never blocks) when the comment for a non-clean report is
202+
// missing the mandated reproduction-packet markers. Prints before the
203+
// submission ack so the reporter sees the nudge while their run is fresh.
204+
printFeedbackLintWarnings({ rating, comment });
205+
// The standalone command runs separately from `render`, so it has no real
206+
// elapsed time to report. Omit it rather than recording a fake duration.
207+
trackRenderFeedback({ rating, comment, ...keys });
208+
}
209+
158210
export default defineCommand({
159211
meta: { name: "feedback", description: "Submit feedback about your experience" },
160212
args: {
161213
rating: {
162214
type: "string",
163-
// Required for a rating report, but --search-miss is a different report
164-
// with no rating to give, so the check moved into the run body.
215+
// Optional: --search-miss and a lone --comment are reports with no
216+
// rating to give, so the check lives in the run body.
165217
description: "Likelihood to recommend (0=not likely, 10=extremely likely)",
166218
},
167219
comment: {
168220
type: "string",
169-
description: "Optional details about your experience",
221+
description: "Details about your experience; alone, a report with no rating",
170222
},
171223
source: {
172224
type: "string",
@@ -220,13 +272,8 @@ export default defineCommand({
220272
return;
221273
}
222274

223-
// `--rating` is no longer required at the arg level, so an absent one
224-
// reaches here as undefined rather than being rejected by the parser.
225-
const rating = args.rating === undefined ? null : parseFeedbackRating(args.rating);
226-
if (rating === null) {
227-
console.error(c.error("Rating must be an integer between 0 and 10"));
228-
failCommand();
229-
}
275+
const comment = normalizeComment(args.comment);
276+
const rating = reportRating(args.rating, comment, args["file-issue"] === true);
230277

231278
const source = parseFeedbackSource(args.source);
232279
if (source === null) {
@@ -239,7 +286,6 @@ export default defineCommand({
239286
return;
240287
}
241288

242-
const comment = normalizeComment(args.comment);
243289
const doctorSummary = await getDoctorSummary();
244290

245291
// Join keys tying this report to the install's PostHog rows — see
@@ -254,14 +300,7 @@ export default defineCommand({
254300
});
255301
const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys;
256302

257-
// Soft-warn (never blocks) when the comment for a non-clean report is
258-
// missing the mandated reproduction-packet markers. Prints before the
259-
// submission ack so the reporter sees the nudge while their run is fresh.
260-
printFeedbackLintWarnings({ rating, comment });
261-
262-
// The standalone command runs separately from `render`, so it has no real
263-
// elapsed time to report. Omit it rather than recording a fake duration.
264-
trackRenderFeedback({
303+
trackFeedback({
265304
rating,
266305
comment,
267306
doctorSummary,
@@ -281,7 +320,7 @@ export default defineCommand({
281320
email: feedbackEmail(),
282321
});
283322

284-
if (args["file-issue"] === true) {
323+
if (args["file-issue"] === true && rating !== undefined) {
285324
await fileGithubIssue({
286325
rating,
287326
comment,

‎packages/cli/src/telemetry/events.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const {
3636
trackCliError,
3737
trackFigmaImport,
3838
trackRenderFeedback,
39+
trackFeedbackComment,
3940
trackRenderPreflightRejected,
4041
trackAuthLoginStarted,
4142
trackAuthLoginCompleted,
@@ -1069,6 +1070,17 @@ describe("render telemetry events", () => {
10691070
});
10701071
});
10711072

1073+
describe("trackFeedbackComment", () => {
1074+
it("is its own event with no rating, so it never counts in the rating metric", () => {
1075+
trackEvent.mockClear();
1076+
trackFeedbackComment({ comment: "MISSING FEATURE: trim", feedbackId: "f1" });
1077+
1078+
const [name, props] = trackEvent.mock.calls[0] as [string, Record<string, unknown>];
1079+
expect(name).toBe("cli_feedback_comment");
1080+
expect(props).toEqual({ comment: "MISSING FEATURE: trim", feedback_id: "f1" });
1081+
});
1082+
});
1083+
10721084
describe("trackRenderFeedback", () => {
10731085
beforeEach(() => {
10741086
trackEvent.mockClear();

‎packages/cli/src/telemetry/events.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -993,6 +993,22 @@ export function trackSkillsInstallSkipped(props: { reason: string }): void {
993993
trackEvent("cli skill install skipped", { reason: props.reason });
994994
}
995995

996+
export function trackFeedbackComment(props: {
997+
comment: string;
998+
doctorSummary?: string;
999+
feedbackId?: string;
1000+
recentRenderIds?: string[];
1001+
}): void {
1002+
trackEvent("cli_feedback_comment", {
1003+
comment: props.comment,
1004+
...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}),
1005+
...(props.feedbackId ? { feedback_id: props.feedbackId } : {}),
1006+
...(props.recentRenderIds?.length
1007+
? { recent_render_ids: props.recentRenderIds.join(",") }
1008+
: {}),
1009+
});
1010+
}
1011+
9961012
export function trackRenderFeedback(props: {
9971013
rating: number;
9981014
renderDurationMs?: number;
@@ -1001,7 +1017,8 @@ export function trackRenderFeedback(props: {
10011017
/**
10021018
* Join key shared with the forwarded feedback report (Slack/backend): the
10031019
* same uuid rides in the report's env string as `fid=…`, so a wild report
1004-
* resolves to exactly one PostHog `cli_render_feedback` event and vice versa.
1020+
* resolves to exactly one PostHog `cli_render_feedback` or
1021+
* `cli_feedback_comment` event and vice versa.
10051022
*/
10061023
feedbackId?: string;
10071024
/** render_job_id values of this install's recent renders (newest last). */

‎packages/cli/src/utils/submitFeedback.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,18 @@ describe("submitFeedback", () => {
1818
vi.unstubAllGlobals();
1919
});
2020

21+
it("leaves the rating and its scale out of a report that has none", async () => {
22+
const fetchMock = vi.fn<typeof fetch>(async () => new Response(null, { status: 202 }));
23+
vi.stubGlobal("fetch", fetchMock);
24+
25+
await submitFeedback({ comment: "MISSING FEATURE: trim", cliVersion: "1.2.3" });
26+
27+
const body = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body));
28+
expect(body).not.toHaveProperty("rating");
29+
expect(body).not.toHaveProperty("rating_scale");
30+
expect(body.comment).toBe("MISSING FEATURE: trim");
31+
});
32+
2133
it("posts feedback to the backend endpoint", async () => {
2234
const fetchMock = vi.fn<typeof fetch>(async () => new Response(null, { status: 202 }));
2335
vi.stubGlobal("fetch", fetchMock);

‎packages/cli/src/utils/submitFeedback.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ function cap(value: string | undefined, max: number): string | undefined {
1515
}
1616

1717
export async function submitFeedback(input: {
18-
rating: number;
18+
rating?: number;
1919
comment?: string;
2020
cliVersion: string;
2121
env?: string;
@@ -28,7 +28,7 @@ export async function submitFeedback(input: {
2828
method: "POST",
2929
body: JSON.stringify({
3030
rating: input.rating,
31-
rating_scale: FEEDBACK_RATING_SCALE,
31+
rating_scale: input.rating === undefined ? undefined : FEEDBACK_RATING_SCALE,
3232
comment: cap(input.comment, MAX_COMMENT),
3333
cli_version: cap(input.cliVersion, MAX_CLI_VERSION),
3434
env: cap(input.env, MAX_ENV),

‎skills-manifest.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"files": 7
3131
},
3232
"hyperframes-cli": {
33-
"hash": "cf031cc9764e451f",
33+
"hash": "599eba83e1bf7245",
3434
"files": 11
3535
},
3636
"hyperframes-core": {

0 commit comments

Comments
 (0)