Skip to content

Commit 775c46f

Browse files
MiaoMintMiaoMint
committed
feat: Implement multimodal input support for AI models and enhance node handle visibility. (#18)
1 parent c7917ac commit 775c46f

File tree

15 files changed

+575
-196
lines changed

15 files changed

+575
-196
lines changed

‎binding/ai/service.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ func (s *Service) SetContext(ctx context.Context) {
2727
// TextRequest defines the parameters for text generation
2828
type TextRequest struct {
2929
Prompt string `json:"prompt"`
30+
Images []string `json:"images,omitempty"`
31+
Videos []string `json:"videos,omitempty"`
32+
Audios []string `json:"audios,omitempty"`
33+
Documents []string `json:"documents,omitempty"`
3034
Model string `json:"model"`
3135
ProviderID int `json:"providerId"`
3236
Temperature *float64 `json:"temperature,omitempty"`
@@ -37,6 +41,9 @@ type TextRequest struct {
3741
// ImageRequest defines the parameters for image generation
3842
type ImageRequest struct {
3943
Prompt string `json:"prompt"`
44+
Images []string `json:"images,omitempty"`
45+
Videos []string `json:"videos,omitempty"`
46+
Audios []string `json:"audios,omitempty"`
4047
Model string `json:"model"`
4148
ProviderID int `json:"providerId"`
4249
Size string `json:"size,omitempty"`
@@ -48,6 +55,9 @@ type ImageRequest struct {
4855
// VideoRequest defines the parameters for video generation
4956
type VideoRequest struct {
5057
Prompt string `json:"prompt"`
58+
Images []string `json:"images,omitempty"`
59+
Videos []string `json:"videos,omitempty"`
60+
Audios []string `json:"audios,omitempty"`
5161
Model string `json:"model"`
5262
ProviderID int `json:"providerId"`
5363
Duration string `json:"duration,omitempty"`
@@ -58,6 +68,9 @@ type VideoRequest struct {
5868
// AudioRequest defines the parameters for audio generation
5969
type AudioRequest struct {
6070
Prompt string `json:"prompt"`
71+
Images []string `json:"images,omitempty"`
72+
Videos []string `json:"videos,omitempty"`
73+
Audios []string `json:"audios,omitempty"`
6174
Model string `json:"model"`
6275
ProviderID int `json:"providerId"`
6376
Voice string `json:"voice,omitempty"`
@@ -93,6 +106,10 @@ func (s *Service) GenerateText(req TextRequest) (*AIResponse, error) {
93106

94107
aiReq := aiservice.TextGenerateRequest{
95108
Prompt: req.Prompt,
109+
Images: req.Images,
110+
Videos: req.Videos,
111+
Audios: req.Audios,
112+
Documents: req.Documents,
96113
Model: req.Model,
97114
Temperature: req.Temperature,
98115
MaxTokens: req.MaxTokens,
@@ -124,6 +141,9 @@ func (s *Service) GenerateImage(req ImageRequest) (*AIResponse, error) {
124141

125142
aiReq := aiservice.ImageGenerateRequest{
126143
Prompt: req.Prompt,
144+
Images: req.Images,
145+
Videos: req.Videos,
146+
Audios: req.Audios,
127147
Model: req.Model,
128148
Size: req.Size,
129149
Quality: req.Quality,
@@ -156,6 +176,9 @@ func (s *Service) GenerateVideo(req VideoRequest) (*AIResponse, error) {
156176

157177
aiReq := aiservice.VideoGenerateRequest{
158178
Prompt: req.Prompt,
179+
Images: req.Images,
180+
Videos: req.Videos,
181+
Audios: req.Audios,
159182
Model: req.Model,
160183
Duration: req.Duration,
161184
Resolution: req.Resolution,
@@ -187,6 +210,9 @@ func (s *Service) GenerateAudio(req AudioRequest) (*AIResponse, error) {
187210

188211
aiReq := aiservice.AudioGenerateRequest{
189212
Prompt: req.Prompt,
213+
Images: req.Images,
214+
Videos: req.Videos,
215+
Audios: req.Audios,
190216
Model: req.Model,
191217
Voice: req.Voice,
192218
Speed: req.Speed,

‎frontend/src/components/canvas-view.tsx‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ import {
2727
Video,
2828
Music,
2929
X,
30+
Download,
31+
Upload,
3032
} from "lucide-react";
3133
import { Card } from "@/components/ui/card";
34+
import { toast } from "sonner";
3235

3336
import {
3437
TextNode,
@@ -227,6 +230,58 @@ function CanvasEditor({ projectId, projectName, onBack }: CanvasViewProps) {
227230
}
228231
}, [setNodes]);
229232

233+
const handleExport = useCallback(() => {
234+
const data = {
235+
nodes,
236+
edges,
237+
};
238+
const jsonString = JSON.stringify(data, null, 2);
239+
navigator.clipboard.writeText(jsonString)
240+
.then(() => toast.success("Project JSON copied to clipboard"))
241+
.catch((err) => {
242+
console.error("Failed to copy:", err);
243+
toast.error("Failed to copy project JSON");
244+
});
245+
}, [nodes, edges]);
246+
247+
const fileInputRef = useRef<HTMLInputElement>(null);
248+
249+
const handleImportClick = () => {
250+
fileInputRef.current?.click();
251+
};
252+
253+
const handleFileChange = useCallback((event: React.ChangeEvent<HTMLInputElement>) => {
254+
const file = event.target.files?.[0];
255+
if (!file) return;
256+
257+
const reader = new FileReader();
258+
reader.onload = (e) => {
259+
try {
260+
const content = e.target?.result as string;
261+
const data = JSON.parse(content);
262+
if (data.nodes && data.edges) {
263+
setNodes(data.nodes);
264+
setEdges(data.edges);
265+
// Update ID counter based on highest ID found to avoid collisions
266+
let maxId = 0;
267+
data.nodes.forEach((n: Node) => {
268+
const idNum = parseInt(n.id.replace(/[^0-9]/g, ''));
269+
if (!isNaN(idNum) && idNum > maxId) maxId = idNum;
270+
});
271+
setNodeIdCounter(maxId + 1);
272+
} else {
273+
alert("Invalid project file format");
274+
}
275+
} catch (err) {
276+
console.error("Import failed:", err);
277+
alert("Failed to parse project file");
278+
}
279+
};
280+
reader.readAsText(file);
281+
// Reset input so same file can be selected again
282+
event.target.value = "";
283+
}, [setNodes, setEdges, setNodeIdCounter]);
284+
230285
return (
231286
<div className="flex h-full relative">
232287
{/* 主要内容区域 - 全屏 */}
@@ -245,7 +300,30 @@ function CanvasEditor({ projectId, projectName, onBack }: CanvasViewProps) {
245300
className="max-w-sm border-none bg-transparent px-2 focus-visible:ring-0 focus-visible:ring-offset-0 font-semibold"
246301
placeholder="项目名称"
247302
/>
248-
<div className="ml-auto">
303+
<div className="ml-auto flex items-center gap-2">
304+
<input
305+
type="file"
306+
ref={fileInputRef}
307+
className="hidden"
308+
accept=".json"
309+
onChange={handleFileChange}
310+
/>
311+
<Button
312+
variant="ghost"
313+
size="icon"
314+
title="Import JSON"
315+
onClick={handleImportClick}
316+
>
317+
<Upload className="h-5 w-5" />
318+
</Button>
319+
<Button
320+
variant="ghost"
321+
size="icon"
322+
title="Copy JSON"
323+
onClick={handleExport}
324+
>
325+
<Download className="h-5 w-5" />
326+
</Button>
249327
<Button
250328
variant="ghost"
251329
size="icon"

‎frontend/src/components/nodes/base-node.tsx‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { Card } from "@/components/ui/card";
33
import { Spinner } from "@/components/ui/spinner";
44
import { NodeParametersPanel } from "./node-parameters-panel";
55
import type { BaseNodeData } from "./types";
6-
import { type LucideIcon } from "lucide-react";
6+
import { PlusCircle, type LucideIcon } from "lucide-react";
77
import { cn } from "@/lib/utils";
88
import { useEffect } from "react";
99

@@ -48,7 +48,7 @@ export function BaseNode({
4848
}, []);
4949

5050
return (
51-
<>
51+
<div className="size-full group">
5252
{isSelected && (
5353
<NodeResizeControl
5454
minWidth={minWidth}
@@ -74,8 +74,8 @@ export function BaseNode({
7474
className="font-semibold text-sm bg-transparent border-none outline-none focus:ring-0 p-0 w-full"
7575
/>
7676
</div>
77-
<Handle type="target" position={Position.Left} />
78-
<Handle type="source" position={Position.Right} />
77+
<Handle type="target" className="opacity-0 group-hover:opacity-100 transition-all" position={Position.Left} />
78+
<Handle type="source" className="opacity-0 group-hover:opacity-100 transition-all" position={Position.Right} />
7979
<Card
8080
className={cn(
8181
"py-0! gap-0 size-full",
@@ -105,6 +105,6 @@ export function BaseNode({
105105
/>
106106
</NodeToolbar>
107107

108-
</ >
108+
</div >
109109
);
110110
}

‎frontend/src/components/nodes/group-node.tsx‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import { ButtonGroup } from "@/components/ui/button-group";
77
import { Play, Ungroup, Square } from "lucide-react";
88
import { Spinner } from "../ui/spinner";
99
import { toast } from "sonner";
10+
import type { BaseNodeData } from "./types";
1011

11-
export const GroupNode = memo(({ id, data, selected }: NodeProps) => {
12+
export const GroupNode = memo(({ id, data, selected, }: NodeProps) => {
13+
const nodeData = data as unknown as BaseNodeData;
1214
const { updateNodeData, deleteElements, getNodes, getEdges, getNode, getIntersectingNodes } = useReactFlow();
1315
const abortRef = useRef(false);
1416

@@ -184,8 +186,11 @@ export const GroupNode = memo(({ id, data, selected }: NodeProps) => {
184186

185187

186188
<div className="p-3 flex items-center gap-2 absolute -top-10 left-0 right-0 nodrag">
189+
{nodeData.processing && (
190+
<Spinner className="h-4 w-4 mr-1" />
191+
)}
187192
<input
188-
value={data.label as string || "Group"}
193+
value={nodeData.label}
189194
onChange={(evt) => updateNodeData(id, { label: evt.target.value })}
190195
className="font-semibold text-sm bg-transparent border-none outline-none focus:ring-0 p-0 w-full"
191196
/>

‎frontend/src/components/nodes/node-parameters-panel.tsx‎

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export function NodeParametersPanel({
3535
};
3636

3737
return (
38-
<Card className="absolute top-full mt-2 left-1/2 -translate-x-1/2 w-175 shadow-lg z-50 flex flex-col py-0! overflow-hidden gap-0! nodrag">
38+
<Card className="absolute top-full mt-2 left-1/2 -translate-x-1/2 w-175 shadow-lg z-9999 flex flex-col py-0! overflow-hidden gap-0! nodrag">
3939
<div className="flex-1 flex flex-col min-h-40">
4040
<Textarea
4141
placeholder={promptPlaceholder}
@@ -44,37 +44,9 @@ export function NodeParametersPanel({
4444
onChange={handlePromptChange}
4545
/>
4646

47-
{(nodeData.input || nodeData.output || nodeData.error) && (
48-
<div className="border-t bg-muted/20 max-h-40 overflow-y-auto">
49-
{nodeData.input && (
50-
<div className="p-2 border-b last:border-0 opacity-80">
51-
<label className="text-xs text-muted-foreground block mb-1">
52-
输入
53-
</label>
54-
<div className="text-xs font-mono text-muted-foreground truncate">
55-
{typeof nodeData.input === "string"
56-
? nodeData.input
57-
: JSON.stringify(nodeData.input)}
58-
</div>
59-
</div>
60-
)}
61-
{nodeData.output && (
62-
<div className="p-2 border-b last:border-0 opacity-80">
63-
<label className="text-xs text-muted-foreground block mb-1">
64-
输出
65-
</label>
66-
<div className="text-xs font-mono text-muted-foreground truncate">
67-
{typeof nodeData.output === "string"
68-
? nodeData.output
69-
: JSON.stringify(nodeData.output)}
70-
</div>
71-
</div>
72-
)}
73-
{nodeData.error && (
74-
<div className="p-2 bg-destructive/10 text-destructive text-xs">
75-
{nodeData.error}
76-
</div>
77-
)}
47+
{nodeData.error && (
48+
<div className="p-2 bg-destructive/10 text-destructive text-xs">
49+
{nodeData.error}
7850
</div>
7951
)}
8052
</div>

‎frontend/src/components/nodes/text-node.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,15 @@ export const TextNode = memo((props: NodeProps) => {
3535
maxWidth={400}
3636
maxHeight={400}
3737
>
38-
<div className="p-4 w-full flex-1">
38+
<div className="p-4 w-full flex-1 overflow-auto">
3939
{nodeData.processing ? (
4040
<div className="w-full space-y-2">
4141
<Skeleton className="h-4 w-full" />
4242
<Skeleton className="h-4 w-3/4" />
4343
<Skeleton className="h-4 w-5/6" />
4444
</div>
4545
) : nodeData.content ? (
46-
<div className="text-sm whitespace-pre-wrap w-full">
46+
<div className="text-sm whitespace-pre-wrap">
4747
{nodeData.content}
4848
</div>
4949
) : (

‎frontend/src/components/nodes/types.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@ export interface BaseNodeData {
66
type: NodeType;
77
modelId?: string;
88
providerId?: number;
9-
input?: any;
10-
output?: any;
119
processing?: boolean;
1210
error?: string;
1311
prompt?: string;

0 commit comments

Comments
 (0)