Skip to content

Commit e664197

Browse files
felixarntzfelixarntz
committed
Implement new helper functions to create a multimodal Content object from a prompt and media file, and to get the base64 data URL for a file.
1 parent 3994156 commit e664197

File tree

3 files changed

+141
-37
lines changed

3 files changed

+141
-37
lines changed

‎includes/Services/API/Helpers.php

+77
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@
1111
use Felix_Arntz\AI_Services\Services\API\Enums\Content_Role;
1212
use Felix_Arntz\AI_Services\Services\API\Types\Candidates;
1313
use Felix_Arntz\AI_Services\Services\API\Types\Content;
14+
use Felix_Arntz\AI_Services\Services\API\Types\Parts;
1415
use Felix_Arntz\AI_Services\Services\API\Types\Parts\Text_Part;
1516
use Felix_Arntz\AI_Services\Services\Util\Formatter;
1617
use Generator;
18+
use WP_Post;
1719

1820
/**
1921
* Class providing static helper methods as part of the public API.
@@ -35,6 +37,56 @@ public static function text_to_content( string $text, string $role = Content_Rol
3537
return Formatter::format_content( $text, $role );
3638
}
3739

40+
/**
41+
* Converts a text string and attachment to a multimodal Content instance.
42+
*
43+
* The text will be included as a prompt as the first part of the content, and the attachment (e.g. an image or
44+
* audio file) will be included as the second part.
45+
*
46+
* @since n.e.x.t
47+
*
48+
* @param string $text The text.
49+
* @param int|WP_Post $attachment The attachment ID or object.
50+
* @param string $role Optional. The role to use for the content. Default 'user'.
51+
* @return Content The content instance.
52+
*/
53+
public static function text_and_attachment_to_content( string $text, $attachment, string $role = Content_Role::USER ): Content {
54+
if ( $attachment instanceof WP_Post ) {
55+
$attachment_id = (int) $attachment->ID;
56+
} else {
57+
$attachment_id = (int) $attachment;
58+
$attachment = get_post( $attachment_id );
59+
}
60+
61+
$file = get_attached_file( $attachment_id );
62+
$large_size = image_get_intermediate_size( $attachment_id, 'large' );
63+
if ( $large_size && isset( $large_size['path'] ) ) {
64+
// To get the absolute path to a sub-size file, we need to prepend the uploads dir.
65+
if ( str_starts_with( $large_size['path'], '/' ) ) {
66+
$file = $large_size['path'];
67+
} else {
68+
$uploads = wp_get_upload_dir();
69+
if ( false === $uploads['error'] ) {
70+
$file = "{$uploads['basedir']}/{$large_size['path']}";
71+
}
72+
}
73+
}
74+
75+
$mime_type = wp_check_filetype( $file );
76+
if ( isset( $mime_type['type'] ) ) {
77+
$mime_type = $mime_type['type'];
78+
} else {
79+
// Fallback that should never be needed.
80+
$mime_type = $attachment->post_mime_type;
81+
}
82+
83+
$parts = new Parts();
84+
$parts->add_text_part( $text );
85+
$parts->add_inline_data_part( $mime_type, self::base64_encode_file( $file, $mime_type ) );
86+
87+
return Formatter::format_content( $parts, $role );
88+
}
89+
3890
/**
3991
* Converts a Content instance to a text string.
4092
*
@@ -149,4 +201,29 @@ public static function get_candidate_contents( Candidates $candidates ): array {
149201
public static function process_candidates_stream( Generator $generator ): Candidates_Stream_Processor { // phpcs:ignore Squiz.Commenting.FunctionComment.IncorrectTypeHint
150202
return new Candidates_Stream_Processor( $generator );
151203
}
204+
205+
/**
206+
* Base64-encodes a file and returns its data URL.
207+
*
208+
* @since n.e.x.t
209+
*
210+
* @param string $file Absolute path to the file, or its URL.
211+
* @param string $mime_type Optional. The MIME type of the file. If provided, the base64-encoded data URL will
212+
* be prefixed with `data:{mime_type};base64,`. Default empty string.
213+
* @return string The base64-encoded file data URL, or empty string on failure.
214+
*/
215+
public static function base64_encode_file( string $file, string $mime_type = '' ): string {
216+
// phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents
217+
$binary_data = file_get_contents( $file );
218+
if ( ! $binary_data ) {
219+
return '';
220+
}
221+
222+
// phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode
223+
$base64 = base64_encode( $binary_data );
224+
if ( '' !== $mime_type ) {
225+
$base64 = "data:$mime_type;base64,$base64";
226+
}
227+
return $base64;
228+
}
152229
}

‎src/ai/helpers.js

+61
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,35 @@ export function textToContent( text, role = ContentRole.USER ) {
2020
};
2121
}
2222

23+
/**
24+
* Converts a text string and attachment to a multimodal Content instance.
25+
*
26+
* The text will be included as a prompt as the first part of the content, and the attachment (e.g. an image or
27+
* audio file) will be included as the second part.
28+
*
29+
* @since n.e.x.t
30+
*
31+
* @param {string} text The text.
32+
* @param {Object} attachment The attachment object.
33+
* @param {string} role Optional. The role to use for the content. Default 'user'.
34+
* @return {Object} The Content object.
35+
*/
36+
export async function textAndAttachmentToContent(
37+
text,
38+
attachment,
39+
role = ContentRole.USER
40+
) {
41+
const mimeType = attachment.mime;
42+
const data = await base64EncodeFile(
43+
attachment.sizes?.large?.url || attachment.url
44+
);
45+
46+
return {
47+
role,
48+
parts: [ { text }, { inlineData: { mimeType, data } } ],
49+
};
50+
}
51+
2352
/**
2453
* Converts a Content object to a text string.
2554
*
@@ -131,3 +160,35 @@ export function getCandidateContents( candidates ) {
131160
export function processCandidatesStream( generator ) {
132161
return new CandidatesStreamProcessor( generator );
133162
}
163+
164+
/**
165+
* Base64-encodes a file and returns its data URL.
166+
*
167+
* @since n.e.x.t
168+
*
169+
* @param {string} file The file URL.
170+
* @param {string} mimeType Optional. The MIME type of the file. If provided, the base64-encoded data URL will
171+
* be prefixed with `data:{mime_type};base64,`. Default empty string.
172+
* @return {string} The base64-encoded file data URL, or empty string on failure.
173+
*/
174+
export async function base64EncodeFile( file, mimeType = '' ) {
175+
const data = await fetch( file );
176+
const blob = await data.blob();
177+
178+
const base64 = await new Promise( ( resolve ) => {
179+
const reader = new window.FileReader();
180+
reader.readAsDataURL( blob );
181+
reader.onloadend = () => {
182+
const base64data = reader.result;
183+
resolve( base64data );
184+
};
185+
} );
186+
187+
if ( mimeType ) {
188+
return base64.replace(
189+
/^data:[a-z]+\/[a-z]+;base64,/,
190+
`data:${ mimeType };base64,`
191+
);
192+
}
193+
return base64;
194+
}

‎src/playground-page/store/messages.js

+3-37
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ const parseContentFromCache = async ( content, attachment ) => {
4646
...part,
4747
inlineData: {
4848
...part.inlineData,
49-
data: await getBase64Representation(
49+
data: await helpers.base64EncodeFile(
5050
attachment.sizes?.large?.url || attachment.url
5151
),
5252
},
@@ -119,45 +119,11 @@ const clearMessages = () => {
119119
window.sessionStorage.removeItem( SESSION_STORAGE_KEY );
120120
};
121121

122-
const getBase64Representation = async ( url ) => {
123-
const data = await fetch( url );
124-
const blob = await data.blob();
125-
return new Promise( ( resolve ) => {
126-
const reader = new window.FileReader();
127-
reader.readAsDataURL( blob );
128-
reader.onloadend = () => {
129-
const base64data = reader.result;
130-
resolve( base64data );
131-
};
132-
} );
133-
};
134-
135122
const formatNewContent = async ( prompt, attachment ) => {
136-
if ( ! attachment ) {
137-
return helpers.textToContent( prompt );
138-
}
139-
140-
const parts = [];
141-
if ( prompt ) {
142-
parts.push( { text: prompt } );
143-
}
144123
if ( attachment ) {
145-
const mimeType = attachment.mime;
146-
const data = await getBase64Representation(
147-
attachment.sizes?.large?.url || attachment.url
148-
);
149-
parts.push( {
150-
inlineData: {
151-
mimeType,
152-
data,
153-
},
154-
} );
124+
return helpers.textAndAttachmentToContent( prompt, attachment );
155125
}
156-
157-
return {
158-
role: enums.ContentRole.USER,
159-
parts,
160-
};
126+
return helpers.textToContent( prompt );
161127
};
162128

163129
const RECEIVE_MESSAGE = 'RECEIVE_MESSAGE';

0 commit comments

Comments
 (0)