Skip to content

Commit 325c4e7

Browse files
cguedescguedes
authored
feat: Adopt HTTP apis for web deployment (#393)
This PR connect the client code (web hosting) to backend APIs via (external) HTTP using standard fetch. A follow-up PR will existing after #417 is merged, that will implement the unified codebase for the desktop+web client code. It's also important to note that for this PR the goal was to not change the existing desktop behaviour, implementing if/else checks with import.meta.env.VITE_IS_WEB. For this PR the main changes where: remove in-memory implementation of fs.tsand stub code. Now the code throws exception. adopt the new project API and therefore, save the projectId in the client to use in every HTTP call. implement web.ts and *API.ts utilities to perform calls to the backend HTTP in a web context. Adjust the code to flow the projectId to every business logic call in order to support listing, opening, upload files, and also references operations (ingestion, list, delete, edit). Adopt the new HTTP api for the chat operation. The rewrite is still using the sidecar command. Notes: The code/files structure wasn't a priority for this PR. That organisation will be implemented in the follow-up PR. (web) the open project will open one of the previously created projects (in the web context) * Send env variables via header "X-..." in the HTTP sidecar interface * Adds a projectsAPI to create new projects in the Web * Support creating new projects for the Web * Fix py lint * WIP: Adopt fs api * Fix settings API import * handle env variable with middleware * Add web.ts with api calls to backend server * Remove outdated tests * Fix `yarn check` issues * Directly use !!import.meta.env.VITE_IS_WEB * Configure project name via API * Adopt HTTP ingestion for web * Add support to delete references * remove app startup utility * Open one of the existing projects instead of open hard-coded * Allow references to be edited (PATCH) * Add support to chat with references via HTTP web * Remove legacy fs.ts and stub * Fix unit tests * fix python tests * Fix tauri-wrapper signature * Fix yarn:check errors * Fix test * Remove unused args
1 parent dfd2a6d commit 325c4e7

23 files changed

Lines changed: 614 additions & 410 deletions

File tree

‎python/sidecar/http.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ async def http_get(project_id: str, reference_id: str) -> Reference | None:
112112
return response
113113

114114

115-
@references_api.put("/{project_id}/{reference_id}")
115+
@references_api.patch("/{project_id}/{reference_id}")
116116
async def http_update(project_id: str, reference_id: str, req: ReferencePatch) -> UpdateStatusResponse:
117117
user_id = "user1"
118118
project_path = projects.get_project_path(user_id, project_id)

‎python/tests/test_http.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ def test_references_update(monkeypatch, tmp_path):
111111
assert ref.citation_key is None
112112

113113
patch = {"data": {"citation_key": "reda2023"}}
114-
response = references_client.put(f"/{project_id}/{ref.id}", json=patch)
114+
response = references_client.patch(f"/{project_id}/{ref.id}", json=patch)
115115

116116
assert response.status_code == 200
117117
assert response.json()["status"] == "ok"

‎python/web.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
api.mount("/api/fs", http.filesystem_api)
1212
api.mount("/api/projects", http.project_api)
1313
api.mount("/api/meta", http.meta_api)
14-
api.mount("/api/settings", http.settings)
14+
api.mount("/api/settings", http.settings_api)
1515

1616

1717
def serve(host: str, port: int):

‎src/api/__tests__/chat.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ describe('chatWithAI', () => {
2020
],
2121
});
2222

23-
const response = await chatWithAI('input text');
23+
const response = await chatWithAI('', 'input text');
2424
expect(response).toStrictEqual(['some reply']);
2525
});
2626

@@ -40,20 +40,20 @@ describe('chatWithAI', () => {
4040
],
4141
});
4242

43-
const response = await chatWithAI('input text');
43+
const response = await chatWithAI('', 'input text');
4444
expect(response).toStrictEqual(['some reply', 'Another reply']);
4545
});
4646

4747
it('should return empty list, and not call sidecar, if provided text is empty', async () => {
48-
const response = await chatWithAI('');
48+
const response = await chatWithAI('', '');
4949
expect(response).toStrictEqual([]);
5050
expect(callSidecar).not.toHaveBeenCalled();
5151
});
5252

5353
it('should return error message reply if exception thrown calling sidecar', async () => {
5454
vi.mocked(callSidecar).mockRejectedValue(new Error('Invalid API Key'));
5555

56-
const response = await chatWithAI('input text');
56+
const response = await chatWithAI('', 'input text');
5757
expect(response.length).toBe(1);
5858
expect(response[0]).toMatch(/Invalid API Key/);
5959
});

‎src/api/__tests__/ingestion.update.test.ts‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,19 @@ describe('ingestion.update', () => {
1818
message: '',
1919
} as UpdateStatusResponse);
2020

21-
await updateReference(ref1.id, {
22-
citationKey: ref1.citationKey + 'xx',
23-
title: ref1.title + ' NEW',
24-
publishedDate: '2023-07-01',
25-
authors: [{ fullName: 'Joe Doe Dundee', lastName: 'Dundee' }],
26-
});
21+
await updateReference(
22+
ref1.filename,
23+
{
24+
citationKey: ref1.citationKey + 'xx',
25+
title: ref1.title + ' NEW',
26+
publishedDate: '2023-07-01',
27+
authors: [{ fullName: 'Joe Doe Dundee', lastName: 'Dundee' }],
28+
},
29+
ref1.id,
30+
);
2731
expect(callSidecar).toHaveBeenCalledTimes(1);
2832
expect(callSidecar).toHaveBeenCalledWith<[string, ReferenceUpdate]>('update', {
29-
reference_id: ref1.id,
33+
reference_id: ref1.filename,
3034
patch: {
3135
data: {
3236
citation_key: 'doe2023xx',
@@ -50,7 +54,7 @@ describe('ingestion.update', () => {
5054
message: '',
5155
} as UpdateStatusResponse);
5256

53-
await updateReference(ref1.filename, {});
57+
await updateReference(ref1.filename, {}, ref1.id);
5458
expect(vi.mocked(callSidecar).mock.calls).toHaveLength(0);
5559
});
5660

@@ -61,6 +65,6 @@ describe('ingestion.update', () => {
6165
message: 'what!',
6266
} as UpdateStatusResponse);
6367

64-
await expect(updateReference(ref1.id, { title: ref1.title + ' updated' })).rejects.toThrow();
68+
await expect(updateReference(ref1.filename, { title: ref1.title + ' updated' }, ref1.id)).rejects.toThrow();
6569
});
6670
});

‎src/api/chat.ts‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,34 @@
11
import { notifyError } from '../notifications/notifications';
22
import { callSidecar } from './sidecar';
3+
import { ChatResponse } from './types';
34

4-
export async function chatWithAI(text: string): Promise<string[]> {
5+
export async function chatWithAI(projectId: string, text: string): Promise<string[]> {
56
try {
67
if (!text) {
78
return [];
89
}
9-
const response = await callSidecar('chat', { text });
1010

11-
if (response.status === 'error') {
12-
notifyError('Chat error', response.message);
13-
return [];
14-
}
11+
if (import.meta.env.VITE_IS_WEB) {
12+
const response = await fetch(`/api/ai/${projectId}/chat`, {
13+
method: 'POST',
14+
headers: {
15+
Accept: 'application/json',
16+
'Content-Type': 'application/json',
17+
},
18+
body: JSON.stringify({ text }),
19+
});
20+
const res = (await response.json()) as ChatResponse;
21+
return res.choices.map((choice) => choice.text);
22+
} else {
23+
const response = await callSidecar('chat', { text });
1524

16-
return response.choices.map((choice) => choice.text);
25+
if (response.status === 'error') {
26+
notifyError('Chat error', response.message);
27+
return [];
28+
}
29+
30+
return response.choices.map((choice) => choice.text);
31+
}
1732
} catch (err) {
1833
return [`Chat error: ${String(err)}`];
1934
}

‎src/api/ingestion.ts‎

Lines changed: 66 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
import { getSystemPath, getUploadsDir, makeUploadPath } from '../io/filesystem';
22
import { ReferenceItem } from '../types/ReferenceItem';
3+
import {
4+
deleteRemoteReferences,
5+
getRemoteReferences,
6+
patchRemoteReference,
7+
startRemoteReferencesIngestion,
8+
} from './referencesAPI';
39
import { callSidecar } from './sidecar';
4-
import { IngestResponse, Reference } from './types';
10+
import { Reference } from './types';
511

6-
function parsePdfIngestionResponse(response: IngestResponse): ReferenceItem[] {
7-
return response.references.map((reference) => ({
12+
function parsePdfIngestionResponse(references: Reference[]): ReferenceItem[] {
13+
return references.map((reference) => ({
814
id: reference.id,
915
source_filename: reference.source_filename,
1016
filepath: makeUploadPath(reference.source_filename),
@@ -21,16 +27,32 @@ function parsePdfIngestionResponse(response: IngestResponse): ReferenceItem[] {
2127
}));
2228
}
2329

24-
export async function runPDFIngestion(): Promise<ReferenceItem[]> {
25-
const uploadsDir = await getSystemPath(getUploadsDir());
26-
const response = await callSidecar('ingest', { pdf_directory: String(uploadsDir) });
27-
return parsePdfIngestionResponse(response);
30+
export async function runPDFIngestion(projectId?: string): Promise<ReferenceItem[]> {
31+
if (import.meta.env.VITE_IS_WEB) {
32+
if (!projectId) {
33+
throw new Error('Project ID is required for web version');
34+
}
35+
const references = await startRemoteReferencesIngestion(projectId);
36+
return parsePdfIngestionResponse(references);
37+
} else {
38+
const uploadsDir = await getSystemPath(getUploadsDir());
39+
const response = await callSidecar('ingest', { pdf_directory: String(uploadsDir) });
40+
return parsePdfIngestionResponse(response.references);
41+
}
2842
}
2943

30-
export async function removeReferences(ids: string[]) {
31-
const response = await callSidecar('delete', { reference_ids: ids });
32-
if (response.status === 'error') {
33-
throw new Error('Error removing references: ' + response.message);
44+
export async function removeReferences(ids: string[], projectId?: string) {
45+
if (import.meta.env.VITE_IS_WEB) {
46+
if (!projectId) {
47+
throw new Error('Project ID is required for web version');
48+
}
49+
await deleteRemoteReferences(projectId, ids);
50+
return;
51+
} else {
52+
const response = await callSidecar('delete', { reference_ids: ids });
53+
if (response.status === 'error') {
54+
throw new Error('Error removing references: ' + response.message);
55+
}
3456
}
3557
}
3658

@@ -42,7 +64,12 @@ function applyPatch(field: keyof ReferenceItem, patch: Partial<ReferenceItem>, g
4264
return {};
4365
}
4466

45-
export async function updateReference(id: string, patch: Partial<ReferenceItem>) {
67+
export async function updateReference(
68+
filename: string,
69+
patch: Partial<ReferenceItem>,
70+
referenceId: string,
71+
projectId?: string,
72+
) {
4673
const referencePatch: Partial<Reference> = {
4774
...applyPatch('citationKey', patch, () => ({ citation_key: patch.citationKey })),
4875
...applyPatch('title', patch, () => ({ title: patch.title })),
@@ -56,21 +83,36 @@ export async function updateReference(id: string, patch: Partial<ReferenceItem>)
5683
return;
5784
}
5885

59-
const response = await callSidecar('update', {
60-
reference_id: id,
61-
patch: {
62-
data: referencePatch,
63-
},
64-
});
65-
if (response.status === 'error') {
66-
throw new Error('Error updating reference: ' + response.message);
86+
if (import.meta.env.VITE_IS_WEB) {
87+
if (!projectId) {
88+
throw new Error('Project ID is required for web version');
89+
}
90+
await patchRemoteReference(projectId, referenceId, referencePatch);
91+
} else {
92+
const response = await callSidecar('update', {
93+
reference_id: filename,
94+
patch: {
95+
data: referencePatch,
96+
},
97+
});
98+
if (response.status === 'error') {
99+
throw new Error('Error updating reference: ' + response.message);
100+
}
67101
}
68102
}
69103

70-
export async function getIngestedReferences(): Promise<ReferenceItem[]> {
71-
const uploadsDir = await getSystemPath(getUploadsDir());
72-
const response = await callSidecar('ingest_references', { pdf_directory: String(uploadsDir) });
73-
return parsePdfIngestionResponse(response);
104+
export async function getIngestedReferences(projectId?: string): Promise<ReferenceItem[]> {
105+
if (import.meta.env.VITE_IS_WEB) {
106+
if (!projectId) {
107+
throw new Error('Project ID is required for web version');
108+
}
109+
const references = await getRemoteReferences(projectId);
110+
return parsePdfIngestionResponse(references);
111+
} else {
112+
const uploadsDir = await getSystemPath(getUploadsDir());
113+
const response = await callSidecar('ingest_references', { pdf_directory: String(uploadsDir) });
114+
return parsePdfIngestionResponse(response.references);
115+
}
74116
}
75117

76118
export async function getIngestionStatus() {

‎src/api/projectsAPI.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/** Utility for calling the backend python backend webserver, for the WEB flow. */
2+
3+
interface ProjectIdResponse {
4+
project_id: string;
5+
project_path: string;
6+
filepaths: string[];
7+
}
8+
9+
type ProjectsResponse = Record<string, { project_name: string; project_path: string }>;
10+
11+
export async function getRemoteProjects() {
12+
const res = await fetch(`/api/projects/`);
13+
const data = res.json() as unknown as ProjectsResponse;
14+
return data;
15+
}
16+
17+
export async function getRemoteProject(projectId: string) {
18+
const res = await fetch(`/api/projects/${projectId}`);
19+
const data = res.json() as unknown as ProjectIdResponse;
20+
return data;
21+
}
22+
23+
export async function postRemoteProject(projectName: string) {
24+
const response = await fetch(`/api/projects/?project_name=${projectName}`, {
25+
method: 'POST',
26+
// We need to update the API definition in order to use BODY
27+
// https://stackoverflow.com/questions/59929028/python-fastapi-error-422-with-post-request-when-sending-json-data
28+
// body: JSON.stringify({ project_name: projectName }),
29+
headers: {
30+
Accept: 'application/json',
31+
},
32+
});
33+
const payload = (await response.json()) as ProjectsResponse;
34+
const projectId = Object.keys(payload)[0];
35+
const { project_path, project_name } = payload[projectId];
36+
return {
37+
projectId,
38+
projectPath: project_path,
39+
projectName: project_name,
40+
};
41+
}

‎src/api/referencesAPI.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import { IngestResponse, Reference } from './types';
2+
3+
export async function startRemoteReferencesIngestion(projectId: string) {
4+
const response = await fetch(`/api/references/${projectId}`, {
5+
method: 'POST',
6+
headers: {
7+
Accept: 'application/json',
8+
},
9+
});
10+
const references = (await response.json()) as IngestResponse;
11+
return references.references;
12+
}
13+
14+
export async function getRemoteReferences(projectId: string) {
15+
const response = await fetch(`/api/references/${projectId}`);
16+
const references = (await response.json()) as Reference[];
17+
return references;
18+
}
19+
20+
export async function deleteRemoteReferences(projectId: string, referenceIds: string[]) {
21+
await fetch(`/api/references/${projectId}/bulk_delete`, {
22+
method: 'DELETE',
23+
body: JSON.stringify({ reference_ids: referenceIds }),
24+
headers: {
25+
'Content-Type': 'application/json',
26+
},
27+
});
28+
}
29+
30+
export async function patchRemoteReference(projectId: string, referenceId: string, patch: Partial<Reference>) {
31+
await fetch(`/api/references/${projectId}/${referenceId}`, {
32+
method: 'PATCH',
33+
body: JSON.stringify({ data: patch }),
34+
headers: {
35+
'Content-Type': 'application/json',
36+
},
37+
});
38+
}

‎src/api/sidecar.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,35 @@ import { CliCommands } from './types';
88
interface SharedCommand {
99
execute: typeof TauriCommand.prototype.execute;
1010
}
11+
interface StubCommandOptions {
12+
env: Record<string, string>;
13+
}
1114

1215
class StubCommand implements SharedCommand {
1316
command: string;
1417
args: string[];
15-
options: unknown;
16-
constructor(command: string, args: string[], options?: unknown) {
18+
options: StubCommandOptions;
19+
constructor(command: string, args: string[], options: StubCommandOptions) {
1720
this.command = command;
1821
this.args = args;
1922
this.options = options;
2023
}
2124

2225
execute: typeof TauriCommand.prototype.execute = async () => {
2326
const [command, body] = this.args;
27+
28+
const envHeaders = Object.entries(this.options.env).reduce(
29+
(acc, [key, value]) => ({ ...acc, [`X-${key.toUpperCase()}`]: value }),
30+
{},
31+
);
32+
2433
const response = await fetch(`/api/sidecar/${command}`, {
2534
method: command === 'ingest_status' ? 'GET' : 'POST',
2635
body,
2736
headers: {
2837
Accept: 'application/json',
2938
'Content-Type': 'application/json',
39+
...envHeaders,
3040
},
3141
});
3242
const responsePayload = await response.text();

0 commit comments

Comments
 (0)