Skip to content

Commit 77ef904

Browse files
gjredagjreda
authored
feat: Add endpoint for creating a new reference with or without PDF (#549)
* small refactor * wip: needs tests * fix * filename * test for create_reference_from_url works * update tests * add tests and clean up * types * linter * default filename * ability to create reference without url * linter * address PR comments * types * rename * Add filepath for legacy ingest * clean up dead code * linter * fix ts issues * Add publication date to search results * types * fix test * initialize storage if it does not exist * add published_date * fixes * fixes * shorten name
1 parent 7985127 commit 77ef904

22 files changed

Lines changed: 1046 additions & 408 deletions

‎python/api.schema.json‎

Lines changed: 104 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -152,12 +152,6 @@
152152
],
153153
"title": "DeleteStatusResponse"
154154
},
155-
"EmptyRequest": {
156-
"properties": {},
157-
"type": "object",
158-
"title": "EmptyRequest",
159-
"description": "Use this to indicate that a request only accepts an empty object ({})"
160-
},
161155
"FileEntry": {
162156
"properties": {
163157
"name": {
@@ -291,6 +285,38 @@
291285
"type": "object",
292286
"title": "HTTPValidationError"
293287
},
288+
"IngestMetadataRequest": {
289+
"properties": {
290+
"type": {
291+
"allOf": [
292+
{
293+
"$ref": "#/definitions/IngestRequestType"
294+
}
295+
],
296+
"default": "metadata"
297+
},
298+
"metadata": {
299+
"$ref": "#/definitions/ReferenceCreate"
300+
},
301+
"url": {
302+
"type": "string"
303+
}
304+
},
305+
"type": "object",
306+
"required": [
307+
"metadata"
308+
],
309+
"title": "IngestMetadataRequest"
310+
},
311+
"IngestRequestType": {
312+
"type": "string",
313+
"enum": [
314+
"uploads",
315+
"metadata"
316+
],
317+
"title": "IngestRequestType",
318+
"description": "An enumeration."
319+
},
294320
"IngestResponse": {
295321
"properties": {
296322
"project_name": {
@@ -320,6 +346,20 @@
320346
"title": "IngestStatus",
321347
"description": "An enumeration."
322348
},
349+
"IngestUploadsRequest": {
350+
"properties": {
351+
"type": {
352+
"allOf": [
353+
{
354+
"$ref": "#/definitions/IngestRequestType"
355+
}
356+
],
357+
"default": "uploads"
358+
}
359+
},
360+
"type": "object",
361+
"title": "IngestUploadsRequest"
362+
},
323363
"ModelProvider": {
324364
"type": "string",
325365
"enum": [
@@ -418,11 +458,14 @@
418458
"id": {
419459
"type": "string"
420460
},
461+
"status": {
462+
"$ref": "#/definitions/IngestStatus"
463+
},
421464
"source_filename": {
422465
"type": "string"
423466
},
424-
"status": {
425-
"$ref": "#/definitions/IngestStatus"
467+
"filepath": {
468+
"type": "string"
426469
},
427470
"citation_key": {
428471
"type": "string"
@@ -465,12 +508,60 @@
465508
"type": "object",
466509
"required": [
467510
"id",
468-
"source_filename",
469511
"status"
470512
],
471513
"title": "Reference",
472514
"description": "A reference for an academic paper / PDF"
473515
},
516+
"ReferenceCreate": {
517+
"properties": {
518+
"title": {
519+
"type": "string"
520+
},
521+
"source_filename": {
522+
"type": "string"
523+
},
524+
"citation_key": {
525+
"type": "string"
526+
},
527+
"doi": {
528+
"type": "string"
529+
},
530+
"abstract": {
531+
"type": "string"
532+
},
533+
"contents": {
534+
"type": "string"
535+
},
536+
"published_date": {
537+
"type": "string",
538+
"format": "date"
539+
},
540+
"authors": {
541+
"items": {
542+
"$ref": "#/definitions/Author"
543+
},
544+
"type": "array",
545+
"default": []
546+
},
547+
"chunks": {
548+
"items": {
549+
"$ref": "#/definitions/Chunk"
550+
},
551+
"type": "array",
552+
"default": []
553+
},
554+
"metadata": {
555+
"type": "object",
556+
"default": {}
557+
}
558+
},
559+
"type": "object",
560+
"required": [
561+
"title"
562+
],
563+
"title": "ReferenceCreate"
564+
},
474565
"ReferencePatch": {
475566
"properties": {
476567
"data": {
@@ -584,6 +675,10 @@
584675
"year": {
585676
"type": "integer"
586677
},
678+
"publicationDate": {
679+
"type": "string",
680+
"format": "date-time"
681+
},
587682
"paperId": {
588683
"type": "string"
589684
},

‎python/main.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import json
33
from argparse import ArgumentParser
44

5+
from sidecar import config
56
from sidecar.typing import CliCommands
67

78

@@ -48,7 +49,7 @@ def get_arg_parser():
4849
if args.command == "serve":
4950
from sidecar.api import serve
5051

51-
serve(host="0.0.0.0", port=1487)
52+
serve(host=config.HOST, port=config.PORT)
5253

5354
else:
5455
raise NotImplementedError(f"Command {args.command} is not implemented.")

‎python/openapi.json‎

Lines changed: 112 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -153,12 +153,6 @@
153153
"title": "DeleteStatusResponse",
154154
"type": "object"
155155
},
156-
"EmptyRequest": {
157-
"description": "Use this to indicate that a request only accepts an empty object ({})",
158-
"properties": {},
159-
"title": "EmptyRequest",
160-
"type": "object"
161-
},
162156
"FileEntry": {
163157
"properties": {
164158
"file_extension": {
@@ -292,6 +286,38 @@
292286
"title": "HTTPValidationError",
293287
"type": "object"
294288
},
289+
"IngestMetadataRequest": {
290+
"properties": {
291+
"metadata": {
292+
"$ref": "#/components/schemas/ReferenceCreate"
293+
},
294+
"type": {
295+
"allOf": [
296+
{
297+
"$ref": "#/components/schemas/IngestRequestType"
298+
}
299+
],
300+
"default": "metadata"
301+
},
302+
"url": {
303+
"type": "string"
304+
}
305+
},
306+
"required": [
307+
"metadata"
308+
],
309+
"title": "IngestMetadataRequest",
310+
"type": "object"
311+
},
312+
"IngestRequestType": {
313+
"description": "An enumeration.",
314+
"enum": [
315+
"uploads",
316+
"metadata"
317+
],
318+
"title": "IngestRequestType",
319+
"type": "string"
320+
},
295321
"IngestResponse": {
296322
"properties": {
297323
"project_name": {
@@ -321,6 +347,20 @@
321347
"title": "IngestStatus",
322348
"type": "string"
323349
},
350+
"IngestUploadsRequest": {
351+
"properties": {
352+
"type": {
353+
"allOf": [
354+
{
355+
"$ref": "#/components/schemas/IngestRequestType"
356+
}
357+
],
358+
"default": "uploads"
359+
}
360+
},
361+
"title": "IngestUploadsRequest",
362+
"type": "object"
363+
},
324364
"ModelProvider": {
325365
"description": "An enumeration.",
326366
"enum": [
@@ -443,6 +483,9 @@
443483
"doi": {
444484
"type": "string"
445485
},
486+
"filepath": {
487+
"type": "string"
488+
},
446489
"id": {
447490
"type": "string"
448491
},
@@ -466,12 +509,60 @@
466509
},
467510
"required": [
468511
"id",
469-
"source_filename",
470512
"status"
471513
],
472514
"title": "Reference",
473515
"type": "object"
474516
},
517+
"ReferenceCreate": {
518+
"properties": {
519+
"abstract": {
520+
"type": "string"
521+
},
522+
"authors": {
523+
"default": [],
524+
"items": {
525+
"$ref": "#/components/schemas/Author"
526+
},
527+
"type": "array"
528+
},
529+
"chunks": {
530+
"default": [],
531+
"items": {
532+
"$ref": "#/components/schemas/Chunk"
533+
},
534+
"type": "array"
535+
},
536+
"citation_key": {
537+
"type": "string"
538+
},
539+
"contents": {
540+
"type": "string"
541+
},
542+
"doi": {
543+
"type": "string"
544+
},
545+
"metadata": {
546+
"default": {},
547+
"type": "object"
548+
},
549+
"published_date": {
550+
"format": "date",
551+
"type": "string"
552+
},
553+
"source_filename": {
554+
"type": "string"
555+
},
556+
"title": {
557+
"type": "string"
558+
}
559+
},
560+
"required": [
561+
"title"
562+
],
563+
"title": "ReferenceCreate",
564+
"type": "object"
565+
},
475566
"ReferencePatch": {
476567
"description": "ReferencePatch is the input type for updating a Reference's metadata.",
477568
"properties": {
@@ -591,6 +682,10 @@
591682
"paperId": {
592683
"type": "string"
593684
},
685+
"publicationDate": {
686+
"format": "date-time",
687+
"type": "string"
688+
},
594689
"title": {
595690
"type": "string"
596691
},
@@ -1441,7 +1536,7 @@
14411536
]
14421537
},
14431538
"post": {
1444-
"description": "Ingests references from PDFs in the project uploads directory",
1539+
"description": "Creates references from a PDF directory or URL.",
14451540
"operationId": "ingest_references_api_references__project_id__post",
14461541
"parameters": [
14471542
{
@@ -1458,7 +1553,15 @@
14581553
"content": {
14591554
"application/json": {
14601555
"schema": {
1461-
"$ref": "#/components/schemas/EmptyRequest"
1556+
"anyOf": [
1557+
{
1558+
"$ref": "#/components/schemas/IngestMetadataRequest"
1559+
},
1560+
{
1561+
"$ref": "#/components/schemas/IngestUploadsRequest"
1562+
}
1563+
],
1564+
"title": "Request"
14621565
}
14631566
}
14641567
},

‎python/sidecar/config.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
load_dotenv()
88

99

10+
HOST = os.environ.get("HOST", "0.0.0.0")
11+
PORT = int(os.environ.get("PORT", 1487))
12+
1013
WEB_STORAGE_URL = Path(os.environ.get("WEB_STORAGE_URL", "/tmp/web-storage-url"))
1114
PROJECT_DIR = Path(os.environ.get("PROJECT_DIR", "/tmp/project-x"))
1215
UPLOADS_DIR = Path(os.path.join(PROJECT_DIR, "uploads"))

0 commit comments

Comments
 (0)