@@ -219,17 +219,41 @@ FaissVectorIndex::~FaissVectorIndex() {
219219 }
220220}
221221
222+ namespace {
223+
224+ Status close_index_output (std::unique_ptr<lucene::store::IndexOutput>& output,
225+ const char * description) {
226+ DCHECK (output != nullptr );
227+ try {
228+ output->close ();
229+ } catch (const CLuceneError& e) {
230+ output.reset ();
231+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR >(" Failed to close {}: {}" ,
232+ description, e.what ());
233+ }
234+ output.reset ();
235+ return Status::OK ();
236+ }
237+
238+ } // namespace
239+
222240struct FaissIndexWriter : faiss::IOWriter {
223241public:
224- FaissIndexWriter () = default ;
225- FaissIndexWriter (lucene::store::IndexOutput* output) : _output(output) {}
242+ explicit FaissIndexWriter (lucene::store::IndexOutput* output) : _output(output) {}
226243 ~FaissIndexWriter () override {
227244 if (_output != nullptr ) {
228- _output->close ();
229- delete _output;
245+ try {
246+ _output->close ();
247+ } catch (const CLuceneError& e) {
248+ // Serialization already failed, so this is cleanup only. A
249+ // destructor must not turn an I/O error into std::terminate.
250+ LOG (WARNING ) << " Failed to close vector index output during cleanup: " << e.what ();
251+ }
230252 }
231253 }
232254
255+ Status close () { return close_index_output (_output, " vector index output" ); }
256+
233257 size_t operator ()(const void * ptr, size_t size, size_t nitems) override {
234258 size_t bytes = size * nitems;
235259 if (bytes > 0 ) {
@@ -252,7 +276,7 @@ struct FaissIndexWriter : faiss::IOWriter {
252276 return nitems;
253277 };
254278
255- lucene::store::IndexOutput* _output = nullptr ;
279+ std::unique_ptr< lucene::store::IndexOutput> _output;
256280};
257281
258282struct FaissIndexReader : faiss::IOReader {
@@ -953,84 +977,88 @@ doris::Status FaissVectorIndex::range_search(const float* query_vec, const float
953977doris::Status FaissVectorIndex::save (lucene::store::Directory* dir) {
954978 auto start_time = std::chrono::high_resolution_clock::now ();
955979
956- if (_index_type == AnnIndexType::IVF_ON_DISK ) {
957- // IVF_ON_DISK: write ivf data to a separate file, then write index metadata.
958- //
959- // Why do we replace invlists here in save() instead of at build() time?
960- // During build/train/add, IndexIVF needs a writable ArrayInvertedLists to
961- // receive vectors via add_entries(). PreadInvertedLists inherits from
962- // ReadOnlyInvertedLists and does not support writes. The original
963- // OnDiskInvertedLists does support writes but requires mmap on a real file,
964- // which is unavailable at build time (Directory is only passed to save()).
965- // So the standard faiss pattern is: build in-memory with ArrayInvertedLists,
966- // then convert to on-disk format at serialization time. The replace_invlists
967- // call in Step 2 is purely a serialization format switch (to emit "ilod"
968- // fourcc instead of "ilar"), not a runtime data structure change.
969- //
970- // Step 1: The in-memory index has ArrayInvertedLists. Write them to ann.ivfdata
971- // by converting to OnDiskInvertedLists format:
972- // For each list: [codes: capacity*code_size][ids: capacity*sizeof(idx_t)]
973- auto * ivf = dynamic_cast <faiss::IndexIVF*>(_index.get ());
974- DCHECK (ivf != nullptr );
975- auto * ails = dynamic_cast <faiss::ArrayInvertedLists*>(ivf->invlists );
976- DCHECK (ails != nullptr );
977-
978- const size_t nlist = ails->nlist ;
979- const size_t code_size = ails->code_size ;
980-
981- // Build OnDiskOneList metadata and write data to ann.ivfdata
982- std::vector<faiss::OnDiskOneList> lists (nlist);
983- lucene::store::IndexOutput* ivfdata_output = dir->createOutput (faiss_ivfdata_file_name);
984- size_t offset = 0 ;
985- for (size_t i = 0 ; i < nlist; i++) {
986- size_t list_size = ails->list_size (i);
987- lists[i].size = list_size;
988- lists[i].capacity = list_size;
989- lists[i].offset = offset;
990-
991- if (list_size > 0 ) {
992- // Write codes
993- const uint8_t * codes = ails->get_codes (i);
994- size_t codes_bytes = list_size * code_size;
995- ivfdata_output->writeBytes (codes, cast_set<Int32>(codes_bytes));
996-
997- // Write ids
998- const faiss::idx_t * ids = ails->get_ids (i);
999- size_t ids_bytes = list_size * sizeof (faiss::idx_t );
1000- ivfdata_output->writeBytes (reinterpret_cast <const uint8_t *>(ids),
1001- cast_set<Int32>(ids_bytes));
1002- }
980+ try {
981+ if (_index_type == AnnIndexType::IVF_ON_DISK ) {
982+ // IVF_ON_DISK: write ivf data to a separate file, then write index metadata.
983+ //
984+ // Why do we replace invlists here in save() instead of at build() time?
985+ // During build/train/add, IndexIVF needs a writable ArrayInvertedLists to
986+ // receive vectors via add_entries(). PreadInvertedLists inherits from
987+ // ReadOnlyInvertedLists and does not support writes. The original
988+ // OnDiskInvertedLists does support writes but requires mmap on a real file,
989+ // which is unavailable at build time (Directory is only passed to save()).
990+ // So the standard faiss pattern is: build in-memory with ArrayInvertedLists,
991+ // then convert to on-disk format at serialization time. The replace_invlists
992+ // call in Step 2 is purely a serialization format switch (to emit "ilod"
993+ // fourcc instead of "ilar"), not a runtime data structure change.
994+ //
995+ // Step 1: The in-memory index has ArrayInvertedLists. Write them to ann.ivfdata
996+ // by converting to OnDiskInvertedLists format:
997+ // For each list: [codes: capacity*code_size][ids: capacity*sizeof(idx_t)]
998+ auto * ivf = dynamic_cast <faiss::IndexIVF*>(_index.get ());
999+ DCHECK (ivf != nullptr );
1000+ auto * ails = dynamic_cast <faiss::ArrayInvertedLists*>(ivf->invlists );
1001+ DCHECK (ails != nullptr );
1002+
1003+ const size_t nlist = ails->nlist ;
1004+ const size_t code_size = ails->code_size ;
1005+
1006+ // Build OnDiskOneList metadata and write data to ann.ivfdata
1007+ std::vector<faiss::OnDiskOneList> lists (nlist);
1008+ auto ivfdata_output = std::unique_ptr<lucene::store::IndexOutput>(
1009+ dir->createOutput (faiss_ivfdata_file_name));
1010+ size_t offset = 0 ;
1011+ for (size_t i = 0 ; i < nlist; i++) {
1012+ size_t list_size = ails->list_size (i);
1013+ lists[i].size = list_size;
1014+ lists[i].capacity = list_size;
1015+ lists[i].offset = offset;
1016+
1017+ if (list_size > 0 ) {
1018+ // Write codes
1019+ const uint8_t * codes = ails->get_codes (i);
1020+ size_t codes_bytes = list_size * code_size;
1021+ ivfdata_output->writeBytes (codes, cast_set<Int32>(codes_bytes));
1022+
1023+ // Write ids
1024+ const faiss::idx_t * ids = ails->get_ids (i);
1025+ size_t ids_bytes = list_size * sizeof (faiss::idx_t );
1026+ ivfdata_output->writeBytes (reinterpret_cast <const uint8_t *>(ids),
1027+ cast_set<Int32>(ids_bytes));
1028+ }
10031029
1004- offset += list_size * (code_size + sizeof (faiss::idx_t ));
1030+ offset += list_size * (code_size + sizeof (faiss::idx_t ));
1031+ }
1032+ size_t totsize = offset;
1033+ RETURN_IF_ERROR (close_index_output (ivfdata_output, " IVF data output" ));
1034+
1035+ // Step 2: Replace ArrayInvertedLists with OnDiskInvertedLists so that
1036+ // write_index serializes in "ilod" format (metadata only).
1037+ auto * od = new faiss::OnDiskInvertedLists ();
1038+ od->nlist = nlist;
1039+ od->code_size = code_size;
1040+ od->lists = std::move (lists);
1041+ od->totsize = totsize;
1042+ od->ptr = nullptr ;
1043+ od->read_only = true ;
1044+ // filename is not used during load (we use separate ivfdata file),
1045+ // but write it for format completeness.
1046+ od->filename = faiss_ivfdata_file_name;
1047+ ivf->replace_invlists (od, true );
1048+
1049+ // Step 3: Write index metadata to ann.faiss (includes "ilod" fourcc)
1050+ FaissIndexWriter writer (dir->createOutput (faiss_index_fila_name));
1051+ RETURN_IF_CATCH_EXCEPTION (faiss::write_index (_index.get (), &writer));
1052+ RETURN_IF_ERROR (writer.close ());
1053+ } else {
1054+ // HNSW / IVF: write the full index to ann.faiss
1055+ FaissIndexWriter writer (dir->createOutput (faiss_index_fila_name));
1056+ RETURN_IF_CATCH_EXCEPTION (faiss::write_index (_index.get (), &writer));
1057+ RETURN_IF_ERROR (writer.close ());
10051058 }
1006- size_t totsize = offset;
1007- ivfdata_output->close ();
1008- delete ivfdata_output;
1009-
1010- // Step 2: Replace ArrayInvertedLists with OnDiskInvertedLists so that
1011- // write_index serializes in "ilod" format (metadata only).
1012- auto * od = new faiss::OnDiskInvertedLists ();
1013- od->nlist = nlist;
1014- od->code_size = code_size;
1015- od->lists = std::move (lists);
1016- od->totsize = totsize;
1017- od->ptr = nullptr ;
1018- od->read_only = true ;
1019- // filename is not used during load (we use separate ivfdata file),
1020- // but write it for format completeness.
1021- od->filename = faiss_ivfdata_file_name;
1022- ivf->replace_invlists (od, true );
1023-
1024- // Step 3: Write index metadata to ann.faiss (includes "ilod" fourcc)
1025- lucene::store::IndexOutput* idx_output = dir->createOutput (faiss_index_fila_name);
1026- auto writer = std::make_unique<FaissIndexWriter>(idx_output);
1027- faiss::write_index (_index.get (), writer.get ());
1028-
1029- } else {
1030- // HNSW / IVF: write the full index to ann.faiss
1031- lucene::store::IndexOutput* idx_output = dir->createOutput (faiss_index_fila_name);
1032- auto writer = std::make_unique<FaissIndexWriter>(idx_output);
1033- faiss::write_index (_index.get (), writer.get ());
1059+ } catch (const CLuceneError& e) {
1060+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR >(
1061+ " Failed to save vector index: {}" , e.what ());
10341062 }
10351063
10361064 auto end_time = std::chrono::high_resolution_clock::now ();
0 commit comments