The BagIt file handler detects and transforms zip files with a BagIt package format into Dataverse DataFiles. The system validates the checksums of the files in the package payload as described in the first manifest file with a hash algorithm that we support. Take a look at `BagChecksumType class <https://github.com/IQSS/dataverse/tree/develop/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagChecksumType.java>`_ for the list of the currently supported hash algorithms.

The handler will not allow packages with checksum errors. The first 5 errors will be displayed to the user. This is configurable though database settings.

The checksum validation uses a thread pool to improve performance. This thread pool can be adjusted to your Dataverse installation requirements.

The BagIt file handler is disabled by default. Use the ``:BagItHandlerEnabled`` database settings to enable it: ``curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:BagItHandlerEnabled``

For more configuration settings see the user guide: https://guides.dataverse.org/en/latest/installation/config.html#bagit-file-handler

.. _BagIt File Handler:

BagIt File Handler

BagIt file handler detects and transforms zip files with a BagIt package format into Dataverse DataFiles. The system validates the checksums of the files in the package payload as described in the first manifest file with a hash algorithm that we support. Take a look at `BagChecksumType class <https://github.com/IQSS/dataverse/tree/develop/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagChecksumType.java>`_ for the list of the currently supported hash algorithms.

The checksum validation uses a thread pool to improve performance. This thread pool can be adjusted to your Dataverse installation requirements.

BagIt file handler configuration settings:

- :ref:`:BagItHandlerEnabled`

- :ref:`:BagValidatorJobPoolSize`

- :ref:`:BagValidatorMaxErrors`

- :ref:`:BagValidatorJobWaitInterval`

:CreateDataFilesMaxErrorsToDisplay

Number of errors to display to the user when creating DataFiles from a file upload. It defaults to 5 errors.

``curl -X PUT -d '1' http://localhost:8080/api/admin/settings/:CreateDataFilesMaxErrorsToDisplay``

.. _:BagItHandlerEnabled:

:BagItHandlerEnabled

Part of the database settings to configure the BagIt file handler. Enables the BagIt file handler. By default, the handler is disabled.

``curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:BagItHandlerEnabled``

.. _:BagValidatorJobPoolSize:

:BagValidatorJobPoolSize

Part of the database settings to configure the BagIt file handler. The number of threads the checksum validation class uses to validate a single zip file. Defaults to 4 threads

``curl -X PUT -d '10' http://localhost:8080/api/admin/settings/:BagValidatorJobPoolSize``

.. _:BagValidatorMaxErrors:

:BagValidatorMaxErrors

Part of the database settings to configure the BagIt file handler. The maximum number of errors allowed before the validation job aborts execution. This is to avoid processing the whole BagIt package. Defaults to 5 errors.

``curl -X PUT -d '2' http://localhost:8080/api/admin/settings/:BagValidatorMaxErrors``

.. _:BagValidatorJobWaitInterval:

:BagValidatorJobWaitInterval

Part of the database settings to configure the BagIt file handler. This is the period in seconds to check for the number of errors during validation. Defaults to 10.

``curl -X PUT -d '60' http://localhost:8080/api/admin/settings/:BagValidatorJobWaitInterval``

package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.util.BundleUtil;

import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;

import javax.ejb.Stateless;

import javax.inject.Inject;

import java.util.List;

import java.util.Optional;

import java.util.stream.Collectors;

public class EditDataFilesPageHelper {

public static final String MAX_ERRORS_TO_DISPLAY_SETTING = ":CreateDataFilesMaxErrorsToDisplay";

public static final Integer MAX_ERRORS_TO_DISPLAY = 5;

@Inject

private SettingsWrapper settingsWrapper;

public String getHtmlErrorMessage(CreateDataFileResult createDataFileResult) {

List<String> errors = createDataFileResult.getErrors();

if(errors == null || errors.isEmpty()) {

return null;

}

Integer maxErrorsToShow = settingsWrapper.getInteger(EditDataFilesPageHelper.MAX_ERRORS_TO_DISPLAY_SETTING, EditDataFilesPageHelper.MAX_ERRORS_TO_DISPLAY);

if(maxErrorsToShow < 1) {

return null;

}

String typeMessage = Optional.ofNullable(BundleUtil.getStringFromBundle(createDataFileResult.getBundleKey())).orElse("Error processing file");

String errorsMessage = errors.stream().limit(maxErrorsToShow).map(text -> String.format("<li>%s</li>", text)).collect(Collectors.joining());

return String.format("%s:<br /><ul>%s</ul>", typeMessage, errorsMessage);

}

}

import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;

@Inject

EditDataFilesPageHelper editDataFilesPageHelper;

CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig);

datafiles = createDataFilesResult.getDataFiles();

errorMessage = editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult);

if(errorMessage != null) {

FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadFailure"), errorMessage));

PrimeFaces.current().ajax().update(":messagePanel");

}

errorMessage = null;

private String errorMessage = null;

// zip file.

CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig);

dFileList = createDataFilesResult.getDataFiles();

errorMessage = editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult);

CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);

datafiles = createDataFilesResult.getDataFiles();

errorMessage = editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult);

}

public Integer getInteger(String settingKey, Integer defaultValue) {

String settingValue = get(settingKey);

if(settingValue != null) {

try {

return Integer.valueOf(settingValue);

} catch (Exception e) {

logger.warning(String.format("action=getInteger result=invalid-integer settingKey=%s settingValue=%s", settingKey, settingValue));

}

}

return defaultValue;

}

import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;

CreateDataFileResult createDataFilesResponse = FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);

dataFiles = createDataFilesResponse.getDataFiles();

import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;

CreateDataFileResult result = FileUtil.createDataFiles(workingVersion,

initialFileList = result.getDataFiles();