Skip to content

Commit 4f17ede

Browse files
walterddrwalterddr
authored
adding retention period to segment delete REST API (#8122)
Add deletion with retention period overwrite for segment deletion
1 parent 45c1062 commit 4f17ede

File tree

3 files changed

+36
-12
lines changed

3 files changed

+36
-12
lines changed

‎pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -571,12 +571,14 @@ public SuccessResponse reloadAllSegmentsDeprecated2(
571571
@ApiOperation(value = "Delete a segment", notes = "Delete a segment")
572572
public SuccessResponse deleteSegment(
573573
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
574-
@ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName) {
574+
@ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
575+
@ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
576+
+ "delete segments without retention") @QueryParam("retention") String retentionPeriod) {
575577
segmentName = URIUtils.decode(segmentName);
576578
TableType tableType = SegmentName.isRealtimeSegmentName(segmentName) ? TableType.REALTIME : TableType.OFFLINE;
577579
String tableNameWithType =
578580
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
579-
deleteSegmentsInternal(tableNameWithType, Collections.singletonList(segmentName));
581+
deleteSegmentsInternal(tableNameWithType, Collections.singletonList(segmentName), retentionPeriod);
580582
return new SuccessResponse("Segment deleted");
581583
}
582584

@@ -587,14 +589,17 @@ public SuccessResponse deleteSegment(
587589
@ApiOperation(value = "Delete all segments", notes = "Delete all segments")
588590
public SuccessResponse deleteAllSegments(
589591
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
590-
@ApiParam(value = "OFFLINE|REALTIME", required = true) @QueryParam("type") String tableTypeStr) {
592+
@ApiParam(value = "OFFLINE|REALTIME", required = true) @QueryParam("type") String tableTypeStr,
593+
@ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
594+
+ "delete segments without retention") @QueryParam("retention") String retentionPeriod) {
591595
TableType tableType = Constants.validateTableType(tableTypeStr);
592596
if (tableType == null) {
593597
throw new ControllerApplicationException(LOGGER, "Table type must not be null", Status.BAD_REQUEST);
594598
}
595599
String tableNameWithType =
596600
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
597-
deleteSegmentsInternal(tableNameWithType, _pinotHelixResourceManager.getSegmentsFor(tableNameWithType, false));
601+
deleteSegmentsInternal(tableNameWithType, _pinotHelixResourceManager.getSegmentsFor(tableNameWithType, false),
602+
retentionPeriod);
598603
return new SuccessResponse("All segments of table " + tableNameWithType + " deleted");
599604
}
600605

@@ -607,6 +612,8 @@ public SuccessResponse deleteAllSegments(
607612
notes = "Delete the segments in the JSON array payload")
608613
public SuccessResponse deleteSegments(
609614
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
615+
@ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
616+
+ "delete segments without retention") @QueryParam("retention") String retentionPeriod,
610617
List<String> segments) {
611618
int numSegments = segments.size();
612619
if (numSegments == 0) {
@@ -622,16 +629,17 @@ public SuccessResponse deleteSegments(
622629
TableType tableType = isRealtimeSegment ? TableType.REALTIME : TableType.OFFLINE;
623630
String tableNameWithType =
624631
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
625-
deleteSegmentsInternal(tableNameWithType, segments);
632+
deleteSegmentsInternal(tableNameWithType, segments, retentionPeriod);
626633
if (numSegments <= 5) {
627634
return new SuccessResponse("Deleted segments: " + segments + " from table: " + tableNameWithType);
628635
} else {
629636
return new SuccessResponse("Deleted " + numSegments + " segments from table: " + tableNameWithType);
630637
}
631638
}
632639

633-
private void deleteSegmentsInternal(String tableNameWithType, List<String> segments) {
634-
PinotResourceManagerResponse response = _pinotHelixResourceManager.deleteSegments(tableNameWithType, segments);
640+
private void deleteSegmentsInternal(String tableNameWithType, List<String> segments, String retentionPeriod) {
641+
PinotResourceManagerResponse response = _pinotHelixResourceManager.deleteSegments(tableNameWithType, segments,
642+
retentionPeriod);
635643
if (!response.isSuccessful()) {
636644
throw new ControllerApplicationException(LOGGER,
637645
"Failed to delete segments from table: " + tableNameWithType + ", error message: " + response.getMessage(),

‎pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@
133133
import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateModel;
134134
import org.apache.pinot.spi.utils.CommonConstants.Server;
135135
import org.apache.pinot.spi.utils.IngestionConfigUtils;
136+
import org.apache.pinot.spi.utils.TimeUtils;
136137
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
137138
import org.apache.pinot.spi.utils.retry.RetryPolicies;
138139
import org.apache.pinot.spi.utils.retry.RetryPolicy;
@@ -694,21 +695,32 @@ public List<SegmentZKMetadata> getSegmentsZKMetadata(String tableNameWithType) {
694695
return ZKMetadataProvider.getSegmentsZKMetadata(_propertyStore, tableNameWithType);
695696
}
696697

698+
public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames) {
699+
return deleteSegments(tableNameWithType, segmentNames, null);
700+
}
701+
697702
/**
698703
* Delete a list of segments from ideal state and remove them from the local storage.
699704
*
700705
* @param tableNameWithType Table name with type suffix
701706
* @param segmentNames List of names of segment to be deleted
707+
* @param retentionPeriod The retention period of the deleted segments.
702708
* @return Request response
703709
*/
704-
public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames) {
710+
public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames,
711+
@Nullable String retentionPeriod) {
705712
try {
706713
LOGGER.info("Trying to delete segments: {} from table: {} ", segmentNames, tableNameWithType);
707714
Preconditions.checkArgument(TableNameBuilder.isTableResource(tableNameWithType),
708715
"Table name: %s is not a valid table name with type suffix", tableNameWithType);
709716
HelixHelper.removeSegmentsFromIdealState(_helixZkManager, tableNameWithType, segmentNames);
710-
TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType);
711-
_segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames, tableConfig);
717+
if (retentionPeriod != null) {
718+
_segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames,
719+
TimeUtils.convertPeriodToMillis(retentionPeriod));
720+
} else {
721+
TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType);
722+
_segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames, tableConfig);
723+
}
712724
return PinotResourceManagerResponse.success("Segment " + segmentNames + " deleted");
713725
} catch (final Exception e) {
714726
LOGGER.error("Caught exception while deleting segment: {} from table: {}", segmentNames, tableNameWithType, e);

‎pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,16 @@ public void stop() {
103103
}
104104

105105
public void deleteSegments(String tableName, Collection<String> segmentIds) {
106-
deleteSegments(tableName, segmentIds, null);
106+
deleteSegments(tableName, segmentIds, (Long) null);
107107
}
108108

109109
public void deleteSegments(String tableName, Collection<String> segmentIds,
110110
@Nullable TableConfig tableConfig) {
111-
Long deletedSegmentsRetentionMs = getRetentionMsFromTableConfig(tableConfig);
111+
deleteSegments(tableName, segmentIds, getRetentionMsFromTableConfig(tableConfig));
112+
}
113+
114+
public void deleteSegments(String tableName, Collection<String> segmentIds,
115+
@Nullable Long deletedSegmentsRetentionMs) {
112116
deleteSegmentsWithDelay(tableName, segmentIds, deletedSegmentsRetentionMs, DEFAULT_DELETION_DELAY_SECONDS);
113117
}
114118

0 commit comments

Comments
 (0)