Skip to content

Commit c28bccc

Browse files
committed
[fix](fe) Clean stored Broker Load profiles by execution query IDs
1 parent 371884e commit c28bccc

2 files changed

Lines changed: 81 additions & 3 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -820,9 +820,9 @@ protected void deleteOutdatedProfilesFromStorage() {
820820
try {
821821
for (ProfileElement profileElement : profilesToBeRemoved) {
822822
profileIdToProfileMap.remove(profileElement.profile.getSummaryProfile().getProfileId());
823-
TUniqueId thriftQueryId = DebugUtil.parseTUniqueIdFromString(
824-
profileElement.profile.getSummaryProfile().getProfileId());
825-
queryIdToExecutionProfiles.remove(thriftQueryId);
823+
for (ExecutionProfile executionProfile : profileElement.profile.getExecutionProfiles()) {
824+
queryIdToExecutionProfiles.remove(executionProfile.getQueryId());
825+
}
826826
}
827827
} finally {
828828
writeLock.unlock();

‎fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
import org.junit.jupiter.api.BeforeEach;
4444
import org.junit.jupiter.api.Test;
4545
import org.junit.jupiter.api.parallel.ResourceLock;
46+
import org.junit.jupiter.params.ParameterizedTest;
47+
import org.junit.jupiter.params.provider.ValueSource;
4648
import org.mockito.Mockito;
4749

4850
import java.io.File;
@@ -703,6 +705,82 @@ void testDeleteOutdatedProfilesFromStorage() throws IOException {
703705
}
704706
}
705707

708+
@ParameterizedTest
709+
@ValueSource(booleans = {false, true})
710+
void testPruneMixedQueryAndBrokerLoadProfiles(boolean archiveEnabled) throws Exception {
711+
boolean originalArchiveEnabled = Config.enable_profile_archive;
712+
String originalArchivePath = Config.profile_archive_path;
713+
int originalMaxSpilled = Config.max_spilled_profile_num;
714+
long originalStorageLimit = Config.spilled_profile_storage_limit_bytes;
715+
int originalBatchSize = Config.profile_archive_batch_size;
716+
int originalPendingTimeout = Config.profile_archive_pending_timeout_seconds;
717+
List<Profile> profiles = new ArrayList<>();
718+
List<File> profileFiles = new ArrayList<>();
719+
try {
720+
Config.enable_profile_archive = archiveEnabled;
721+
Config.profile_archive_path = "";
722+
Config.max_spilled_profile_num = 1;
723+
Config.max_query_profile_num = 3;
724+
Config.spilled_profile_storage_limit_bytes = Long.MAX_VALUE;
725+
Config.profile_archive_batch_size = 100;
726+
Config.profile_archive_pending_timeout_seconds = 3600;
727+
long finishTime = System.currentTimeMillis();
728+
for (int i = 0; i < 3; i++) {
729+
boolean brokerLoad = i != 1;
730+
TUniqueId queryId = new TUniqueId(0x22040L, i * 2 + 1);
731+
Profile profile = ProfilePersistentTest.constructRandomProfile(0);
732+
profile.getSummaryProfile().getSummary().getInfoStrings().put(SummaryProfile.PROFILE_ID,
733+
brokerLoad ? String.valueOf(123456 + i) : DebugUtil.printId(queryId));
734+
profile.isQueryFinished = true;
735+
profile.setQueryFinishTimestamp(finishTime + i);
736+
profile.addExecutionProfile(new ExecutionProfile(queryId, Lists.newArrayList(0)));
737+
if (brokerLoad) {
738+
profile.addExecutionProfile(new ExecutionProfile(
739+
new TUniqueId(queryId.hi, queryId.lo + 1), Lists.newArrayList(1)));
740+
}
741+
for (ExecutionProfile execution : profile.getExecutionProfiles()) {
742+
profileManager.addExecutionProfile(execution);
743+
}
744+
profileManager.pushProfile(profile);
745+
profile.writeToStorage(ProfileManager.PROFILE_STORAGE_PATH);
746+
File profileFile = new File(profile.getProfileStoragePath());
747+
Assertions.assertTrue(profileFile.exists());
748+
profileFiles.add(profileFile);
749+
profiles.add(profile);
750+
}
751+
List<ProfileElement> selected = profileManager.getProfilesToBeRemoved();
752+
Assertions.assertEquals(2, selected.size());
753+
Assertions.assertSame(profiles.get(0), selected.get(0).profile);
754+
Assertions.assertSame(profiles.get(1), selected.get(1).profile);
755+
756+
profileManager.profileLoadStatus.set(ProfileLoadStatus.LOADED);
757+
profileManager.deleteOutdatedProfilesFromStorage();
758+
759+
Assertions.assertEquals(1, profileManager.profileIdToProfileMap.size());
760+
Assertions.assertEquals(2, profileManager.queryIdToExecutionProfiles.size());
761+
for (int i = 0; i < profiles.size(); i++) {
762+
Profile profile = profiles.get(i);
763+
boolean retained = i == 2;
764+
Assertions.assertEquals(retained, profileManager.profileIdToProfileMap.containsKey(profile.getId()));
765+
Assertions.assertEquals(retained, profileFiles.get(i).exists());
766+
for (ExecutionProfile execution : profile.getExecutionProfiles()) {
767+
Assertions.assertEquals(retained,
768+
profileManager.queryIdToExecutionProfiles.containsKey(execution.getQueryId()));
769+
}
770+
if (!retained && archiveEnabled) {
771+
Assertions.assertTrue(new File(tempDir, "archive/pending/" + profileFiles.get(i).getName()).exists());
772+
}
773+
}
774+
} finally {
775+
Config.enable_profile_archive = originalArchiveEnabled;
776+
Config.profile_archive_path = originalArchivePath;
777+
Config.max_spilled_profile_num = originalMaxSpilled;
778+
Config.spilled_profile_storage_limit_bytes = originalStorageLimit;
779+
Config.profile_archive_batch_size = originalBatchSize;
780+
Config.profile_archive_pending_timeout_seconds = originalPendingTimeout;
781+
}
782+
}
783+
706784
@Test
707785
void testGetBrokenProfiles() throws IOException {
708786
// Create normal profiles

0 commit comments

Comments
 (0)