//

// This file is part of the aMule Project.

//

// Copyright (c) 2003-2026 aMule Team ( https://amule-org.github.io )

// Copyright (c) 2002-2011 Merkur ( [email protected] / http://www.emule-project.net )

//

// Any parts of this program derived from the xMule, lMule or eMule project,

// or contributed by third-party developers are copyrighted by their

// respective authors.

//

// This program is free software; you can redistribute it and/or modify

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation; either version 2 of the License, or

// (at your option) any later version.

//

// This program is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with this program; if not, write to the Free Software

// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA

//

#include "KnownFileList.h" // Interface declarations

#include "GuiEvents.h" // Notify_KnownFileBeingDestroyed

#include <common/DataFileVersion.h>

#include <algorithm>

#include <map>

#include <memory> // Do_not_auto_remove (lionel's Mac, 10.3)

#include <set>

#include <vector>

#include "DownloadQueue.h" // Needed for theApp->downloadqueue access

#include "PartFile.h" // Needed for CPartFile

#include "amule.h"

#include "Logger.h"

#include "MemFile.h"

#include "ScopedPtr.h"

#include "SearchList.h" // Needed for UpdateSearchFileByHash

#include "SharedFileList.h"

#include <common/Format.h>

#include "Preferences.h" // Needed for thePrefs

// Max duplicate-list records retained per hash. Unique hashes always

// keep their live m_knownFileMap entry; this caps only the historical

// (name/date) variants in m_duplicateFileList. 8 covers daily-touch /

// weekly-snapshot / monthly-backup cycles while bounding the on-disk

// known.met at unique_hashes × (1 + cap).

#define KNOWN_DUPLICATE_HASH_CAP 8

// TTL after which a record (live or duplicate) whose lastSeen hasn't

// been refreshed gets dropped. A real file currently on disk has its

// lastSeen bumped by FindKnownFile / IsOnDuplicates / Append every

// share-scan; anything left stale for this long either lost its file

// or had its mtime/name change in a way that won't recur (mtime is

// monotone-forward in practice, so a duplicate captured at an older

// mtime will not match again). 30 days catches most pathological

// touch loops on the next save without losing legitimate intermittent

// matches.

#define KNOWN_DUPLICATE_TTL_SECS (30 * 24 * 60 * 60)

// This function is inlined for performance

inline bool CKnownFileList::KnownFileMatches(

CKnownFile *knownFile,

const CPath& filename,

uint32 in_date,

uint64 in_size) const

{

return

(knownFile->GetLastChangeDatetime() == (time_t)in_date) &&

(knownFile->GetFileSize() == in_size) &&

(knownFile->GetFileName() == filename);

}

CKnownFileList::CKnownFileList()

{

accepted = 0;

requested = 0;

transferred = 0;

m_filename = "known.met";

m_knownSizeMap = NULL;

m_duplicateSizeMap = NULL;

m_initialShareScanComplete = false;

Init();

}

CKnownFileList::~CKnownFileList()

{

Clear();

}

bool CKnownFileList::Init()

{

CFile file;

CPath fullpath = CPath(thePrefs::GetConfigDir() + m_filename);

if (!fullpath.FileExists()) {

// This is perfectly normal. The file was probably either

// deleted, or this is the first time running aMule.

return false;

}

if (!file.Open(fullpath)) {

AddLogLineC(CFormat(_("WARNING: %s cannot be opened.")) % m_filename);

return false;

}

try {

uint8 version = file.ReadUInt8();

if ((version != MET_HEADER) && (version != MET_HEADER_WITH_LARGEFILES)) {

AddLogLineC(_("WARNING: Known file list corrupted, contains invalid header."));

return false;

}

wxMutexLocker sLock(list_mut);

uint32 RecordsNumber = file.ReadUInt32();

AddDebugLogLineN(logKnownFiles, CFormat("Reading %i known files from file format 0x%2.2x.")

% RecordsNumber % version);

// Keep the size-map index live during the load. Append() is O(log N)

// on every record, but on each MD4 hash collision (real-world

// libraries hit these whenever the same content was indexed under

// two paths/names) it falls back to IsOnDuplicates(name, date, size).

// Without a duplicate-size index, IsOnDuplicates scans

// m_duplicateFileList linearly, so the dedup cost grows with each

// duplicate appended — O(N^2) over the whole load. Prebuilding the

// (empty) index here lets Append maintain it incrementally, giving

// the O(log N) equal_range fast path on every collision check.

// Issue #562 startup gap, ~36 s on a 200 k-file library.

PrepareIndex();

for (uint32 i = 0; i < RecordsNumber; i++) {

CScopedPtr<CKnownFile> record;

if (record->LoadFromFile(&file)) {

AddDebugLogLineN(logKnownFiles,

CFormat("Known file read: %s") % record->GetFileName());

Append(record.release());

} else {

AddLogLineC(_("Failed to load entry in known file list, file may be corrupt"));

}

}

ReleaseIndex();

AddDebugLogLineN(logKnownFiles, "Finished reading known files");

return true;

} catch (const CInvalidPacket& e) {

ReleaseIndex();

AddLogLineC(_("Invalid entry in known file list, file may be corrupt: ") + e.what());

} catch (const CSafeIOException& e) {

ReleaseIndex();

AddLogLineC(CFormat(_("IO error while reading %s file: %s")) % m_filename % e.what());

}

return false;

}

void CKnownFileList::Save()

{

// Acquire the lock before opening the .new file. Save() is called

// from both the main thread (on shutdown / scheduled persist) and

// the hashing worker thread (CHashingTask::OnLastTask). If two

// callers raced past the open, both would create known.met.new at

// the same path, the first to Close() would rename it away, and

// the second's rename would fail with ENOENT -- producing the

// "Impossible to get permissions for file 'known.met.new'" /

// "couldn't be renamed 'known.met.new' -> 'known.met'" pair seen

// in #86. Holding list_mut around the whole save serialises the

// .new lifecycle. The list itself is read-only inside, so this

// doesn't widen the existing critical section meaningfully.

wxMutexLocker sLock(list_mut);

// Snapshot the in-use set under our own lock. Taking the snapshot

// before locking left a TOCTOU window where the main thread could

// add a CKnownFile to sharedfiles between snapshot and prune; the

// prune then deleted a file that sharedfiles still indexed,

// leaving a dangling pointer that the EC encoder map kept feeding

// to Get_EC_Response_GetUpdate -> use-after-free crash (#685).

//

// Brief overlap of knownfiles -> sharedfiles / downloadqueue locks

// is safe: no code path acquires those in the reverse order while

// holding the first. sharedfiles never calls into knownfiles under

// its own lock, and downloadqueue never calls into knownfiles at

// all.

std::unordered_set<CKnownFile*> inUse;

if (theApp && theApp->sharedfiles) {

std::vector<CKnownFile*> sharedSnapshot;

theApp->sharedfiles->CopyFileList(sharedSnapshot);

inUse.insert(sharedSnapshot.begin(), sharedSnapshot.end());

}

if (theApp && theApp->downloadqueue) {

std::vector<CPartFile*> dqSnapshot;

theApp->downloadqueue->CopyFileList(dqSnapshot, true);

inUse.insert(dqSnapshot.begin(), dqSnapshot.end());

}

PruneDuplicates(inUse);

CFile file(thePrefs::GetConfigDir() + m_filename, CFile::write_safe);

if (!file.IsOpened()) {

return;

}

AddDebugLogLineN(logKnownFiles, CFormat("start saving %s") % m_filename);

try {

// Kry - This is the version, but we don't know it till

// we know if any largefile is saved. This allows the list

// to be compatible with previous versions.

bool bContainsAnyLargeFiles = false;

file.WriteUInt8(0);

file.WriteUInt32(m_knownFileMap.size() + m_duplicateFileList.size());

// Duplicates handling. Duplicates needs to be saved first,

// since it is the last entry that gets used.

KnownFileList::iterator itDup = m_duplicateFileList.begin();

for ( ; itDup != m_duplicateFileList.end(); ++itDup ) {

(*itDup)->WriteToFile(&file);

if ((*itDup)->IsLargeFile()) {

bContainsAnyLargeFiles = true;

}

}

CKnownFileMap::iterator it = m_knownFileMap.begin();

for (; it != m_knownFileMap.end(); ++it) {

it->second->WriteToFile(&file);

if (it->second->IsLargeFile()) {

bContainsAnyLargeFiles = true;

}

}

file.Seek(0);

file.WriteUInt8(bContainsAnyLargeFiles ? MET_HEADER_WITH_LARGEFILES : MET_HEADER);

file.Close();

} catch (const CIOFailureException& e) {

AddLogLineC(CFormat(_("Error while saving %s file: %s")) % m_filename % e.what());

}

AddDebugLogLineN(logKnownFiles, CFormat("finished saving %s") % m_filename);

}

bool CKnownFileList::IsKnownFile(const CKnownFile* file) const

{

// Pointer-value scan over the canonical map; safe to call with a

// possibly-freed `file` pointer (no deref). Used by

// OnFinishedHashing / OnFinishedAICHHashing to validate the

// owner pointer survived hashing. m_knownFileMap is hash-keyed

// not pointer-keyed, so we walk it — linear in shareset size but

// invoked only on rare events (hash completion). For 100 k+

// sharesets this is the cost; a per-pointer index would speed

// it up but isn't justified for the call rate.

wxMutexLocker sLock(list_mut);

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();

it != m_knownFileMap.end(); ++it) {

if (it->second == file) {

return true;

}

}

return false;

}

void CKnownFileList::Clear()

{

wxMutexLocker sLock(list_mut);

// Fire Notify_KnownFileBeingDestroyed for every file we're about

// to delete, so subscribers (list ctrls, dialogs, AICH static

// list, write-thread flushList, EC client-side m_uploadingfile /

// m_reqfile fields, etc.) strip their references before the

// `delete`. Pointer-value comparison only; the objects are still

// alive at the time of the notify, but subscribers must not

// deref them on the main-thread dispatch (which may run after

// DeleteContents has freed them). See MuleNotify::

// KnownFileBeingDestroyed (GuiEvents.cpp).

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();

it != m_knownFileMap.end(); ++it) {

Notify_KnownFileBeingDestroyed(it->second);

}

for (KnownFileList::const_iterator it = m_duplicateFileList.begin();

it != m_duplicateFileList.end(); ++it) {

Notify_KnownFileBeingDestroyed(*it);

}

DeleteContents(m_knownFileMap);

DeleteContents(m_duplicateFileList);

ReleaseIndex();

m_pinnedDuplicates.clear();

m_initialShareScanComplete = false;

}

void CKnownFileList::MarkInitialShareScanComplete()

{

wxMutexLocker sLock(list_mut);

m_initialShareScanComplete = true;

}

void CKnownFileList::CollectLiveAICHRoots(

std::unordered_set<CAICHHash> & out)

{

wxMutexLocker sLock(list_mut);

out.reserve(out.size() + m_knownFileMap.size() + m_duplicateFileList.size());

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();

it != m_knownFileMap.end(); ++it) {

const CKnownFile * f = it->second;

if (f && f->HasProperAICHHashSet()) {

out.insert(f->GetAICHHashset()->GetMasterHash());

}

}

// Duplicate-list records can also be the only owner of an AICH

// master hash, because a hash-collision demote in Append parks

// the previous record (including its hashset) on the duplicate

// list while the new record takes over m_knownFileMap. If we

// drop the duplicate's AICH from known2_64.met and then the

// duplicate later gets re-promoted (mtime restore) we'd

// silently lose its hashset; cheap to keep both sets here.

for (KnownFileList::const_iterator it = m_duplicateFileList.begin();

it != m_duplicateFileList.end(); ++it) {

const CKnownFile * f = *it;

if (f && f->HasProperAICHHashSet()) {

out.insert(f->GetAICHHashset()->GetMasterHash());

}

}

}

CKnownFile* CKnownFileList::FindKnownFile(

const CPath& filename,

time_t in_date,

uint64 in_size)

{

wxMutexLocker sLock(list_mut);

const uint32 now = (uint32) time(NULL);

if (m_knownSizeMap) {

const auto key = std::make_pair((uint32) in_size, (uint32) in_date);

std::pair<KnownFileSizeMap::const_iterator, KnownFileSizeMap::const_iterator> p;

p = m_knownSizeMap->equal_range(key);

for (KnownFileSizeMap::const_iterator it = p.first; it != p.second; ++it) {

CKnownFile *cur_file = it->second;

if (KnownFileMatches(cur_file, filename, in_date, in_size)) {

cur_file->SetLastSeen(now);

return cur_file;

}

}

} else {

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();

it != m_knownFileMap.end(); ++it) {

CKnownFile *cur_file = it->second;

if (KnownFileMatches(cur_file, filename, in_date, in_size)) {

cur_file->SetLastSeen(now);

return cur_file;

}

}

}

// Pin any duplicate-list match against this session's prune so a

// real on-disk file's record isn't dropped just because its hash

// is also held by a more-recent live entry (the dual-content-copy

// case: same hash in two shared paths -- one becomes m_Files_map,

// the other only ever appears here).

CKnownFile * dup = IsOnDuplicates(filename, in_date, in_size);

if (dup) {

dup->SetLastSeen(now);

m_pinnedDuplicates.insert(dup);

}

return dup;

}

CKnownFile *CKnownFileList::IsOnDuplicates(

const CPath& filename,

uint32 in_date,

uint64 in_size) const

{

if (m_duplicateSizeMap) {

const auto key = std::make_pair((uint32) in_size, (uint32) in_date);

std::pair<KnownFileSizeMap::const_iterator, KnownFileSizeMap::const_iterator> p;

p = m_duplicateSizeMap->equal_range(key);

for (KnownFileSizeMap::const_iterator it = p.first; it != p.second; ++it) {

CKnownFile *cur_file = it->second;

if (KnownFileMatches(cur_file, filename, in_date, in_size)) {

return cur_file;

}

}

} else {

for (KnownFileList::const_iterator it = m_duplicateFileList.begin();

it != m_duplicateFileList.end(); ++it) {

CKnownFile *cur_file = *it;

if (KnownFileMatches(cur_file, filename, in_date, in_size)) {

return cur_file;

}

}

}

return NULL;

}

CKnownFile* CKnownFileList::FindKnownFileByID(const CMD4Hash& hash)

{

wxMutexLocker sLock(list_mut);

if (!hash.IsEmpty()) {

if (m_knownFileMap.find(hash) != m_knownFileMap.end()) {

return m_knownFileMap[hash];

} else {

return NULL;

}

}

return NULL;

}

bool CKnownFileList::SafeAddKFile(CKnownFile* toadd, bool afterHashing)

{

bool ret;

{

wxMutexLocker sLock(list_mut);

ret = Append(toadd, afterHashing);

}

if (ret) {

theApp->searchlist->UpdateSearchFileByHash(toadd->GetFileHash());

}

return ret;

}

bool CKnownFileList::Append(CKnownFile *Record, bool afterHashing)

{

if (Record->GetFileSize() > 0) {

// sanity check if the number of part hashes is correct here

if (Record->GetHashCount() != Record->GetED2KPartHashCount()) {

AddDebugLogLineC(logKnownFiles, CFormat("%s with size %d should have %d part hashes, but only %d are available")

% Record->GetFileName().GetPrintable() % Record->GetFileSize() % Record->GetED2KPartHashCount() % Record->GetHashCount());

return false;

}

const uint32 now = (uint32) time(NULL);

const CMD4Hash& tkey = Record->GetFileHash();

CKnownFileMap::iterator it = m_knownFileMap.find(tkey);

if (it == m_knownFileMap.end()) {

// Only stamp lastSeen=now when this is a confirmed

// sighting of the file on disk (post-hash via

// CHashingTask, or any other afterHashing=true path).

// During known.met load Append is called with

// afterHashing=false; touching lastSeen here would

// overwrite either the FT_LASTSEEN tag we just loaded

// or the m_lastDateChanged fallback that

// CKnownFile::LoadFromFile substitutes when the tag

// is absent. The latter is the only thing that lets

// the TTL prune do useful migration work on an old

// known.met -- if we trample it, every loaded record

// looks "fresh" and TTL never evicts anything (this

// is exactly what bit ngosang on the first PR test).

if (afterHashing) {

Record->SetLastSeen(now);

}

m_knownFileMap[tkey] = Record;

if (m_knownSizeMap) {

m_knownSizeMap->insert(

std::make_pair(

std::make_pair((uint32) Record->GetFileSize(),

(uint32) Record->GetLastChangeDatetime()),

Record));

}

return true;

} else {

CKnownFile *existing = it->second;

if (KnownFileMatches(Record, existing->GetFileName(), existing->GetLastChangeDatetime(), existing->GetFileSize())) {

// The file is already on the list, ignore it.

AddDebugLogLineN(logKnownFiles, CFormat("%s is already on the list") % Record->GetFileName().GetPrintable());

if (afterHashing) {

existing->SetLastSeen(now);

}

return false;

} else if (CKnownFile * dup = IsOnDuplicates(

Record->GetFileName(), Record->GetLastChangeDatetime(),

Record->GetFileSize())) {

// The file is on the duplicates list, ignore it.

// Should not happen, at least not after hashing. Or why did it get hashed in the first place then?

AddDebugLogLineN(logKnownFiles, CFormat("%s is on the duplicates list") % Record->GetFileName().GetPrintable());

// Pin the duplicate only when this branch was hit

// because we just hashed a real on-disk file (the

// only case the comment above describes anyway).

// During load Append doesn't represent a fresh

// sighting -- pinning here would falsely protect

// stale records from the cap/TTL prune.

if (afterHashing) {

dup->SetLastSeen(now);

m_pinnedDuplicates.insert(dup);

}

return false;

} else {

if (afterHashing && existing->GetFileSize() == Record->GetFileSize()) {

// We just hashed a "new" shared file and find it's already known under a different name or date.

// Guess what - it was probably renamed or touched.

// So copy over all properties from the existing known file and just keep name/date.

time_t newDate = Record->GetLastChangeDatetime();

CPath newName = Record->GetFileName();

CMemFile f;

existing->WriteToFile(&f);

f.Reset();

if (!Record->LoadFromFile(&f)) {

// this also shouldn't happen

AddDebugLogLineC(logKnownFiles, CFormat("error copying known file: existing: %s %d %d %d Record: %s %d %d %d")

% existing->GetFileName().GetPrintable() % existing->GetFileSize() % existing->GetED2KPartHashCount() % existing->GetHashCount()

% Record->GetFileName().GetPrintable() % Record->GetFileSize() % Record->GetED2KPartHashCount() % Record->GetHashCount());

return false;

}

Record->SetLastChangeDatetime(newDate);

Record->SetFileName(newName);

}

// The file is a duplicated hash. Add THE OLD ONE to the duplicates list.

// (This is used when reading the known file list where the duplicates are stored in front.)

m_duplicateFileList.push_back(existing);

if (m_duplicateSizeMap) {

m_duplicateSizeMap->insert(

std::make_pair(

std::make_pair((uint32) existing->GetFileSize(),

(uint32) existing->GetLastChangeDatetime()),

existing));

}

if (theApp->sharedfiles) {

// Removing the old kad keywords created with the old filename

theApp->sharedfiles->RemoveKeywords(existing);

}

if (m_knownSizeMap) {

// existing is leaving m_knownFileMap for m_duplicateFileList;

// drop its size-map entry so FindKnownFile doesn't return a

// pointer that no longer belongs to the live map.

const auto existingKey = std::make_pair(

(uint32) existing->GetFileSize(),

(uint32) existing->GetLastChangeDatetime());

std::pair<KnownFileSizeMap::iterator,

KnownFileSizeMap::iterator> p =

m_knownSizeMap->equal_range(existingKey);

for (KnownFileSizeMap::iterator hit = p.first;

hit != p.second; ++hit) {

if (hit->second == existing) {

m_knownSizeMap->erase(hit);

break;

}

}

m_knownSizeMap->insert(

std::make_pair(

std::make_pair((uint32) Record->GetFileSize(),

(uint32) Record->GetLastChangeDatetime()),

Record));

}

// On the afterHashing path the copy-existing-tags

// block above pulled the prior FT_LASTSEEN into

// Record; refresh it so the new live entry isn't

// born aged-out of the TTL window. During load

// (afterHashing=false) keep Record's own loaded

// lastSeen instead -- demoting an entry to the

// duplicate list must not stamp the replacement

// as "fresh now" or the migration-driven TTL pass

// loses its signal (every live entry would look

// load-time-fresh and never be evicted).

if (afterHashing) {

Record->SetLastSeen(now);

}

m_knownFileMap[tkey] = Record;

return true;

}

}

} else {

AddDebugLogLineN(logGeneral,

CFormat("%s is 0-size, not added") %

Record->GetFileName());

return false;

}

}

// Make a (size, mtime) index to speed up FindKnownFile. Size + mtime

// modulo 2^32 is enough here — same precision the rest of the file

// uses for FindKnownFile's inputs and KnownFileMatches' comparisons.

void CKnownFileList::PrepareIndex()

{

ReleaseIndex();

m_knownSizeMap = new KnownFileSizeMap;

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin(); it != m_knownFileMap.end(); ++it) {

m_knownSizeMap->insert(std::make_pair(

std::make_pair((uint32) it->second->GetFileSize(),

(uint32) it->second->GetLastChangeDatetime()),

it->second));

}

m_duplicateSizeMap = new KnownFileSizeMap;

for (KnownFileList::const_iterator it = m_duplicateFileList.begin(); it != m_duplicateFileList.end(); ++it) {

m_duplicateSizeMap->insert(std::make_pair(

std::make_pair((uint32) (*it)->GetFileSize(),

(uint32) (*it)->GetLastChangeDatetime()),

*it));

}

}

void CKnownFileList::ReleaseIndex()

{

delete m_knownSizeMap;

delete m_duplicateSizeMap;

m_knownSizeMap = NULL;

m_duplicateSizeMap = NULL;

}

void CKnownFileList::PruneDuplicates(

const std::unordered_set<CKnownFile*> & inUse)

{

// Caller must hold list_mut.

// Gate on a full share-scan having run this session -- before that,

// inUse is empty and m_pinnedDuplicates hasn't been populated by

// FindKnownFile yet, so a prune here would drop records the next

// scan would have legitimately pinned. Set true by

// MarkInitialShareScanComplete() from CSharedFileList::Reload.

if (!m_initialShareScanComplete) {

return;

}

const uint32 now = (uint32) time(NULL);

const uint32 ttlCutoff =

(now > KNOWN_DUPLICATE_TTL_SECS)

? (now - KNOWN_DUPLICATE_TTL_SECS) : 0;

auto isProtected = [&](CKnownFile * r) {

return inUse.count(r) > 0 || m_pinnedDuplicates.count(r) > 0;

};

auto eraseFromSizeMap = [&](KnownFileSizeMap * sizeMap,

CKnownFile * dead) {

if (!sizeMap) {

return;

}

const auto key = std::make_pair(

(uint32) dead->GetFileSize(),

(uint32) dead->GetLastChangeDatetime());

std::pair<KnownFileSizeMap::iterator,

KnownFileSizeMap::iterator> p =

sizeMap->equal_range(key);

for (KnownFileSizeMap::iterator hit = p.first;

hit != p.second; ++hit) {

if (hit->second == dead) {

sizeMap->erase(hit);

return;

}

}

};

// Pass 1: live entries (m_knownFileMap) past TTL. A non-refreshed

// live entry means no share-scan in the last TTL window produced a

// (name, date, size) match -- the file is no longer accessible to

// us. The whole hash is dead; we'll also wipe its duplicates.

// std::set (not unordered_set) because CMD4Hash provides operator<

// but no std::hash specialization.

std::set<CMD4Hash> deadHashes;

for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();

it != m_knownFileMap.end(); ++it) {

CKnownFile * live = it->second;

if (isProtected(live)) {

continue;

}

if (live->GetLastSeen() < ttlCutoff) {

deadHashes.insert(it->first);

}

}

// Pass 2: duplicates -- drop if their hash is dead, or their own

// lastSeen is past TTL. Bucket survivors by hash for the cap pass.

std::map<CMD4Hash, std::vector<KnownFileList::iterator> > survivors;

size_t droppedDupTTL = 0;

for (KnownFileList::iterator it = m_duplicateFileList.begin();

it != m_duplicateFileList.end(); ) {

CKnownFile * record = *it;

if (isProtected(record)) {

survivors[record->GetFileHash()].push_back(it);

++it;

continue;

}

const bool hashDead =

deadHashes.count(record->GetFileHash()) > 0;

const bool ownStale =

record->GetLastSeen() < ttlCutoff;

if (hashDead || ownStale) {

eraseFromSizeMap(m_duplicateSizeMap, record);

KnownFileList::iterator victim = it++;

Notify_KnownFileBeingDestroyed(record);

delete record;

m_duplicateFileList.erase(victim);

++droppedDupTTL;

} else {

survivors[record->GetFileHash()].push_back(it);

++it;

}

}

// Pass 3: drop the dead live entries (and their size-map index).

size_t droppedLive = 0;

for (std::set<CMD4Hash>::const_iterator it = deadHashes.begin();

it != deadHashes.end(); ++it) {

CKnownFileMap::iterator kit = m_knownFileMap.find(*it);

if (kit == m_knownFileMap.end()) {

continue;

}

CKnownFile * dead = kit->second;

// Final paranoid re-check: even though Save() now snapshots

// inUse under our own lock, the snapshot's sharedfiles /

// downloadqueue locks were released before the prune body

// ran. A concurrent SafeAddKFile (main thread) or RemoveFile

// (UploadDiskIOThread) could have changed membership in

// between. Re-query under the owner's lock, immediately

// before delete, to make this point-in-time correct (#685).

if (theApp && theApp->sharedfiles &&

theApp->sharedfiles->GetFileByID(*it) != NULL) {

continue;

}

if (theApp && theApp->downloadqueue &&

theApp->downloadqueue->GetFileByID(*it) != NULL) {

continue;

}

eraseFromSizeMap(m_knownSizeMap, dead);

Notify_KnownFileBeingDestroyed(dead);

delete dead;

m_knownFileMap.erase(kit);

++droppedLive;

}

// Pass 4: per-hash cap on whatever duplicate survivors remain.

size_t droppedDupCap = 0;

for (std::map<CMD4Hash,

std::vector<KnownFileList::iterator> >::iterator

bucket = survivors.begin();

bucket != survivors.end(); ++bucket) {

std::vector<KnownFileList::iterator> & iters = bucket->second;

if (iters.size() <= KNOWN_DUPLICATE_HASH_CAP) {

continue;

}

// Partition out protected entries first; the cap counts

// only the prunable remainder so protected records don't

// crowd legitimate survivors out.

std::vector<KnownFileList::iterator> prunable;

prunable.reserve(iters.size());

for (size_t i = 0; i < iters.size(); ++i) {

if (!isProtected(*iters[i])) {

prunable.push_back(iters[i]);

}

}

if (prunable.size() <= KNOWN_DUPLICATE_HASH_CAP) {

continue;

}

// Newest mtime survives the cap. Older mtimes for the same

// hash are unlikely to match again in practice (mtime is

// monotone-forward outside explicit-restore tooling).

std::sort(prunable.begin(), prunable.end(),

[](const KnownFileList::iterator & a,

const KnownFileList::iterator & b) {

return (*a)->GetLastChangeDatetime()

> (*b)->GetLastChangeDatetime();

});

for (size_t i = KNOWN_DUPLICATE_HASH_CAP;

i < prunable.size(); ++i) {

CKnownFile * dead = *prunable[i];

eraseFromSizeMap(m_duplicateSizeMap, dead);

m_duplicateFileList.erase(prunable[i]);

Notify_KnownFileBeingDestroyed(dead);

delete dead;

++droppedDupCap;

}

}

if (droppedLive || droppedDupTTL || droppedDupCap) {

AddDebugLogLineN(logKnownFiles,

CFormat("known.met prune: dropped %u live + %u dup (TTL %u days) + %u dup (cap %u)")

% (unsigned)droppedLive

% (unsigned)droppedDupTTL

% (unsigned)(KNOWN_DUPLICATE_TTL_SECS / (24 * 60 * 60))

% (unsigned)droppedDupCap

% KNOWN_DUPLICATE_HASH_CAP);

}

}

// File_checked_for_headers