//

// This file is part of the aMule Project.

//

// Copyright (c) 2004-2011 Angel Vidal ( [email protected] )

// Copyright (c) 2003-2026 aMule Team ( https://amule-org.github.io )

// Copyright (c) 2002-2011 Merkur ( [email protected] / http://www.emule-project.net )

//

// Any parts of this program derived from the xMule, lMule or eMule project,

// or contributed by third-party developers are copyrighted by their

// respective authors.

//

// This program is free software; you can redistribute it and/or modify

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation; either version 2 of the License, or

// (at your option) any later version.

//

// This program is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with this program; if not, write to the Free Software

// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA

//

#include <wx/file.h>

#include "SHAHashSet.h"

#include "amule.h"

#include "MemFile.h"

#include "Preferences.h"

#include "SHA.h"

#include "updownclient.h"

#include "DownloadQueue.h"

#include "PartFile.h"

#include "Logger.h"

#include <common/Format.h>

// for this version the limits are set very high, they might be lowered later

// to make a hash trustworthy, at least 10 unique Ips (255.255.128.0) must have sent it

// and if we have received more than one hash for the file, one hash has to be sent by more than 95% of all

// unique IPs

#define MINUNIQUEIPS_TOTRUST 10 // how many unique IPs have to send us a hash to make it trustworthy

#define MINPERCENTAGE_TOTRUST \

92 // how many percentage of clients have to send the same hash to make it trustworthy

CAICHRequestedDataList CAICHHashSet::m_liRequestedData;

// Lazily-built index mapping root hash → file offset in known2.met.

// See SaveHashSet (dedup-on-append) and LoadHashSet (O(1) lookup for

// incoming AICH requests) for usage.

wxMutex CAICHHashSet::s_rootHashCacheMutex;

std::unordered_map<CAICHHash, uint64> CAICHHashSet::s_rootHashCache;

bool CAICHHashSet::s_rootHashCacheLoaded = false;

/////////////////////////////////////////////////////////////////////////////////////////

/// CAICHHash

wxString CAICHHash::GetString() const

{

return EncodeBase32(m_abyBuffer, HASHSIZE);

}

void CAICHHash::Read(CFileDataIO *file)

{

file->Read(m_abyBuffer, HASHSIZE);

}

void CAICHHash::Write(CFileDataIO *file) const

{

file->Write(m_abyBuffer, HASHSIZE);

}

unsigned int CAICHHash::DecodeBase32(const wxString &base32)

{

return ::DecodeBase32(base32, HASHSIZE, m_abyBuffer);

}

/////////////////////////////////////////////////////////////////////////////////////////

/// CAICHHashTree

CAICHHashTree::CAICHHashTree(uint64 nDataSize, bool bLeftBranch, uint64 nBaseSize)

{

m_nDataSize = nDataSize;

m_nBaseSize = nBaseSize;

m_bIsLeftBranch = bLeftBranch;

m_pLeftTree = NULL;

m_pRightTree = NULL;

m_bHashValid = false;

}

CAICHHashTree::~CAICHHashTree()

{

delete m_pLeftTree;

delete m_pRightTree;

}

// recursive

CAICHHashTree *CAICHHashTree::FindHash(uint64 nStartPos, uint64 nSize, uint8 *nLevel)

{

(*nLevel)++;

wxCHECK(*nLevel <= 22, NULL);

wxCHECK(nStartPos + nSize <= m_nDataSize, NULL);

wxCHECK(nSize <= m_nDataSize, NULL);

if (nStartPos == 0 && nSize == m_nDataSize) {

// this is the searched hash

return this;

} else if (m_nDataSize <= m_nBaseSize) { // sanity

// this is already the last level, can't go deeper

wxFAIL;

return NULL;

} else {

uint64 nBlocks = m_nDataSize / m_nBaseSize + ((m_nDataSize % m_nBaseSize != 0) ? 1 : 0);

uint64 nLeft = (((m_bIsLeftBranch) ? nBlocks + 1 : nBlocks) / 2) * m_nBaseSize;

uint64 nRight = m_nDataSize - nLeft;

if (nStartPos < nLeft) {

if (nStartPos + nSize > nLeft) { // sanity

wxFAIL;

return NULL;

}

if (m_pLeftTree == NULL) {

m_pLeftTree = new CAICHHashTree(

nLeft, true, (nLeft <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pLeftTree->m_nDataSize == nLeft);

}

return m_pLeftTree->FindHash(nStartPos, nSize, nLevel);

} else {

nStartPos -= nLeft;

if (nStartPos + nSize > nRight) { // sanity

wxFAIL;

return NULL;

}

if (m_pRightTree == NULL) {

m_pRightTree = new CAICHHashTree(

nRight, false, (nRight <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pRightTree->m_nDataSize == nRight);

}

return m_pRightTree->FindHash(nStartPos, nSize, nLevel);

}

}

}

// recursive

// calculates missing hash from the existing ones

// overwrites existing hashs

// fails if no hash is found for any branch

bool CAICHHashTree::ReCalculateHash(CAICHHashAlgo *hashalg, bool bDontReplace)

{

if (m_pLeftTree && m_pRightTree) {

if (!m_pLeftTree->ReCalculateHash(hashalg, bDontReplace) ||

!m_pRightTree->ReCalculateHash(hashalg, bDontReplace)) {

return false;

}

if (bDontReplace && m_bHashValid) {

return true;

}

if (m_pRightTree->m_bHashValid && m_pLeftTree->m_bHashValid) {

hashalg->Reset();

hashalg->Add(m_pLeftTree->m_Hash.GetRawHash(), HASHSIZE);

hashalg->Add(m_pRightTree->m_Hash.GetRawHash(), HASHSIZE);

hashalg->Finish(m_Hash);

m_bHashValid = true;

return true;

} else {

return m_bHashValid;

}

} else if (m_pLeftTree == NULL && m_pRightTree == NULL) {

return true;

} else {

AddDebugLogLineN(logSHAHashSet, "ReCalculateHash failed - Hashtree incomplete");

return false;

}

}

bool CAICHHashTree::VerifyHashTree(CAICHHashAlgo *hashalg, bool bDeleteBadTrees)

{

if (!m_bHashValid) {

wxFAIL;

if (bDeleteBadTrees) {

if (m_pLeftTree) {

delete m_pLeftTree;

m_pLeftTree = NULL;

}

if (m_pRightTree) {

delete m_pRightTree;

m_pRightTree = NULL;

}

}

AddDebugLogLineN(logSHAHashSet, "VerifyHashTree - No masterhash available");

return false;

}

// calculated missing hashs without overwriting anything

if (m_pLeftTree && !m_pLeftTree->m_bHashValid) {

m_pLeftTree->ReCalculateHash(hashalg, true);

}

if (m_pRightTree && !m_pRightTree->m_bHashValid) {

m_pRightTree->ReCalculateHash(hashalg, true);

}

if ((m_pRightTree && m_pRightTree->m_bHashValid) ^ (m_pLeftTree && m_pLeftTree->m_bHashValid)) {

// one branch can never be verified

if (bDeleteBadTrees) {

if (m_pLeftTree) {

delete m_pLeftTree;

m_pLeftTree = NULL;

}

if (m_pRightTree) {

delete m_pRightTree;

m_pRightTree = NULL;

}

}

AddDebugLogLineN(logSHAHashSet, "VerifyHashSet failed - Hashtree incomplete");

return false;

}

if ((m_pRightTree && m_pRightTree->m_bHashValid) && (m_pLeftTree && m_pLeftTree->m_bHashValid)) {

// check verify the hashs of both child nodes against my hash

CAICHHash CmpHash;

hashalg->Reset();

hashalg->Add(m_pLeftTree->m_Hash.GetRawHash(), HASHSIZE);

hashalg->Add(m_pRightTree->m_Hash.GetRawHash(), HASHSIZE);

hashalg->Finish(CmpHash);

if (m_Hash != CmpHash) {

if (bDeleteBadTrees) {

if (m_pLeftTree) {

delete m_pLeftTree;

m_pLeftTree = NULL;

}

if (m_pRightTree) {

delete m_pRightTree;

m_pRightTree = NULL;

}

}

return false;

}

return m_pLeftTree->VerifyHashTree(hashalg, bDeleteBadTrees) &&

m_pRightTree->VerifyHashTree(hashalg, bDeleteBadTrees);

} else {

// last hash in branch - nothing below to verify

return true;

}

}

void CAICHHashTree::SetBlockHash(uint64 nSize, uint64 nStartPos, CAICHHashAlgo *pHashAlg)

{

wxASSERT(nSize <= EMBLOCKSIZE);

CAICHHashTree *pToInsert = FindHash(nStartPos, nSize);

if (pToInsert == NULL) { // sanity

wxFAIL;

AddDebugLogLineN(

logSHAHashSet, "Critical Error: Failed to Insert SHA-HashBlock, FindHash() failed!");

return;

}

// sanity

if (pToInsert->m_nBaseSize != EMBLOCKSIZE || pToInsert->m_nDataSize != nSize) {

wxFAIL;

AddDebugLogLineN(

logSHAHashSet, "Critical Error: Logical error on values in SetBlockHashFromData");

return;

}

pHashAlg->Finish(pToInsert->m_Hash);

pToInsert->m_bHashValid = true;

}

bool CAICHHashTree::CreatePartRecoveryData(

uint64 nStartPos, uint64 nSize, CFileDataIO *fileDataOut, uint32 wHashIdent, bool b32BitIdent)

{

wxCHECK(nStartPos + nSize <= m_nDataSize, false);

wxCHECK(nSize <= m_nDataSize, false);

if (nStartPos == 0 && nSize == m_nDataSize) {

// this is the searched part, now write all blocks of this part

// hashident for this level will be adjusted by WriteLowestLevelHash

return WriteLowestLevelHashs(fileDataOut, wHashIdent, false, b32BitIdent);

} else if (m_nDataSize <= m_nBaseSize) { // sanity

// this is already the last level, can't go deeper

wxFAIL;

return false;

} else {

wHashIdent <<= 1;

wHashIdent |= (m_bIsLeftBranch) ? 1 : 0;

uint64 nBlocks = m_nDataSize / m_nBaseSize + ((m_nDataSize % m_nBaseSize != 0) ? 1 : 0);

uint64 nLeft = (((m_bIsLeftBranch) ? nBlocks + 1 : nBlocks) / 2) * m_nBaseSize;

uint64 nRight = m_nDataSize - nLeft;

if (m_pLeftTree == NULL || m_pRightTree == NULL) {

wxFAIL;

return false;

}

if (nStartPos < nLeft) {

if (nStartPos + nSize > nLeft || !m_pRightTree->m_bHashValid) { // sanity

wxFAIL;

return false;

}

m_pRightTree->WriteHash(fileDataOut, wHashIdent, b32BitIdent);

return m_pLeftTree->CreatePartRecoveryData(

nStartPos, nSize, fileDataOut, wHashIdent, b32BitIdent);

} else {

nStartPos -= nLeft;

if (nStartPos + nSize > nRight || !m_pLeftTree->m_bHashValid) { // sanity

wxFAIL;

return false;

}

m_pLeftTree->WriteHash(fileDataOut, wHashIdent, b32BitIdent);

return m_pRightTree->CreatePartRecoveryData(

nStartPos, nSize, fileDataOut, wHashIdent, b32BitIdent);

}

}

}

void CAICHHashTree::WriteHash(CFileDataIO *fileDataOut, uint32 wHashIdent, bool b32BitIdent) const

{

wxASSERT(m_bHashValid);

wHashIdent <<= 1;

wHashIdent |= (m_bIsLeftBranch) ? 1 : 0;

if (!b32BitIdent) {

wxASSERT(wHashIdent <= 0xFFFF);

fileDataOut->WriteUInt16((uint16)wHashIdent);

} else {

fileDataOut->WriteUInt32(wHashIdent);

}

m_Hash.Write(fileDataOut);

}

// write lowest level hashs into file, ordered from left to right optional without identifier

bool CAICHHashTree::WriteLowestLevelHashs(

CFileDataIO *fileDataOut, uint32 wHashIdent, bool bNoIdent, bool b32BitIdent) const

{

wHashIdent <<= 1;

wHashIdent |= (m_bIsLeftBranch) ? 1 : 0;

if (m_pLeftTree == NULL && m_pRightTree == NULL) {

if (m_nDataSize <= m_nBaseSize && m_bHashValid) {

if (!bNoIdent && !b32BitIdent) {

wxASSERT(wHashIdent <= 0xFFFF);

fileDataOut->WriteUInt16((uint16)wHashIdent);

} else if (!bNoIdent && b32BitIdent) {

fileDataOut->WriteUInt32(wHashIdent);

}

m_Hash.Write(fileDataOut);

return true;

} else {

wxFAIL;

return false;

}

} else if (m_pLeftTree == NULL || m_pRightTree == NULL) {

wxFAIL;

return false;

} else {

return m_pLeftTree->WriteLowestLevelHashs(fileDataOut, wHashIdent, bNoIdent, b32BitIdent) &&

m_pRightTree->WriteLowestLevelHashs(fileDataOut, wHashIdent, bNoIdent, b32BitIdent);

}

}

// recover all low level hashs from given data. hashs are assumed to be ordered in left to right - no

// identifier used

bool CAICHHashTree::LoadLowestLevelHashs(CFileDataIO *fileInput)

{

if (m_nDataSize <= m_nBaseSize) { // sanity

// lowest level, read hash

m_Hash.Read(fileInput);

m_bHashValid = true;

return true;

} else {

uint64 nBlocks = m_nDataSize / m_nBaseSize + ((m_nDataSize % m_nBaseSize != 0) ? 1 : 0);

uint64 nLeft = (((m_bIsLeftBranch) ? nBlocks + 1 : nBlocks) / 2) * m_nBaseSize;

uint64 nRight = m_nDataSize - nLeft;

if (m_pLeftTree == NULL) {

m_pLeftTree =

new CAICHHashTree(nLeft, true, (nLeft <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pLeftTree->m_nDataSize == nLeft);

}

if (m_pRightTree == NULL) {

m_pRightTree = new CAICHHashTree(

nRight, false, (nRight <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pRightTree->m_nDataSize == nRight);

}

return m_pLeftTree->LoadLowestLevelHashs(fileInput) &&

m_pRightTree->LoadLowestLevelHashs(fileInput);

}

}

// write the hash, specified by wHashIdent, with Data from fileInput.

bool CAICHHashTree::SetHash(CFileDataIO *fileInput, uint32 wHashIdent, sint8 nLevel, bool bAllowOverwrite)

{

if (nLevel == (-1)) {

// first call, check how many level we need to go

uint8 i = 0;

for (; i != 32 && (wHashIdent & 0x80000000) == 0; ++i) {

wHashIdent <<= 1;

}

if (i > 31) {

AddDebugLogLineN(

logSHAHashSet, "CAICHHashTree::SetHash - found invalid HashIdent (0)");

return false;

} else {

nLevel = 31 - i;

}

}

if (nLevel == 0) {

// this is the searched hash

if (m_bHashValid && !bAllowOverwrite) {

// not allowed to overwrite this hash, however move the filepointer as if we read a

// hash

fileInput->Seek(HASHSIZE, wxFromCurrent);

return true;

}

m_Hash.Read(fileInput);

m_bHashValid = true;

return true;

} else if (m_nDataSize <= m_nBaseSize) { // sanity

// this is already the last level, can't go deeper

wxFAIL;

return false;

} else {

// adjust ident to point the path to the next node

wHashIdent <<= 1;

nLevel--;

uint64 nBlocks = m_nDataSize / m_nBaseSize + ((m_nDataSize % m_nBaseSize != 0) ? 1 : 0);

uint64 nLeft = (((m_bIsLeftBranch) ? nBlocks + 1 : nBlocks) / 2) * m_nBaseSize;

uint64 nRight = m_nDataSize - nLeft;

if ((wHashIdent & 0x80000000) > 0) {

if (m_pLeftTree == NULL) {

m_pLeftTree = new CAICHHashTree(

nLeft, true, (nLeft <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pLeftTree->m_nDataSize == nLeft);

}

return m_pLeftTree->SetHash(fileInput, wHashIdent, nLevel);

} else {

if (m_pRightTree == NULL) {

m_pRightTree = new CAICHHashTree(

nRight, false, (nRight <= PARTSIZE) ? EMBLOCKSIZE : PARTSIZE);

} else {

wxASSERT(m_pRightTree->m_nDataSize == nRight);

}

return m_pRightTree->SetHash(fileInput, wHashIdent, nLevel);

}

}

}

/////////////////////////////////////////////////////////////////////////////////////////

/// CAICHUntrustedHash

bool CAICHUntrustedHash::AddSigningIP(uint32 dwIP)

{

dwIP &= 0x00F0FFFF; // we use only the 20 most significant bytes for unique IPs

return m_adwIpsSigning.insert(dwIP).second;

}

/////////////////////////////////////////////////////////////////////////////////////////

/// CAICHHashSet

CAICHHashSet::CAICHHashSet(CKnownFile *pOwner)

: m_pHashTree(0, true, PARTSIZE)

{

m_eStatus = AICH_EMPTY;

m_pOwner = pOwner;

}

CAICHHashSet::~CAICHHashSet(void)

{

FreeHashSet();

}

bool CAICHHashSet::CreatePartRecoveryData(uint64 nPartStartPos, CFileDataIO *fileDataOut, bool bDbgDontLoad)

{

wxASSERT(m_pOwner);

if (m_pOwner->IsPartFile() || m_eStatus != AICH_HASHSETCOMPLETE) {

wxFAIL;

return false;

}

if (m_pHashTree.m_nDataSize <= EMBLOCKSIZE) {

wxFAIL;

return false;

}

if (!bDbgDontLoad) {

if (!LoadHashSet()) {

AddDebugLogLineN(logSHAHashSet,

CFormat("Created RecoveryData error: failed to load hashset. File: %s") %

m_pOwner->GetFileName());

SetStatus(AICH_ERROR);

return false;

}

}

bool bResult;

uint8 nLevel = 0;

uint32 nPartSize = min<uint64>(PARTSIZE, m_pOwner->GetFileSize() - nPartStartPos);

m_pHashTree.FindHash(nPartStartPos, nPartSize, &nLevel);

uint16 nHashsToWrite =

(nLevel - 1) + nPartSize / EMBLOCKSIZE + ((nPartSize % EMBLOCKSIZE != 0) ? 1 : 0);

const bool bUse32BitIdentifier = m_pOwner->IsLargeFile();

if (bUse32BitIdentifier) {

fileDataOut->WriteUInt16(0); // no 16bit hashs to write

}

fileDataOut->WriteUInt16(nHashsToWrite);

uint64 nCheckFilePos = fileDataOut->GetPosition();

if (m_pHashTree.CreatePartRecoveryData(

nPartStartPos, nPartSize, fileDataOut, 0, bUse32BitIdentifier)) {

if (nHashsToWrite * (HASHSIZE + (bUse32BitIdentifier ? 4u : 2u)) !=

fileDataOut->GetPosition() - nCheckFilePos) {

wxFAIL;

AddDebugLogLineN(logSHAHashSet,

CFormat("Created RecoveryData has wrong length. File: %s") %

m_pOwner->GetFileName());

bResult = false;

SetStatus(AICH_ERROR);

} else {

bResult = true;

}

} else {

AddDebugLogLineN(logSHAHashSet,

CFormat("Failed to create RecoveryData for '%s'") % m_pOwner->GetFileName());

bResult = false;

SetStatus(AICH_ERROR);

}

if (!bUse32BitIdentifier) {

fileDataOut->WriteUInt16(0); // no 32bit hashs to write

}

if (!bDbgDontLoad) {

FreeHashSet();

}

return bResult;

}

bool CAICHHashSet::ReadRecoveryData(uint64 nPartStartPos, CMemFile *fileDataIn)

{

if (/*eMule TODO !m_pOwner->IsPartFile() ||*/ !(

m_eStatus == AICH_VERIFIED || m_eStatus == AICH_TRUSTED)) {

wxFAIL;

return false;

}

/* V2 AICH Hash Packet:

<count1 uint16>

16bit-hashs-to-read

(<identifier uint16><hash HASHSIZE>)[count1] AICH hashs

<count2 uint16>

32bit-hashs-to-read

(<identifier uint32><hash HASHSIZE>)[count2] AICH hashs

*/

// at this time we check the recoverydata for the correct ammounts of hashs only

// all hash are then taken into the tree, depending on there hashidentifier (except the masterhash)

uint8 nLevel = 0;

uint32 nPartSize = min<uint64>(PARTSIZE, m_pOwner->GetFileSize() - nPartStartPos);

m_pHashTree.FindHash(nPartStartPos, nPartSize, &nLevel);

uint16 nHashsToRead =

(nLevel - 1) + nPartSize / EMBLOCKSIZE + ((nPartSize % EMBLOCKSIZE != 0) ? 1 : 0);

// read hashs with 16 bit identifier

uint16 nHashsAvailable = fileDataIn->ReadUInt16();

if (fileDataIn->GetLength() - fileDataIn->GetPosition() < nHashsToRead * (HASHSIZE + 2u) ||

(nHashsToRead != nHashsAvailable && nHashsAvailable != 0)) {

// this check is redundant, CSafememfile would catch such an error too

AddDebugLogLineN(logSHAHashSet,

CFormat("Failed to read RecoveryData for '%s' - Received datasize/amounts of hashs "

"was invalid") %

m_pOwner->GetFileName());

return false;

}

for (uint32 i = 0; i != nHashsAvailable; i++) {

uint16 wHashIdent = fileDataIn->ReadUInt16();

if (wHashIdent == 1 /*never allow masterhash to be overwritten*/

|| !m_pHashTree.SetHash(fileDataIn, wHashIdent, (-1), false)) {

AddDebugLogLineN(logSHAHashSet,

CFormat("Failed to read RecoveryData for '%s' - Error when trying to read "

"hash into tree") %

m_pOwner->GetFileName());

VerifyHashTree(true); // remove invalid hashes which we have already written

return false;

}

}

// read hashs with 32bit identifier

if (nHashsAvailable == 0 && fileDataIn->GetLength() - fileDataIn->GetPosition() >= 2) {

nHashsAvailable = fileDataIn->ReadUInt16();

if (fileDataIn->GetLength() - fileDataIn->GetPosition() < nHashsToRead * (HASHSIZE + 4u) ||

(nHashsToRead != nHashsAvailable && nHashsAvailable != 0)) {

// this check is redundant, CSafememfile would catch such an error too

// TODO: theApp->QueueDebugLogLine(/*DLP_VERYHIGH,*/ false,

// _T("Failed to read RecoveryData for %s - Received datasize/amounts of hashs was

// invalid (2)"), m_pOwner->GetFileName() );

return false;

}

// TODO: DEBUG_ONLY( theApp->QueueDebugLogLine(/*DLP_VERYHIGH,*/ false, _T("read RecoveryData

// for %s - Received packet with %u 32bit hash identifiers)"), m_pOwner->GetFileName(),

// nHashsAvailable ) );

for (uint32 i = 0; i != nHashsToRead; i++) {

uint32 wHashIdent = fileDataIn->ReadUInt32();

if (wHashIdent == 1 /*never allow masterhash to be overwritten*/

|| wHashIdent > 0x400000 ||

!m_pHashTree.SetHash(fileDataIn, wHashIdent, (-1), false)) {

// TODO: theApp->QueueDebugLogLine(/*DLP_VERYHIGH,*/ false,

// _T("Failed to read RecoveryData for %s - Error when trying to read hash

// into tree (2)"), m_pOwner->GetFileName() );

VerifyHashTree(true); // remove invalid hashes which we have already written

return false;

}

}

}

if (nHashsAvailable == 0) {

// TODO: theApp->QueueDebugLogLine(/*DLP_VERYHIGH,*/ false, _T("Failed to read

// RecoveryData for %s - Packet didn't contained any hashs"), m_pOwner->GetFileName() );

return false;

}

if (VerifyHashTree(true)) {

// some final check if all hashs we wanted are there

for (uint32 nPartPos = 0; nPartPos < nPartSize; nPartPos += EMBLOCKSIZE) {

CAICHHashTree *phtToCheck = m_pHashTree.FindHash(

nPartStartPos + nPartPos, min<uint64>(EMBLOCKSIZE, nPartSize - nPartPos));

if (phtToCheck == NULL || !phtToCheck->m_bHashValid) {

AddDebugLogLineN(logSHAHashSet,

CFormat("Failed to read RecoveryData for '%s' - Error while "

"verifying presence of all lowest level hashes") %

m_pOwner->GetFileName());

return false;

}

}

// all done

return true;

} else {

AddDebugLogLineN(logSHAHashSet,

CFormat("Failed to read RecoveryData for '%s' - Verifying received hashtree failed") %

m_pOwner->GetFileName());

return false;

}

}

// this function is only allowed to be called right after successfully calculating the hashset (!)

void CAICHHashSet::InvalidateRootHashCache()

{

wxMutexLocker lock(s_rootHashCacheMutex);

s_rootHashCache.clear();

s_rootHashCacheLoaded = false;

}

void CAICHHashSet::LoadRootHashCacheLocked()

{

// Walk known2.met once, collecting every root hash. Replaces the

// per-call in-file linear scan that turned bulk-hashing N files into

// O(N^2) on-disk work (issue #579).

s_rootHashCache.clear();

s_rootHashCacheLoaded = true; // marked early so a partial read still ends the loop

const wxString fullpath = thePrefs::GetConfigDir() + KNOWN2_MET_FILENAME;

if (!wxFile::Exists(fullpath)) {

return;

}

CFile file(fullpath, CFile::read);

if (!file.IsOpened()) {

// Couldn't open: don't claim the cache is valid; SaveHashSet

// will retry on the next invocation.

s_rootHashCacheLoaded = false;

return;

}

try {

const uint64 nFileSize = file.GetLength();

if (nFileSize == 0) {

return;

}

const uint8 header = file.ReadUInt8();

if (header != KNOWN2_MET_VERSION) {

AddDebugLogLineC(logSHAHashSet,

"AICH cache load: unexpected known2.met version, leaving cache empty");

return;

}

while (file.GetPosition() < nFileSize) {

// Remember the byte offset of the root-hash position so

// LoadHashSet can seek straight here later instead of

// rewalking the file.

const uint64 entryOffset = file.GetPosition();

CAICHHash rootHash;

rootHash.Read(&file);

const uint32 nHashCount = file.ReadUInt32();

const uint64 skipBytes = static_cast<uint64>(nHashCount) * HASHSIZE;

if (file.GetPosition() + skipBytes > nFileSize) {

// known2.met is truncated past this entry; stop here.

// CAICHSyncTask will handle the actual truncation/recovery.

AddDebugLogLineC(

logSHAHashSet, "AICH cache load: known2.met truncated mid-entry");

break;

}

file.Seek(static_cast<wxFileOffset>(skipBytes), wxFromCurrent);

s_rootHashCache.emplace(rootHash, entryOffset);

}

} catch (const CSafeIOException &e) {

AddDebugLogLineC(logSHAHashSet, "IO error walking known2.met for AICH cache: " + e.what());

// Keep whatever we collected; stay marked loaded so we don't

// re-scan on every SaveHashSet call (and risk the same error).

}

}

bool CAICHHashSet::SaveHashSet()

{

if (m_eStatus != AICH_HASHSETCOMPLETE) {

wxFAIL;

return false;

}

if (!m_pHashTree.m_bHashValid || m_pHashTree.m_nDataSize != m_pOwner->GetFileSize()) {

wxFAIL;

return false;

}

wxMutexLocker cacheLock(s_rootHashCacheMutex);

if (!s_rootHashCacheLoaded) {

LoadRootHashCacheLocked();

}

// O(1) dedup — replaces the linear file walk that used to make this

// O(N) per call and O(N^2) over a bulk-hashing batch.

if (s_rootHashCache.find(m_pHashTree.m_Hash) != s_rootHashCache.end()) {

return true;

}

// Byte offset at which the new entry's root hash will be appended.

// Captured inside the try block and used after it to update the

// cache once the write has succeeded.

uint64 newEntryOffset = 0;

try {

const wxString fullpath = thePrefs::GetConfigDir() + KNOWN2_MET_FILENAME;

const bool exists = wxFile::Exists(fullpath);

CFile file(fullpath, exists ? CFile::read_write : CFile::write);

if (!file.IsOpened()) {

AddDebugLogLineC(logSHAHashSet, "Failed to save HashSet: opening met file failed!");

return false;

}

uint64 nExistingSize = file.GetLength();

if (nExistingSize) {

uint8 header = file.ReadUInt8();

if (header != KNOWN2_MET_VERSION) {

AddDebugLogLineC(

logSHAHashSet, "Saving failed: Current file is not a met-file!");

return false;

}

// Skip the in-file dedup walk; the cache already confirmed

// our root hash isn't present.

file.Seek(static_cast<wxFileOffset>(nExistingSize), wxFromStart);

} else {

file.WriteUInt8(KNOWN2_MET_VERSION);

// Update the recorded size, in order for the sanity check below to work.

nExistingSize += 1;

}

// This is the byte offset at which the new entry's root hash

// will land — capture it for the cache so LoadHashSet can

// seek straight here later.

newEntryOffset = nExistingSize;

// write hashset

m_pHashTree.m_Hash.Write(&file);

uint32 nHashCount = (PARTSIZE / EMBLOCKSIZE + ((PARTSIZE % EMBLOCKSIZE != 0) ? 1 : 0)) *

(m_pHashTree.m_nDataSize / PARTSIZE);

if (m_pHashTree.m_nDataSize % PARTSIZE != 0) {

nHashCount += (m_pHashTree.m_nDataSize % PARTSIZE) / EMBLOCKSIZE +

(((m_pHashTree.m_nDataSize % PARTSIZE) % EMBLOCKSIZE != 0) ? 1 : 0);

}

file.WriteUInt32(nHashCount);

if (!m_pHashTree.WriteLowestLevelHashs(&file, 0, true, true)) {

// that's bad... really

file.SetLength(nExistingSize);

AddDebugLogLineC(

logSHAHashSet, "Failed to save HashSet: WriteLowestLevelHashs() failed!");

return false;

}

if (file.GetLength() != nExistingSize + (nHashCount + 1) * HASHSIZE + 4) {

// that's even worse

file.SetLength(nExistingSize);

AddDebugLogLineC(logSHAHashSet,

"Failed to save HashSet: Calculated and real size of hashset differ!");

return false;

}

AddDebugLogLineN(logSHAHashSet,

CFormat("Successfully saved eMuleAC Hashset, %u Hashs + 1 Masterhash written") %

nHashCount);

} catch (const CSafeIOException &e) {

AddDebugLogLineC(logSHAHashSet, "IO error while saving AICH HashSet: " + e.what());

return false;

}

// Append succeeded — record offset in the cache so the next

// SaveHashSet for this root hash dedups in O(1), and LoadHashSet

// can seek straight to it.

s_rootHashCache.emplace(m_pHashTree.m_Hash, newEntryOffset);

return true;

}

bool CAICHHashSet::LoadHashSet()

{

if (m_eStatus != AICH_HASHSETCOMPLETE) {

wxFAIL;

return false;

}

if (!m_pHashTree.m_bHashValid || m_pHashTree.m_nDataSize != m_pOwner->GetFileSize() ||

m_pHashTree.m_nDataSize == 0) {

wxFAIL;

return false;

}

// O(1) cache lookup: ask the offset index where this root hash

// lives in known2.met. The cache was the dedup-on-write index

// before; here we reuse it to skip the linear scan that used to

// happen on every incoming OP_AICHREQUEST (issue #166). If the

// cache miss-and-cold-load path is taken we still pay the one-shot

// walk, but only once across all subsequent calls.

uint64 cachedOffset = 0;

bool haveCachedOffset = false;

{

wxMutexLocker lock(s_rootHashCacheMutex);

if (!s_rootHashCacheLoaded) {

LoadRootHashCacheLocked();

}

auto it = s_rootHashCache.find(m_pHashTree.m_Hash);

if (it != s_rootHashCache.end()) {

cachedOffset = it->second;

haveCachedOffset = true;

} else if (s_rootHashCacheLoaded) {

// Cache is authoritative and the hash isn't there.

// known2.met genuinely doesn't contain it; skip the I/O.

return false;

}

}

wxString fullpath = thePrefs::GetConfigDir() + KNOWN2_MET_FILENAME;

CFile file(fullpath, CFile::read);

if (!file.IsOpened()) {

if (wxFileExists(fullpath)) {

wxString strError("Failed to load " KNOWN2_MET_FILENAME " file");

AddDebugLogLineC(logSHAHashSet, strError);

}

return false;

}

try {

uint8 header = file.ReadUInt8();

if (header != KNOWN2_MET_VERSION) {

AddDebugLogLineC(logSHAHashSet, "Loading failed: Current file is not a met-file!");

return false;

}

uint64 nExistingSize = file.GetLength();

// Fast path: seek straight to the cached entry. If the offset

// turns out to be stale -- past EOF, or first read at that

// position doesn't match our root hash -- rewind once to just

// past the version header and fall through to a true linear

// scan from the top as defensive recovery against external

// modification of known2.met between cache load and now.

if (haveCachedOffset) {

if (cachedOffset >= nExistingSize) {

haveCachedOffset = false;

} else {

file.Seek(static_cast<wxFileOffset>(cachedOffset), wxFromStart);

}

}

CAICHHash CurrentHash;

uint32 nHashCount;

bool cacheFallbackTriggered = false;

while (file.GetPosition() < nExistingSize) {

// Position of the root-hash at the start of the entry we're

// about to examine — captured pre-read so we can stamp it

// back into the cache when a stale-cache rewind succeeds.

const uint64 entryStartPos = file.GetPosition();

CurrentHash.Read(&file);

if (m_pHashTree.m_Hash == CurrentHash) {

// found Hashset

uint32 nExpectedCount =

(PARTSIZE / EMBLOCKSIZE + ((PARTSIZE % EMBLOCKSIZE != 0) ? 1 : 0)) *

(m_pHashTree.m_nDataSize / PARTSIZE);

if (m_pHashTree.m_nDataSize % PARTSIZE != 0) {

nExpectedCount +=

(m_pHashTree.m_nDataSize % PARTSIZE) / EMBLOCKSIZE +

(((m_pHashTree.m_nDataSize % PARTSIZE) % EMBLOCKSIZE != 0)

? 1

: 0);

}

nHashCount = file.ReadUInt32();

if (nHashCount != nExpectedCount) {

AddDebugLogLineC(logSHAHashSet,

"Failed to load HashSet: Available Hashs and expected "

"hashcount differ!");

return false;

}

if (!m_pHashTree.LoadLowestLevelHashs(&file)) {

AddDebugLogLineC(logSHAHashSet,

"Failed to load HashSet: LoadLowestLevelHashs failed!");

return false;

}

if (!ReCalculateHash(false)) {

AddDebugLogLineC(logSHAHashSet,

"Failed to load HashSet: Calculating loaded hashs failed!");

return false;

}

if (CurrentHash != m_pHashTree.m_Hash) {

AddDebugLogLineC(logSHAHashSet,

"Failed to load HashSet: Calculated Masterhash differs from "

"given Masterhash - hashset corrupt!");

return false;

}

// Self-heal: if we got here via the stale-cache rewind,

// update the cache so future lookups for this root hash

// go straight to the new correct offset instead of

// paying the linear-scan penalty every time.

if (cacheFallbackTriggered) {

wxMutexLocker lock(s_rootHashCacheMutex);

s_rootHashCache[m_pHashTree.m_Hash] = entryStartPos;

}

return true;

}

// First read after seeking to the cached offset didn't match

// our root hash. The cache must be stale -- known2.met was

// modified externally between cache load and now. Rewind once

// to just past the version header and restart as a true linear

// scan from the top.

if (haveCachedOffset && !cacheFallbackTriggered) {

cacheFallbackTriggered = true;

haveCachedOffset = false;

file.Seek(1, wxFromStart);

continue;

}

nHashCount = file.ReadUInt32();

if (file.GetPosition() + nHashCount * HASHSIZE > nExistingSize) {

AddDebugLogLineC(logSHAHashSet,

"Saving failed: File contains fewer entries than specified!");

return false;

}

// skip the rest of this hashset

file.Seek(nHashCount * HASHSIZE, wxFromCurrent);

}

AddDebugLogLineC(logSHAHashSet, "Failed to load HashSet: HashSet not found!");

} catch (const CSafeIOException &e) {

AddDebugLogLineC(logSHAHashSet, "IO error while loading AICH HashSet: " + e.what());

}

return false;

}

// delete the hashset except the masterhash (we dont keep aich hashsets in memory to save resources)

void CAICHHashSet::FreeHashSet()

{

if (m_pHashTree.m_pLeftTree) {

delete m_pHashTree.m_pLeftTree;

m_pHashTree.m_pLeftTree = NULL;

}

if (m_pHashTree.m_pRightTree) {

delete m_pHashTree.m_pRightTree;

m_pHashTree.m_pRightTree = NULL;

}

}

void CAICHHashSet::SetMasterHash(const CAICHHash &Hash, EAICHStatus eNewStatus)

{

m_pHashTree.m_Hash = Hash;

m_pHashTree.m_bHashValid = true;

SetStatus(eNewStatus);

}

CAICHHashAlgo *CAICHHashSet::GetNewHashAlgo()

{

return new CSHA();

}

bool CAICHHashSet::ReCalculateHash(bool bDontReplace)

{

CAICHHashAlgo *hashalg = GetNewHashAlgo();

bool bResult = m_pHashTree.ReCalculateHash(hashalg, bDontReplace);

delete hashalg;

return bResult;

}

bool CAICHHashSet::VerifyHashTree(bool bDeleteBadTrees)

{