Fixed CORE-6501: Segfault when shutting down database which got encrypted by another process

AlexPeshkoff · AlexPeshkoff · commit 18019a629053 · 2021-03-03T18:55:34.000+03:00
diff --git a/src/jrd/CryptoManager.cpp b/src/jrd/CryptoManager.cpp
@@ -357,7 +357,9 @@ namespace Jrd {
 		crypt = hdr->hdr_flags & Ods::hdr_encrypted;
 		process = hdr->hdr_flags & Ods::hdr_crypt_process;
 
-		if ((crypt || process) && !cryptPlugin)
+		// tdbb w/o attachment comes when database is shutting down in the end of detachDatabase()
+		// the only needed here page is header, i.e. we can live w/o cryptPlugin
+		if ((crypt || process) && (!cryptPlugin) && tdbb->getAttachment())
 		{
 			ClumpletWriter hc(ClumpletWriter::UnTagged, hdr->hdr_page_size);
 			hdr.getClumplets(hc);
@@ -381,7 +383,7 @@ namespace Jrd {
 				hash = valid;
 		}
 
-		if (flags & CRYPT_HDR_INIT)
+		if (cryptPlugin && (flags & CRYPT_HDR_INIT))
 			checkDigitalSignature(tdbb, hdr);
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -357,7 +357,9 @@ namespace Jrd {`
`357`	`357`	`crypt = hdr->hdr_flags & Ods::hdr_encrypted;`
`358`	`358`	`process = hdr->hdr_flags & Ods::hdr_crypt_process;`
`359`	`359`
`360`		`- if ((crypt \|\| process) && !cryptPlugin)`
	`360`	`+ // tdbb w/o attachment comes when database is shutting down in the end of detachDatabase()`
	`361`	`+ // the only needed here page is header, i.e. we can live w/o cryptPlugin`
	`362`	`+ if ((crypt \|\| process) && (!cryptPlugin) && tdbb->getAttachment())`
`361`	`363`	`{`
`362`	`364`	`ClumpletWriter hc(ClumpletWriter::UnTagged, hdr->hdr_page_size);`
`363`	`365`	`hdr.getClumplets(hc);`
`@@ -381,7 +383,7 @@ namespace Jrd {`
`381`	`383`	`hash = valid;`
`382`	`384`	`}`
`383`	`385`
`384`		`- if (flags & CRYPT_HDR_INIT)`
	`386`	`+ if (cryptPlugin && (flags & CRYPT_HDR_INIT))`
`385`	`387`	`checkDigitalSignature(tdbb, hdr);`
`386`	`388`	`}`
`387`	`389`