Backported CORE-6501: Segfault when shutting down database which got encrypted by another process

AlexPeshkoff · AlexPeshkoff · commit 64fe26fb101e · 2021-03-03T18:56:12.000+03:00
diff --git a/src/jrd/CryptoManager.cpp b/src/jrd/CryptoManager.cpp
@@ -365,7 +365,9 @@ namespace Jrd {
 		crypt = hdr->hdr_flags & Ods::hdr_encrypted;
 		process = hdr->hdr_flags & Ods::hdr_crypt_process;
 
-		if ((crypt || process) && !cryptPlugin)
+		// tdbb w/o attachment comes when database is shutting down in the end of detachDatabase()
+		// the only needed here page is header, i.e. we can live w/o cryptPlugin
+		if ((crypt || process) && (!cryptPlugin) && tdbb->getAttachment())
 		{
 			ClumpletWriter hc(ClumpletWriter::UnTagged, hdr->hdr_page_size);
 			hdr.getClumplets(hc);
@@ -388,7 +390,7 @@ namespace Jrd {
 				hash = valid;
 		}
 
-		if (flags & CRYPT_HDR_INIT)
+		if (cryptPlugin && (flags & CRYPT_HDR_INIT))
 			checkDigitalSignature(tdbb, hdr);
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -365,7 +365,9 @@ namespace Jrd {`
`365`	`365`	`crypt = hdr->hdr_flags & Ods::hdr_encrypted;`
`366`	`366`	`process = hdr->hdr_flags & Ods::hdr_crypt_process;`
`367`	`367`
`368`		`- if ((crypt \|\| process) && !cryptPlugin)`
	`368`	`+ // tdbb w/o attachment comes when database is shutting down in the end of detachDatabase()`
	`369`	`+ // the only needed here page is header, i.e. we can live w/o cryptPlugin`
	`370`	`+ if ((crypt \|\| process) && (!cryptPlugin) && tdbb->getAttachment())`
`369`	`371`	`{`
`370`	`372`	`ClumpletWriter hc(ClumpletWriter::UnTagged, hdr->hdr_page_size);`
`371`	`373`	`hdr.getClumplets(hc);`
`@@ -388,7 +390,7 @@ namespace Jrd {`
`388`	`390`	`hash = valid;`
`389`	`391`	`}`
`390`	`392`
`391`		`- if (flags & CRYPT_HDR_INIT)`
	`393`	`+ if (cryptPlugin && (flags & CRYPT_HDR_INIT))`
`392`	`394`	`checkDigitalSignature(tdbb, hdr);`
`393`	`395`	`}`
`394`	`396`