Add explicit prefetching to the new AES-GCM code

ebiggers · Boringssl LUCI CQ · commit 13840dd094f9 · 2025-01-16T11:45:18.000-08:00
Add explicit prefetching to the main loop of the new AES-GCM code, following the same rationale as change I6312e01ff0da70cc52f09194846b82cc6b69d37a. For now the same prefetch distance of 512 bytes is used. Change-Id: Ib57affb414e88675f3a4c8e124728a0cf412bc0a Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/75267 Reviewed-by: David Benjamin <davidben@google.com> Commit-Queue: David Benjamin <davidben@google.com>
diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl b/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl
@@ -1118,6 +1118,13 @@ sub _aes_gcm_update {
 .Laes128$local_label_suffix:
 ___
 
+    # Prefetch the source data 512 bytes ahead into the L1 data cache, to
+    # improve performance when the hardware prefetcher is disabled.  Assumes the
+    # L1 data cache line size is 64 bytes (de facto standard on x86_64).
+    for ( my $i = 0 ; $i < 4 * $VL ; $i += 64 ) {
+        $code .= "prefetcht0  512+$i($SRC)\n";
+    }
+
     # Finish the AES encryption of the counter blocks in V0-V3, interleaved
     # with the GHASH update of the ciphertext blocks in GHASHDATA[0-3].
     for my $i ( reverse 1 .. 9 ) {
diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl b/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl
@@ -805,6 +805,12 @@ sub _aes_gcm_update {
 .Laes128$local_label_suffix:
 ___
 
+    # Prefetch the source data 512 bytes ahead into the L1 data cache, to
+    # improve performance when the hardware prefetcher is disabled.  Assumes the
+    # L1 data cache line size is 64 bytes (de facto standard on x86_64).
+    $code .= "prefetcht0 512($SRC)\n";
+    $code .= "prefetcht0 512+64($SRC)\n";
+
     # Finish the AES encryption of the counter blocks in AESDATA[0-3],
     # interleaved with the GHASH update of the ciphertext blocks.
     for my $i ( reverse 1 .. 9 ) {
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-apple.S b/gen/bcm/aes-gcm-avx10-x86_64-apple.S
@@ -512,6 +512,10 @@ L$aes192__func1:
 	vaesenc	%zmm9,%zmm3,%zmm3
 
 L$aes128__func1:
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
 	vpshufb	%zmm8,%zmm4,%zmm4
 	vpxord	%zmm10,%zmm4,%zmm4
 	vpshufb	%zmm8,%zmm5,%zmm5
@@ -953,6 +957,10 @@ L$aes192__func2:
 	vaesenc	%zmm9,%zmm3,%zmm3
 
 L$aes128__func2:
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
 	vpshufb	%zmm8,%zmm4,%zmm4
 	vpxord	%zmm10,%zmm4,%zmm4
 	vpshufb	%zmm8,%zmm5,%zmm5
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-linux.S b/gen/bcm/aes-gcm-avx10-x86_64-linux.S
@@ -514,6 +514,10 @@ _CET_ENDBR
 	vaesenc	%zmm9,%zmm3,%zmm3
 
 .Laes128__func1:
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
 	vpshufb	%zmm8,%zmm4,%zmm4
 	vpxord	%zmm10,%zmm4,%zmm4
 	vpshufb	%zmm8,%zmm5,%zmm5
@@ -957,6 +961,10 @@ _CET_ENDBR
 	vaesenc	%zmm9,%zmm3,%zmm3
 
 .Laes128__func2:
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
 	vpshufb	%zmm8,%zmm4,%zmm4
 	vpxord	%zmm10,%zmm4,%zmm4
 	vpshufb	%zmm8,%zmm5,%zmm5
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-win.asm b/gen/bcm/aes-gcm-avx10-x86_64-win.asm
@@ -579,6 +579,10 @@ $L$aes192__func1:
 	vaesenc	zmm3,zmm3,zmm9
 
 $L$aes128__func1:
+	prefetcht0	[((512+0))+rcx]
+	prefetcht0	[((512+64))+rcx]
+	prefetcht0	[((512+128))+rcx]
+	prefetcht0	[((512+192))+rcx]
 	vpshufb	zmm4,zmm4,zmm8
 	vpxord	zmm4,zmm4,zmm10
 	vpshufb	zmm5,zmm5,zmm8
@@ -1061,6 +1065,10 @@ $L$aes192__func2:
 	vaesenc	zmm3,zmm3,zmm9
 
 $L$aes128__func2:
+	prefetcht0	[((512+0))+rcx]
+	prefetcht0	[((512+64))+rcx]
+	prefetcht0	[((512+128))+rcx]
+	prefetcht0	[((512+192))+rcx]
 	vpshufb	zmm4,zmm4,zmm8
 	vpxord	zmm4,zmm4,zmm10
 	vpshufb	zmm5,zmm5,zmm8
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-apple.S b/gen/bcm/aes-gcm-avx2-x86_64-apple.S
@@ -498,6 +498,8 @@ L$aes192__func1:
 	vaesenc	%ymm2,%ymm15,%ymm15
 
 L$aes128__func1:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
 
 	vmovdqu	0(%rsi),%ymm3
 	vpshufb	%ymm0,%ymm3,%ymm3
@@ -983,6 +985,8 @@ L$aes192__func2:
 	vaesenc	%ymm2,%ymm15,%ymm15
 
 L$aes128__func2:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
 
 	vmovdqu	0(%rdi),%ymm3
 	vpshufb	%ymm0,%ymm3,%ymm3
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-linux.S b/gen/bcm/aes-gcm-avx2-x86_64-linux.S
@@ -500,6 +500,8 @@ _CET_ENDBR
 	vaesenc	%ymm2,%ymm15,%ymm15
 
 .Laes128__func1:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
 
 	vmovdqu	0(%rsi),%ymm3
 	vpshufb	%ymm0,%ymm3,%ymm3
@@ -987,6 +989,8 @@ _CET_ENDBR
 	vaesenc	%ymm2,%ymm15,%ymm15
 
 .Laes128__func2:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
 
 	vmovdqu	0(%rdi),%ymm3
 	vpshufb	%ymm0,%ymm3,%ymm3
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-win.asm b/gen/bcm/aes-gcm-avx2-x86_64-win.asm
@@ -559,6 +559,8 @@ $L$aes192__func1:
 	vaesenc	ymm15,ymm15,ymm2
 
 $L$aes128__func1:
+	prefetcht0	[512+rcx]
+	prefetcht0	[((512+64))+rcx]
 
 	vmovdqu	ymm3,YMMWORD[rdx]
 	vpshufb	ymm3,ymm3,ymm0
@@ -1085,6 +1087,8 @@ $L$aes192__func2:
 	vaesenc	ymm15,ymm15,ymm2
 
 $L$aes128__func2:
+	prefetcht0	[512+rcx]
+	prefetcht0	[((512+64))+rcx]
 
 	vmovdqu	ymm3,YMMWORD[rcx]
 	vpshufb	ymm3,ymm3,ymm0
