Skip to content

Commit e634a6a

Browse files
juenglinjuenglin
committed
ci: log TLS chain fingerprints, certificate copies and native clients in the trust probe
On GitHub-hosted runners the chain .NET builds during TLS shows the eMudhra intermediate as NoError even when the same certificate, checked on its own, is explicitly distrusted; on self-hosted runners the TLS chain flags it as Revoked,ExplicitDistrust. Add diagnostics to find out why: - TLS chain rows now give the SHA-256 of each certificate Windows used and of each certificate the server sent, marked when it is in the impacted list. - Search the certificate stores and the CryptnetUrlCache (certificates downloaded through AIA) for certificates that carry a compromised key or an eMudhra G3 intermediate name, and report any that are not in the impacted list. Runs at baseline and after the experiment. - Connect with the inbox curl.exe (Schannel, --ssl-no-revoke) and with gh (Go crypto/x509 over Windows chain building), at baseline and after each experiment step, so the result can be compared with a client that does not use .NET. gh runs with a placeholder token and an empty config directory. - Record pwsh, .NET, crypt32.dll, curl, gh and gh's Go toolchain versions; Go releases with the GO-2026-6809 fix reject the intermediates on their own.
1 parent 1c25840 commit e634a6a

1 file changed

Lines changed: 229 additions & 13 deletions

File tree

‎ci/tools/runner_trust_probe.ps1‎

Lines changed: 229 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ $ErrorActionPreference = 'Stop'
99

1010
$EmudhraHelperPath = Join-Path $PSScriptRoot '..\..\.github\actions\upgrade-ca-certificates\emudhra_intermediates.ps1'
1111
. $EmudhraHelperPath
12+
# SHA-256 (upper-case hex) -> label of each certificate in the impacted list.
13+
$EmudhraListed = @{}
14+
foreach ($entry in (Get-Content -LiteralPath $EmudhraIntermediatesPath -Raw | ConvertFrom-Json)) {
15+
$EmudhraListed[$entry.sha256.ToUpperInvariant()] = "$($entry.name) (issued by $($entry.issued_by))"
16+
}
1217

1318
$EmSignG3Hex = '86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B'
1419
# One host per path: via emSign ECC Root CA - G3, and via the Sectigo
@@ -169,16 +174,17 @@ function Measure-ExperimentState([string]$Label) {
169174
}
170175
Record-Check "Experiment: $Label, Invoke-WebRequest ``$endpoint``" 'INFO' "$($outcome.Kind): $($outcome.Detail)" | Out-Host
171176
$chainCheck = "Experiment: $Label, TLS chain ``$endpoint``"
172-
Invoke-Diagnostic $chainCheck { Record-Check $chainCheck 'INFO' (Get-TlsChainFresh $endpoint) } | Out-Host
177+
Invoke-Diagnostic $chainCheck { Record-Check $chainCheck 'INFO' (Format-TlsChain (Get-TlsChainFresh $endpoint)) } | Out-Host
173178
}
179+
Record-NativeClients "Experiment: $Label, " | Out-Host
174180
return [pscustomobject]@{ Count = $count; TlsTrusted = $tlsTrusted }
175181
}
176182

177183
# Runs a function defined in this script in a fresh pwsh process, so no TLS or
178184
# chain state cached by this process carries over, and returns its result.
179185
function Invoke-FunctionFresh([string]$Name, [string]$Argument) {
180186
$command = "function $Name { $((Get-Item -LiteralPath "function:$Name").ScriptBlock) }; " +
181-
"$Name '$Argument' | ConvertTo-Json -Compress"
187+
"$Name '$Argument' | ConvertTo-Json -Compress -Depth 5"
182188
$encoded = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($command))
183189
$output = @(& (Get-Process -Id $PID).Path -NoProfile -NonInteractive -EncodedCommand $encoded 2>&1 | ForEach-Object { "$_" })
184190
if ($LASTEXITCODE -ne 0 -or $output.Count -ne 1) {
@@ -258,26 +264,34 @@ function Get-TlsOutcomeFresh([string]$Endpoint) {
258264
return Invoke-FunctionFresh 'Get-TlsOutcome' $Endpoint
259265
}
260266

261-
# Connects to an endpoint with SslStream and describes the chain .NET built
262-
# for the server certificate: the policy errors, the chain status and each
263-
# element with its status flags. The validation callback accepts every
264-
# certificate, so this observes the chain without gating on it.
267+
# Connects to an endpoint with SslStream and returns the chain .NET built for
268+
# the server certificate (policy errors, chain status, and each element with
269+
# its status flags and SHA-256) together with the certificates the server sent
270+
# (the chain's ExtraStore). The validation callback accepts
271+
# every certificate, so this observes the chain without gating on it.
265272
function Get-TlsChain {
266273
param([string]$Endpoint)
267274
$uri = [Uri]$Endpoint
268-
$lines = [System.Collections.Generic.List[string]]::new()
275+
$result = @{}
269276
$callback = {
270277
param($source, $certificate, $chain, $errors)
271278
$flagsOf = { param($statuses) $f = @($statuses | ForEach-Object { $_.Status }) -join ','; if ($f) { $f } else { 'NoError' } }
272-
$lines.Add("policy errors $errors; chain status $(& $flagsOf $chain.ChainStatus)")
273-
foreach ($element in $chain.ChainElements) {
274-
$cert = $element.Certificate
279+
$describe = {
280+
param($cert)
275281
$name = $cert.GetNameInfo('SimpleName', $false)
276282
if ($cert.Subject -ne $cert.Issuer) {
277283
$name += " (issued by $($cert.GetNameInfo('SimpleName', $true)))"
278284
}
279-
$lines.Add("$name [$(& $flagsOf $element.ChainElementStatus)]")
285+
[pscustomobject]@{ Name = $name; Sha256 = $cert.GetCertHashString('SHA256'); Flags = '' }
280286
}
287+
$result.Errors = "$errors"
288+
$result.Status = & $flagsOf $chain.ChainStatus
289+
$result.Built = @(foreach ($element in $chain.ChainElements) {
290+
$d = & $describe $element.Certificate
291+
$d.Flags = & $flagsOf $element.ChainElementStatus
292+
$d
293+
})
294+
$result.Sent = @($chain.ChainPolicy.ExtraStore | ForEach-Object { & $describe $_ })
281295
return $true
282296
}.GetNewClosure()
283297
$tcp = [System.Net.Sockets.TcpClient]::new($uri.Host, $uri.Port)
@@ -291,13 +305,211 @@ function Get-TlsChain {
291305
} finally {
292306
$tcp.Dispose()
293307
}
294-
return ($lines -join ' -> ')
308+
return [pscustomobject]$result
295309
}
296310

297311
function Get-TlsChainFresh([string]$Endpoint) {
298312
return Invoke-FunctionFresh 'Get-TlsChain' $Endpoint
299313
}
300314

315+
function Format-CertRef($Cert) {
316+
$text = "$($Cert.Name) sha256 $($Cert.Sha256.Substring(0, 16))"
317+
if ($EmudhraListed.ContainsKey($Cert.Sha256)) {
318+
$text += ' (listed)'
319+
}
320+
return $text
321+
}
322+
323+
function Format-TlsChain($Chain) {
324+
$built = @($Chain.Built | ForEach-Object { "$(Format-CertRef $_) [$($_.Flags)]" }) -join ' -> '
325+
$sent = @($Chain.Sent | ForEach-Object { Format-CertRef $_ }) -join ', '
326+
if (-not $sent) {
327+
$sent = 'none'
328+
}
329+
return "policy errors $($Chain.Errors); chain status $($Chain.Status); built: $built; server sent: $sent"
330+
}
331+
332+
# Looks for certificates that carry one of the compromised keys or an eMudhra
333+
# G3 intermediate name, in the certificate stores and in the CryptnetUrlCache
334+
# where Windows keeps certificates downloaded through AIA. A copy that is not
335+
# in the impacted list would let chain building route around the blocks.
336+
function Record-EmudhraCopies([string]$Name) {
337+
$listedCerts = @(Get-EmudhraIntermediates)
338+
$keys = @{}
339+
foreach ($cert in $listedCerts) {
340+
$keys[[Convert]::ToBase64String($cert.GetPublicKey())] = $true
341+
$cert.Dispose()
342+
}
343+
$found = [System.Collections.Generic.List[string]]::new()
344+
$inspect = {
345+
param($cert, $where)
346+
$simple = $cert.GetNameInfo('SimpleName', $false)
347+
if ($keys.ContainsKey([Convert]::ToBase64String($cert.GetPublicKey())) -or $simple -cmatch '^EM .+ CA - G3') {
348+
$sha = $cert.GetCertHashString('SHA256')
349+
$tag = if ($EmudhraListed.ContainsKey($sha)) { 'listed' } else { 'NOT listed' }
350+
$found.Add("${where}: $simple (issued by $($cert.GetNameInfo('SimpleName', $true))), sha256 $($sha.Substring(0, 16)), " +
351+
"serial $($cert.SerialNumber), $($cert.NotBefore.ToString('yyyy-MM-dd'))..$($cert.NotAfter.ToString('yyyy-MM-dd')), $tag")
352+
}
353+
}
354+
355+
$storeCount = 0
356+
foreach ($location in 'CurrentUser', 'LocalMachine') {
357+
foreach ($storeName in 'Root', 'AuthRoot', 'CA', 'TrustedPeople', 'TrustedPublisher', 'Disallowed') {
358+
$store = [System.Security.Cryptography.X509Certificates.X509Store]::new($storeName, $location)
359+
try {
360+
$store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]'ReadOnly, OpenExistingOnly')
361+
foreach ($cert in $store.Certificates) {
362+
$storeCount++
363+
& $inspect $cert "$location\$storeName"
364+
$cert.Dispose()
365+
}
366+
} catch [System.Security.Cryptography.CryptographicException] {
367+
# OpenExistingOnly: the store does not exist for this location.
368+
} finally {
369+
$store.Dispose()
370+
}
371+
}
372+
}
373+
374+
$cacheDirs = @(
375+
(Join-Path $env:USERPROFILE 'AppData\LocalLow\Microsoft\CryptnetUrlCache\Content'),
376+
(Join-Path $env:SystemRoot 'System32\config\systemprofile\AppData\LocalLow\Microsoft\CryptnetUrlCache\Content'),
377+
(Join-Path $env:SystemRoot 'ServiceProfiles\LocalService\AppData\LocalLow\Microsoft\CryptnetUrlCache\Content'),
378+
(Join-Path $env:SystemRoot 'ServiceProfiles\NetworkService\AppData\LocalLow\Microsoft\CryptnetUrlCache\Content')
379+
)
380+
$cacheFiles = 0
381+
$cacheCerts = 0
382+
$unreadable = @()
383+
foreach ($dir in $cacheDirs) {
384+
if (-not (Test-Path -LiteralPath $dir)) {
385+
continue
386+
}
387+
try {
388+
$files = @(Get-ChildItem -LiteralPath $dir -File -Force)
389+
} catch {
390+
$unreadable += "$dir ($($_.Exception.Message))"
391+
continue
392+
}
393+
foreach ($file in $files) {
394+
$cacheFiles++
395+
$collection = [System.Security.Cryptography.X509Certificates.X509Certificate2Collection]::new()
396+
try {
397+
$collection.Import($file.FullName)
398+
} catch [System.Security.Cryptography.CryptographicException] {
399+
# CRLs, CTLs and OCSP responses share the cache; only certificates are of interest.
400+
continue
401+
}
402+
foreach ($cert in $collection) {
403+
$cacheCerts++
404+
& $inspect $cert "CryptnetUrlCache $($file.Name)"
405+
$cert.Dispose()
406+
}
407+
}
408+
}
409+
410+
$details = "$($found.Count) found among $storeCount store certificates and $cacheCerts certificates in $cacheFiles cache files"
411+
if ($found.Count -gt 0) {
412+
$details += ": $($found -join '; ')"
413+
}
414+
if ($unreadable.Count -gt 0) {
415+
$details += "; unreadable cache directories: $($unreadable -join '; ')"
416+
}
417+
Record-Check $Name 'INFO' $details
418+
}
419+
420+
# Connects with the inbox curl.exe, which uses Schannel and Windows chain
421+
# building without .NET. --ssl-no-revoke keeps an online revocation check from
422+
# masking the local trust decision.
423+
function Get-CurlOutcome([string]$Endpoint) {
424+
$curl = Join-Path $env:SystemRoot 'System32\curl.exe'
425+
$output = @(& $curl --ssl-no-revoke --silent --show-error --output NUL --write-out 'HTTP %{http_code}' --max-time 15 $Endpoint 2>&1 | ForEach-Object { "$_" })
426+
$exitCode = $LASTEXITCODE
427+
$text = ($output -join ' ').Trim()
428+
if ($exitCode -eq 0) {
429+
return "Trusted: $text"
430+
}
431+
# 60 is CURLE_PEER_FAILED_VERIFICATION; Schannel also reports chain errors as 35 (CURLE_SSL_CONNECT_ERROR).
432+
if ($exitCode -eq 60 -or ($exitCode -eq 35 -and $text -match 'SEC_E_UNTRUSTED_ROOT|SEC_E_CERT|CERT_E_|CRYPT_E_|certificate')) {
433+
return "Rejected (exit $exitCode): $text"
434+
}
435+
return "Other (exit $exitCode): $text"
436+
}
437+
438+
# Connects with gh, whose Go crypto/x509 uses Windows chain building on
439+
# Windows. gh api refuses to run without a token, so it gets a placeholder and
440+
# an empty config directory; no real credential is available to send to the
441+
# test host. Go releases with the GO-2026-6809 fix reject the compromised
442+
# intermediates on their own, so read this together with gh's Go version.
443+
function Get-GhOutcome([string]$Endpoint) {
444+
$names = 'GH_TOKEN', 'GITHUB_TOKEN', 'GH_ENTERPRISE_TOKEN', 'GITHUB_ENTERPRISE_TOKEN', 'GH_HOST', 'GH_CONFIG_DIR'
445+
$saved = @{}
446+
foreach ($n in $names) {
447+
$saved[$n] = [Environment]::GetEnvironmentVariable($n)
448+
Remove-Item -LiteralPath "Env:$n" -ErrorAction SilentlyContinue
449+
}
450+
$configDir = Join-Path ([System.IO.Path]::GetTempPath()) "gh-probe-$PID"
451+
New-Item -ItemType Directory -Path $configDir -Force | Out-Null
452+
try {
453+
$env:GH_TOKEN = 'placeholder-not-a-token'
454+
$env:GH_CONFIG_DIR = $configDir
455+
$output = @(& gh api $Endpoint 2>&1 | ForEach-Object { "$_" })
456+
$exitCode = $LASTEXITCODE
457+
} finally {
458+
foreach ($n in $names) {
459+
if ($null -eq $saved[$n]) {
460+
Remove-Item -LiteralPath "Env:$n" -ErrorAction SilentlyContinue
461+
} else {
462+
Set-Item -LiteralPath "Env:$n" -Value $saved[$n]
463+
}
464+
}
465+
Remove-Item -LiteralPath $configDir -Recurse -Force
466+
}
467+
$certLine = $output | Where-Object { $_ -match 'x509:|certificate' } | Select-Object -First 1
468+
if ($certLine) {
469+
return "Rejected (exit $exitCode): $($certLine.Trim())"
470+
}
471+
$httpLine = $output | Where-Object { $_ -match '\bHTTP \d{3}\b' } | Select-Object -First 1
472+
if ($exitCode -eq 0) {
473+
return 'Trusted: response received'
474+
}
475+
if ($httpLine) {
476+
return "Trusted (exit $exitCode): $($httpLine.Trim())"
477+
}
478+
$first = ($output | Select-Object -First 1)
479+
return "Other (exit $exitCode): $first"
480+
}
481+
482+
function Record-NativeClients([string]$Prefix) {
483+
$hasGh = [bool](Get-Command gh -ErrorAction SilentlyContinue)
484+
foreach ($endpoint in $TestEndpoints) {
485+
Invoke-Diagnostic "${Prefix}curl.exe ``$endpoint``" { Record-Check "${Prefix}curl.exe ``$endpoint``" 'INFO' (Get-CurlOutcome $endpoint) }
486+
if ($hasGh) {
487+
Invoke-Diagnostic "${Prefix}gh api ``$endpoint``" { Record-Check "${Prefix}gh api ``$endpoint``" 'INFO' (Get-GhOutcome $endpoint) }
488+
}
489+
}
490+
}
491+
492+
function Get-ClientVersions {
493+
$parts = @(
494+
"pwsh $($PSVersionTable.PSVersion)",
495+
[System.Runtime.InteropServices.RuntimeInformation]::FrameworkDescription,
496+
"crypt32.dll $((Get-Item -LiteralPath (Join-Path $env:SystemRoot 'System32\crypt32.dll')).VersionInfo.FileVersion)",
497+
((& (Join-Path $env:SystemRoot 'System32\curl.exe') -V | Select-Object -First 1) -replace '\s+Release-Date.*$', '')
498+
)
499+
$gh = Get-Command gh -ErrorAction SilentlyContinue
500+
if (-not $gh) {
501+
$parts += 'gh not found'
502+
} else {
503+
$parts += (& gh --version | Select-Object -First 1)
504+
if (Get-Command go -ErrorAction SilentlyContinue) {
505+
$parts += "gh built with $((& go version -m $gh.Source | Select-Object -First 1) -replace '^.*:\s*', '')"
506+
} else {
507+
$parts += "gh Go version unknown (go not found)"
508+
}
509+
}
510+
return ($parts -join '; ')
511+
}
512+
301513
$GatingFailures = 0
302514

303515
Append-Summary "### $ProbeLabel"
@@ -323,6 +535,7 @@ Invoke-Diagnostic 'Windows version' {
323535
$cv = Get-ItemProperty -LiteralPath 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'
324536
Record-Check 'Windows version' 'INFO' "$($cv.ProductName) $($cv.DisplayVersion), build $($cv.CurrentBuild).$($cv.UBR), runner environment $env:RUNNER_ENVIRONMENT"
325537
}
538+
Invoke-Diagnostic 'Client versions' { Record-Check 'Client versions' 'INFO' (Get-ClientVersions) }
326539
Invoke-Diagnostic 'CTL auto-update policy' {
327540
Record-Check 'CTL auto-update policy' 'INFO' ("DisableRootAutoUpdate=$(Get-RegistryValueText $CtlPolicyPath 'DisableRootAutoUpdate'), " +
328541
"EnableDisallowedCertAutoUpdate=$(Get-RegistryValueText $CtlPolicyPath 'EnableDisallowedCertAutoUpdate')")
@@ -371,8 +584,10 @@ foreach ($endpoint in $TestEndpoints) {
371584
}
372585
}
373586
foreach ($endpoint in $TestEndpoints) {
374-
Invoke-Diagnostic "TLS chain ``$endpoint``" { Record-Check "TLS chain ``$endpoint``" 'INFO' (Get-TlsChain $endpoint) }
587+
Invoke-Diagnostic "TLS chain ``$endpoint``" { Record-Check "TLS chain ``$endpoint``" 'INFO' (Format-TlsChain (Get-TlsChain $endpoint)) }
375588
}
589+
Record-NativeClients ''
590+
Invoke-Diagnostic 'Certificate copies' { Record-EmudhraCopies 'Certificate copies' }
376591

377592
# 5. Disallowed CTL currently published on Windows Update (informational). -f
378593
# forces a download; this runs after the gating checks so it cannot change
@@ -404,6 +619,7 @@ if ($env:RUNNER_ENVIRONMENT -ne 'github-hosted') {
404619
Remove-ItemProperty -LiteralPath $CtlPolicyPath -Name 'DisableRootAutoUpdate'
405620
[void](Measure-ExperimentState 'DisableRootAutoUpdate removed')
406621
}
622+
Record-EmudhraCopies 'Experiment: certificate copies after all steps'
407623
}
408624
}
409625

0 commit comments

Comments
 (0)