SBRP validation tests #817
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System; | ||
| using System.Reflection; | ||
| using System.Reflection.Metadata; | ||
| using System.Reflection.Metadata.Ecma335; | ||
|
|
||
| namespace SbrpTests; | ||
|
|
||
| internal class DummyAttributeTypeProvider : ICustomAttributeTypeProvider<Type?> | ||
| { | ||
| public static readonly DummyAttributeTypeProvider Instance = new(); | ||
|
|
||
| public Type? GetPrimitiveType(PrimitiveTypeCode typeCode) => default(Type); | ||
|
|
||
| public Type? GetSystemType() => default(Type); | ||
|
|
||
| public Type? GetSZArrayType(Type? elementType) => default(Type); | ||
|
|
||
| public Type? GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => default(Type); | ||
|
|
||
| public Type? GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => default(Type); | ||
|
|
||
| public Type? GetTypeFromSerializedName(string name) => default(Type); | ||
|
|
||
| public PrimitiveTypeCode GetUnderlyingEnumType(Type? type) => default(PrimitiveTypeCode); | ||
|
|
||
| public bool IsSystemType(Type? type) => default(bool); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Runtime.CompilerServices; | ||
|
|
||
| namespace SbrpTests; | ||
|
|
||
| internal static class PathUtilities | ||
| { | ||
| public static string GetRepoRoot () => | ||
| (string)AppContext.GetData("SbrpTests.RepoRoot")!; | ||
|
|
||
| public static string GetSourceBuildRepoRoot () | ||
| { | ||
| var artifactsDir = (string)AppContext.GetData ("SbrpTests.ArtifactsDir")!; | ||
| return Path.Combine(artifactsDir, "sb", "src"); | ||
| } | ||
|
|
||
| public static string GetSourceBuildPackagesShippingDir () | ||
| { | ||
| var configuration = (string)AppContext.GetData ("SbrpTests.Configuration")!; | ||
| return Path.Combine(GetSourceBuildRepoRoot(), "artifacts", "packages", configuration, "Shipping"); | ||
| } | ||
| } |
|
Member
There was a problem hiding this comment. The file name feels a little odd to me. Any reason it can be simplified to SbrpTests.csproj?
Member
There was a problem hiding this comment. The That said, |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>$(NetCurrent)</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="NuGet.Packaging" Version="$(NuGetPackagingVersion)" /> | ||
| <PackageReference Include="Xunit.SkippableFact" Version="$(XunitSkippableFactVersion)" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <RuntimeHostConfigurationOption Include="SbrpTests.RepoRoot"> | ||
| <Value>$(RepoRoot)</Value> | ||
| </RuntimeHostConfigurationOption> | ||
|
|
||
| <RuntimeHostConfigurationOption Include="SbrpTests.Configuration"> | ||
| <Value>$(Configuration)</Value> | ||
| </RuntimeHostConfigurationOption> | ||
|
|
||
| <RuntimeHostConfigurationOption Include="SbrpTests.ArtifactsDir"> | ||
| <Value>$(ArtifactsDir)</Value> | ||
| </RuntimeHostConfigurationOption> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.IO.Compression; | ||
| using System.Linq; | ||
| using System.Reflection; | ||
| using System.Reflection.Metadata; | ||
| using System.Reflection.PortableExecutable; | ||
| using System.Runtime.InteropServices; | ||
| using System.Text.RegularExpressions; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using NuGet.Packaging; | ||
| using NuGet.Packaging.Signing; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace SbrpTests; | ||
|
|
||
| public class ValidationTests | ||
| { | ||
| private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute"; | ||
| private const string SbrpRepoIdentifier = "source-build-reference-packages"; | ||
| private const string VersionPattern = @"(\.\d)+([\-\w])*"; | ||
| public ITestOutputHelper Output { get; set; } | ||
|
|
||
| public ValidationTests(ITestOutputHelper output) | ||
| { | ||
| Output = output; | ||
| Skip.If(RuntimeInformation.IsOSPlatform(OSPlatform.Windows), "Validation tests are not supported on Windows."); | ||
| } | ||
|
|
||
| [SkippableFact] | ||
| public void ValidateSbrpAttribute() | ||
| { | ||
| string[] packages = GetPackages(); | ||
|
|
||
| HashSet<string> targetAndTextOnlyPacks = new ( | ||
| Directory.GetDirectories(Path.Combine(PathUtilities.GetSourceBuildRepoRoot(), "src/targetPacks/ILsrc")) | ||
| .Union(Directory.GetDirectories(Path.Combine(PathUtilities.GetSourceBuildRepoRoot(), "src/textOnlyPackages/src"))) | ||
| .Select(x => Path.GetFileName(x).ToLower()) | ||
| ); | ||
|
|
||
| var filteredPackages = packages | ||
| .Where(package => | ||
| { | ||
| string packageName = Path.GetFileNameWithoutExtension(package).ToLower(); | ||
| packageName = Regex.Replace(packageName, VersionPattern, string.Empty); | ||
| return !targetAndTextOnlyPacks.Contains(packageName); | ||
| }); | ||
|
|
||
| Output.WriteLine($"Checking {filteredPackages.Count()} packages for SBRP attribute."); | ||
|
|
||
| foreach (var package in filteredPackages) | ||
| { | ||
| string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName(), Path.GetFileNameWithoutExtension(package)); | ||
|
|
||
| try | ||
| { | ||
| Directory.CreateDirectory(tempDirectory); | ||
| ZipFile.ExtractToDirectory(package, tempDirectory); | ||
| var dlls = Directory.GetFiles(tempDirectory, "*.dll", SearchOption.AllDirectories); | ||
|
|
||
| foreach (var dll in dlls) | ||
| { | ||
| using FileStream stream = new (dll, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); | ||
| using PEReader peReader = new (stream); | ||
| MetadataReader reader = peReader.GetMetadataReader(); | ||
|
|
||
| Assert.True(HasSbrpAttribute(reader), $"{package}/{Path.GetRelativePath(tempDirectory, dll)} does not contain the {SbrpAttributeType} attribute with key='source' and value='{SbrpRepoIdentifier}'."); | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| Directory.Delete(tempDirectory, true); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| [SkippableFact] | ||
| public async Task ValidateSignatures() | ||
| { | ||
| string[] packages = GetPackages(); | ||
|
|
||
| ISignatureVerificationProvider[] trustProviders = [new SignatureTrustAndValidityVerificationProvider()]; | ||
| PackageSignatureVerifier verifier = new (trustProviders); | ||
| var settings = SignedPackageVerifierSettings.GetDefault(); | ||
|
|
||
| Output.WriteLine($"Checking {packages.Count()} packages for signatures."); | ||
|
|
||
| foreach (var package in packages) | ||
| { | ||
| bool isSigned = await IsPackageSignedAsync(package, verifier, settings); | ||
| Assert.False(isSigned, $"{package} is signed. Signed packages are not allowed in {SbrpRepoIdentifier}."); | ||
| } | ||
| } | ||
|
|
||
| private static string[] GetPackages() | ||
| { | ||
| string buildPackagesDirectory = PathUtilities.GetSourceBuildPackagesShippingDir(); | ||
|
|
||
| string[] packages = Directory.GetFiles(buildPackagesDirectory, "*.nupkg", SearchOption.AllDirectories); | ||
|
|
||
| if (packages.Length == 0) | ||
| { | ||
| throw new FileNotFoundException($"No packages found in {buildPackagesDirectory}"); | ||
| } | ||
| return packages; | ||
| } | ||
|
|
||
| private static bool HasSbrpAttribute(MetadataReader reader) => | ||
| reader.CustomAttributes | ||
| .Select(attrHandle => reader.GetCustomAttribute(attrHandle)) | ||
| .Any(attr => IsAttributeSbrp(reader, attr)); | ||
|
|
||
| private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr) | ||
| { | ||
| string attributeType = string.Empty; | ||
|
|
||
| if (attr.Constructor.Kind == HandleKind.MemberReference) | ||
| { | ||
| var mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor); | ||
| if (mref.Parent.Kind == HandleKind.TypeReference) | ||
| { | ||
| var tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent); | ||
| attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}"; | ||
| } | ||
| else if (mref.Parent.Kind == HandleKind.TypeDefinition) | ||
| { | ||
| var tdef = reader.GetTypeDefinition((TypeDefinitionHandle)mref.Parent); | ||
| attributeType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}"; | ||
| } | ||
| } | ||
| else if (attr.Constructor.Kind == HandleKind.MethodDefinition) | ||
| { | ||
| var mdef = reader.GetMethodDefinition((MethodDefinitionHandle)attr.Constructor); | ||
| var tdef = reader.GetTypeDefinition(mdef.GetDeclaringType()); | ||
| attributeType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}"; | ||
| } | ||
|
|
||
| if (attributeType == SbrpAttributeType) | ||
| { | ||
| var decodedValue = attr.DecodeValue(DummyAttributeTypeProvider.Instance); | ||
| try | ||
| { | ||
| return decodedValue.FixedArguments[0].Value?.ToString() == "source" && decodedValue.FixedArguments[1].Value?.ToString() == SbrpRepoIdentifier; | ||
| } | ||
| catch | ||
| { | ||
| throw new InvalidOperationException($"{SbrpAttributeType} is not formatted properly with a key, value pair."); | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static async Task<bool> IsPackageSignedAsync(string packagePath, PackageSignatureVerifier verifier, SignedPackageVerifierSettings settings) | ||
| { | ||
| using PackageArchiveReader packageReader = new (packagePath); | ||
| var result = await verifier.VerifySignaturesAsync(packageReader, settings, CancellationToken.None); | ||
| return result.IsSigned; | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's absolutely fine to use SkippableFact here (as you don't want to run these tests in source build without prebuilts). Just wanted to let you know that we already have in-built support for this and more in Arcade via Microsoft.DotNet.XUnitExtensions.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is great to know! Thanks for pointing this out.