Goals & Motivation:

In order to enable prebuild detection on repo level we need to enhance reference package generator. And the primary bottleneck is Microsoft.CCI dependency of GenAPI, which no longer supported. Microsoft.CCI is used for reconstructing source tree out of dynamic library symbols and as it is no longer supported, it does not support latest language features out of the box. Runtime team tries to support language features they needed via Microsoft.CCi.Extension, but usually it takes much time to reverse engineer, implement and test.

We are going to implement a Roslyn-based backend (using Microsoft.CodeAnalysis).

We are not going to make any changes to Microsoft.CCI or Microsoft.CCI.Extenstions.

Diagram of interfaces/classes in Microsoft.CCi.Extension project

Repository and porject structure:

Repository: dotnet/arcade

• Microsoft.DotNet.GenAPI\Shared - shared codebase for both MSBuild task and CLI tool
• Microsoft.DotNet.GenAPI\Task - MSBuild task
• Microsoft.DotNet.GenAPI\Tool - CLI tool

and corresponding namespaces: Microsoft.DotNet.GenAPI.Shared, Microsoft.DotNet.GenAPI.Task, Microsoft.DotNet.GenAPI.Tool

GenAPIv2 supported options

List of all options currently supported by the GenAPI listed in GenAPITask.cs.

In GenAPIv2 we'll support options:

• Assembly - [required] Path for an specific assembly or a directory to get all assemblies.
• GenAPILibPath - Delimited (',' or ';') set of paths to use for resolving assembly references.
• ApiList - Specify a api list in the DocId format of which APIs to include.
• ExcludeApiList - Specify a api list in the DocId format of which APIs to exclude.
• OutputPath - Output path. Default is the console. Can specify an existing directory as well and then a file will be created for each assembly with the matching name of the assembly
• HeaderFile - Specify a file with an alternate header content to prepend to output.
• ExceptionMessage - Method bodies should throw PlatformNotSupportedException.
• ExcludeAttributesList - Specify a list in the DocId format of which attributes should be excluded from being applied on apis.
• FollowTypeForwards - Resolve type forwards and include its members.

Often we are force to generate reference assemblies for a list of internal classes. So we'll rethink/reimplement options:

• All - Include all API's not just public APIs. Default is public only.
• RespectInternals - Include both internal and public APIs if assembly contains an InternalsVisibleTo attribute. Otherwise, include only public APIs.

We won't support options:

• ApiOnly - Include only API's not CS code that compiles.
• ExcludeCompilerGenerated - Exclude APIs marked with a CompilerGenerated attribute.
• MemberHeadings - Include member headings for each type of member.
• HighlightBaseMembers - Highlight overridden base members.
• HighlightInterfaceMembers - Highlight interface implementation members.
• AlwaysIncludeBase - Include base types, interfaces, and attributes, even when those types are filtered.
• ConditionalTypeLists - Specify one or more namespace+type list(s) in the DocId format of types that should be wrapped inside an #if with the symbol specified in the Symbol metadata item. If the Symbol metadata item is empty the types won't be wrapped but will still be output after all other types, this can be used in combination with DefaultCondition to wrap all other types.
• DefaultCondition - #if condition to apply to all types not included in ConditionalTypeLists.
• ExcludeMembers - Exclude members when return value or parameter types are excluded.
• LangVersion - Language Version to target.

Interfaces

/// <summary>
/// Interface responsible for creating Compilation Factory and loading <see cref="IAssemblySymbol"/> out of binaries.
/// </summary>
public interface IAssemblySymbolLoader
{
    /// <summary>
    /// Loads an assembly from the provided path.
    /// </summary>
    /// <param name="path">The full path to the assembly.</param>
    /// <returns><see cref="IAssemblySymbol"/> representing the loaded assembly.</returns>
    IAssemblySymbol? LoadAssembly(string path);
    
    /// <summary>
    /// Loads an assembly from a given <see cref="Stream"/>.
    /// </summary>
    /// <param name="stream">The stream to read the metadata from.</param>
    /// <returns><see cref="IAssemblySymbol"/> respresenting the given <paramref name="stream"/>.</returns>
    IAssemblySymbol? LoadAssembly(Stream stream);
}

/// <summary>
/// Defines interface for implementing filtering of attributes, namespaces, types, members, filter out/allow private, internals.
/// </summary>
public interface IAssemblySymbolFilter
{
    bool Include(INamespaceSymbol ns);
    bool Include(AttributeData at);
    bool Include(ITypeSymbol type);
    bool Include(ISymbol member);
}

public class AndFilter : IAssemblySymbolFilter
{
    private List<IAssemblySymbolFilter> _filters = new List<IAssemblySymbolFilter>();

    public static AndFilter operator &(AndFilter a, IAssemblySymbolFilter b)
    {
        a._filters.Add(b);
        return a;
    }

    public bool Include(INamespaceSymbol ns) { /*iterate _filters*/ }
    public bool Include(AttributeData at) { /*iterate _filters*/ }
    public bool Include(ITypeSymbol type) { /*iterate _filters*/ }
    public bool Include(ISymbol member) { /*iterate _filters*/ }
}

public class OrFilter : IAssemblySymbolFilter
{
    private List<IAssemblySymbolFilter> _filters = new List<IAssemblySymbolFilter>();

    public static OrFilter operator |(OrFilter a, IAssemblySymbolFilter b)
    {
        a._filters.Add(b);
        return a;
    }

    public bool Include(INamespaceSymbol ns) { /*iterate _filters*/ }
    public bool Include(AttributeData at) { /*iterate _filters*/ }
    public bool Include(ITypeSymbol type) { /*iterate _filters*/ }
    public bool Include(ISymbol member) { /*iterate _filters*/ }
}

/// <summary>
/// Interface provides ordering for namespaces, types and members.
/// </summary>
public interface IAssemblySymbolOrderProvider
{
    /// <summary>
    /// Sorts the elements of a INamespaceSymbol.
    /// </summary>
    /// <param name="namespaces">List of namespaces to be sorted.</param>
    /// <returns>Returns namespaces in sorted order.</returns>
    IEnumerable<INamespaceSymbol> OrderNamespaces(IEnumerable<INamespaceSymbol> namespaces);
    /// <summary>
    /// Sorts the elements of a ITypeSymbol.
    /// </summary>
    /// <param name="namespaces">List of TypeMembers to be sorted.</param>
    /// <returns>Returns TypeMembers in sorted order.</returns>
    IEnumerable<T> OrderTypes<T>(IEnumerable<T> symbols) where T : ITypeSymbol;
    /// <summary>
    /// Sorts the elements of a ISymbol.
    /// </summary>
    /// <param name="namespaces">List of Members to be sorted.</param>
    /// <returns>Returns Members in sorted order.</returns>
    IEnumerable<ISymbol> OrderMembers(IEnumerable<ISymbol> members);
}

Syntax writers

The main idea here is that we'd like to incapsulate and hide implementation details of how we work with underlying System.IO.TextWriter, customer 'd call interface and don't need to care about indentaion etc.

I'll reuse interfaces and implementation from CCi.Extensions responsible for indentation and writing into file.

public interface ISyntaxWriter
{
    void Write(string str);
    void WriteSymbol(string symbol);
    void WriteIdentifier(string id);
    void WriteKeyword(string keyword);
    void WriteTypeName(string typeName);
    void WriteLine();
    int IndentLevel { get; set; }
}

Diagram:

Roslyn API

/TMP Placeholder/

Rikki G.: Heads up that I'm working with Cyrus and Jared separately to determine what the path forward is for utilities which generate code from symbols. It's possible we will recommend you to use some different API other than SymbolDisplay. I'll let you know when I figure out more.

/End of TMP Placeholder/

Tests

/Work in progress/

To validate new GenAPI we' need to regenerate all existing reference packages in source-build-reference-packages one by one.

Project Microsoft.CCi.Extensions does not have UTs or integration tests written, in contrast we'd like to have some integration. TestData as a source code --> dll --> reconstruct source tree --> ApiDiff (round trip)

namespace Microsoft.DotNet.GenAPI
{
public class GenAPITask : BuildTask
{
/// List of options

public override bool Execute()
{
    /// main logic here. load assemblies and choose appropriate writer: CSharpWriter, XMLWriter, etc.
}
}

  public enum BackendType
  {
      CCI,
      Roslyn,
  }

class GenAPIContext
{
public string Assembly { get; set; }
public string LibPath { get; set; }
// ...
}

class BackendFactory
{
    static IGenAPIBackend Create(BackendType type)
    {
        switch (type)
        {
            /// ...
        }
    }
}

class CompilationFactory
{
    public static IAssemblySymbol GetCompilation(string filePath)
    {
        /// ...
        return compilation.GetAssemblyOrModuleSymbol();
    }
}

namespace Microsoft.CodeAnalysis.Extensions
{

public interface IAssemblyWriter
{
    void WriteAssemblies(IEnumerable<Microsoft.CodeAnalysis.AssemblySymbol> assemblies);
}

public class SimpleTypeMemberTraverser
{
    public virtual void Visit(IEnumerable<IAssemblySymbol> assemblies);

    public virtual void Visit(IAssemblySymbol assembly);

    public virtual void Visit(IEnumerable<INamespaceSymbol> nss);

    public virtual void Visit(INamespaceDefinition ns);

    public virtual void Visit(IEnumerable<INamedTypeSymbol> ntss);

    public virtual void Visit(INamedTypeSymbol nts);
}

public class CSharpWriter : SimpleTypeMemberTraverser, IAssemblyWriter, IDisposable
{
/// implement/override IAssemblyWriter, SimpleTypeMemberTraverser interface/class
}

public class DocumentIdWriter : SimpleTypeMemberTraverser, IAssemblyWriter
{
/// implement/override IAssemblyWriter, SimpleTypeMemberTraverser interface/class
}

}

public interface ICciFilter
{
    bool Include(INamespaceDefinition ns);
    bool Include(ITypeDefinition type);
    bool Include(ITypeDefinitionMember member);
    bool Include(ICustomAttribute attribute);
}

public interface IFilter
{
    bool Include(INamespaceSymbol);
    bool Include(INamedTypeSymbol );
    bool Include(IMethodSymbol);
    bool Include(AttributeData); //?
}