introduction of nuget packages as plugins support

Torch.API/WebAPI/Plugins/IPackageItem.cs

@@ -0,0 +1,11 @@
+#nullable enable
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public interface IPackageItem
+{
+    Task<Stream> OpenFileAsync();
+    public string FileName { get; }
+}

Torch.API/WebAPI/Plugins/IPackageReader.cs

@@ -0,0 +1,12 @@
+#nullable enable
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public interface IPackageReader
+{
+    Task<(IEnumerable<IPackageItem> Root, IReadOnlyDictionary<PackageDependency, IEnumerable<IPackageItem>> Dependencies)> GetItemsAsync();
+}

Torch.API/WebAPI/Plugins/IPackageResolver.cs

@@ -0,0 +1,11 @@
+#nullable enable
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public interface IPackageResolver
+{
+    Task<IEnumerable<Package>> ResolvePackagesAsync(IReadOnlyDictionary<string, string> packages);
+    Task<IPackageReader> GetPackageAsync(Package package);
+}

Torch.API/WebAPI/Plugins/NLogLogger.cs

@@ -0,0 +1,87 @@
+#nullable enable
+using System;
+using System.Threading.Tasks;
+using NuGet.Common;
+
+namespace Torch.API.WebAPI.Plugins;
+
+internal class NLogLogger : ILogger
+{
+    private readonly NLog.ILogger _logger;
+
+    public NLogLogger(NLog.ILogger logger)
+    {
+        _logger = logger;
+    }
+
+    public void LogDebug(string data)
+    {
+        _logger.Debug(data);
+    }
+
+    public void LogVerbose(string data)
+    {
+        _logger.Trace(data);
+    }
+
+    public void LogInformation(string data)
+    {
+        _logger.Info(data);
+    }
+
+    public void LogMinimal(string data)
+    {
+        _logger.Debug(data);
+    }
+
+    public void LogWarning(string data)
+    {
+        _logger.Warn(data);
+    }
+
+    public void LogError(string data)
+    {
+        _logger.Error(data);
+    }
+
+    public void LogInformationSummary(string data)
+    {
+        _logger.Info(data);
+    }
+
+    public void Log(LogLevel level, string data)
+    {
+        _logger.Log(ToNLogLevel(level), data);
+    }
+
+    private static NLog.LogLevel ToNLogLevel(LogLevel level)
+    {
+        return level switch
+        {
+            LogLevel.Debug => NLog.LogLevel.Debug,
+            LogLevel.Verbose => NLog.LogLevel.Trace,
+            LogLevel.Information => NLog.LogLevel.Info,
+            LogLevel.Minimal => NLog.LogLevel.Debug,
+            LogLevel.Warning => NLog.LogLevel.Warn,
+            LogLevel.Error => NLog.LogLevel.Error,
+            _ => throw new ArgumentOutOfRangeException(nameof(level), level, null)
+        };
+    }
+
+    public Task LogAsync(LogLevel level, string data)
+    {
+        Log(level, data);
+        return Task.CompletedTask;
+    }
+
+    public void Log(ILogMessage message)
+    {
+        _logger.Log(ToNLogLevel(message.Level), message.FormatWithCode);
+    }
+
+    public Task LogAsync(ILogMessage message)
+    {
+        Log(message);
+        return Task.CompletedTask;
+    }
+}

Torch.API/WebAPI/Plugins/Package.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using NuGet.DependencyResolver;
+using SemanticVersioning;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public record Package(string Name, Version Version, IReadOnlySet<PackageDependency> Dependencies)
+{
+    internal GraphItem<RemoteResolveResult> Graph { get; init; }
+}

Torch.API/WebAPI/Plugins/PackageDependency.cs

@@ -0,0 +1,9 @@
+using NuGet.DependencyResolver;
+using SemanticVersioning;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public record PackageDependency(string Name, Version Version, PackageDependencyKind Kind)
+{
+    internal RemoteMatch Match { get; init; }
+}

Torch.API/WebAPI/Plugins/PackageDependencyKind.cs

@@ -0,0 +1,8 @@
+namespace Torch.API.WebAPI.Plugins;
+
+public enum PackageDependencyKind
+{
+    None,
+    Transitive,
+    Direct
+}

Torch.API/WebAPI/Plugins/PackageReader.cs

@@ -0,0 +1,91 @@
+#nullable enable
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using NuGet.Common;
+using NuGet.DependencyResolver;
+using NuGet.Frameworks;
+using NuGet.Packaging;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public class PackageReader : IPackageReader
+{
+    private readonly Package _package;
+    private readonly SourceCacheContext _cacheContext;
+    private readonly ILogger _logger;
+    private readonly NuGetFramework _framework;
+    private readonly IFrameworkCompatibilityProvider _compatibilityProvider;
+    private readonly DirectoryInfo _packagesDirectory;
+
+    public PackageReader(Package package, SourceCacheContext cacheContext, ILogger logger, NuGetFramework framework,
+                         IFrameworkCompatibilityProvider compatibilityProvider, DirectoryInfo packagesDirectory)
+    {
+        _package = package;
+        _cacheContext = cacheContext;
+        _logger = logger;
+        _framework = framework;
+        _compatibilityProvider = compatibilityProvider;
+        _packagesDirectory = packagesDirectory;
+    }
+
+    public async Task<(IEnumerable<IPackageItem> Root, IReadOnlyDictionary<PackageDependency, IEnumerable<IPackageItem>>
+            Dependencies)>
+        GetItemsAsync()
+    {
+        async Task<IEnumerable<IPackageItem>> GetPackageItemsAsync(string id, NuGetVersion version,
+                                                                   IRemoteDependencyProvider provider)
+        {
+            var downloader =
+                await provider.GetPackageDownloaderAsync(new(id, version), _cacheContext, _logger,
+                                                         CancellationToken.None);
+
+            await downloader.CopyNupkgFileToAsync(Path.Combine(_packagesDirectory.FullName, $"{id}.{version}.nupkg"),
+                                                  CancellationToken.None);
+
+            var frameworks = await downloader.ContentReader.GetReferenceItemsAsync(CancellationToken.None);
+            var items = frameworks.Where(b => _compatibilityProvider.IsCompatible(_framework, b.TargetFramework))
+                                  .MaxBy(b => b.TargetFramework.Version)?.Items;
+
+            return items?.Select(b => new PackageItem(b, downloader)) ?? ImmutableArray<PackageItem>.Empty;
+        }
+
+        var rootIdentity = _package.Graph.Key;
+        return (await GetPackageItemsAsync(rootIdentity.Name, rootIdentity.Version, _package.Graph.Data.Match.Provider),
+            await _package.Dependencies.ToAsyncEnumerable().SelectManyAwait(async b =>
+                                                                                (await GetPackageItemsAsync(
+                                                                                    b.Match.Library.Name,
+                                                                                    b.Match.Library.Version,
+                                                                                    b.Match.Provider))
+                                                                                .ToAsyncEnumerable()
+                                                                                .Select(c => (b, c)))
+                          .GroupBy(b => b.b, b => b.c)
+                          .ToDictionaryAwaitAsync<IAsyncGrouping<PackageDependency, IPackageItem>, PackageDependency,
+                              IEnumerable<IPackageItem>>(b => ValueTask.FromResult(b.Key),
+                                                         async b => await b.ToArrayAsync()));
+    }
+}
+
+file class PackageItem : IPackageItem
+{
+    private readonly string _path;
+    private readonly IPackageDownloader _downloader;
+
+    public string FileName => Path.GetFileName(_path);
+
+    public PackageItem(string path, IPackageDownloader downloader)
+    {
+        _path = path;
+        _downloader = downloader;
+    }
+
+    public Task<Stream> OpenFileAsync()
+    {
+        return _downloader.CoreReader.GetStreamAsync(_path, CancellationToken.None);
+    }
+}

Torch.API/WebAPI/Plugins/PackageResolver.cs

@@ -0,0 +1,104 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using NLog;
+using NuGet.Commands;
+using NuGet.Configuration;
+using NuGet.DependencyResolver;
+using NuGet.Frameworks;
+using NuGet.LibraryModel;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
+using Version = SemanticVersioning.Version;
+
+namespace Torch.API.WebAPI.Plugins;
+
+public class PackageResolver : IPackageResolver
+{
+    private static readonly ILogger Log = LogManager.GetCurrentClassLogger();
+
+    private readonly NuGetFramework _framework = NuGetFramework.Parse("net7.0-windows7.0");
+    private readonly NLogLogger _logger = new(Log);
+    private readonly SourceCacheContext _sourceCacheContext = new();
+    private readonly RemoteWalkContext _remoteWalkContext;
+    private readonly DirectoryInfo _packagesDirectory;
+    private readonly IFrameworkCompatibilityProvider _compatibilityProvider = DefaultCompatibilityProvider.Instance;
+
+    public PackageResolver(IEnumerable<PackageSource> sources, DirectoryInfo packagesDirectory)
+    {
+        _packagesDirectory = packagesDirectory;
+        IReadOnlySet<PackageSource> packageSources = sources.Where(b => b.Type is PackageSourceType.NuGet).ToImmutableHashSet();
+
+        var mapping = new PackageSourceMapping(packageSources.ToDictionary(b => b.Name, b => b.Patterns));
+        _remoteWalkContext = new RemoteWalkContext(_sourceCacheContext, mapping, _logger);
+
+        foreach (var (name, url, _, _) in packageSources)
+        {
+            var packageSource = new NuGet.Configuration.PackageSource(url, name);
+            var sourceRepository = new SourceRepository(packageSource, new INuGetResourceProvider[]
+            {
+                new DownloadResourceV3Provider(),
+                new DependencyInfoResourceV3Provider(),
+                new ServiceIndexResourceV3Provider(),
+                new RemoteV3FindPackageByIdResourceProvider(),
+                new V3FeedListResourceProvider(),
+                new HttpSourceResourceProvider(),
+                new RegistrationResourceV3Provider(),
+                new HttpHandlerResourceV3Provider()
+            }.Select(b => new Lazy<INuGetResourceProvider>(b)), FeedType.HttpV3);
+
+            _remoteWalkContext.RemoteLibraryProviders.Add(
+                new SourceRepositoryDependencyProvider(sourceRepository, _logger, _sourceCacheContext, true, false));
+        }
+    }
+
+    public async Task<IEnumerable<Package>> ResolvePackagesAsync(
+        IReadOnlyDictionary<string, string> packages)
+    {
+        Log.Info("Restoring {0} packages", packages.Count);
+
+        var graphs = await Task.WhenAll(packages.Select(b =>
+        {
+            var (key, versionRange) = b;
+            var libraryRange = new LibraryRange(key, VersionRange.Parse(versionRange), LibraryDependencyTarget.All);
+            return ResolverUtility.FindLibraryEntryAsync(libraryRange, _framework, "win-x64",
+                                                         _remoteWalkContext, CancellationToken.None);
+        }));
+
+        return await graphs.ToAsyncEnumerable().SelectAwait(async graph =>
+        {
+            return new Package(graph.Key.Name, Version.Parse(graph.Key.Version.ToFullString()),
+                               await graph.Data.Dependencies
+                                          .ToAsyncEnumerable()
+                                          .SelectAwait(async b =>
+                                          {
+                                              var match = await ResolverUtility.FindLibraryByVersionAsync(
+                                                  b.LibraryRange, _framework, _remoteWalkContext.RemoteLibraryProviders,
+                                                  _sourceCacheContext, _logger, CancellationToken.None);
+
+                                              return new PackageDependency(
+                                                  b.Name, Version.Parse(match.Library.Version.ToFullString()),
+                                                  (PackageDependencyKind)b.ReferenceType)
+                                              {
+                                                  Match = match
+                                              };
+                                          })
+                                          .ToHashSetAsync())
+            {
+                Graph = graph
+            };
+        }).ToArrayAsync();
+    }
+
+    public Task<IPackageReader> GetPackageAsync(Package package)
+    {
+        var reader = new PackageReader(package, _sourceCacheContext, _logger, _framework, _compatibilityProvider, _packagesDirectory);
+        return Task.FromResult<IPackageReader>(reader);
+    }
+}

Torch.API/WebAPI/Plugins/PackageSource.cs

@@ -0,0 +1,8 @@
+using System.Collections.Generic;
+
+namespace Torch.API.WebAPI.Plugins;
+
+#nullable enable
+public record PackageSource
+#nullable restore
+    (string Name, string Url, IReadOnlyList<string> Patterns, PackageSourceType Type);

Torch.API/WebAPI/Plugins/PackageSourceType.cs

@@ -0,0 +1,8 @@
+#nullable enable
+namespace Torch.API.WebAPI.Plugins;
+
+public enum PackageSourceType
+{
+    NuGet,
+    LegacyTorch
+}