|
| 1 | +using System.Text.RegularExpressions; |
| 2 | + |
| 3 | +namespace MLoop.Core.DeepLearning; |
| 4 | + |
| 5 | +/// <summary> |
| 6 | +/// What a user is told when ML.NET cannot fetch the pretrained weights a deep-learning task trains |
| 7 | +/// from. |
| 8 | +/// </summary> |
| 9 | +/// <remarks> |
| 10 | +/// <para> |
| 11 | +/// The TorchSharp trainers (text classification, sentence similarity, NER, question answering, |
| 12 | +/// object detection) download their weights on first use into <c>mlnet</c> under the context's temp |
| 13 | +/// folder — <see cref="Path.GetTempPath"/>, since MLoop does not set one — and reuse the file after |
| 14 | +/// that. The download fails on this project's machines with "The wait completed due to an abandoned |
| 15 | +/// mutex": ML.NET's downloader takes a named mutex on one thread and releases it on another, an open |
| 16 | +/// upstream defect (dotnet/machinelearning#6980). Large files hit it reliably (measured twice on the |
| 17 | +/// 496MB RoBERTa encoder), and a changed TEMP hits it again because the cache follows TEMP. |
| 18 | +/// </para> |
| 19 | +/// <para> |
| 20 | +/// ML.NET's own message names the URL but not where the file is expected, and MLoop's generic advice |
| 21 | +/// after it suggested commands unrelated to the failure. Fetching the file into that folder by hand is |
| 22 | +/// enough — the trainer finds it and skips the download. |
| 23 | +/// </para> |
| 24 | +/// </remarks> |
| 25 | +public static partial class PretrainedWeights |
| 26 | +{ |
| 27 | + private const string DownloadFailureMarker = "Error downloading resource"; |
| 28 | + |
| 29 | + /// <summary>The folder the trainers look for their weights in.</summary> |
| 30 | + public static string CacheDirectory => Path.Combine(Path.GetTempPath(), "mlnet"); |
| 31 | + |
| 32 | + /// <summary> |
| 33 | + /// Runs <paramref name="fit"/>, rethrowing a failed weights download as a |
| 34 | + /// <see cref="PretrainedWeightsUnavailableException"/> that says what to fetch and where to put it. |
| 35 | + /// Any other failure passes through untouched. |
| 36 | + /// </summary> |
| 37 | + public static T Guard<T>(Func<T> fit) |
| 38 | + { |
| 39 | + try |
| 40 | + { |
| 41 | + return fit(); |
| 42 | + } |
| 43 | + catch (Exception ex) when (FindDownloadFailure(ex) is { } failure) |
| 44 | + { |
| 45 | + throw new PretrainedWeightsUnavailableException(Explain(failure.Message), ex); |
| 46 | + } |
| 47 | + } |
| 48 | + |
| 49 | + private static Exception? FindDownloadFailure(Exception? ex) |
| 50 | + { |
| 51 | + for (; ex != null; ex = ex.InnerException) |
| 52 | + { |
| 53 | + if (ex.Message.Contains(DownloadFailureMarker, StringComparison.Ordinal)) |
| 54 | + return ex; |
| 55 | + if (ex is AggregateException aggregate) |
| 56 | + foreach (var inner in aggregate.InnerExceptions) |
| 57 | + if (FindDownloadFailure(inner) is { } found) |
| 58 | + return found; |
| 59 | + } |
| 60 | + return null; |
| 61 | + } |
| 62 | + |
| 63 | + private static string Explain(string mlnetMessage) |
| 64 | + { |
| 65 | + const string cause = |
| 66 | + "ML.NET could not download the pretrained weights this task trains from. Its downloader fails on large " |
| 67 | + + "files with an abandoned-mutex error (dotnet/machinelearning#6980)."; |
| 68 | + |
| 69 | + if (UrlPattern().Match(mlnetMessage) is not { Success: true } match) |
| 70 | + return $"{cause} Fetch the file named in the error below into '{CacheDirectory}' and train again: {mlnetMessage}"; |
| 71 | + |
| 72 | + var url = match.Groups[1].Value; |
| 73 | + var file = Path.Combine(CacheDirectory, url.Split('/').Last()); |
| 74 | + return $"{cause} Download it yourself and training will use it:\n" |
| 75 | + + $" curl -L -o \"{file}\" {url}\n" |
| 76 | + + $"The file is looked for in '{CacheDirectory}', the temp folder — a different TEMP looks somewhere else."; |
| 77 | + } |
| 78 | + |
| 79 | + [GeneratedRegex("from '([^']+)'")] |
| 80 | + private static partial Regex UrlPattern(); |
| 81 | +} |
| 82 | + |
| 83 | +/// <summary> |
| 84 | +/// The pretrained weights a deep-learning task trains from could not be fetched. The message says |
| 85 | +/// which file, where it is looked for and how to fetch it — a typed failure so the CLI answers with |
| 86 | +/// that remedy rather than its generic training advice. |
| 87 | +/// </summary> |
| 88 | +public sealed class PretrainedWeightsUnavailableException(string message, Exception inner) |
| 89 | + : InvalidOperationException(message, inner); |
0 commit comments