Skip to content

Commit 87daa23

Browse files
committed
fix: say how to fetch pretrained weights ML.NET could not download
ML.NET's downloader fails on large files (dotnet/machinelearning#6980). Its error named the URL but not where the file is expected, and the advice after it pointed at listing and promoting models because "download" matched the model-load rule. The TorchSharp trainers' fit now rethrows that failure as PretrainedWeightsUnavailableException naming the file, the folder it is looked for in and the command that fetches it, and the CLI suggests only that. A training error also no longer prints its cause a second time as an inner exception, the rule the general error display already applied.
1 parent 9fac347 commit 87daa23

7 files changed

Lines changed: 205 additions & 8 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
1919
`exact_match` stops; the guide's Question Answering section explains the limit.
2020

2121
### Fixed
22+
- **A deep-learning task whose pretrained weights cannot be downloaded says how to fetch them.**
23+
ML.NET's downloader fails on large files (dotnet/machinelearning#6980), and its error named the URL
24+
but not where the file is expected; the advice after it pointed at listing and promoting models.
25+
Training now names the file, the folder it is looked for in (`mlnet` in the temp folder) and the
26+
command that fetches it, and suggests only that. A training error also no longer prints its cause a
27+
second time as an "Inner exception".
2228
- **Question answering trains, predicts and evaluates.** The trainer was given only the passage and
2329
the question, so it read the answer and its position from columns that had to be named
2430
`TrainingAnswer` and `AnswerStart`; training reported no metric, so the quality gate had nothing to

‎src/MLoop.Core.DeepLearning/Handlers/DeepLearningHandlers.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public static async Task<AutoMLResult> RunTextClassificationAsync(
8888

8989
ITransformer model;
9090
using (EpochProgress.Attach(mlContext, progress, "Text classification (NAS-BERT)", log))
91-
model = pipeline.Fit(trainSet);
91+
model = PretrainedWeights.Guard(() => pipeline.Fit(trainSet));
9292
var predictions = model.Transform(testSet);
9393
var metrics = mlContext.MulticlassClassification.Evaluate(predictions, labelColumnName: "Label");
9494

@@ -137,7 +137,7 @@ public static async Task<AutoMLResult> RunSentenceSimilarityAsync(
137137

138138
ITransformer model;
139139
using (EpochProgress.Attach(mlContext, progress, "Sentence similarity (NAS-BERT)", log))
140-
model = pipeline.Fit(trainSet);
140+
model = PretrainedWeights.Guard(() => pipeline.Fit(trainSet));
141141
var predictions = model.Transform(testSet);
142142
var metrics = mlContext.Regression.Evaluate(predictions, labelColumnName: config.LabelColumn);
143143

@@ -194,7 +194,7 @@ public static async Task<AutoMLResult> RunNerAsync(
194194

195195
ITransformer model;
196196
using (EpochProgress.Attach(mlContext, progress, "NER (NAS-BERT)", log))
197-
model = pipeline.Fit(preparedTrain);
197+
model = PretrainedWeights.Guard(() => pipeline.Fit(preparedTrain));
198198

199199
var (micro, macro) = NerTagAccuracy.Measure(mlContext, model.Transform(testSet), config.LabelColumn);
200200
// Named as multiclass names them, and for the same reason: the tag average is what a model
@@ -265,7 +265,7 @@ public static async Task<AutoMLResult> RunObjectDetectionAsync(
265265
// report said accuracy=0, which for a detector reads as "found nothing".
266266
ITransformer model;
267267
using (EpochProgress.Attach(mlContext, progress, "Object detection (AutoFormerV2)", log))
268-
model = pipeline.Fit(trainSet);
268+
model = PretrainedWeights.Guard(() => pipeline.Fit(trainSet));
269269
var predictions = model.Transform(testSet);
270270

271271
return new AutoMLResult
@@ -326,7 +326,7 @@ public static async Task<AutoMLResult> RunQuestionAnsweringAsync(
326326

327327
ITransformer model;
328328
using (EpochProgress.Attach(mlContext, progress, "Question answering", log))
329-
model = pipeline.Fit(prepared);
329+
model = PretrainedWeights.Guard(() => pipeline.Fit(prepared));
330330

331331
var (exact, f1) = AnswerOverlap.Measure(model.Transform(testSet), answerCol);
332332
var metricsDict = new Dictionary<string, double>
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
using System.Text.RegularExpressions;
2+
3+
namespace MLoop.Core.DeepLearning;
4+
5+
/// <summary>
6+
/// What a user is told when ML.NET cannot fetch the pretrained weights a deep-learning task trains
7+
/// from.
8+
/// </summary>
9+
/// <remarks>
10+
/// <para>
11+
/// The TorchSharp trainers (text classification, sentence similarity, NER, question answering,
12+
/// object detection) download their weights on first use into <c>mlnet</c> under the context's temp
13+
/// folder — <see cref="Path.GetTempPath"/>, since MLoop does not set one — and reuse the file after
14+
/// that. The download fails on this project's machines with "The wait completed due to an abandoned
15+
/// mutex": ML.NET's downloader takes a named mutex on one thread and releases it on another, an open
16+
/// upstream defect (dotnet/machinelearning#6980). Large files hit it reliably (measured twice on the
17+
/// 496MB RoBERTa encoder), and a changed TEMP hits it again because the cache follows TEMP.
18+
/// </para>
19+
/// <para>
20+
/// ML.NET's own message names the URL but not where the file is expected, and MLoop's generic advice
21+
/// after it suggested commands unrelated to the failure. Fetching the file into that folder by hand is
22+
/// enough — the trainer finds it and skips the download.
23+
/// </para>
24+
/// </remarks>
25+
public static partial class PretrainedWeights
26+
{
27+
private const string DownloadFailureMarker = "Error downloading resource";
28+
29+
/// <summary>The folder the trainers look for their weights in.</summary>
30+
public static string CacheDirectory => Path.Combine(Path.GetTempPath(), "mlnet");
31+
32+
/// <summary>
33+
/// Runs <paramref name="fit"/>, rethrowing a failed weights download as a
34+
/// <see cref="PretrainedWeightsUnavailableException"/> that says what to fetch and where to put it.
35+
/// Any other failure passes through untouched.
36+
/// </summary>
37+
public static T Guard<T>(Func<T> fit)
38+
{
39+
try
40+
{
41+
return fit();
42+
}
43+
catch (Exception ex) when (FindDownloadFailure(ex) is { } failure)
44+
{
45+
throw new PretrainedWeightsUnavailableException(Explain(failure.Message), ex);
46+
}
47+
}
48+
49+
private static Exception? FindDownloadFailure(Exception? ex)
50+
{
51+
for (; ex != null; ex = ex.InnerException)
52+
{
53+
if (ex.Message.Contains(DownloadFailureMarker, StringComparison.Ordinal))
54+
return ex;
55+
if (ex is AggregateException aggregate)
56+
foreach (var inner in aggregate.InnerExceptions)
57+
if (FindDownloadFailure(inner) is { } found)
58+
return found;
59+
}
60+
return null;
61+
}
62+
63+
private static string Explain(string mlnetMessage)
64+
{
65+
const string cause =
66+
"ML.NET could not download the pretrained weights this task trains from. Its downloader fails on large "
67+
+ "files with an abandoned-mutex error (dotnet/machinelearning#6980).";
68+
69+
if (UrlPattern().Match(mlnetMessage) is not { Success: true } match)
70+
return $"{cause} Fetch the file named in the error below into '{CacheDirectory}' and train again: {mlnetMessage}";
71+
72+
var url = match.Groups[1].Value;
73+
var file = Path.Combine(CacheDirectory, url.Split('/').Last());
74+
return $"{cause} Download it yourself and training will use it:\n"
75+
+ $" curl -L -o \"{file}\" {url}\n"
76+
+ $"The file is looked for in '{CacheDirectory}', the temp folder — a different TEMP looks somewhere else.";
77+
}
78+
79+
[GeneratedRegex("from '([^']+)'")]
80+
private static partial Regex UrlPattern();
81+
}
82+
83+
/// <summary>
84+
/// The pretrained weights a deep-learning task trains from could not be fetched. The message says
85+
/// which file, where it is looked for and how to fetch it — a typed failure so the CLI answers with
86+
/// that remedy rather than its generic training advice.
87+
/// </summary>
88+
public sealed class PretrainedWeightsUnavailableException(string message, Exception inner)
89+
: InvalidOperationException(message, inner);
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using MLoop.Core.DeepLearning;
2+
3+
namespace MLoop.Core.DeepLearning.Tests.Handlers;
4+
5+
public class PretrainedWeightsTests
6+
{
7+
// The shape ML.NET's TorchSharp trainers throw when their weights cannot be fetched
8+
// (ResourceManagerUtils.GetErrorMessage + the trainer's own suffix), as observed.
9+
private const string Url = "https://aka.ms/mlnet-resources/models/pretrained_Roberta_encoder.tsm";
10+
private static readonly Exception DownloadFailure = new InvalidOperationException(
11+
$"Error downloading resource from '{Url}': DownloadFailed with exception One or more errors occurred. "
12+
+ "(The wait completed due to an abandoned mutex.)\n\nmodel file could not be downloaded!");
13+
14+
[Fact]
15+
public void A_failed_download_is_told_with_the_file_the_folder_and_how_to_fetch_it()
16+
{
17+
var ex = Assert.Throws<PretrainedWeightsUnavailableException>(() =>
18+
PretrainedWeights.Guard<int>(() => throw new AggregateException(DownloadFailure)));
19+
20+
var cache = Path.Combine(Path.GetTempPath(), "mlnet");
21+
Assert.Contains(Url, ex.Message);
22+
Assert.Contains(Path.Combine(cache, "pretrained_Roberta_encoder.tsm"), ex.Message);
23+
Assert.Contains("curl -L -o", ex.Message);
24+
Assert.Contains("dotnet/machinelearning#6980", ex.Message);
25+
Assert.Same(DownloadFailure, ex.InnerException?.InnerException);
26+
}
27+
28+
[Fact]
29+
public void Any_other_failure_passes_through_untouched()
30+
{
31+
var other = new InvalidOperationException("Could not find input column 'context'");
32+
33+
var ex = Assert.Throws<InvalidOperationException>(() => PretrainedWeights.Guard<int>(() => throw other));
34+
35+
Assert.Same(other, ex);
36+
}
37+
38+
[Fact]
39+
public void A_result_passes_through() => Assert.Equal(7, PretrainedWeights.Guard(() => 7));
40+
}

‎tests/MLoop.Tests/Diagnostics/ErrorConsoleTests.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,21 @@ public void DisplayTrainingError_NumbersItsSuggestionsInOrder()
115115
Assert.Contains(" 2. ", output);
116116
}
117117

118+
[Fact]
119+
public void DisplayTrainingError_DoesNotRepeatAnInnerMessageTheErrorAlreadySays()
120+
{
121+
// The training engine wraps its failure as "Training failed for experiment …: {inner}", so the
122+
// inner message printed again below it is the same sentence twice. The general display already
123+
// skipped such an inner; the training display did not.
124+
var inner = new InvalidOperationException("The weights could not be fetched.");
125+
var ex = new InvalidOperationException($"Training failed for experiment default/exp-005: {inner.Message}", inner);
126+
127+
var output = CaptureStdErr(() => ErrorSuggestions.DisplayTrainingError(ex, "default"));
128+
129+
Assert.DoesNotContain("Inner exception", output);
130+
Assert.Single(System.Text.RegularExpressions.Regex.Matches(output, "could not be fetched"));
131+
}
132+
118133
[Fact]
119134
public void PathNotFoundTip_NamesTheProjectRoot()
120135
{

‎tests/MLoop.Tests/Diagnostics/ErrorSuggestionsTests.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,39 @@ public void GetSuggestions_FileNotFound_SuggestsCheckPath()
1515
Assert.Contains(suggestions, s => s.Contains("file path"));
1616
}
1717

18+
[Fact]
19+
public void GetSuggestions_AFailedDownloadIsNotAMissingModel()
20+
{
21+
// "download" contains "load", and a weights URL contains "models/" — together they read as a
22+
// model that failed to load, and the reader was sent to list, train and promote.
23+
var ex = new InvalidOperationException(
24+
"ML.NET could not download the pretrained weights this task trains from: "
25+
+ "curl -L -o w.tsm https://aka.ms/mlnet-resources/models/w.tsm");
26+
27+
Assert.DoesNotContain(ErrorSuggestions.GetSuggestions(ex, "training"), s => s.Contains("mloop promote"));
28+
}
29+
30+
[Fact]
31+
public void GetSuggestions_UnavailableWeightsAnswerWithTheirRemedyNotTheGenericTrainingAdvice()
32+
{
33+
var weights = new MLoop.Core.DeepLearning.PretrainedWeightsUnavailableException(
34+
"ML.NET could not download the pretrained weights …", new InvalidOperationException("abandoned mutex"));
35+
var ex = new InvalidOperationException($"Training failed for experiment default/exp-006: {weights.Message}", weights);
36+
37+
var suggestions = ErrorSuggestions.GetSuggestions(ex, "training");
38+
39+
Assert.Contains(suggestions, s => s.Contains("Fetch the file"));
40+
Assert.DoesNotContain(suggestions, s => s.Contains("--time 30"));
41+
}
42+
43+
[Fact]
44+
public void GetSuggestions_AModelThatFailedToLoadStillPointsAtTheModels()
45+
{
46+
var ex = new InvalidOperationException("Failed to load model from models/default/production/model.zip");
47+
48+
Assert.Contains(ErrorSuggestions.GetSuggestions(ex), s => s.Contains("mloop list"));
49+
}
50+
1851
[Fact]
1952
public void GetSuggestions_FileNotFoundInTraining_SuggestsDataFile()
2053
{

‎tools/MLoop.CLI/Infrastructure/Diagnostics/ErrorSuggestions.cs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System.Reflection;
2+
using System.Text.RegularExpressions;
23
using MLoop.Core.AutoML;
34
using Spectre.Console;
45

@@ -8,7 +9,7 @@ namespace MLoop.CLI.Infrastructure.Diagnostics;
89
/// Provides actionable error suggestions based on exception types and messages.
910
/// Helps users understand and resolve common issues quickly.
1011
/// </summary>
11-
public static class ErrorSuggestions
12+
public static partial class ErrorSuggestions
1213
{
1314
/// <summary>
1415
/// Displays an error with actionable suggestions based on context.
@@ -181,6 +182,12 @@ public static List<string> GetSuggestions(Exception ex, string context = "")
181182
suggestions.Add("If the log showed native allocation warnings (e.g. [[LightGBM]] bad allocation), memory is the likely cause");
182183
}
183184

185+
// Pretrained weights that could not be fetched: the message already carries the command that
186+
// fetches them, and the generic training fallback below ("a smaller time limit") does not apply
187+
// to a fixed-epoch deep-learning trainer that never started.
188+
if (HasInChain<MLoop.Core.DeepLearning.PretrainedWeightsUnavailableException>(ex))
189+
suggestions.Add("Fetch the file with the command above, then run the same [cyan]mloop train[/] again");
190+
184191
// Memory and resource errors. The type test walks the inner chain: AutoML wraps OOM in an
185192
// AggregateException and the CLI wraps again ("Training failed for experiment {id}: …"), so a
186193
// bare `ex is OutOfMemoryException` never matched in production. Native trainers report the
@@ -216,7 +223,9 @@ public static List<string> GetSuggestions(Exception ex, string context = "")
216223
// Model errors. A missing label column names the model it was looking for ("… not found in
217224
// data for model 'default'"), which is not a missing model — those suggestions would send the
218225
// reader after experiments when the problem is a column.
219-
if (!labelNotFound && message.Contains("model") && (message.Contains("not found") || message.Contains("load")))
226+
// "load" as a word start: "download" is not a model failing to load, and a weights URL under
227+
// ".../models/" otherwise completed the match.
228+
if (!labelNotFound && message.Contains("model") && (message.Contains("not found") || LoadWord().IsMatch(message)))
220229
{
221230
suggestions.Add("See the models and their experiments: [cyan]mloop list[/]");
222231
suggestions.Add("Train a model first: [cyan]mloop train[/]");
@@ -379,7 +388,9 @@ public static void DisplayTrainingError(Exception ex, string modelName, string?
379388
// through this path reported no error event whatsoever.
380389
var machineParts = new List<string> { $"Training failed for model '{modelName}': {ex.Message}" };
381390

382-
if (ex.InnerException != null)
391+
// The training engine wraps its failure with the inner message in it — repeating it says the
392+
// same sentence twice (the same rule DisplayError applies).
393+
if (ex.InnerException != null && AddsInformation(ex.Message, ex.InnerException.Message))
383394
{
384395
err.WriteLine();
385396
err.MarkupLine("[grey]Inner exception:[/]");
@@ -410,4 +421,7 @@ public static void DisplayTrainingError(Exception ex, string modelName, string?
410421
}
411422
err.MarkupLine(" [cyan]mloop status[/] - Check project status");
412423
}
424+
425+
[GeneratedRegex(@"\bload")]
426+
private static partial Regex LoadWord();
413427
}

0 commit comments

Comments
 (0)