Skip to content

Commit b625557

Browse files
tamasvajktamasvajk
authored
Merge pull request #17253 from tamasvajk/impr/add-retry-logic-to-file-download
C#: Add retry logic to file (nuget.exe, dotnet-install.sh) downloads
2 parents 68dfca4 + 73674e7 commit b625557

8 files changed

Lines changed: 53 additions & 17 deletions

File tree

‎csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using Microsoft.Build.Construction;
88
using Semmle.Util;
99
using Semmle.Autobuild.Shared;
10+
using Semmle.Util.Logging;
1011

1112
namespace Semmle.Autobuild.CSharp.Tests
1213
{
@@ -203,7 +204,7 @@ public void CreateDirectory(string path)
203204
throw new ArgumentException($"Missing CreateDirectory, {path}");
204205
}
205206

206-
public void DownloadFile(string address, string fileName)
207+
public void DownloadFile(string address, string fileName, ILogger logger)
207208
{
208209
if (!DownloadFiles.Contains((address, fileName)))
209210
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

‎csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using Microsoft.Build.Construction;
88
using System.Xml;
99
using System.IO;
10+
using Semmle.Util.Logging;
1011

1112
namespace Semmle.Autobuild.Cpp.Tests
1213
{
@@ -189,7 +190,7 @@ public void CreateDirectory(string path)
189190
throw new ArgumentException($"Missing CreateDirectory, {path}");
190191
}
191192

192-
public void DownloadFile(string address, string fileName)
193+
public void DownloadFile(string address, string fileName, ILogger logger)
193194
{
194195
if (!DownloadFiles.Contains((address, fileName)))
195196
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

‎csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAut
157157
BuildScript.DownloadFile(
158158
FileUtils.NugetExeUrl,
159159
path,
160-
e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
160+
e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),
161+
builder.Logger)
161162
&
162163
BuildScript.Create(_ =>
163164
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,8 @@ BuildScript GetInstall(string pwsh) =>
248248
var downloadDotNetInstallSh = BuildScript.DownloadFile(
249249
"https://dot.net/v1/dotnet-install.sh",
250250
dotnetInstallPath,
251-
e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
251+
e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"),
252+
logger);
252253

253254
var chmod = new CommandBuilder(actions).
254255
RunCommand("chmod").

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ private string DownloadNugetExe(string sourceDir)
145145

146146
Directory.CreateDirectory(directory);
147147
logger.LogInfo("Attempting to download nuget.exe");
148-
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
148+
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
149149
logger.LogInfo($"Downloaded nuget.exe to {nuget}");
150150
return nuget;
151151
}

‎csharp/extractor/Semmle.Util/BuildActions.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
using System.Linq;
77
using System.Runtime.InteropServices;
88
using System.Xml;
9-
using Semmle.Util;
9+
using Semmle.Util.Logging;
1010

1111
namespace Semmle.Util
1212
{
@@ -165,7 +165,7 @@ public interface IBuildActions
165165
/// <summary>
166166
/// Downloads the resource with the specified URI to a local file.
167167
/// </summary>
168-
void DownloadFile(string address, string fileName);
168+
void DownloadFile(string address, string fileName, ILogger logger);
169169

170170
/// <summary>
171171
/// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ XmlDocument IBuildActions.LoadXml(string filename)
280280

281281
public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
282282

283-
public void DownloadFile(string address, string fileName) =>
284-
FileUtils.DownloadFile(address, fileName);
283+
public void DownloadFile(string address, string fileName, ILogger logger) =>
284+
FileUtils.DownloadFile(address, fileName, logger);
285285

286286
public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
287287

‎csharp/extractor/Semmle.Util/BuildScript.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.IO;
4+
using Semmle.Util.Logging;
45

56
namespace Semmle.Util
67
{
@@ -275,14 +276,14 @@ public static BuildScript Create(Func<IBuildActions, int> func) =>
275276
/// <summary>
276277
/// Creates a build script that downloads the specified file.
277278
/// </summary>
278-
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
279+
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
279280
Create(actions =>
280281
{
281282
if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
282283
actions.CreateDirectory(dir);
283284
try
284285
{
285-
actions.DownloadFile(address, fileName);
286+
actions.DownloadFile(address, fileName, logger);
286287
return 0;
287288
}
288289
catch (Exception e)

‎csharp/extractor/Semmle.Util/FileUtils.cs‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Net.Http;
55
using System.Security.Cryptography;
66
using System.Text;
7+
using System.Threading;
78
using System.Threading.Tasks;
89
using Semmle.Util.Logging;
910

@@ -99,19 +100,49 @@ public static string ComputeFileHash(string filePath)
99100
return hex.ToString();
100101
}
101102

102-
private static async Task DownloadFileAsync(string address, string filename)
103+
private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
103104
{
104-
using var httpClient = new HttpClient();
105-
using var contentStream = await httpClient.GetStreamAsync(address);
105+
using var contentStream = await httpClient.GetStreamAsync(address, token);
106106
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
107-
await contentStream.CopyToAsync(stream);
107+
await contentStream.CopyToAsync(stream, token);
108+
}
109+
110+
private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
111+
{
112+
logger.LogDebug($"Downloading {address} to {fileName}.");
113+
using HttpClient client = new();
114+
115+
for (var i = 0; i < tryCount; i++)
116+
{
117+
logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
118+
using var cts = new CancellationTokenSource();
119+
cts.CancelAfter(timeoutMilliSeconds);
120+
try
121+
{
122+
DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
123+
logger.LogDebug($"Downloaded {address} to {fileName}.");
124+
return;
125+
}
126+
catch (Exception exc)
127+
{
128+
logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
129+
timeoutMilliSeconds *= 2;
130+
131+
if (i == tryCount - 1)
132+
{
133+
logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
134+
// Rethrowing the last exception
135+
throw;
136+
}
137+
}
138+
}
108139
}
109140

110141
/// <summary>
111142
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
112143
/// </summary>
113-
public static void DownloadFile(string address, string fileName) =>
114-
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
144+
public static void DownloadFile(string address, string fileName, ILogger logger) =>
145+
DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
115146

116147
public static string ConvertPathToSafeRelativePath(string path)
117148
{

0 commit comments

Comments
 (0)