Add retries to client create and GetPipelineCacheArtifactAsync (#3539)

microsoft · Sep 22, 2021 · d9928d9 · d9928d9
1 parent 0299fb6
commit d9928d9
Showing 1 changed file with 33 additions and 5 deletions.
diff --git a/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs b/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs
@@ -41,7 +41,7 @@ internal async Task UploadAsync(
             VssConnection connection = context.VssConnection;
             var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
                 .CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
-            PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
+            PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken);
 
             using (clientTelemetry)
             {
@@ -122,13 +122,23 @@ internal async Task DownloadAsync(
             VssConnection connection = context.VssConnection;
             var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
                 .CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
-            PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
+            PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken);
 
             using (clientTelemetry)
             {
                 PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                         new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
-                PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
+
+                PipelineCacheArtifact result = await AsyncHttpRetryHelper.InvokeAsync(
+                    async () =>
+                    {
+                        return await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
+                    },
+                    maxRetries: 3,
+                    tracer: tracer,
+                    canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
+                    cancellationToken: cancellationToken,
+                    continueOnCapturedContext: false);
 
                 // Send results to CustomerIntelligence
                 context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
@@ -181,14 +191,32 @@ await clientTelemetry.MeasureActionAsync(
             }
         }
 
-        private PipelineCacheClient CreateClient(
+
+        private Task<PipelineCacheClient> CreateClientWithRetryAsync(
+            BlobStoreClientTelemetry blobStoreClientTelemetry,
+            AgentTaskPluginExecutionContext context,
+            VssConnection connection,
+            CancellationToken cancellationToken)
+        {
+            // this uses location service so needs http retries.
+            return AsyncHttpRetryHelper.InvokeAsync(
+                async () => await this.CreateClientAsync(blobStoreClientTelemetry, context, connection),
+                maxRetries: 3,
+                tracer: tracer,
+                canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
+                cancellationToken: cancellationToken,
+                continueOnCapturedContext: false);
+        }
+
+        private async Task<PipelineCacheClient> CreateClientAsync(
             BlobStoreClientTelemetry blobStoreClientTelemetry,
             AgentTaskPluginExecutionContext context,
             VssConnection connection)
         {
+
             var tracer = context.CreateArtifactsTracer();
             IClock clock = UtcClock.Instance;
-            var pipelineCacheHttpClient = connection.GetClient<PipelineCacheHttpClient>();
+            var pipelineCacheHttpClient = await connection.GetClientAsync<PipelineCacheHttpClient>();
             var pipelineCacheClient = new PipelineCacheClient(blobStoreClientTelemetry, pipelineCacheHttpClient, clock, tracer);
 
             return pipelineCacheClient;