using System.Net; using Microsoft.SharePoint.Client; using Serilog; using SharepointToolbox.Web.Core.Models; namespace SharepointToolbox.Web.Core.Helpers; public static class ExecuteQueryRetryHelper { private const int MaxRetries = 5; public static async Task ExecuteQueryRetryAsync( ClientContext ctx, IProgress? progress = null, CancellationToken ct = default) { int attempt = 0; while (true) { ct.ThrowIfCancellationRequested(); try { await ctx.ExecuteQueryAsync(); return; } catch (Exception ex) when (IsThrottleException(ex) && attempt < MaxRetries) { attempt++; int delaySeconds = (int)Math.Pow(2, attempt) * 5; progress?.Report(OperationProgress.Indeterminate( $"Throttled by SharePoint — retrying in {delaySeconds}s (attempt {attempt}/{MaxRetries})…")); await Task.Delay(TimeSpan.FromSeconds(delaySeconds), ct); } catch (Exception ex) { throw EnrichException(ctx, ex); } } } // CSOM surfaces a 403 as a bare "The remote server returned an error: (403) FORBIDDEN." WebException, // which hides the actual SharePoint reason. Pull the server's response body / correlation id so the // root cause (token scope, tenant policy, access-denied vs endpoint-blocked) is visible. private static Exception EnrichException(ClientContext ctx, Exception ex) { var detail = new System.Text.StringBuilder(); detail.Append(ex.Message); detail.Append($" [site={ctx.Url}]"); if (ex is ServerException se) { detail.Append($" [serverErrorType={se.ServerErrorTypeName}; value={se.ServerErrorValue}; " + $"correlationId={se.ServerErrorTraceCorrelationId}; details={se.ServerErrorDetails}]"); } // Walk inner exceptions for a WebException carrying the raw HTTP response body. for (Exception? cur = ex; cur is not null; cur = cur.InnerException) { if (cur is WebException we && we.Response is HttpWebResponse resp) { detail.Append($" [httpStatus={(int)resp.StatusCode} {resp.StatusCode}]"); try { using var stream = resp.GetResponseStream(); using var reader = new StreamReader(stream); var body = reader.ReadToEnd(); if (!string.IsNullOrWhiteSpace(body)) detail.Append($" [responseBody={body.Trim()}]"); } catch { /* body already consumed or unavailable */ } break; } } var enriched = detail.ToString(); Log.Error("CSOM ExecuteQuery failed: {Detail}", enriched); // Preserve access-denied as a typed exception so the elevation coordinator can // detect it, take site-collection admin ownership, and retry. Everything else // stays a generic InvalidOperationException carrying the enriched diagnostic. if (IsAccessDenied(ex)) return new SharePointAccessDeniedException(enriched, ctx.Url, ex); return new InvalidOperationException(enriched, ex); } // Access-denied reaches us in two shapes: a typed CSOM ServerException // (ServerErrorTypeName = System.UnauthorizedAccessException), and — notably on // Microsoft 365 Group / Teams-connected sites — a bare HTTP (403) FORBIDDEN // WebException carrying "Access is denied ... 0x80070005 (E_ACCESSDENIED)". // Both are ownership issues elevation can fix, so classify either as access-denied. private static bool IsAccessDenied(Exception ex) { for (Exception? cur = ex; cur is not null; cur = cur.InnerException) { if (cur is ServerUnauthorizedAccessException) return true; if (cur is ServerException se && string.Equals(se.ServerErrorTypeName, "System.UnauthorizedAccessException", StringComparison.Ordinal)) return true; if (cur is WebException we && we.Response is HttpWebResponse resp && resp.StatusCode == HttpStatusCode.Forbidden) return true; if (cur.Message.Contains("0x80070005", StringComparison.OrdinalIgnoreCase) || cur.Message.Contains("E_ACCESSDENIED", StringComparison.OrdinalIgnoreCase)) return true; } return false; } internal static bool IsThrottleException(Exception ex) { var msg = ex.Message; return msg.Contains("429") || msg.Contains("503") || msg.Contains("throttl", StringComparison.OrdinalIgnoreCase); } }