SharepointToolbox-Web/Core/Helpers/ExecuteQueryRetryHelper.cs

using System.Net;
using Microsoft.SharePoint.Client;
using Serilog;
using SharepointToolbox.Web.Core.Models;

namespace SharepointToolbox.Web.Core.Helpers;

public static class ExecuteQueryRetryHelper
{
    private const int MaxRetries = 5;

    public static async Task ExecuteQueryRetryAsync(
        ClientContext ctx,
        IProgress<OperationProgress>? progress = null,
        CancellationToken ct = default)
    {
        int attempt = 0;
        while (true)
        {
            ct.ThrowIfCancellationRequested();
            try
            {
                await ctx.ExecuteQueryAsync();
                return;
            }
            catch (Exception ex) when (IsThrottleException(ex) && attempt < MaxRetries)
            {
                attempt++;
                int delaySeconds = (int)Math.Pow(2, attempt) * 5;
                progress?.Report(OperationProgress.Indeterminate(
                    $"Throttled by SharePoint — retrying in {delaySeconds}s (attempt {attempt}/{MaxRetries})…"));
                await Task.Delay(TimeSpan.FromSeconds(delaySeconds), ct);
            }
            catch (Exception ex)
            {
                throw EnrichException(ctx, ex);
            }
        }
    }

    // CSOM surfaces a 403 as a bare "The remote server returned an error: (403) FORBIDDEN." WebException,
    // which hides the actual SharePoint reason. Pull the server's response body / correlation id so the
    // root cause (token scope, tenant policy, access-denied vs endpoint-blocked) is visible.
    private static Exception EnrichException(ClientContext ctx, Exception ex)
    {
        var detail = new System.Text.StringBuilder();
        detail.Append(ex.Message);
        detail.Append($"  [site={ctx.Url}]");

        if (ex is ServerException se)
        {
            detail.Append($"  [serverErrorType={se.ServerErrorTypeName}; value={se.ServerErrorValue}; " +
                          $"correlationId={se.ServerErrorTraceCorrelationId}; details={se.ServerErrorDetails}]");
        }

        // Walk inner exceptions for a WebException carrying the raw HTTP response body.
        for (Exception? cur = ex; cur is not null; cur = cur.InnerException)
        {
            if (cur is WebException we && we.Response is HttpWebResponse resp)
            {
                detail.Append($"  [httpStatus={(int)resp.StatusCode} {resp.StatusCode}]");
                try
                {
                    using var stream = resp.GetResponseStream();
                    using var reader = new StreamReader(stream);
                    var body = reader.ReadToEnd();
                    if (!string.IsNullOrWhiteSpace(body))
                        detail.Append($"  [responseBody={body.Trim()}]");
                }
                catch { /* body already consumed or unavailable */ }
                break;
            }
        }

        var enriched = detail.ToString();
        Log.Error("CSOM ExecuteQuery failed: {Detail}", enriched);

        // Preserve access-denied as a typed exception so the elevation coordinator can
        // detect it, take site-collection admin ownership, and retry. Everything else
        // stays a generic InvalidOperationException carrying the enriched diagnostic.
        if (IsAccessDenied(ex))
            return new SharePointAccessDeniedException(enriched, ctx.Url, ex);

        return new InvalidOperationException(enriched, ex);
    }

    // Access-denied reaches us in two shapes: a typed CSOM ServerException
    // (ServerErrorTypeName = System.UnauthorizedAccessException), and — notably on
    // Microsoft 365 Group / Teams-connected sites — a bare HTTP (403) FORBIDDEN
    // WebException carrying "Access is denied ... 0x80070005 (E_ACCESSDENIED)".
    // Both are ownership issues elevation can fix, so classify either as access-denied.
    private static bool IsAccessDenied(Exception ex)
    {
        for (Exception? cur = ex; cur is not null; cur = cur.InnerException)
        {
            if (cur is ServerUnauthorizedAccessException)
                return true;
            if (cur is ServerException se &&
                string.Equals(se.ServerErrorTypeName, "System.UnauthorizedAccessException", StringComparison.Ordinal))
                return true;
            if (cur is WebException we && we.Response is HttpWebResponse resp &&
                resp.StatusCode == HttpStatusCode.Forbidden)
                return true;
            if (cur.Message.Contains("0x80070005", StringComparison.OrdinalIgnoreCase) ||
                cur.Message.Contains("E_ACCESSDENIED", StringComparison.OrdinalIgnoreCase))
                return true;
        }
        return false;
    }

    internal static bool IsThrottleException(Exception ex)
    {
        var msg = ex.Message;
        return msg.Contains("429") || msg.Contains("503") ||
               msg.Contains("throttl", StringComparison.OrdinalIgnoreCase);
    }
}