Sharepoint-Toolbox/SharepointToolbox/Core/Helpers/SharePointPaginationHelper.cs

using System.Runtime.CompilerServices;
using Microsoft.SharePoint.Client;
using SharepointToolbox.Core.Models;

namespace SharepointToolbox.Core.Helpers;

public static class SharePointPaginationHelper
{
    // Max page size SharePoint honors with Paged='TRUE' (threshold bypass).
    private const int DefaultRowLimit = 5000;

    /// <summary>
    /// Enumerates all items in a SharePoint list, bypassing the 5,000-item threshold.
    /// Uses CamlQuery with Paged='TRUE' RowLimit and ListItemCollectionPosition for pagination.
    /// Never call ExecuteQuery directly on a list — always use this helper.
    /// </summary>
    public static async IAsyncEnumerable<ListItem> GetAllItemsAsync(
        ClientContext ctx,
        List list,
        CamlQuery? baseQuery = null,
        [EnumeratorCancellation] CancellationToken ct = default)
    {
        var query = baseQuery ?? CamlQuery.CreateAllItemsQuery();
        query.ViewXml = BuildPagedViewXml(query.ViewXml, DefaultRowLimit);
        query.ListItemCollectionPosition = null;

        do
        {
            ct.ThrowIfCancellationRequested();
            var items = list.GetItems(query);
            ctx.Load(items);
            await ctx.ExecuteQueryAsync();

            foreach (var item in items)
                yield return item;

            query.ListItemCollectionPosition = items.ListItemCollectionPosition;
        }
        while (query.ListItemCollectionPosition != null);
    }

    /// <summary>
    /// Enumerates items within a specific folder (direct children by default, or
    /// recursive when <paramref name="recursive"/> is true). Uses paginated CAML
    /// with no WHERE clause so it works on libraries above the 5,000-item threshold.
    /// Callers filter by FSObjType client-side via the returned ListItem fields.
    /// </summary>
    public static async IAsyncEnumerable<ListItem> GetItemsInFolderAsync(
        ClientContext ctx,
        List list,
        string folderServerRelativeUrl,
        bool recursive,
        string[]? viewFields = null,
        [EnumeratorCancellation] CancellationToken ct = default)
    {
        var fields = viewFields ?? new[]
        {
            "FSObjType", "FileRef", "FileLeafRef", "FileDirRef", "File_x0020_Size"
        };

        var viewFieldsXml = string.Join(string.Empty,
            fields.Select(f => $"<FieldRef Name='{f}' />"));

        var scope = recursive ? " Scope='RecursiveAll'" : string.Empty;
        var viewXml =
            $"<View{scope}>" +
              "<Query></Query>" +
              $"<ViewFields>{viewFieldsXml}</ViewFields>" +
              $"<RowLimit Paged='TRUE'>{DefaultRowLimit}</RowLimit>" +
            "</View>";

        var query = new CamlQuery
        {
            ViewXml = viewXml,
            FolderServerRelativeUrl = folderServerRelativeUrl,
            ListItemCollectionPosition = null
        };

        do
        {
            ct.ThrowIfCancellationRequested();
            var items = list.GetItems(query);
            ctx.Load(items);
            await ctx.ExecuteQueryAsync();

            foreach (var item in items)
                yield return item;

            query.ListItemCollectionPosition = items.ListItemCollectionPosition;
        }
        while (query.ListItemCollectionPosition != null);
    }

    internal static string BuildPagedViewXml(string? existingXml, int rowLimit)
    {
        if (string.IsNullOrWhiteSpace(existingXml))
            return $"<View><RowLimit Paged='TRUE'>{rowLimit}</RowLimit></View>";

        // Replace any existing <RowLimit ...>n</RowLimit> with paged form.
        if (System.Text.RegularExpressions.Regex.IsMatch(
                existingXml, @"<RowLimit[^>]*>\d+</RowLimit>",
                System.Text.RegularExpressions.RegexOptions.IgnoreCase))
        {
            return System.Text.RegularExpressions.Regex.Replace(
                existingXml, @"<RowLimit[^>]*>\d+</RowLimit>",
                $"<RowLimit Paged='TRUE'>{rowLimit}</RowLimit>",
                System.Text.RegularExpressions.RegexOptions.IgnoreCase);
        }
        return existingXml.Replace("</View>",
            $"<RowLimit Paged='TRUE'>{rowLimit}</RowLimit></View>",
            StringComparison.OrdinalIgnoreCase);
    }
}