using System.IO; using Microsoft.SharePoint.Client; using Serilog; using SharepointToolbox.Web.Core.Helpers; using SharepointToolbox.Web.Core.Models; using SharepointToolbox.Web.Services.Audit; namespace SharepointToolbox.Web.Services; public class FileTransferService : IFileTransferService { private const int ListViewThresholdItemCount = 5000; private readonly IAuditService _audit; public FileTransferService(IAuditService audit) { _audit = audit; } public async Task> TransferAsync( ClientContext sourceCtx, ClientContext destCtx, TransferJob job, IProgress progress, CancellationToken ct) { var srcItemCount = await TryGetListItemCountAsync(sourceCtx, job.SourceLibrary, progress, ct); var dstItemCount = await TryGetListItemCountAsync(destCtx, job.DestinationLibrary, progress, ct); Log.Information("Transfer pre-flight: source={SrcLib} ({SrcCount} items), dest={DstLib} ({DstCount} items)", job.SourceLibrary, srcItemCount, job.DestinationLibrary, dstItemCount); if (srcItemCount > ListViewThresholdItemCount || dstItemCount > ListViewThresholdItemCount) progress.Report(OperationProgress.Indeterminate($"Large library detected (source: {srcItemCount}, dest: {dstItemCount}). Using paged enumeration.")); IReadOnlyList files = job.CopyFolderContents ? await EnumerateFilesAsync(sourceCtx, job, srcItemCount, progress, ct) : Array.Empty(); if (files.Count == 0 && !job.IncludeSourceFolder) { progress.Report(new OperationProgress(0, 0, "No files found to transfer.")); return new BulkOperationSummary(new List>()); } var srcBasePath = await ResolveLibraryPathAsync(sourceCtx, job.SourceLibrary, job.SourceFolderPath, progress, ct); var dstBasePath = await ResolveLibraryPathAsync(destCtx, job.DestinationLibrary, job.DestinationFolderPath, progress, ct); var ensuredFolders = new HashSet(StringComparer.OrdinalIgnoreCase); if (job.IncludeSourceFolder) { var srcFolderName = !string.IsNullOrEmpty(job.SourceFolderPath) ? Path.GetFileName(job.SourceFolderPath.TrimEnd('/')) : job.SourceLibrary; if (!string.IsNullOrEmpty(srcFolderName)) { dstBasePath = $"{dstBasePath}/{srcFolderName}"; await EnsureFolderCachedAsync(destCtx, dstBasePath, ensuredFolders, progress, ct); } } var result = await BulkOperationRunner.RunAsync(files, async (fileRelUrl, idx, token) => { var relativePart = fileRelUrl; if (fileRelUrl.StartsWith(srcBasePath, StringComparison.OrdinalIgnoreCase)) relativePart = fileRelUrl[srcBasePath.Length..].TrimStart('/'); var destFolderRelative = dstBasePath; var fileFolder = Path.GetDirectoryName(relativePart)?.Replace('\\', '/'); if (!string.IsNullOrEmpty(fileFolder)) { destFolderRelative = $"{dstBasePath}/{fileFolder}"; await EnsureFolderCachedAsync(destCtx, destFolderRelative, ensuredFolders, progress, token); } var destFileUrl = $"{destFolderRelative}/{Path.GetFileName(relativePart)}"; await TransferSingleFileAsync(sourceCtx, destCtx, fileRelUrl, destFileUrl, job, progress, token); Log.Information("Transferred: {Source} -> {Dest}", fileRelUrl, destFileUrl); }, progress, ct); await _audit.LogAsync("FileTransfer", sourceCtx.Url, new[] { sourceCtx.Url, destCtx.Url }, $"{result.SuccessCount} files transferred ({job.Mode}), {(result.TotalCount - result.SuccessCount)} failed"); return result; } private async Task TransferSingleFileAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress progress, CancellationToken ct) { try { await ServerSideTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); } catch (ServerException ex) when (IsListViewThresholdException(ex)) { Log.Warning("Server-side transfer hit LVT — falling back to stream copy for {File}.", srcFileUrl); await StreamTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); } } private async Task ServerSideTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress progress, CancellationToken ct) { var srcAbs = ToAbsoluteUrl(sourceCtx, srcFileUrl); var dstAbs = ToAbsoluteUrl(destCtx, dstFileUrl); var srcPath = ResourcePath.FromDecodedUrl(srcAbs); var dstPath = ResourcePath.FromDecodedUrl(dstAbs); bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite; var options = new MoveCopyOptions { KeepBoth = job.ConflictPolicy == ConflictPolicy.Rename, ResetAuthorAndCreatedOnCopy = false }; try { if (job.Mode == TransferMode.Copy) { MoveCopyUtil.CopyFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); } else { MoveCopyUtil.MoveFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); } } catch (ServerException ex) when (job.ConflictPolicy == ConflictPolicy.Skip && ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase)) { Log.Warning("Skipped (already exists): {File}", srcFileUrl); } } private async Task StreamTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress progress, CancellationToken ct) { var effectiveDestUrl = await ResolveDestinationOnConflictAsync(destCtx, dstFileUrl, job, progress, ct); if (effectiveDestUrl == null) { Log.Warning("Skipped (already exists, stream fallback): {File}", srcFileUrl); return; } bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite; ct.ThrowIfCancellationRequested(); var srcFile = sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl); var streamResult = srcFile.OpenBinaryStream(); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); using var buffer = new MemoryStream(); await streamResult.Value.CopyToAsync(buffer, 81920, ct); buffer.Position = 0; var slash = effectiveDestUrl.LastIndexOf('/'); var destFolder = destCtx.Web.GetFolderByServerRelativeUrl(effectiveDestUrl[..slash]); destFolder.Files.Add(new FileCreationInformation { Url = effectiveDestUrl[(slash + 1)..], Overwrite = overwrite, ContentStream = buffer }); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(destCtx, progress, ct); if (job.Mode == TransferMode.Move) { sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl).DeleteObject(); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); } } private static async Task ResolveDestinationOnConflictAsync(ClientContext destCtx, string dstFileUrl, TransferJob job, IProgress progress, CancellationToken ct) { if (job.ConflictPolicy == ConflictPolicy.Overwrite) return dstFileUrl; bool exists = await FileExistsAsync(destCtx, dstFileUrl, progress, ct); if (!exists) return dstFileUrl; if (job.ConflictPolicy == ConflictPolicy.Skip) return null; var dir = dstFileUrl[..dstFileUrl.LastIndexOf('/')]; var leaf = dstFileUrl[(dstFileUrl.LastIndexOf('/') + 1)..]; var stem = Path.GetFileNameWithoutExtension(leaf); var ext = Path.GetExtension(leaf); for (int n = 1; n <= 999; n++) { var candidate = $"{dir}/{stem} ({n}){ext}"; if (!await FileExistsAsync(destCtx, candidate, progress, ct)) return candidate; } throw new InvalidOperationException($"Could not find unused filename for {dstFileUrl} after 999 attempts."); } private static async Task FileExistsAsync(ClientContext ctx, string fileServerRelativeUrl, IProgress progress, CancellationToken ct) { try { var file = ctx.Web.GetFileByServerRelativeUrl(fileServerRelativeUrl); ctx.Load(file, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return file.Exists; } catch { return false; } } internal static bool IsListViewThresholdException(Exception ex) { var msg = ex.Message ?? string.Empty; return msg.Contains("list view threshold", StringComparison.OrdinalIgnoreCase) || msg.Contains("seuil d'affichage", StringComparison.OrdinalIgnoreCase) || msg.Contains("Listenansichtsschwellenwert", StringComparison.OrdinalIgnoreCase); } private async Task> EnumerateFilesAsync(ClientContext ctx, TransferJob job, int sourceItemCount, IProgress progress, CancellationToken ct) { var list = ctx.Web.Lists.GetByTitle(job.SourceLibrary); var rootFolder = list.RootFolder; ctx.Load(rootFolder, f => f.ServerRelativeUrl); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); var libraryRoot = rootFolder.ServerRelativeUrl.TrimEnd('/'); if (job.SelectedFilePaths.Count > 0) return job.SelectedFilePaths.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => $"{libraryRoot}/{p.TrimStart('/')}").ToList(); var baseFolderUrl = libraryRoot; if (!string.IsNullOrEmpty(job.SourceFolderPath)) baseFolderUrl = $"{baseFolderUrl}/{job.SourceFolderPath.TrimStart('/')}"; var files = new List(); await foreach (var item in SharePointPaginationHelper.GetItemsInFolderAsync(ctx, list, baseFolderUrl, recursive: true, viewFields: new[] { "FSObjType", "FileRef", "FileDirRef" }, ct: ct)) { ct.ThrowIfCancellationRequested(); if (item["FSObjType"]?.ToString() != "0") continue; var fileRef = item["FileRef"]?.ToString(); if (string.IsNullOrEmpty(fileRef)) continue; var dir = item["FileDirRef"]?.ToString() ?? string.Empty; if (HasSystemFolderSegment(dir, baseFolderUrl)) continue; files.Add(fileRef); } return files; } private static bool HasSystemFolderSegment(string fileDirRef, string baseFolderUrl) { if (string.IsNullOrEmpty(fileDirRef)) return false; var baseTrim = baseFolderUrl.TrimEnd('/'); if (!fileDirRef.StartsWith(baseTrim, StringComparison.OrdinalIgnoreCase)) return false; var tail = fileDirRef[baseTrim.Length..].Trim('/'); if (string.IsNullOrEmpty(tail)) return false; foreach (var seg in tail.Split('/', StringSplitOptions.RemoveEmptyEntries)) if (seg.StartsWith("_") || seg.Equals("Forms", StringComparison.OrdinalIgnoreCase)) return true; return false; } private async Task TryGetListItemCountAsync(ClientContext ctx, string libraryTitle, IProgress progress, CancellationToken ct) { try { var list = ctx.Web.Lists.GetByTitle(libraryTitle); ctx.Load(list, l => l.ItemCount); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return list.ItemCount; } catch (Exception ex) { Log.Warning("Failed to read ItemCount for {Library}: {Error}", libraryTitle, ex.Message); return -1; } } private async Task EnsureFolderCachedAsync(ClientContext ctx, string folderServerRelativeUrl, HashSet cache, IProgress progress, CancellationToken ct) { var normalized = folderServerRelativeUrl.TrimEnd('/'); if (!cache.Add(normalized)) return; await EnsureFolderAsync(ctx, normalized, progress, ct); } private async Task EnsureFolderAsync(ClientContext ctx, string folderServerRelativeUrl, IProgress progress, CancellationToken ct) { folderServerRelativeUrl = folderServerRelativeUrl.TrimEnd('/'); try { var existing = ctx.Web.GetFolderByServerRelativeUrl(folderServerRelativeUrl); ctx.Load(existing, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); if (existing.Exists) return; } catch { } int slash = folderServerRelativeUrl.LastIndexOf('/'); if (slash <= 0) return; var parentUrl = folderServerRelativeUrl[..slash]; var leafName = folderServerRelativeUrl[(slash + 1)..]; if (string.IsNullOrEmpty(leafName)) return; await EnsureFolderAsync(ctx, parentUrl, progress, ct); ctx.Web.GetFolderByServerRelativeUrl(parentUrl).Folders.Add(leafName); try { await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); } catch (Exception ex) { Log.Warning("EnsureFolder failed at {Parent}/{Leaf}: {Error}", parentUrl, leafName, ex.Message); throw; } } private static string ToAbsoluteUrl(ClientContext ctx, string pathOrUrl) { if (pathOrUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || pathOrUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return pathOrUrl; var uri = new Uri(ctx.Url); return $"{uri.Scheme}://{uri.Host}{(pathOrUrl.StartsWith("/") ? "" : "/")}{pathOrUrl}"; } private static async Task ResolveLibraryPathAsync(ClientContext ctx, string libraryTitle, string relativeFolderPath, IProgress progress, CancellationToken ct) { var list = ctx.Web.Lists.GetByTitle(libraryTitle); ctx.Load(list, l => l.RootFolder.ServerRelativeUrl); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); var basePath = list.RootFolder.ServerRelativeUrl.TrimEnd('/'); if (!string.IsNullOrEmpty(relativeFolderPath)) basePath = $"{basePath}/{relativeFolderPath.TrimStart('/')}"; return basePath; } }