using System.IO; using Microsoft.SharePoint.Client; using Serilog; using SharepointToolbox.Core.Helpers; using SharepointToolbox.Core.Models; namespace SharepointToolbox.Services; /// /// Orchestrates server-side file copy/move between two SharePoint libraries /// (same or different tenants). Uses for the /// transfer itself so bytes never round-trip through the local machine. /// Folder creation and enumeration are done via CSOM; all ambient retries /// flow through . /// public class FileTransferService : IFileTransferService { /// /// Runs the configured . Enumerates source files /// (unless the job is folder-only), pre-creates destination folders, then /// copies or moves each file according to /// and . Returns a per-item /// summary where failures are reported individually — the method does /// not abort on first error so partial transfers are recoverable. /// public async Task> TransferAsync( ClientContext sourceCtx, ClientContext destCtx, TransferJob job, IProgress progress, CancellationToken ct) { // 1. Enumerate files from source (unless contents are suppressed). IReadOnlyList files; if (job.CopyFolderContents) { progress.Report(new OperationProgress(0, 0, "Enumerating source files...")); files = await EnumerateFilesAsync(sourceCtx, job, progress, ct); } else { files = Array.Empty(); } // When CopyFolderContents is off, the job is folder-only: ensure the // destination folder is created below (IncludeSourceFolder branch) and // return without iterating any files. if (files.Count == 0 && !job.IncludeSourceFolder) { progress.Report(new OperationProgress(0, 0, "No files found to transfer.")); return new BulkOperationSummary(new List>()); } // 2. Build source and destination base paths. Resolve library roots via // CSOM — constructing from title breaks for localized libraries whose // URL segment differs (e.g. title "Documents" → URL "Shared Documents"), // causing "Access denied" when CSOM tries to touch a non-existent path. var srcBasePath = await ResolveLibraryPathAsync( sourceCtx, job.SourceLibrary, job.SourceFolderPath, progress, ct); var dstBasePath = await ResolveLibraryPathAsync( destCtx, job.DestinationLibrary, job.DestinationFolderPath, progress, ct); // When IncludeSourceFolder is set, recreate the source folder name under // destination so dest/srcFolderName/... mirrors the source tree. When // no SourceFolderPath is set, fall back to the source library name. // Also pre-create the folder itself — per-file EnsureFolder only fires // for nested paths, so flat files at the root of the source folder // would otherwise copy into a missing parent and fail. if (job.IncludeSourceFolder) { var srcFolderName = !string.IsNullOrEmpty(job.SourceFolderPath) ? Path.GetFileName(job.SourceFolderPath.TrimEnd('/')) : job.SourceLibrary; if (!string.IsNullOrEmpty(srcFolderName)) { dstBasePath = $"{dstBasePath}/{srcFolderName}"; await EnsureFolderAsync(destCtx, dstBasePath, progress, ct); } } // 3. Transfer each file using BulkOperationRunner return await BulkOperationRunner.RunAsync( files, async (fileRelUrl, idx, token) => { // Compute destination path by replacing source base with dest base var relativePart = fileRelUrl; if (fileRelUrl.StartsWith(srcBasePath, StringComparison.OrdinalIgnoreCase)) relativePart = fileRelUrl.Substring(srcBasePath.Length).TrimStart('/'); // Ensure destination folder exists var destFolderRelative = dstBasePath; var fileFolder = Path.GetDirectoryName(relativePart)?.Replace('\\', '/'); if (!string.IsNullOrEmpty(fileFolder)) { destFolderRelative = $"{dstBasePath}/{fileFolder}"; await EnsureFolderAsync(destCtx, destFolderRelative, progress, token); } var fileName = Path.GetFileName(relativePart); var destFileUrl = $"{destFolderRelative}/{fileName}"; await TransferSingleFileAsync(sourceCtx, destCtx, fileRelUrl, destFileUrl, job, progress, token); Log.Information("Transferred: {Source} -> {Dest}", fileRelUrl, destFileUrl); }, progress, ct); } private async Task TransferSingleFileAsync( ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress progress, CancellationToken ct) { // MoveCopyUtil.CopyFileByPath expects absolute URLs (scheme + host), // not server-relative paths. Passing "/sites/..." silently fails or // returns no error yet copies nothing — especially across site // collections. Prefix with the owning site's scheme+host. var srcAbs = ToAbsoluteUrl(sourceCtx, srcFileUrl); var dstAbs = ToAbsoluteUrl(destCtx, dstFileUrl); var srcPath = ResourcePath.FromDecodedUrl(srcAbs); var dstPath = ResourcePath.FromDecodedUrl(dstAbs); bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite; var options = new MoveCopyOptions { KeepBoth = job.ConflictPolicy == ConflictPolicy.Rename, ResetAuthorAndCreatedOnCopy = false, // best-effort metadata preservation }; try { if (job.Mode == TransferMode.Copy) { MoveCopyUtil.CopyFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); } else // Move { MoveCopyUtil.MoveFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); } } catch (ServerException ex) when (job.ConflictPolicy == ConflictPolicy.Skip && ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase)) { Log.Warning("Skipped (already exists): {File}", srcFileUrl); } } private async Task> EnumerateFilesAsync( ClientContext ctx, TransferJob job, IProgress progress, CancellationToken ct) { var list = ctx.Web.Lists.GetByTitle(job.SourceLibrary); var rootFolder = list.RootFolder; ctx.Load(rootFolder, f => f.ServerRelativeUrl); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); var libraryRoot = rootFolder.ServerRelativeUrl.TrimEnd('/'); // Explicit per-file selection overrides folder enumeration. Paths are // library-relative (e.g. "SubFolder/file.docx") and get resolved to // full server-relative URLs here. if (job.SelectedFilePaths.Count > 0) { return job.SelectedFilePaths .Where(p => !string.IsNullOrWhiteSpace(p)) .Select(p => $"{libraryRoot}/{p.TrimStart('/')}") .ToList(); } var baseFolderUrl = libraryRoot; if (!string.IsNullOrEmpty(job.SourceFolderPath)) baseFolderUrl = $"{baseFolderUrl}/{job.SourceFolderPath.TrimStart('/')}"; // Paginated recursive CAML query — Folder.Files / Folder.Folders lazy // loading hits the list-view threshold on libraries > 5,000 items. var files = new List(); await foreach (var item in SharePointPaginationHelper.GetItemsInFolderAsync( ctx, list, baseFolderUrl, recursive: true, viewFields: new[] { "FSObjType", "FileRef", "FileDirRef" }, ct: ct)) { ct.ThrowIfCancellationRequested(); if (item["FSObjType"]?.ToString() != "0") continue; // files only var fileRef = item["FileRef"]?.ToString(); if (string.IsNullOrEmpty(fileRef)) continue; // Skip files under SharePoint system folders (e.g. "Forms", "_*"). var dir = item["FileDirRef"]?.ToString() ?? string.Empty; if (HasSystemFolderSegment(dir, baseFolderUrl)) continue; files.Add(fileRef); } return files; } private static bool HasSystemFolderSegment(string fileDirRef, string baseFolderUrl) { if (string.IsNullOrEmpty(fileDirRef)) return false; var baseTrim = baseFolderUrl.TrimEnd('/'); if (!fileDirRef.StartsWith(baseTrim, StringComparison.OrdinalIgnoreCase)) return false; var tail = fileDirRef.Substring(baseTrim.Length).Trim('/'); if (string.IsNullOrEmpty(tail)) return false; foreach (var seg in tail.Split('/', StringSplitOptions.RemoveEmptyEntries)) { if (seg.StartsWith("_", StringComparison.Ordinal) || seg.Equals("Forms", StringComparison.OrdinalIgnoreCase)) return true; } return false; } private async Task EnsureFolderAsync( ClientContext ctx, string folderServerRelativeUrl, IProgress progress, CancellationToken ct) { folderServerRelativeUrl = folderServerRelativeUrl.TrimEnd('/'); // Already there? try { var existing = ctx.Web.GetFolderByServerRelativeUrl(folderServerRelativeUrl); ctx.Load(existing, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); if (existing.Exists) return; } catch { /* not present — fall through to creation */ } // Walk the path, creating each missing segment. `Web.Folders.Add(url)` is // ambiguous across CSOM versions (some treat the arg as relative to Web, // others server-relative), which produces bogus paths + "Access denied". // Resolve the parent explicitly and add only the leaf name instead. int slash = folderServerRelativeUrl.LastIndexOf('/'); if (slash <= 0) return; var parentUrl = folderServerRelativeUrl.Substring(0, slash); var leafName = folderServerRelativeUrl.Substring(slash + 1); if (string.IsNullOrEmpty(leafName)) return; // Recurse to guarantee the parent exists first. await EnsureFolderAsync(ctx, parentUrl, progress, ct); var parent = ctx.Web.GetFolderByServerRelativeUrl(parentUrl); parent.Folders.Add(leafName); try { await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); } catch (Exception ex) { Log.Warning("EnsureFolder failed at {Parent}/{Leaf}: {Error}", parentUrl, leafName, ex.Message); throw; } } private static string ToAbsoluteUrl(ClientContext ctx, string pathOrUrl) { if (pathOrUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || pathOrUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return pathOrUrl; var uri = new Uri(ctx.Url); return $"{uri.Scheme}://{uri.Host}{(pathOrUrl.StartsWith("/") ? "" : "/")}{pathOrUrl}"; } private static async Task ResolveLibraryPathAsync( ClientContext ctx, string libraryTitle, string relativeFolderPath, IProgress progress, CancellationToken ct) { var list = ctx.Web.Lists.GetByTitle(libraryTitle); ctx.Load(list, l => l.RootFolder.ServerRelativeUrl); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); var basePath = list.RootFolder.ServerRelativeUrl.TrimEnd('/'); if (!string.IsNullOrEmpty(relativeFolderPath)) basePath = $"{basePath}/{relativeFolderPath.TrimStart('/')}"; return basePath; } }