222 lines
14 KiB
C#
222 lines
14 KiB
C#
using System.IO;
|
|
using Microsoft.SharePoint.Client;
|
|
using Serilog;
|
|
using SharepointToolbox.Web.Core.Helpers;
|
|
using SharepointToolbox.Web.Core.Models;
|
|
using SharepointToolbox.Web.Services.Audit;
|
|
|
|
namespace SharepointToolbox.Web.Services;
|
|
|
|
public class FileTransferService : IFileTransferService
|
|
{
|
|
private const int ListViewThresholdItemCount = 5000;
|
|
private readonly IAuditService _audit;
|
|
|
|
public FileTransferService(IAuditService audit) { _audit = audit; }
|
|
|
|
public async Task<BulkOperationSummary<string>> TransferAsync(
|
|
ClientContext sourceCtx, ClientContext destCtx,
|
|
TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var srcItemCount = await TryGetListItemCountAsync(sourceCtx, job.SourceLibrary, progress, ct);
|
|
var dstItemCount = await TryGetListItemCountAsync(destCtx, job.DestinationLibrary, progress, ct);
|
|
Log.Information("Transfer pre-flight: source={SrcLib} ({SrcCount} items), dest={DstLib} ({DstCount} items)", job.SourceLibrary, srcItemCount, job.DestinationLibrary, dstItemCount);
|
|
|
|
if (srcItemCount > ListViewThresholdItemCount || dstItemCount > ListViewThresholdItemCount)
|
|
progress.Report(OperationProgress.Indeterminate($"Large library detected (source: {srcItemCount}, dest: {dstItemCount}). Using paged enumeration."));
|
|
|
|
IReadOnlyList<string> files = job.CopyFolderContents
|
|
? await EnumerateFilesAsync(sourceCtx, job, srcItemCount, progress, ct)
|
|
: Array.Empty<string>();
|
|
|
|
if (files.Count == 0 && !job.IncludeSourceFolder)
|
|
{
|
|
progress.Report(new OperationProgress(0, 0, "No files found to transfer."));
|
|
return new BulkOperationSummary<string>(new List<BulkItemResult<string>>());
|
|
}
|
|
|
|
var srcBasePath = await ResolveLibraryPathAsync(sourceCtx, job.SourceLibrary, job.SourceFolderPath, progress, ct);
|
|
var dstBasePath = await ResolveLibraryPathAsync(destCtx, job.DestinationLibrary, job.DestinationFolderPath, progress, ct);
|
|
var ensuredFolders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
if (job.IncludeSourceFolder)
|
|
{
|
|
var srcFolderName = !string.IsNullOrEmpty(job.SourceFolderPath) ? Path.GetFileName(job.SourceFolderPath.TrimEnd('/')) : job.SourceLibrary;
|
|
if (!string.IsNullOrEmpty(srcFolderName)) { dstBasePath = $"{dstBasePath}/{srcFolderName}"; await EnsureFolderCachedAsync(destCtx, dstBasePath, ensuredFolders, progress, ct); }
|
|
}
|
|
|
|
var result = await BulkOperationRunner.RunAsync(files,
|
|
async (fileRelUrl, idx, token) =>
|
|
{
|
|
var relativePart = fileRelUrl;
|
|
if (fileRelUrl.StartsWith(srcBasePath, StringComparison.OrdinalIgnoreCase))
|
|
relativePart = fileRelUrl[srcBasePath.Length..].TrimStart('/');
|
|
var destFolderRelative = dstBasePath;
|
|
var fileFolder = Path.GetDirectoryName(relativePart)?.Replace('\\', '/');
|
|
if (!string.IsNullOrEmpty(fileFolder)) { destFolderRelative = $"{dstBasePath}/{fileFolder}"; await EnsureFolderCachedAsync(destCtx, destFolderRelative, ensuredFolders, progress, token); }
|
|
var destFileUrl = $"{destFolderRelative}/{Path.GetFileName(relativePart)}";
|
|
await TransferSingleFileAsync(sourceCtx, destCtx, fileRelUrl, destFileUrl, job, progress, token);
|
|
Log.Information("Transferred: {Source} -> {Dest}", fileRelUrl, destFileUrl);
|
|
},
|
|
progress, ct);
|
|
await _audit.LogAsync("FileTransfer",
|
|
sourceCtx.Url,
|
|
new[] { sourceCtx.Url, destCtx.Url },
|
|
$"{result.SuccessCount} files transferred ({job.Mode}), {(result.TotalCount - result.SuccessCount)} failed");
|
|
return result;
|
|
}
|
|
|
|
private async Task TransferSingleFileAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
try { await ServerSideTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); }
|
|
catch (ServerException ex) when (IsListViewThresholdException(ex)) { Log.Warning("Server-side transfer hit LVT — falling back to stream copy for {File}.", srcFileUrl); await StreamTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); }
|
|
}
|
|
|
|
private async Task ServerSideTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var srcAbs = ToAbsoluteUrl(sourceCtx, srcFileUrl);
|
|
var dstAbs = ToAbsoluteUrl(destCtx, dstFileUrl);
|
|
var srcPath = ResourcePath.FromDecodedUrl(srcAbs);
|
|
var dstPath = ResourcePath.FromDecodedUrl(dstAbs);
|
|
bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite;
|
|
var options = new MoveCopyOptions { KeepBoth = job.ConflictPolicy == ConflictPolicy.Rename, ResetAuthorAndCreatedOnCopy = false };
|
|
try
|
|
{
|
|
if (job.Mode == TransferMode.Copy) { MoveCopyUtil.CopyFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
|
else { MoveCopyUtil.MoveFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
|
}
|
|
catch (ServerException ex) when (job.ConflictPolicy == ConflictPolicy.Skip && ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase)) { Log.Warning("Skipped (already exists): {File}", srcFileUrl); }
|
|
}
|
|
|
|
private async Task StreamTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var effectiveDestUrl = await ResolveDestinationOnConflictAsync(destCtx, dstFileUrl, job, progress, ct);
|
|
if (effectiveDestUrl == null) { Log.Warning("Skipped (already exists, stream fallback): {File}", srcFileUrl); return; }
|
|
bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite;
|
|
ct.ThrowIfCancellationRequested();
|
|
var srcFile = sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl);
|
|
var streamResult = srcFile.OpenBinaryStream();
|
|
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct);
|
|
using var buffer = new MemoryStream();
|
|
await streamResult.Value.CopyToAsync(buffer, 81920, ct);
|
|
buffer.Position = 0;
|
|
var slash = effectiveDestUrl.LastIndexOf('/');
|
|
var destFolder = destCtx.Web.GetFolderByServerRelativeUrl(effectiveDestUrl[..slash]);
|
|
destFolder.Files.Add(new FileCreationInformation { Url = effectiveDestUrl[(slash + 1)..], Overwrite = overwrite, ContentStream = buffer });
|
|
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(destCtx, progress, ct);
|
|
if (job.Mode == TransferMode.Move) { sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl).DeleteObject(); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
|
}
|
|
|
|
private static async Task<string?> ResolveDestinationOnConflictAsync(ClientContext destCtx, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
if (job.ConflictPolicy == ConflictPolicy.Overwrite) return dstFileUrl;
|
|
bool exists = await FileExistsAsync(destCtx, dstFileUrl, progress, ct);
|
|
if (!exists) return dstFileUrl;
|
|
if (job.ConflictPolicy == ConflictPolicy.Skip) return null;
|
|
var dir = dstFileUrl[..dstFileUrl.LastIndexOf('/')];
|
|
var leaf = dstFileUrl[(dstFileUrl.LastIndexOf('/') + 1)..];
|
|
var stem = Path.GetFileNameWithoutExtension(leaf);
|
|
var ext = Path.GetExtension(leaf);
|
|
for (int n = 1; n <= 999; n++) { var candidate = $"{dir}/{stem} ({n}){ext}"; if (!await FileExistsAsync(destCtx, candidate, progress, ct)) return candidate; }
|
|
throw new InvalidOperationException($"Could not find unused filename for {dstFileUrl} after 999 attempts.");
|
|
}
|
|
|
|
private static async Task<bool> FileExistsAsync(ClientContext ctx, string fileServerRelativeUrl, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
try { var file = ctx.Web.GetFileByServerRelativeUrl(fileServerRelativeUrl); ctx.Load(file, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return file.Exists; }
|
|
catch { return false; }
|
|
}
|
|
|
|
internal static bool IsListViewThresholdException(Exception ex)
|
|
{
|
|
var msg = ex.Message ?? string.Empty;
|
|
return msg.Contains("list view threshold", StringComparison.OrdinalIgnoreCase) || msg.Contains("seuil d'affichage", StringComparison.OrdinalIgnoreCase) || msg.Contains("Listenansichtsschwellenwert", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private async Task<IReadOnlyList<string>> EnumerateFilesAsync(ClientContext ctx, TransferJob job, int sourceItemCount, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var list = ctx.Web.Lists.GetByTitle(job.SourceLibrary);
|
|
var rootFolder = list.RootFolder;
|
|
ctx.Load(rootFolder, f => f.ServerRelativeUrl);
|
|
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct);
|
|
var libraryRoot = rootFolder.ServerRelativeUrl.TrimEnd('/');
|
|
|
|
if (job.SelectedFilePaths.Count > 0)
|
|
return job.SelectedFilePaths.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => $"{libraryRoot}/{p.TrimStart('/')}").ToList();
|
|
|
|
var baseFolderUrl = libraryRoot;
|
|
if (!string.IsNullOrEmpty(job.SourceFolderPath)) baseFolderUrl = $"{baseFolderUrl}/{job.SourceFolderPath.TrimStart('/')}";
|
|
|
|
var files = new List<string>();
|
|
await foreach (var item in SharePointPaginationHelper.GetItemsInFolderAsync(ctx, list, baseFolderUrl, recursive: true, viewFields: new[] { "FSObjType", "FileRef", "FileDirRef" }, ct: ct))
|
|
{
|
|
ct.ThrowIfCancellationRequested();
|
|
if (item["FSObjType"]?.ToString() != "0") continue;
|
|
var fileRef = item["FileRef"]?.ToString();
|
|
if (string.IsNullOrEmpty(fileRef)) continue;
|
|
var dir = item["FileDirRef"]?.ToString() ?? string.Empty;
|
|
if (HasSystemFolderSegment(dir, baseFolderUrl)) continue;
|
|
files.Add(fileRef);
|
|
}
|
|
return files;
|
|
}
|
|
|
|
private static bool HasSystemFolderSegment(string fileDirRef, string baseFolderUrl)
|
|
{
|
|
if (string.IsNullOrEmpty(fileDirRef)) return false;
|
|
var baseTrim = baseFolderUrl.TrimEnd('/');
|
|
if (!fileDirRef.StartsWith(baseTrim, StringComparison.OrdinalIgnoreCase)) return false;
|
|
var tail = fileDirRef[baseTrim.Length..].Trim('/');
|
|
if (string.IsNullOrEmpty(tail)) return false;
|
|
foreach (var seg in tail.Split('/', StringSplitOptions.RemoveEmptyEntries))
|
|
if (seg.StartsWith("_") || seg.Equals("Forms", StringComparison.OrdinalIgnoreCase)) return true;
|
|
return false;
|
|
}
|
|
|
|
private async Task<int> TryGetListItemCountAsync(ClientContext ctx, string libraryTitle, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
try { var list = ctx.Web.Lists.GetByTitle(libraryTitle); ctx.Load(list, l => l.ItemCount); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return list.ItemCount; }
|
|
catch (Exception ex) { Log.Warning("Failed to read ItemCount for {Library}: {Error}", libraryTitle, ex.Message); return -1; }
|
|
}
|
|
|
|
private async Task EnsureFolderCachedAsync(ClientContext ctx, string folderServerRelativeUrl, HashSet<string> cache, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var normalized = folderServerRelativeUrl.TrimEnd('/');
|
|
if (!cache.Add(normalized)) return;
|
|
await EnsureFolderAsync(ctx, normalized, progress, ct);
|
|
}
|
|
|
|
private async Task EnsureFolderAsync(ClientContext ctx, string folderServerRelativeUrl, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
folderServerRelativeUrl = folderServerRelativeUrl.TrimEnd('/');
|
|
try { var existing = ctx.Web.GetFolderByServerRelativeUrl(folderServerRelativeUrl); ctx.Load(existing, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); if (existing.Exists) return; }
|
|
catch { }
|
|
int slash = folderServerRelativeUrl.LastIndexOf('/');
|
|
if (slash <= 0) return;
|
|
var parentUrl = folderServerRelativeUrl[..slash];
|
|
var leafName = folderServerRelativeUrl[(slash + 1)..];
|
|
if (string.IsNullOrEmpty(leafName)) return;
|
|
await EnsureFolderAsync(ctx, parentUrl, progress, ct);
|
|
ctx.Web.GetFolderByServerRelativeUrl(parentUrl).Folders.Add(leafName);
|
|
try { await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); }
|
|
catch (Exception ex) { Log.Warning("EnsureFolder failed at {Parent}/{Leaf}: {Error}", parentUrl, leafName, ex.Message); throw; }
|
|
}
|
|
|
|
private static string ToAbsoluteUrl(ClientContext ctx, string pathOrUrl)
|
|
{
|
|
if (pathOrUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || pathOrUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return pathOrUrl;
|
|
var uri = new Uri(ctx.Url);
|
|
return $"{uri.Scheme}://{uri.Host}{(pathOrUrl.StartsWith("/") ? "" : "/")}{pathOrUrl}";
|
|
}
|
|
|
|
private static async Task<string> ResolveLibraryPathAsync(ClientContext ctx, string libraryTitle, string relativeFolderPath, IProgress<OperationProgress> progress, CancellationToken ct)
|
|
{
|
|
var list = ctx.Web.Lists.GetByTitle(libraryTitle);
|
|
ctx.Load(list, l => l.RootFolder.ServerRelativeUrl);
|
|
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct);
|
|
var basePath = list.RootFolder.ServerRelativeUrl.TrimEnd('/');
|
|
if (!string.IsNullOrEmpty(relativeFolderPath)) basePath = $"{basePath}/{relativeFolderPath.TrimStart('/')}";
|
|
return basePath;
|
|
}
|
|
}
|