Initial commit
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
using System.IO;
|
||||
using Microsoft.SharePoint.Client;
|
||||
using Serilog;
|
||||
using SharepointToolbox.Web.Core.Helpers;
|
||||
using SharepointToolbox.Web.Core.Models;
|
||||
using SharepointToolbox.Web.Services.Audit;
|
||||
|
||||
namespace SharepointToolbox.Web.Services;
|
||||
|
||||
public class FileTransferService : IFileTransferService
|
||||
{
|
||||
private const int ListViewThresholdItemCount = 5000;
|
||||
private readonly IAuditService _audit;
|
||||
|
||||
public FileTransferService(IAuditService audit) { _audit = audit; }
|
||||
|
||||
public async Task<BulkOperationSummary<string>> TransferAsync(
|
||||
ClientContext sourceCtx, ClientContext destCtx,
|
||||
TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var srcItemCount = await TryGetListItemCountAsync(sourceCtx, job.SourceLibrary, progress, ct);
|
||||
var dstItemCount = await TryGetListItemCountAsync(destCtx, job.DestinationLibrary, progress, ct);
|
||||
Log.Information("Transfer pre-flight: source={SrcLib} ({SrcCount} items), dest={DstLib} ({DstCount} items)", job.SourceLibrary, srcItemCount, job.DestinationLibrary, dstItemCount);
|
||||
|
||||
if (srcItemCount > ListViewThresholdItemCount || dstItemCount > ListViewThresholdItemCount)
|
||||
progress.Report(OperationProgress.Indeterminate($"Large library detected (source: {srcItemCount}, dest: {dstItemCount}). Using paged enumeration."));
|
||||
|
||||
IReadOnlyList<string> files = job.CopyFolderContents
|
||||
? await EnumerateFilesAsync(sourceCtx, job, srcItemCount, progress, ct)
|
||||
: Array.Empty<string>();
|
||||
|
||||
if (files.Count == 0 && !job.IncludeSourceFolder)
|
||||
{
|
||||
progress.Report(new OperationProgress(0, 0, "No files found to transfer."));
|
||||
return new BulkOperationSummary<string>(new List<BulkItemResult<string>>());
|
||||
}
|
||||
|
||||
var srcBasePath = await ResolveLibraryPathAsync(sourceCtx, job.SourceLibrary, job.SourceFolderPath, progress, ct);
|
||||
var dstBasePath = await ResolveLibraryPathAsync(destCtx, job.DestinationLibrary, job.DestinationFolderPath, progress, ct);
|
||||
var ensuredFolders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (job.IncludeSourceFolder)
|
||||
{
|
||||
var srcFolderName = !string.IsNullOrEmpty(job.SourceFolderPath) ? Path.GetFileName(job.SourceFolderPath.TrimEnd('/')) : job.SourceLibrary;
|
||||
if (!string.IsNullOrEmpty(srcFolderName)) { dstBasePath = $"{dstBasePath}/{srcFolderName}"; await EnsureFolderCachedAsync(destCtx, dstBasePath, ensuredFolders, progress, ct); }
|
||||
}
|
||||
|
||||
var result = await BulkOperationRunner.RunAsync(files,
|
||||
async (fileRelUrl, idx, token) =>
|
||||
{
|
||||
var relativePart = fileRelUrl;
|
||||
if (fileRelUrl.StartsWith(srcBasePath, StringComparison.OrdinalIgnoreCase))
|
||||
relativePart = fileRelUrl[srcBasePath.Length..].TrimStart('/');
|
||||
var destFolderRelative = dstBasePath;
|
||||
var fileFolder = Path.GetDirectoryName(relativePart)?.Replace('\\', '/');
|
||||
if (!string.IsNullOrEmpty(fileFolder)) { destFolderRelative = $"{dstBasePath}/{fileFolder}"; await EnsureFolderCachedAsync(destCtx, destFolderRelative, ensuredFolders, progress, token); }
|
||||
var destFileUrl = $"{destFolderRelative}/{Path.GetFileName(relativePart)}";
|
||||
await TransferSingleFileAsync(sourceCtx, destCtx, fileRelUrl, destFileUrl, job, progress, token);
|
||||
Log.Information("Transferred: {Source} -> {Dest}", fileRelUrl, destFileUrl);
|
||||
},
|
||||
progress, ct);
|
||||
await _audit.LogAsync("FileTransfer",
|
||||
sourceCtx.Url,
|
||||
new[] { sourceCtx.Url, destCtx.Url },
|
||||
$"{result.SuccessCount} files transferred ({job.Mode}), {(result.TotalCount - result.SuccessCount)} failed");
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task TransferSingleFileAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
try { await ServerSideTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); }
|
||||
catch (ServerException ex) when (IsListViewThresholdException(ex)) { Log.Warning("Server-side transfer hit LVT — falling back to stream copy for {File}.", srcFileUrl); await StreamTransferAsync(sourceCtx, destCtx, srcFileUrl, dstFileUrl, job, progress, ct); }
|
||||
}
|
||||
|
||||
private async Task ServerSideTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var srcAbs = ToAbsoluteUrl(sourceCtx, srcFileUrl);
|
||||
var dstAbs = ToAbsoluteUrl(destCtx, dstFileUrl);
|
||||
var srcPath = ResourcePath.FromDecodedUrl(srcAbs);
|
||||
var dstPath = ResourcePath.FromDecodedUrl(dstAbs);
|
||||
bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite;
|
||||
var options = new MoveCopyOptions { KeepBoth = job.ConflictPolicy == ConflictPolicy.Rename, ResetAuthorAndCreatedOnCopy = false };
|
||||
try
|
||||
{
|
||||
if (job.Mode == TransferMode.Copy) { MoveCopyUtil.CopyFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
||||
else { MoveCopyUtil.MoveFileByPath(sourceCtx, srcPath, dstPath, overwrite, options); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
||||
}
|
||||
catch (ServerException ex) when (job.ConflictPolicy == ConflictPolicy.Skip && ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase)) { Log.Warning("Skipped (already exists): {File}", srcFileUrl); }
|
||||
}
|
||||
|
||||
private async Task StreamTransferAsync(ClientContext sourceCtx, ClientContext destCtx, string srcFileUrl, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var effectiveDestUrl = await ResolveDestinationOnConflictAsync(destCtx, dstFileUrl, job, progress, ct);
|
||||
if (effectiveDestUrl == null) { Log.Warning("Skipped (already exists, stream fallback): {File}", srcFileUrl); return; }
|
||||
bool overwrite = job.ConflictPolicy == ConflictPolicy.Overwrite;
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var srcFile = sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl);
|
||||
var streamResult = srcFile.OpenBinaryStream();
|
||||
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct);
|
||||
using var buffer = new MemoryStream();
|
||||
await streamResult.Value.CopyToAsync(buffer, 81920, ct);
|
||||
buffer.Position = 0;
|
||||
var slash = effectiveDestUrl.LastIndexOf('/');
|
||||
var destFolder = destCtx.Web.GetFolderByServerRelativeUrl(effectiveDestUrl[..slash]);
|
||||
destFolder.Files.Add(new FileCreationInformation { Url = effectiveDestUrl[(slash + 1)..], Overwrite = overwrite, ContentStream = buffer });
|
||||
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(destCtx, progress, ct);
|
||||
if (job.Mode == TransferMode.Move) { sourceCtx.Web.GetFileByServerRelativeUrl(srcFileUrl).DeleteObject(); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(sourceCtx, progress, ct); }
|
||||
}
|
||||
|
||||
private static async Task<string?> ResolveDestinationOnConflictAsync(ClientContext destCtx, string dstFileUrl, TransferJob job, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
if (job.ConflictPolicy == ConflictPolicy.Overwrite) return dstFileUrl;
|
||||
bool exists = await FileExistsAsync(destCtx, dstFileUrl, progress, ct);
|
||||
if (!exists) return dstFileUrl;
|
||||
if (job.ConflictPolicy == ConflictPolicy.Skip) return null;
|
||||
var dir = dstFileUrl[..dstFileUrl.LastIndexOf('/')];
|
||||
var leaf = dstFileUrl[(dstFileUrl.LastIndexOf('/') + 1)..];
|
||||
var stem = Path.GetFileNameWithoutExtension(leaf);
|
||||
var ext = Path.GetExtension(leaf);
|
||||
for (int n = 1; n <= 999; n++) { var candidate = $"{dir}/{stem} ({n}){ext}"; if (!await FileExistsAsync(destCtx, candidate, progress, ct)) return candidate; }
|
||||
throw new InvalidOperationException($"Could not find unused filename for {dstFileUrl} after 999 attempts.");
|
||||
}
|
||||
|
||||
private static async Task<bool> FileExistsAsync(ClientContext ctx, string fileServerRelativeUrl, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
try { var file = ctx.Web.GetFileByServerRelativeUrl(fileServerRelativeUrl); ctx.Load(file, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return file.Exists; }
|
||||
catch { return false; }
|
||||
}
|
||||
|
||||
internal static bool IsListViewThresholdException(Exception ex)
|
||||
{
|
||||
var msg = ex.Message ?? string.Empty;
|
||||
return msg.Contains("list view threshold", StringComparison.OrdinalIgnoreCase) || msg.Contains("seuil d'affichage", StringComparison.OrdinalIgnoreCase) || msg.Contains("Listenansichtsschwellenwert", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<string>> EnumerateFilesAsync(ClientContext ctx, TransferJob job, int sourceItemCount, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var list = ctx.Web.Lists.GetByTitle(job.SourceLibrary);
|
||||
var rootFolder = list.RootFolder;
|
||||
ctx.Load(rootFolder, f => f.ServerRelativeUrl);
|
||||
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct);
|
||||
var libraryRoot = rootFolder.ServerRelativeUrl.TrimEnd('/');
|
||||
|
||||
if (job.SelectedFilePaths.Count > 0)
|
||||
return job.SelectedFilePaths.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => $"{libraryRoot}/{p.TrimStart('/')}").ToList();
|
||||
|
||||
var baseFolderUrl = libraryRoot;
|
||||
if (!string.IsNullOrEmpty(job.SourceFolderPath)) baseFolderUrl = $"{baseFolderUrl}/{job.SourceFolderPath.TrimStart('/')}";
|
||||
|
||||
var files = new List<string>();
|
||||
await foreach (var item in SharePointPaginationHelper.GetItemsInFolderAsync(ctx, list, baseFolderUrl, recursive: true, viewFields: new[] { "FSObjType", "FileRef", "FileDirRef" }, ct: ct))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (item["FSObjType"]?.ToString() != "0") continue;
|
||||
var fileRef = item["FileRef"]?.ToString();
|
||||
if (string.IsNullOrEmpty(fileRef)) continue;
|
||||
var dir = item["FileDirRef"]?.ToString() ?? string.Empty;
|
||||
if (HasSystemFolderSegment(dir, baseFolderUrl)) continue;
|
||||
files.Add(fileRef);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private static bool HasSystemFolderSegment(string fileDirRef, string baseFolderUrl)
|
||||
{
|
||||
if (string.IsNullOrEmpty(fileDirRef)) return false;
|
||||
var baseTrim = baseFolderUrl.TrimEnd('/');
|
||||
if (!fileDirRef.StartsWith(baseTrim, StringComparison.OrdinalIgnoreCase)) return false;
|
||||
var tail = fileDirRef[baseTrim.Length..].Trim('/');
|
||||
if (string.IsNullOrEmpty(tail)) return false;
|
||||
foreach (var seg in tail.Split('/', StringSplitOptions.RemoveEmptyEntries))
|
||||
if (seg.StartsWith("_") || seg.Equals("Forms", StringComparison.OrdinalIgnoreCase)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<int> TryGetListItemCountAsync(ClientContext ctx, string libraryTitle, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
try { var list = ctx.Web.Lists.GetByTitle(libraryTitle); ctx.Load(list, l => l.ItemCount); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); return list.ItemCount; }
|
||||
catch (Exception ex) { Log.Warning("Failed to read ItemCount for {Library}: {Error}", libraryTitle, ex.Message); return -1; }
|
||||
}
|
||||
|
||||
private async Task EnsureFolderCachedAsync(ClientContext ctx, string folderServerRelativeUrl, HashSet<string> cache, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var normalized = folderServerRelativeUrl.TrimEnd('/');
|
||||
if (!cache.Add(normalized)) return;
|
||||
await EnsureFolderAsync(ctx, normalized, progress, ct);
|
||||
}
|
||||
|
||||
private async Task EnsureFolderAsync(ClientContext ctx, string folderServerRelativeUrl, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
folderServerRelativeUrl = folderServerRelativeUrl.TrimEnd('/');
|
||||
try { var existing = ctx.Web.GetFolderByServerRelativeUrl(folderServerRelativeUrl); ctx.Load(existing, f => f.Exists); await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); if (existing.Exists) return; }
|
||||
catch { }
|
||||
int slash = folderServerRelativeUrl.LastIndexOf('/');
|
||||
if (slash <= 0) return;
|
||||
var parentUrl = folderServerRelativeUrl[..slash];
|
||||
var leafName = folderServerRelativeUrl[(slash + 1)..];
|
||||
if (string.IsNullOrEmpty(leafName)) return;
|
||||
await EnsureFolderAsync(ctx, parentUrl, progress, ct);
|
||||
ctx.Web.GetFolderByServerRelativeUrl(parentUrl).Folders.Add(leafName);
|
||||
try { await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct); }
|
||||
catch (Exception ex) { Log.Warning("EnsureFolder failed at {Parent}/{Leaf}: {Error}", parentUrl, leafName, ex.Message); throw; }
|
||||
}
|
||||
|
||||
private static string ToAbsoluteUrl(ClientContext ctx, string pathOrUrl)
|
||||
{
|
||||
if (pathOrUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || pathOrUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return pathOrUrl;
|
||||
var uri = new Uri(ctx.Url);
|
||||
return $"{uri.Scheme}://{uri.Host}{(pathOrUrl.StartsWith("/") ? "" : "/")}{pathOrUrl}";
|
||||
}
|
||||
|
||||
private static async Task<string> ResolveLibraryPathAsync(ClientContext ctx, string libraryTitle, string relativeFolderPath, IProgress<OperationProgress> progress, CancellationToken ct)
|
||||
{
|
||||
var list = ctx.Web.Lists.GetByTitle(libraryTitle);
|
||||
ctx.Load(list, l => l.RootFolder.ServerRelativeUrl);
|
||||
await ExecuteQueryRetryHelper.ExecuteQueryRetryAsync(ctx, progress, ct);
|
||||
var basePath = list.RootFolder.ServerRelativeUrl.TrimEnd('/');
|
||||
if (!string.IsNullOrEmpty(relativeFolderPath)) basePath = $"{basePath}/{relativeFolderPath.TrimStart('/')}";
|
||||
return basePath;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user