Sharepoint-Toolbox/.planning/milestones/v1.0-phases/04-bulk-operations-and-provisioning/04-02-PLAN.md

phase, plan, title, status, wave, depends_on, files_modified, autonomous, requirements, must_haves

phase	plan	title	status	wave	depends_on	files_modified	autonomous	requirements	must_haves
04	02	CsvValidationService + TemplateRepository	pending	1	04-01	SharepointToolbox/Services/CsvValidationService.cs SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs SharepointToolbox.Tests/Services/CsvValidationServiceTests.cs SharepointToolbox.Tests/Services/TemplateRepositoryTests.cs	true	BULK-05 TMPL-03 TMPL-04 FOLD-02	truths artifacts key_links CsvValidationService parses CSV with CsvHelper, auto-detects delimiter (comma or semicolon), detects BOM Each row is validated individually — invalid rows get error messages, valid rows get parsed records TemplateRepository saves/loads SiteTemplate as JSON with atomic write (tmp + File.Move) TemplateRepository supports GetAll, GetById, Save, Delete, Rename All previously-skipped tests now pass path provides exports SharepointToolbox/Services/CsvValidationService.cs CSV parsing and validation for all bulk operation types CsvValidationService path provides exports SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs JSON persistence for site templates TemplateRepository from to via pattern CsvValidationService.cs CsvHelper CsvReader with DetectDelimiter and BOM detection CsvReader from to via pattern TemplateRepository.cs SiteTemplate.cs System.Text.Json serialization JsonSerializer

Plan 04-02: CsvValidationService + TemplateRepository

Goal

Implement CsvValidationService (CsvHelper-based CSV parsing with type mapping, validation, and preview generation) and TemplateRepository (JSON persistence for site templates using the same atomic write pattern as SettingsRepository). Activate the test scaffolds from Plan 04-01.

Context

ICsvValidationService and models (BulkMemberRow, BulkSiteRow, FolderStructureRow, CsvValidationRow<T>) are defined in Plan 04-01. SettingsRepository in Infrastructure/Persistence/ provides the atomic JSON write pattern to follow.

Existing example CSVs in /examples/:

• bulk_add_members.csv — Email column only (will be extended with GroupName, GroupUrl, Role)
• bulk_create_sites.csv — semicolon-delimited: Name;Alias;Type;Template;Owners;Members
• folder_structure.csv — semicolon-delimited: Level1;Level2;Level3;Level4

Tasks

Task 1: Implement CsvValidationService + unit tests

Files:

• SharepointToolbox/Services/CsvValidationService.cs
• SharepointToolbox.Tests/Services/CsvValidationServiceTests.cs

Action:

Create CsvValidationService.cs:

using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using CsvHelper;
using CsvHelper.Configuration;
using SharepointToolbox.Core.Models;

namespace SharepointToolbox.Services;

public class CsvValidationService : ICsvValidationService
{
    private static readonly Regex EmailRegex = new(
        @"^[^@\s]+@[^@\s]+\.[^@\s]+$", RegexOptions.Compiled);

    public List<CsvValidationRow<T>> ParseAndValidate<T>(Stream csvStream) where T : class
    {
        using var reader = new StreamReader(csvStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
        using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture)
        {
            HasHeaderRecord = true,
            MissingFieldFound = null,
            HeaderValidated = null,
            DetectDelimiter = true,
            TrimOptions = TrimOptions.Trim,
        });

        var rows = new List<CsvValidationRow<T>>();
        csv.Read();
        csv.ReadHeader();
        while (csv.Read())
        {
            try
            {
                var record = csv.GetRecord<T>();
                if (record == null)
                {
                    rows.Add(CsvValidationRow<T>.ParseError(csv.Context.Parser.RawRecord, "Failed to parse row"));
                    continue;
                }
                rows.Add(new CsvValidationRow<T>(record, new List<string>()));
            }
            catch (Exception ex)
            {
                rows.Add(CsvValidationRow<T>.ParseError(csv.Context.Parser.RawRecord, ex.Message));
            }
        }
        return rows;
    }

    public List<CsvValidationRow<BulkMemberRow>> ParseAndValidateMembers(Stream csvStream)
    {
        var rows = ParseAndValidate<BulkMemberRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateMemberRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    public List<CsvValidationRow<BulkSiteRow>> ParseAndValidateSites(Stream csvStream)
    {
        var rows = ParseAndValidate<BulkSiteRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateSiteRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    public List<CsvValidationRow<FolderStructureRow>> ParseAndValidateFolders(Stream csvStream)
    {
        var rows = ParseAndValidate<FolderStructureRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateFolderRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    private static List<string> ValidateMemberRow(BulkMemberRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Email))
            errors.Add("Email is required");
        else if (!EmailRegex.IsMatch(row.Email.Trim()))
            errors.Add($"Invalid email format: {row.Email}");

        if (string.IsNullOrWhiteSpace(row.GroupName) && string.IsNullOrWhiteSpace(row.GroupUrl))
            errors.Add("GroupName or GroupUrl is required");

        if (!string.IsNullOrWhiteSpace(row.Role) &&
            !row.Role.Equals("Member", StringComparison.OrdinalIgnoreCase) &&
            !row.Role.Equals("Owner", StringComparison.OrdinalIgnoreCase))
            errors.Add($"Role must be 'Member' or 'Owner', got: {row.Role}");

        return errors;
    }

    private static List<string> ValidateSiteRow(BulkSiteRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Name))
            errors.Add("Name is required");

        if (string.IsNullOrWhiteSpace(row.Type))
            errors.Add("Type is required");
        else if (!row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
                 !row.Type.Equals("Communication", StringComparison.OrdinalIgnoreCase))
            errors.Add($"Type must be 'Team' or 'Communication', got: {row.Type}");

        // Team sites require at least one owner (Pitfall 6 from research)
        if (row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
            string.IsNullOrWhiteSpace(row.Owners))
            errors.Add("Team sites require at least one owner");

        // Team sites need an alias
        if (row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
            string.IsNullOrWhiteSpace(row.Alias))
            errors.Add("Team sites require an alias");

        return errors;
    }

    private static List<string> ValidateFolderRow(FolderStructureRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Level1))
            errors.Add("Level1 is required (root folder)");
        return errors;
    }
}

Replace the skipped tests in CsvValidationServiceTests.cs with real tests:

using System.IO;
using System.Text;
using SharepointToolbox.Core.Models;
using SharepointToolbox.Services;

namespace SharepointToolbox.Tests.Services;

public class CsvValidationServiceTests
{
    private readonly CsvValidationService _service = new();

    private static Stream ToStream(string content)
    {
        return new MemoryStream(Encoding.UTF8.GetBytes(content));
    }

    private static Stream ToStreamWithBom(string content)
    {
        var preamble = Encoding.UTF8.GetPreamble();
        var bytes = Encoding.UTF8.GetBytes(content);
        var combined = new byte[preamble.Length + bytes.Length];
        preamble.CopyTo(combined, 0);
        bytes.CopyTo(combined, preamble.Length);
        return new MemoryStream(combined);
    }

    [Fact]
    public void ParseAndValidateMembers_ValidCsv_ReturnsValidRows()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,user@test.com,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("user@test.com", rows[0].Record!.Email);
    }

    [Fact]
    public void ParseAndValidateMembers_InvalidEmail_ReturnsErrors()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,not-an-email,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("Invalid email"));
    }

    [Fact]
    public void ParseAndValidateMembers_MissingGroup_ReturnsError()
    {
        var csv = "GroupName,GroupUrl,Email,Role\n,,user@test.com,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("GroupName or GroupUrl"));
    }

    [Fact]
    public void ParseAndValidateSites_TeamWithoutOwner_ReturnsError()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Team;;;\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("owner"));
    }

    [Fact]
    public void ParseAndValidateSites_ValidTeam_ReturnsValid()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Team;;admin@test.com;user@test.com\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("Site A", rows[0].Record!.Name);
    }

    [Fact]
    public void ParseAndValidateFolders_ValidCsv_ReturnsValidRows()
    {
        var csv = "Level1;Level2;Level3;Level4\nAdmin;HR;;\n";
        var rows = _service.ParseAndValidateFolders(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("Admin", rows[0].Record!.Level1);
        Assert.Equal("HR", rows[0].Record!.Level2);
    }

    [Fact]
    public void ParseAndValidateFolders_MissingLevel1_ReturnsError()
    {
        var csv = "Level1;Level2;Level3;Level4\n;SubFolder;;\n";
        var rows = _service.ParseAndValidateFolders(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("Level1"));
    }

    [Fact]
    public void ParseAndValidate_BomDetection_WorksWithAndWithoutBom()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,user@test.com,Member\n";
        var rowsNoBom = _service.ParseAndValidateMembers(ToStream(csv));
        var rowsWithBom = _service.ParseAndValidateMembers(ToStreamWithBom(csv));

        Assert.Single(rowsNoBom);
        Assert.Single(rowsWithBom);
        Assert.True(rowsNoBom[0].IsValid);
        Assert.True(rowsWithBom[0].IsValid);
    }

    [Fact]
    public void ParseAndValidate_SemicolonDelimiter_DetectedAutomatically()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Communication;;;;\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.Equal("Site A", rows[0].Record!.Name);
        Assert.Equal("Communication", rows[0].Record!.Type);
    }
}

Verify:

dotnet test SharepointToolbox.Tests --no-build --filter "FullyQualifiedName~CsvValidationService" -q

Done: All 9 CsvValidationService tests pass. CSV parses both comma and semicolon delimiters, detects BOM, validates member/site/folder rows individually.

Task 2: Implement TemplateRepository + unit tests

Files:

• SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs
• SharepointToolbox.Tests/Services/TemplateRepositoryTests.cs

Action:

Create TemplateRepository.cs following the SettingsRepository pattern (atomic write with .tmp + File.Move, SemaphoreSlim for thread safety):

using System.IO;
using System.Text;
using System.Text.Json;
using SharepointToolbox.Core.Models;

namespace SharepointToolbox.Infrastructure.Persistence;

public class TemplateRepository
{
    private readonly string _directoryPath;
    private readonly SemaphoreSlim _writeLock = new(1, 1);
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        WriteIndented = true,
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        PropertyNameCaseInsensitive = true,
    };

    public TemplateRepository(string directoryPath)
    {
        _directoryPath = directoryPath;
    }

    public async Task<List<SiteTemplate>> GetAllAsync()
    {
        if (!Directory.Exists(_directoryPath))
            return new List<SiteTemplate>();

        var templates = new List<SiteTemplate>();
        foreach (var file in Directory.GetFiles(_directoryPath, "*.json"))
        {
            try
            {
                var json = await File.ReadAllTextAsync(file, Encoding.UTF8);
                var template = JsonSerializer.Deserialize<SiteTemplate>(json, JsonOptions);
                if (template != null)
                    templates.Add(template);
            }
            catch (JsonException)
            {
                // Skip corrupted template files
            }
        }
        return templates.OrderByDescending(t => t.CapturedAt).ToList();
    }

    public async Task<SiteTemplate?> GetByIdAsync(string id)
    {
        var filePath = GetFilePath(id);
        if (!File.Exists(filePath))
            return null;

        var json = await File.ReadAllTextAsync(filePath, Encoding.UTF8);
        return JsonSerializer.Deserialize<SiteTemplate>(json, JsonOptions);
    }

    public async Task SaveAsync(SiteTemplate template)
    {
        await _writeLock.WaitAsync();
        try
        {
            if (!Directory.Exists(_directoryPath))
                Directory.CreateDirectory(_directoryPath);

            var json = JsonSerializer.Serialize(template, JsonOptions);
            var filePath = GetFilePath(template.Id);
            var tmpPath = filePath + ".tmp";

            await File.WriteAllTextAsync(tmpPath, json, Encoding.UTF8);

            // Validate round-trip before replacing
            JsonDocument.Parse(await File.ReadAllTextAsync(tmpPath, Encoding.UTF8)).Dispose();

            File.Move(tmpPath, filePath, overwrite: true);
        }
        finally
        {
            _writeLock.Release();
        }
    }

    public Task DeleteAsync(string id)
    {
        var filePath = GetFilePath(id);
        if (File.Exists(filePath))
            File.Delete(filePath);
        return Task.CompletedTask;
    }

    public async Task RenameAsync(string id, string newName)
    {
        var template = await GetByIdAsync(id);
        if (template == null)
            throw new InvalidOperationException($"Template not found: {id}");

        template.Name = newName;
        await SaveAsync(template);
    }

    private string GetFilePath(string id) => Path.Combine(_directoryPath, $"{id}.json");
}

Replace the skipped tests in TemplateRepositoryTests.cs:

using System.IO;
using SharepointToolbox.Core.Models;
using SharepointToolbox.Infrastructure.Persistence;

namespace SharepointToolbox.Tests.Services;

public class TemplateRepositoryTests : IDisposable
{
    private readonly string _tempDir;
    private readonly TemplateRepository _repo;

    public TemplateRepositoryTests()
    {
        _tempDir = Path.Combine(Path.GetTempPath(), $"sptoolbox_test_{Guid.NewGuid():N}");
        _repo = new TemplateRepository(_tempDir);
    }

    public void Dispose()
    {
        if (Directory.Exists(_tempDir))
            Directory.Delete(_tempDir, true);
    }

    private static SiteTemplate CreateTestTemplate(string name = "Test Template")
    {
        return new SiteTemplate
        {
            Id = Guid.NewGuid().ToString(),
            Name = name,
            SourceUrl = "https://contoso.sharepoint.com/sites/test",
            CapturedAt = DateTime.UtcNow,
            SiteType = "Team",
            Options = new SiteTemplateOptions(),
            Settings = new TemplateSettings { Title = "Test", Description = "Desc", Language = 1033 },
            Libraries = new List<TemplateLibraryInfo>
            {
                new() { Name = "Documents", BaseType = "DocumentLibrary", BaseTemplate = 101 }
            },
        };
    }

    [Fact]
    public async Task SaveAndLoad_RoundTrips_Correctly()
    {
        var template = CreateTestTemplate();
        await _repo.SaveAsync(template);

        var loaded = await _repo.GetByIdAsync(template.Id);

        Assert.NotNull(loaded);
        Assert.Equal(template.Name, loaded!.Name);
        Assert.Equal(template.SiteType, loaded.SiteType);
        Assert.Equal(template.SourceUrl, loaded.SourceUrl);
        Assert.Single(loaded.Libraries);
        Assert.Equal("Documents", loaded.Libraries[0].Name);
    }

    [Fact]
    public async Task GetAll_ReturnsAllSavedTemplates()
    {
        await _repo.SaveAsync(CreateTestTemplate("Template A"));
        await _repo.SaveAsync(CreateTestTemplate("Template B"));
        await _repo.SaveAsync(CreateTestTemplate("Template C"));

        var all = await _repo.GetAllAsync();

        Assert.Equal(3, all.Count);
    }

    [Fact]
    public async Task Delete_RemovesTemplate()
    {
        var template = CreateTestTemplate();
        await _repo.SaveAsync(template);
        Assert.NotNull(await _repo.GetByIdAsync(template.Id));

        await _repo.DeleteAsync(template.Id);

        Assert.Null(await _repo.GetByIdAsync(template.Id));
    }

    [Fact]
    public async Task Rename_UpdatesTemplateName()
    {
        var template = CreateTestTemplate("Old Name");
        await _repo.SaveAsync(template);

        await _repo.RenameAsync(template.Id, "New Name");

        var loaded = await _repo.GetByIdAsync(template.Id);
        Assert.Equal("New Name", loaded!.Name);
    }

    [Fact]
    public async Task GetAll_EmptyDirectory_ReturnsEmptyList()
    {
        var all = await _repo.GetAllAsync();
        Assert.Empty(all);
    }

    [Fact]
    public async Task GetById_NonExistent_ReturnsNull()
    {
        var result = await _repo.GetByIdAsync("nonexistent-id");
        Assert.Null(result);
    }
}

Verify:

dotnet build SharepointToolbox.slnx --no-restore -q && dotnet test SharepointToolbox.Tests --no-build --filter "FullyQualifiedName~CsvValidationService|FullyQualifiedName~TemplateRepository" -q

Done: CsvValidationService tests pass (9 tests). TemplateRepository tests pass (6 tests). Both services compile and function correctly.

Commit: feat(04-02): implement CsvValidationService and TemplateRepository with tests