Sharepoint-Toolbox/.planning/milestones/v1.0-phases/04-bulk-operations-and-provisioning/04-02-PLAN.md

---
phase: 04
plan: 02
title: CsvValidationService + TemplateRepository
status: pending
wave: 1
depends_on:
  - 04-01
files_modified:
  - SharepointToolbox/Services/CsvValidationService.cs
  - SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs
  - SharepointToolbox.Tests/Services/CsvValidationServiceTests.cs
  - SharepointToolbox.Tests/Services/TemplateRepositoryTests.cs
autonomous: true
requirements:
  - BULK-05
  - TMPL-03
  - TMPL-04
  - FOLD-02

must_haves:
  truths:
    - "CsvValidationService parses CSV with CsvHelper, auto-detects delimiter (comma or semicolon), detects BOM"
    - "Each row is validated individually — invalid rows get error messages, valid rows get parsed records"
    - "TemplateRepository saves/loads SiteTemplate as JSON with atomic write (tmp + File.Move)"
    - "TemplateRepository supports GetAll, GetById, Save, Delete, Rename"
    - "All previously-skipped tests now pass"
  artifacts:
    - path: "SharepointToolbox/Services/CsvValidationService.cs"
      provides: "CSV parsing and validation for all bulk operation types"
      exports: ["CsvValidationService"]
    - path: "SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs"
      provides: "JSON persistence for site templates"
      exports: ["TemplateRepository"]
  key_links:
    - from: "CsvValidationService.cs"
      to: "CsvHelper"
      via: "CsvReader with DetectDelimiter and BOM detection"
      pattern: "CsvReader"
    - from: "TemplateRepository.cs"
      to: "SiteTemplate.cs"
      via: "System.Text.Json serialization"
      pattern: "JsonSerializer"
---

# Plan 04-02: CsvValidationService + TemplateRepository

## Goal

Implement `CsvValidationService` (CsvHelper-based CSV parsing with type mapping, validation, and preview generation) and `TemplateRepository` (JSON persistence for site templates using the same atomic write pattern as SettingsRepository). Activate the test scaffolds from Plan 04-01.

## Context

`ICsvValidationService` and models (`BulkMemberRow`, `BulkSiteRow`, `FolderStructureRow`, `CsvValidationRow<T>`) are defined in Plan 04-01. `SettingsRepository` in `Infrastructure/Persistence/` provides the atomic JSON write pattern to follow.

Existing example CSVs in `/examples/`:
- `bulk_add_members.csv` — Email column only (will be extended with GroupName, GroupUrl, Role)
- `bulk_create_sites.csv` — semicolon-delimited: Name;Alias;Type;Template;Owners;Members
- `folder_structure.csv` — semicolon-delimited: Level1;Level2;Level3;Level4

## Tasks

### Task 1: Implement CsvValidationService + unit tests

**Files:**
- `SharepointToolbox/Services/CsvValidationService.cs`
- `SharepointToolbox.Tests/Services/CsvValidationServiceTests.cs`

**Action:**

Create `CsvValidationService.cs`:
```csharp
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using CsvHelper;
using CsvHelper.Configuration;
using SharepointToolbox.Core.Models;

namespace SharepointToolbox.Services;

public class CsvValidationService : ICsvValidationService
{
    private static readonly Regex EmailRegex = new(
        @"^[^@\s]+@[^@\s]+\.[^@\s]+$", RegexOptions.Compiled);

    public List<CsvValidationRow<T>> ParseAndValidate<T>(Stream csvStream) where T : class
    {
        using var reader = new StreamReader(csvStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
        using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture)
        {
            HasHeaderRecord = true,
            MissingFieldFound = null,
            HeaderValidated = null,
            DetectDelimiter = true,
            TrimOptions = TrimOptions.Trim,
        });

        var rows = new List<CsvValidationRow<T>>();
        csv.Read();
        csv.ReadHeader();
        while (csv.Read())
        {
            try
            {
                var record = csv.GetRecord<T>();
                if (record == null)
                {
                    rows.Add(CsvValidationRow<T>.ParseError(csv.Context.Parser.RawRecord, "Failed to parse row"));
                    continue;
                }
                rows.Add(new CsvValidationRow<T>(record, new List<string>()));
            }
            catch (Exception ex)
            {
                rows.Add(CsvValidationRow<T>.ParseError(csv.Context.Parser.RawRecord, ex.Message));
            }
        }
        return rows;
    }

    public List<CsvValidationRow<BulkMemberRow>> ParseAndValidateMembers(Stream csvStream)
    {
        var rows = ParseAndValidate<BulkMemberRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateMemberRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    public List<CsvValidationRow<BulkSiteRow>> ParseAndValidateSites(Stream csvStream)
    {
        var rows = ParseAndValidate<BulkSiteRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateSiteRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    public List<CsvValidationRow<FolderStructureRow>> ParseAndValidateFolders(Stream csvStream)
    {
        var rows = ParseAndValidate<FolderStructureRow>(csvStream);
        foreach (var row in rows.Where(r => r.IsValid && r.Record != null))
        {
            var errors = ValidateFolderRow(row.Record!);
            row.Errors.AddRange(errors);
        }
        return rows;
    }

    private static List<string> ValidateMemberRow(BulkMemberRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Email))
            errors.Add("Email is required");
        else if (!EmailRegex.IsMatch(row.Email.Trim()))
            errors.Add($"Invalid email format: {row.Email}");

        if (string.IsNullOrWhiteSpace(row.GroupName) && string.IsNullOrWhiteSpace(row.GroupUrl))
            errors.Add("GroupName or GroupUrl is required");

        if (!string.IsNullOrWhiteSpace(row.Role) &&
            !row.Role.Equals("Member", StringComparison.OrdinalIgnoreCase) &&
            !row.Role.Equals("Owner", StringComparison.OrdinalIgnoreCase))
            errors.Add($"Role must be 'Member' or 'Owner', got: {row.Role}");

        return errors;
    }

    private static List<string> ValidateSiteRow(BulkSiteRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Name))
            errors.Add("Name is required");

        if (string.IsNullOrWhiteSpace(row.Type))
            errors.Add("Type is required");
        else if (!row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
                 !row.Type.Equals("Communication", StringComparison.OrdinalIgnoreCase))
            errors.Add($"Type must be 'Team' or 'Communication', got: {row.Type}");

        // Team sites require at least one owner (Pitfall 6 from research)
        if (row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
            string.IsNullOrWhiteSpace(row.Owners))
            errors.Add("Team sites require at least one owner");

        // Team sites need an alias
        if (row.Type.Equals("Team", StringComparison.OrdinalIgnoreCase) &&
            string.IsNullOrWhiteSpace(row.Alias))
            errors.Add("Team sites require an alias");

        return errors;
    }

    private static List<string> ValidateFolderRow(FolderStructureRow row)
    {
        var errors = new List<string>();
        if (string.IsNullOrWhiteSpace(row.Level1))
            errors.Add("Level1 is required (root folder)");
        return errors;
    }
}
```

Replace the skipped tests in `CsvValidationServiceTests.cs` with real tests:
```csharp
using System.IO;
using System.Text;
using SharepointToolbox.Core.Models;
using SharepointToolbox.Services;

namespace SharepointToolbox.Tests.Services;

public class CsvValidationServiceTests
{
    private readonly CsvValidationService _service = new();

    private static Stream ToStream(string content)
    {
        return new MemoryStream(Encoding.UTF8.GetBytes(content));
    }

    private static Stream ToStreamWithBom(string content)
    {
        var preamble = Encoding.UTF8.GetPreamble();
        var bytes = Encoding.UTF8.GetBytes(content);
        var combined = new byte[preamble.Length + bytes.Length];
        preamble.CopyTo(combined, 0);
        bytes.CopyTo(combined, preamble.Length);
        return new MemoryStream(combined);
    }

    [Fact]
    public void ParseAndValidateMembers_ValidCsv_ReturnsValidRows()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,user@test.com,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("user@test.com", rows[0].Record!.Email);
    }

    [Fact]
    public void ParseAndValidateMembers_InvalidEmail_ReturnsErrors()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,not-an-email,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("Invalid email"));
    }

    [Fact]
    public void ParseAndValidateMembers_MissingGroup_ReturnsError()
    {
        var csv = "GroupName,GroupUrl,Email,Role\n,,user@test.com,Member\n";
        var rows = _service.ParseAndValidateMembers(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("GroupName or GroupUrl"));
    }

    [Fact]
    public void ParseAndValidateSites_TeamWithoutOwner_ReturnsError()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Team;;;\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("owner"));
    }

    [Fact]
    public void ParseAndValidateSites_ValidTeam_ReturnsValid()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Team;;admin@test.com;user@test.com\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("Site A", rows[0].Record!.Name);
    }

    [Fact]
    public void ParseAndValidateFolders_ValidCsv_ReturnsValidRows()
    {
        var csv = "Level1;Level2;Level3;Level4\nAdmin;HR;;\n";
        var rows = _service.ParseAndValidateFolders(ToStream(csv));

        Assert.Single(rows);
        Assert.True(rows[0].IsValid);
        Assert.Equal("Admin", rows[0].Record!.Level1);
        Assert.Equal("HR", rows[0].Record!.Level2);
    }

    [Fact]
    public void ParseAndValidateFolders_MissingLevel1_ReturnsError()
    {
        var csv = "Level1;Level2;Level3;Level4\n;SubFolder;;\n";
        var rows = _service.ParseAndValidateFolders(ToStream(csv));

        Assert.Single(rows);
        Assert.False(rows[0].IsValid);
        Assert.Contains(rows[0].Errors, e => e.Contains("Level1"));
    }

    [Fact]
    public void ParseAndValidate_BomDetection_WorksWithAndWithoutBom()
    {
        var csv = "GroupName,GroupUrl,Email,Role\nTeam A,https://site,user@test.com,Member\n";
        var rowsNoBom = _service.ParseAndValidateMembers(ToStream(csv));
        var rowsWithBom = _service.ParseAndValidateMembers(ToStreamWithBom(csv));

        Assert.Single(rowsNoBom);
        Assert.Single(rowsWithBom);
        Assert.True(rowsNoBom[0].IsValid);
        Assert.True(rowsWithBom[0].IsValid);
    }

    [Fact]
    public void ParseAndValidate_SemicolonDelimiter_DetectedAutomatically()
    {
        var csv = "Name;Alias;Type;Template;Owners;Members\nSite A;site-a;Communication;;;;\n";
        var rows = _service.ParseAndValidateSites(ToStream(csv));

        Assert.Single(rows);
        Assert.Equal("Site A", rows[0].Record!.Name);
        Assert.Equal("Communication", rows[0].Record!.Type);
    }
}
```

**Verify:**
```bash
dotnet test SharepointToolbox.Tests --no-build --filter "FullyQualifiedName~CsvValidationService" -q
```

**Done:** All 9 CsvValidationService tests pass. CSV parses both comma and semicolon delimiters, detects BOM, validates member/site/folder rows individually.

### Task 2: Implement TemplateRepository + unit tests

**Files:**
- `SharepointToolbox/Infrastructure/Persistence/TemplateRepository.cs`
- `SharepointToolbox.Tests/Services/TemplateRepositoryTests.cs`

**Action:**

Create `TemplateRepository.cs` following the SettingsRepository pattern (atomic write with .tmp + File.Move, SemaphoreSlim for thread safety):
```csharp
using System.IO;
using System.Text;
using System.Text.Json;
using SharepointToolbox.Core.Models;

namespace SharepointToolbox.Infrastructure.Persistence;

public class TemplateRepository
{
    private readonly string _directoryPath;
    private readonly SemaphoreSlim _writeLock = new(1, 1);
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        WriteIndented = true,
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        PropertyNameCaseInsensitive = true,
    };

    public TemplateRepository(string directoryPath)
    {
        _directoryPath = directoryPath;
    }

    public async Task<List<SiteTemplate>> GetAllAsync()
    {
        if (!Directory.Exists(_directoryPath))
            return new List<SiteTemplate>();

        var templates = new List<SiteTemplate>();
        foreach (var file in Directory.GetFiles(_directoryPath, "*.json"))
        {
            try
            {
                var json = await File.ReadAllTextAsync(file, Encoding.UTF8);
                var template = JsonSerializer.Deserialize<SiteTemplate>(json, JsonOptions);
                if (template != null)
                    templates.Add(template);
            }
            catch (JsonException)
            {
                // Skip corrupted template files
            }
        }
        return templates.OrderByDescending(t => t.CapturedAt).ToList();
    }

    public async Task<SiteTemplate?> GetByIdAsync(string id)
    {
        var filePath = GetFilePath(id);
        if (!File.Exists(filePath))
            return null;

        var json = await File.ReadAllTextAsync(filePath, Encoding.UTF8);
        return JsonSerializer.Deserialize<SiteTemplate>(json, JsonOptions);
    }

    public async Task SaveAsync(SiteTemplate template)
    {
        await _writeLock.WaitAsync();
        try
        {
            if (!Directory.Exists(_directoryPath))
                Directory.CreateDirectory(_directoryPath);

            var json = JsonSerializer.Serialize(template, JsonOptions);
            var filePath = GetFilePath(template.Id);
            var tmpPath = filePath + ".tmp";

            await File.WriteAllTextAsync(tmpPath, json, Encoding.UTF8);

            // Validate round-trip before replacing
            JsonDocument.Parse(await File.ReadAllTextAsync(tmpPath, Encoding.UTF8)).Dispose();

            File.Move(tmpPath, filePath, overwrite: true);
        }
        finally
        {
            _writeLock.Release();
        }
    }

    public Task DeleteAsync(string id)
    {
        var filePath = GetFilePath(id);
        if (File.Exists(filePath))
            File.Delete(filePath);
        return Task.CompletedTask;
    }

    public async Task RenameAsync(string id, string newName)
    {
        var template = await GetByIdAsync(id);
        if (template == null)
            throw new InvalidOperationException($"Template not found: {id}");

        template.Name = newName;
        await SaveAsync(template);
    }

    private string GetFilePath(string id) => Path.Combine(_directoryPath, $"{id}.json");
}
```

Replace the skipped tests in `TemplateRepositoryTests.cs`:
```csharp
using System.IO;
using SharepointToolbox.Core.Models;
using SharepointToolbox.Infrastructure.Persistence;

namespace SharepointToolbox.Tests.Services;

public class TemplateRepositoryTests : IDisposable
{
    private readonly string _tempDir;
    private readonly TemplateRepository _repo;

    public TemplateRepositoryTests()
    {
        _tempDir = Path.Combine(Path.GetTempPath(), $"sptoolbox_test_{Guid.NewGuid():N}");
        _repo = new TemplateRepository(_tempDir);
    }

    public void Dispose()
    {
        if (Directory.Exists(_tempDir))
            Directory.Delete(_tempDir, true);
    }

    private static SiteTemplate CreateTestTemplate(string name = "Test Template")
    {
        return new SiteTemplate
        {
            Id = Guid.NewGuid().ToString(),
            Name = name,
            SourceUrl = "https://contoso.sharepoint.com/sites/test",
            CapturedAt = DateTime.UtcNow,
            SiteType = "Team",
            Options = new SiteTemplateOptions(),
            Settings = new TemplateSettings { Title = "Test", Description = "Desc", Language = 1033 },
            Libraries = new List<TemplateLibraryInfo>
            {
                new() { Name = "Documents", BaseType = "DocumentLibrary", BaseTemplate = 101 }
            },
        };
    }

    [Fact]
    public async Task SaveAndLoad_RoundTrips_Correctly()
    {
        var template = CreateTestTemplate();
        await _repo.SaveAsync(template);

        var loaded = await _repo.GetByIdAsync(template.Id);

        Assert.NotNull(loaded);
        Assert.Equal(template.Name, loaded!.Name);
        Assert.Equal(template.SiteType, loaded.SiteType);
        Assert.Equal(template.SourceUrl, loaded.SourceUrl);
        Assert.Single(loaded.Libraries);
        Assert.Equal("Documents", loaded.Libraries[0].Name);
    }

    [Fact]
    public async Task GetAll_ReturnsAllSavedTemplates()
    {
        await _repo.SaveAsync(CreateTestTemplate("Template A"));
        await _repo.SaveAsync(CreateTestTemplate("Template B"));
        await _repo.SaveAsync(CreateTestTemplate("Template C"));

        var all = await _repo.GetAllAsync();

        Assert.Equal(3, all.Count);
    }

    [Fact]
    public async Task Delete_RemovesTemplate()
    {
        var template = CreateTestTemplate();
        await _repo.SaveAsync(template);
        Assert.NotNull(await _repo.GetByIdAsync(template.Id));

        await _repo.DeleteAsync(template.Id);

        Assert.Null(await _repo.GetByIdAsync(template.Id));
    }

    [Fact]
    public async Task Rename_UpdatesTemplateName()
    {
        var template = CreateTestTemplate("Old Name");
        await _repo.SaveAsync(template);

        await _repo.RenameAsync(template.Id, "New Name");

        var loaded = await _repo.GetByIdAsync(template.Id);
        Assert.Equal("New Name", loaded!.Name);
    }

    [Fact]
    public async Task GetAll_EmptyDirectory_ReturnsEmptyList()
    {
        var all = await _repo.GetAllAsync();
        Assert.Empty(all);
    }

    [Fact]
    public async Task GetById_NonExistent_ReturnsNull()
    {
        var result = await _repo.GetByIdAsync("nonexistent-id");
        Assert.Null(result);
    }
}
```

**Verify:**
```bash
dotnet build SharepointToolbox.slnx --no-restore -q && dotnet test SharepointToolbox.Tests --no-build --filter "FullyQualifiedName~CsvValidationService|FullyQualifiedName~TemplateRepository" -q
```

**Done:** CsvValidationService tests pass (9 tests). TemplateRepository tests pass (6 tests). Both services compile and function correctly.

**Commit:** `feat(04-02): implement CsvValidationService and TemplateRepository with tests`