349 lines
8.4 KiB
Go
349 lines
8.4 KiB
Go
// Package job tracks long-running migrations and streams their progress to
|
|
// the UI. A job is a tree: job -> per-container item -> per-operation step,
|
|
// with byte counters on the steps that move data.
|
|
package job
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
// State is the lifecycle of a job, item or step.
|
|
type State string
|
|
|
|
const (
|
|
StatePending State = "pending"
|
|
StateRunning State = "running"
|
|
StateSucceeded State = "succeeded"
|
|
StateFailed State = "failed"
|
|
StateSkipped State = "skipped"
|
|
StateCanceled State = "canceled"
|
|
)
|
|
|
|
// Terminal reports whether no further transitions are expected.
|
|
func (s State) Terminal() bool {
|
|
switch s {
|
|
case StateSucceeded, StateFailed, StateSkipped, StateCanceled:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Kind distinguishes the two migration modes.
|
|
type Kind string
|
|
|
|
const (
|
|
KindSSH Kind = "ssh"
|
|
KindPackage Kind = "package"
|
|
KindRestore Kind = "restore"
|
|
)
|
|
|
|
// Level classifies a log line.
|
|
type Level string
|
|
|
|
const (
|
|
LevelInfo Level = "info"
|
|
LevelWarn Level = "warn"
|
|
LevelError Level = "error"
|
|
LevelCmd Level = "cmd" // a command that was (or would be) run on a host
|
|
)
|
|
|
|
// LogEntry is one line in the job log.
|
|
type LogEntry struct {
|
|
Seq int64 `json:"seq"`
|
|
At time.Time `json:"at"`
|
|
Level Level `json:"level"`
|
|
Item string `json:"item,omitempty"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
// Step is one unit of work inside an item, e.g. "transfer volume pgdata".
|
|
type Step struct {
|
|
ID string `json:"id"`
|
|
Label string `json:"label"`
|
|
State State `json:"state"`
|
|
BytesDone int64 `json:"bytesDone"`
|
|
BytesTotal int64 `json:"bytesTotal"` // -1 when unknown
|
|
Error string `json:"error,omitempty"`
|
|
StartedAt *time.Time `json:"startedAt,omitempty"`
|
|
EndedAt *time.Time `json:"endedAt,omitempty"`
|
|
}
|
|
|
|
// Item is the migration of one container.
|
|
type Item struct {
|
|
ID string `json:"id"` // container id on the source
|
|
Name string `json:"name"`
|
|
State State `json:"state"`
|
|
Error string `json:"error,omitempty"`
|
|
Steps []*Step `json:"steps"`
|
|
Warnings []string `json:"warnings,omitempty"`
|
|
}
|
|
|
|
// Snapshot is the serializable view of a job handed to the UI.
|
|
type Snapshot struct {
|
|
ID string `json:"id"`
|
|
Kind Kind `json:"kind"`
|
|
Title string `json:"title"`
|
|
State State `json:"state"`
|
|
DryRun bool `json:"dryRun"`
|
|
Error string `json:"error,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
StartedAt *time.Time `json:"startedAt,omitempty"`
|
|
EndedAt *time.Time `json:"endedAt,omitempty"`
|
|
Items []*Item `json:"items"`
|
|
Log []LogEntry `json:"log"`
|
|
BytesDone int64 `json:"bytesDone"`
|
|
BytesTotal int64 `json:"bytesTotal"`
|
|
// Artifact is the produced package path, for package jobs.
|
|
Artifact string `json:"artifact,omitempty"`
|
|
// ArtifactBytes is the package size on disk.
|
|
ArtifactBytes int64 `json:"artifactBytes,omitempty"`
|
|
Revision int64 `json:"revision"`
|
|
}
|
|
|
|
// Job is a running or finished migration.
|
|
type Job struct {
|
|
mu sync.RWMutex
|
|
snap Snapshot
|
|
seq int64
|
|
revision int64
|
|
maxLog int
|
|
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
|
|
subsMu sync.Mutex
|
|
subs map[int]chan struct{}
|
|
nextID int
|
|
}
|
|
|
|
func newJob(id string, kind Kind, title string, dryRun bool) *Job {
|
|
return &Job{
|
|
snap: Snapshot{
|
|
ID: id, Kind: kind, Title: title, State: StatePending,
|
|
DryRun: dryRun, CreatedAt: time.Now(), Items: []*Item{}, Log: []LogEntry{},
|
|
BytesTotal: 0,
|
|
},
|
|
maxLog: 5000,
|
|
done: make(chan struct{}),
|
|
subs: map[int]chan struct{}{},
|
|
}
|
|
}
|
|
|
|
// ID returns the job identifier.
|
|
func (j *Job) ID() string { return j.snap.ID }
|
|
|
|
// Done is closed once the job reaches a terminal state.
|
|
func (j *Job) Done() <-chan struct{} { return j.done }
|
|
|
|
// Snapshot returns a deep-enough copy for JSON serialization.
|
|
func (j *Job) Snapshot() Snapshot {
|
|
j.mu.RLock()
|
|
defer j.mu.RUnlock()
|
|
s := j.snap
|
|
s.Items = make([]*Item, len(j.snap.Items))
|
|
var done, total int64
|
|
for i, it := range j.snap.Items {
|
|
cp := *it
|
|
cp.Steps = make([]*Step, len(it.Steps))
|
|
for k, st := range it.Steps {
|
|
sc := *st
|
|
cp.Steps[k] = &sc
|
|
done += sc.BytesDone
|
|
if sc.BytesTotal > 0 {
|
|
total += sc.BytesTotal
|
|
}
|
|
}
|
|
s.Items[i] = &cp
|
|
}
|
|
s.Log = append([]LogEntry(nil), j.snap.Log...)
|
|
s.BytesDone, s.BytesTotal = done, total
|
|
s.Revision = atomic.LoadInt64(&j.revision)
|
|
return s
|
|
}
|
|
|
|
// Subscribe returns a channel that receives a signal whenever the job changes,
|
|
// plus a function to unsubscribe.
|
|
func (j *Job) Subscribe() (<-chan struct{}, func()) {
|
|
j.subsMu.Lock()
|
|
defer j.subsMu.Unlock()
|
|
id := j.nextID
|
|
j.nextID++
|
|
ch := make(chan struct{}, 1)
|
|
j.subs[id] = ch
|
|
return ch, func() {
|
|
j.subsMu.Lock()
|
|
defer j.subsMu.Unlock()
|
|
if c, ok := j.subs[id]; ok {
|
|
delete(j.subs, id)
|
|
close(c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (j *Job) touch() {
|
|
atomic.AddInt64(&j.revision, 1)
|
|
j.subsMu.Lock()
|
|
for _, ch := range j.subs {
|
|
select {
|
|
case ch <- struct{}{}:
|
|
default: // a signal is already pending; the reader will see the latest state
|
|
}
|
|
}
|
|
j.subsMu.Unlock()
|
|
}
|
|
|
|
// Logf appends a line to the job log.
|
|
func (j *Job) Logf(level Level, item, format string, args ...any) {
|
|
j.mu.Lock()
|
|
j.seq++
|
|
e := LogEntry{Seq: j.seq, At: time.Now(), Level: level, Item: item, Message: fmt.Sprintf(format, args...)}
|
|
j.snap.Log = append(j.snap.Log, e)
|
|
if len(j.snap.Log) > j.maxLog {
|
|
j.snap.Log = j.snap.Log[len(j.snap.Log)-j.maxLog:]
|
|
}
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// AddItem registers a container in the job and returns its handle.
|
|
func (j *Job) AddItem(id, name string) *Item {
|
|
j.mu.Lock()
|
|
it := &Item{ID: id, Name: name, State: StatePending, Steps: []*Step{}}
|
|
j.snap.Items = append(j.snap.Items, it)
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
return it
|
|
}
|
|
|
|
// AddStep registers a unit of work under an item. bytesTotal may be -1 when
|
|
// the size is not known ahead of time.
|
|
func (j *Job) AddStep(it *Item, id, label string, bytesTotal int64) *Step {
|
|
j.mu.Lock()
|
|
st := &Step{ID: id, Label: label, State: StatePending, BytesTotal: bytesTotal}
|
|
it.Steps = append(it.Steps, st)
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
return st
|
|
}
|
|
|
|
// StartStep marks a step as running.
|
|
func (j *Job) StartStep(st *Step) {
|
|
now := time.Now()
|
|
j.mu.Lock()
|
|
st.State = StateRunning
|
|
st.StartedAt = &now
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// FinishStep closes a step, recording an error when one occurred.
|
|
func (j *Job) FinishStep(st *Step, err error) {
|
|
now := time.Now()
|
|
j.mu.Lock()
|
|
st.EndedAt = &now
|
|
if err != nil {
|
|
st.State = StateFailed
|
|
st.Error = err.Error()
|
|
} else {
|
|
st.State = StateSucceeded
|
|
if st.BytesTotal < 0 {
|
|
st.BytesTotal = st.BytesDone
|
|
}
|
|
}
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// SkipStep marks a step as deliberately not performed.
|
|
func (j *Job) SkipStep(st *Step, reason string) {
|
|
now := time.Now()
|
|
j.mu.Lock()
|
|
st.State = StateSkipped
|
|
st.EndedAt = &now
|
|
st.Error = reason
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// AddBytes advances a step's byte counter. It is safe to call at high rates
|
|
// from the transfer goroutine.
|
|
func (j *Job) AddBytes(st *Step, n int64) {
|
|
j.mu.Lock()
|
|
st.BytesDone += n
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// SetItemState transitions an item.
|
|
func (j *Job) SetItemState(it *Item, s State, err error) {
|
|
j.mu.Lock()
|
|
it.State = s
|
|
if err != nil {
|
|
it.Error = err.Error()
|
|
}
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
// AddItemWarning attaches a non-fatal note to an item.
|
|
func (j *Job) AddItemWarning(it *Item, format string, args ...any) {
|
|
msg := fmt.Sprintf(format, args...)
|
|
j.mu.Lock()
|
|
it.Warnings = append(it.Warnings, msg)
|
|
j.mu.Unlock()
|
|
j.Logf(LevelWarn, it.ID, "%s", msg)
|
|
}
|
|
|
|
// SetArtifact records the produced package.
|
|
func (j *Job) SetArtifact(path string, bytes int64) {
|
|
j.mu.Lock()
|
|
j.snap.Artifact = path
|
|
j.snap.ArtifactBytes = bytes
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
func (j *Job) start() {
|
|
now := time.Now()
|
|
j.mu.Lock()
|
|
j.snap.State = StateRunning
|
|
j.snap.StartedAt = &now
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
}
|
|
|
|
func (j *Job) finish(err error) {
|
|
now := time.Now()
|
|
j.mu.Lock()
|
|
if j.snap.State.Terminal() {
|
|
j.mu.Unlock()
|
|
return
|
|
}
|
|
j.snap.EndedAt = &now
|
|
switch {
|
|
case err == nil:
|
|
j.snap.State = StateSucceeded
|
|
case err == context.Canceled:
|
|
j.snap.State = StateCanceled
|
|
j.snap.Error = "canceled by operator"
|
|
default:
|
|
j.snap.State = StateFailed
|
|
j.snap.Error = err.Error()
|
|
}
|
|
j.mu.Unlock()
|
|
j.touch()
|
|
close(j.done)
|
|
}
|
|
|
|
// Cancel asks the job to stop. Work already in flight unwinds through context
|
|
// cancellation.
|
|
func (j *Job) Cancel() {
|
|
if j.cancel != nil {
|
|
j.cancel()
|
|
}
|
|
}
|