168 lines
4.2 KiB
Go
168 lines
4.2 KiB
Go
package dregarnuhr
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
// Queue owns atomic priority-ordered claims and persisted completion. ErrNoJob
|
|
// means no eligible job. The initial adapter is single-writer-process, not HA.
|
|
type Queue interface {
|
|
Settings(context.Context) (Settings, error)
|
|
Claim(context.Context, time.Time) (*Job, error)
|
|
Cancellation(context.Context, string) (bool, error)
|
|
Started(context.Context, *Job) error
|
|
Finish(context.Context, *Job, error, time.Time) error
|
|
}
|
|
|
|
type RunnerOptions struct {
|
|
Clock Clock
|
|
// Active is an optional compatibility counter for an existing monitor.
|
|
Active *atomic.Int64
|
|
Foreground func() bool
|
|
Wake func()
|
|
Report func(stage string, job *Job, err error)
|
|
}
|
|
|
|
type Runner struct {
|
|
queue Queue
|
|
registry *Registry
|
|
opts RunnerOptions
|
|
mu sync.Mutex
|
|
stopping bool
|
|
stopCtx context.Context
|
|
cancel context.CancelFunc
|
|
workers sync.WaitGroup
|
|
stopOnce sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func NewRunner(queue Queue, registry *Registry, opts RunnerOptions) *Runner {
|
|
if opts.Clock == nil {
|
|
opts.Clock = SystemClock{}
|
|
}
|
|
if opts.Active == nil {
|
|
opts.Active = new(atomic.Int64)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
return &Runner{queue: queue, registry: registry, opts: opts, stopCtx: ctx, cancel: cancel, done: make(chan struct{})}
|
|
}
|
|
|
|
// Stop closes admission permanently, cancels handlers and waits for persisted
|
|
// completions. A timeout does not mean workers have exited: callers must keep
|
|
// ownership/database resources alive and may call Stop again to await them.
|
|
func (r *Runner) Stop(ctx context.Context) error {
|
|
r.stopOnce.Do(func() {
|
|
r.cancel()
|
|
go func() {
|
|
r.mu.Lock()
|
|
r.stopping = true
|
|
r.mu.Unlock()
|
|
r.workers.Wait()
|
|
close(r.done)
|
|
}()
|
|
})
|
|
select {
|
|
case <-r.done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (r *Runner) report(stage string, job *Job, err error) {
|
|
if err != nil && r.opts.Report != nil {
|
|
r.opts.Report(stage, job, err)
|
|
}
|
|
}
|
|
|
|
func (r *Runner) Running() int64 { return r.opts.Active.Load() }
|
|
|
|
// Dispatch serializes admission plus claim across concurrent callers. Each
|
|
// invocation is bounded even if workers finish immediately and refill slots.
|
|
func (r *Runner) Dispatch(ctx context.Context) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.stopping || r.stopCtx.Err() != nil {
|
|
return
|
|
}
|
|
for n := 0; n < 32 && ctx.Err() == nil && r.stopCtx.Err() == nil; n++ {
|
|
settings, err := r.queue.Settings(ctx)
|
|
if err != nil {
|
|
r.report("settings", nil, err)
|
|
return
|
|
}
|
|
foreground := r.opts.Foreground != nil && r.opts.Foreground()
|
|
if !CanStart(settings, r.Running(), foreground) {
|
|
return
|
|
}
|
|
job, err := r.queue.Claim(ctx, r.opts.Clock.Now())
|
|
if errors.Is(err, ErrNoJob) || errors.Is(err, context.Canceled) {
|
|
return
|
|
}
|
|
if err != nil {
|
|
r.report("claim", nil, err)
|
|
return
|
|
}
|
|
if job == nil {
|
|
r.report("claim", nil, errors.New("queue returned nil job"))
|
|
return
|
|
}
|
|
r.opts.Active.Add(1)
|
|
r.report("start-log", job, r.queue.Started(ctx, job))
|
|
workerCtx, cancel := context.WithCancel(ctx)
|
|
stop := context.AfterFunc(r.stopCtx, cancel)
|
|
r.workers.Add(1)
|
|
go func() {
|
|
defer r.workers.Done()
|
|
defer cancel()
|
|
defer stop()
|
|
r.execute(workerCtx, job)
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (r *Runner) execute(ctx context.Context, job *Job) {
|
|
defer func() {
|
|
r.opts.Active.Add(-1)
|
|
if r.opts.Wake != nil {
|
|
r.opts.Wake()
|
|
}
|
|
}()
|
|
runErr := r.invoke(ctx, job)
|
|
// Completion must still be attempted when the worker context is cancelled.
|
|
finishCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
r.report("finish", job, r.queue.Finish(finishCtx, job, runErr, r.opts.Clock.Now()))
|
|
r.report("execute", job, runErr)
|
|
}
|
|
|
|
func (r *Runner) invoke(ctx context.Context, job *Job) (err error) {
|
|
defer func() {
|
|
if recover() != nil {
|
|
err = errors.New("event handler panicked")
|
|
}
|
|
}()
|
|
cancelled, err := r.queue.Cancellation(ctx, job.UUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cancelled {
|
|
return errors.New("cancelled before execution")
|
|
}
|
|
if err = r.registry.Run(ctx, job); err != nil {
|
|
return err
|
|
}
|
|
cancelled, err = r.queue.Cancellation(ctx, job.UUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cancelled {
|
|
return errors.New("cancelled during execution")
|
|
}
|
|
return nil
|
|
}
|