diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index dfb7d32..d458a6e 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -7,6 +7,7 @@ package scheduler import ( "context" "fmt" + "sync" "time" "git.smarteching.com/goffee/core/logger" @@ -37,13 +38,15 @@ const ( // QueueItem is a task waiting to be executed. // Any external program can insert a row into this table to enqueue work. // -// INSERT INTO queue_items (task_type, payload, priority) VALUES (?, ?, ?); +// INSERT INTO queue_items (task_type, payload, priority, max_runs, thread) VALUES (?, ?, ?, ?, ?); type QueueItem struct { ID uint `gorm:"primaryKey;autoIncrement"` TaskType string `gorm:"not null;index"` // e.g. "send_email", "sync_records" Payload string `gorm:"type:text"` // JSON or plain string, interpreted by the handler Status QueueStatus `gorm:"not null;default:'pending';index"` Priority int `gorm:"not null;default:0;index"` // higher = runs first within a tick + MaxRuns int `gorm:"not null;default:1"` // 1 = run once, -1 = repeat indefinitely, N = run N times + Thread int `gorm:"not null;default:0"` // 0 = sequential, 1 = concurrent per type CreatedAt time.Time UpdatedAt time.Time } @@ -122,12 +125,20 @@ func NewStore(db *gorm.DB) (*Store, error) { } // Enqueue inserts a new pending task. -func (s *Store) Enqueue(ctx context.Context, taskType, payload string, priority int) (*QueueItem, error) { +// maxRuns controls how many times the task executes: 1 = once (default), -1 = repeat indefinitely. +// thread controls concurrency: 0 = sequential per type, 1 = concurrent per type. +// Any value of maxRuns < -1 is clamped to 1. +func (s *Store) Enqueue(ctx context.Context, taskType, payload string, priority, maxRuns, thread int) (*QueueItem, error) { + if maxRuns < -1 { + maxRuns = 1 + } item := &QueueItem{ TaskType: taskType, Payload: payload, Status: StatusPending, Priority: priority, + MaxRuns: maxRuns, + Thread: thread, } if err := s.db.WithContext(ctx).Create(item).Error; err != nil { return nil, fmt.Errorf("enqueue %s: %w", taskType, err) @@ -172,8 +183,9 @@ func (s *Store) ClaimBatch(ctx context.Context, limit int) ([]*QueueItem, error) return items, nil } -// RecordResult writes a ProcessedItem and deletes the original QueueItem -// in a single transaction. +// RecordResult writes a ProcessedItem, then either deletes the QueueItem (if +// MaxRuns is exhausted) or decrements MaxRuns and resets the status to pending +// for re-execution on the next tick. A maxRuns of -1 means repeat indefinitely. func (s *Store) RecordResult(ctx context.Context, item *QueueItem, startedAt time.Time, execErr error) error { finishedAt := time.Now() @@ -196,12 +208,36 @@ func (s *Store) RecordResult(ctx context.Context, item *QueueItem, startedAt tim } return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // 1. Insert the immutable result record. if err := tx.Create(processed).Error; err != nil { return fmt.Errorf("insert processed_item: %w", err) } - if err := tx.Delete(&QueueItem{}, item.ID).Error; err != nil { - return fmt.Errorf("delete queue_item %d: %w", item.ID, err) + + // 2. Decide: delete the queue item or re-queue it. + // MaxRuns == 1 means "run once" — delete. + // MaxRuns == -1 means "repeat indefinitely" — always re-queue. + // MaxRuns > 1 means "run N times" — decrement and re-queue. + if item.MaxRuns == 1 { + // Exhausted — delete. + if err := tx.Delete(&QueueItem{}, item.ID).Error; err != nil { + return fmt.Errorf("delete queue_item %d: %w", item.ID, err) + } + } else { + // Decrement MaxRuns (if not -1/infinite) and set back to pending. + newMaxRuns := item.MaxRuns + if newMaxRuns > 1 { + newMaxRuns-- + } + if err := tx.Model(&QueueItem{}). + Where("id = ?", item.ID). + Updates(map[string]interface{}{ + "status": StatusPending, + "max_runs": newMaxRuns, + }).Error; err != nil { + return fmt.Errorf("re-queue queue_item %d: %w", item.ID, err) + } } + return nil }) } @@ -361,10 +397,28 @@ func (s *Scheduler) tick(ctx context.Context, t time.Time) { logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] tick %s — executing %d task(s)", ts, len(items))) - for i, item := range items { + // Split items into sequential vs concurrent groups. + var sequential []*QueueItem + concurrentGroups := make(map[string][]*QueueItem) // taskType -> items + + for _, item := range items { + if item.Thread != 0 { + concurrentGroups[item.TaskType] = append(concurrentGroups[item.TaskType], item) + } else { + sequential = append(sequential, item) + } + } + + // Execute sequential items one at a time. + for i, item := range sequential { s.execute(ctx, item, i+1, len(items)) } + // Execute concurrent groups in parallel per type. + for _, group := range concurrentGroups { + s.executeGroup(ctx, group, len(items)) + } + if remaining, err := s.store.PendingCount(ctx); err == nil { logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] tick %s — done (%d task(s) still queued)", ts, remaining)) } @@ -389,3 +443,22 @@ func (s *Scheduler) execute(ctx context.Context, item *QueueItem, pos, total int logger.ResolveLogger().Error(fmt.Sprintf("[scheduler] [%d/%d] failed to record result for id=%d: %v", pos, total, item.ID, err)) } } + +// executeGroup runs all items in a group concurrently using goroutines. +// Each item's result is recorded independently. +func (s *Scheduler) executeGroup(ctx context.Context, items []*QueueItem, total int) { + var wg sync.WaitGroup + wg.Add(len(items)) + + for _, item := range items { + item := item // capture range variable + go func() { + defer wg.Done() + // Find the item's position among all items for logging context. + // Since we don't have global position here, use a local index. + s.execute(ctx, item, 0, total) + }() + } + + wg.Wait() +}