diff --git a/config.go b/config.go index aeffbd7..921cb24 100644 --- a/config.go +++ b/config.go @@ -1,13 +1,10 @@ // Copyright 2021 Harran Ali . All rights reserved. // Copyright (c) 2024 Zeni Kim -// Copyright (c) 2026 Jose Cely // Use of this source code is governed by MIT-style // license that can be found in the LICENSE file. package core -import "time" - type EnvFileConfig struct { UseDotEnvFile bool } @@ -31,12 +28,6 @@ type QueueConfig struct { Queues map[string]int } -type SchedulerConfig struct { - EnableScheduler bool - SchedulerInterval time.Duration - SchedulerRateLimit int -} - type CacheConfig struct { EnableCache bool } diff --git a/scheduler.go b/scheduler.go deleted file mode 100644 index 90f6f57..0000000 --- a/scheduler.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2026 Jose Cely -// Use of this source code is governed by MIT-style -// license that can be found in the LICENSE file. - -package core - -import ( - "context" - "fmt" - "log" - "time" - - "git.smarteching.com/goffee/core/logger" - "git.smarteching.com/goffee/core/scheduler" -) - -// Default values for SchedulerConfig -var DefaultSchedulerInterval = 10 * time.Second -var DefaultSchedulerRateLimit = 10 - -// globalSchedulerMux holds the active Schedulermux instance so controllers -// can access it without requiring a direct import chain. -var globalSchedulerMux *Schedulermux - -// Schedulermux wraps the scheduler Registry and manages the lifecycle -// of a simple database-backed task scheduler. -type Schedulermux struct { - registry *scheduler.Registry - store *scheduler.Store - semaphore *scheduler.Semaphore - sched *scheduler.Scheduler -} - -// SchedulerInit initializes the scheduler internals (registry and semaphore). -// The store must be set separately via SetStore before starting. -func (s *Schedulermux) SchedulerInit() { - s.registry = scheduler.NewRegistry() - s.semaphore = scheduler.NewSemaphore(true) - globalSchedulerMux = s -} - -// SetStore sets the store (backed by *gorm.DB) on the scheduler. -// This also runs AutoMigrate for the queue_items and processed_items tables. -func (s *Schedulermux) SetStore(st *scheduler.Store) { - s.store = st -} - -// AddWork registers a task handler function for a given task type. -func (s *Schedulermux) AddWork(taskType string, handler scheduler.HandlerFunc) { - s.registry.Register(taskType, handler) -} - -// GetStore returns the scheduler store, or nil if not initialized. -func (s *Schedulermux) GetStore() *scheduler.Store { - return s.store -} - -// RunScheduler starts the scheduler loop with the given configuration. -// It creates the internal Scheduler instance and launches it in a goroutine. -func (s *Schedulermux) RunScheduler(config SchedulerConfig) { - if s.store == nil { - log.Fatal("scheduler: store is not set — call SetStore before RunScheduler") - } - - interval := config.SchedulerInterval - if interval <= 0 { - interval = DefaultSchedulerInterval - } - - rateLimit := config.SchedulerRateLimit - if rateLimit < 0 { - rateLimit = DefaultSchedulerRateLimit - } - - s.sched = scheduler.NewScheduler( - scheduler.SchedulerConfig{ - Interval: interval, - RateLimit: rateLimit, - }, - s.semaphore, - s.store, - s.registry, - ) - - // Recover stuck items from a previous crash before starting. - ctx := context.Background() - recovered, err := s.store.RecoverStuck(ctx) - if err != nil { - logger.ResolveLogger().Error(fmt.Sprintf("[scheduler] recover stuck error: %v", err)) - } else if recovered > 0 { - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] recovered %d stuck task(s) back to pending", recovered)) - } - - s.sched.Start(ctx) -} - -// StopScheduler gracefully stops the scheduler loop. -func (s *Schedulermux) StopScheduler() { - if s.sched != nil { - s.sched.Stop() - } -} - -// SemaphoreSetGreen allows task execution to proceed. -func (s *Schedulermux) SemaphoreSetGreen() { - if s.semaphore != nil { - s.semaphore.SetGreen() - } -} - -// SemaphoreSetRed blocks task execution. -func (s *Schedulermux) SemaphoreSetRed() { - if s.semaphore != nil { - s.semaphore.SetRed() - } -} - -// ResolveSchedulerStore returns the store from the global scheduler mux instance. -// Returns nil if the scheduler was never initialized. -func ResolveSchedulerStore() *scheduler.Store { - if globalSchedulerMux == nil { - return nil - } - return globalSchedulerMux.GetStore() -} - -// SchedulerSemaphoreSetGreen enables task execution by setting the semaphore to green. -func SchedulerSemaphoreSetGreen() { - if globalSchedulerMux != nil { - globalSchedulerMux.SemaphoreSetGreen() - } -} - -// SchedulerSemaphoreSetRed blocks task execution by setting the semaphore to red. -func SchedulerSemaphoreSetRed() { - if globalSchedulerMux != nil { - globalSchedulerMux.SemaphoreSetRed() - } -} - -// SchedulerSemaphoreIsGreen returns whether the scheduler semaphore is currently green. -func SchedulerSemaphoreIsGreen() bool { - if globalSchedulerMux == nil || globalSchedulerMux.semaphore == nil { - return false - } - return globalSchedulerMux.semaphore.IsGreen() -} diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go deleted file mode 100644 index dfb7d32..0000000 --- a/scheduler/scheduler.go +++ /dev/null @@ -1,391 +0,0 @@ -// Copyright (c) 2026 Jose Cely -// Use of this source code is governed by MIT-style -// license that can be found in the LICENSE file. - -package scheduler - -import ( - "context" - "fmt" - "time" - - "git.smarteching.com/goffee/core/logger" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -// ─── Status types ──────────────────────────────────────────────────────────── - -// QueueStatus represents the lifecycle state of a queued item. -type QueueStatus string - -const ( - StatusPending QueueStatus = "pending" - StatusProcessing QueueStatus = "processing" // claimed by a scheduler tick -) - -// ProcessedStatus is the final outcome recorded in processed_items. -type ProcessedStatus string - -const ( - ProcessedOK ProcessedStatus = "ok" - ProcessedError ProcessedStatus = "error" -) - -// ─── Models ────────────────────────────────────────────────────────────────── - -// QueueItem is a task waiting to be executed. -// Any external program can insert a row into this table to enqueue work. -// -// INSERT INTO queue_items (task_type, payload, priority) VALUES (?, ?, ?); -type QueueItem struct { - ID uint `gorm:"primaryKey;autoIncrement"` - TaskType string `gorm:"not null;index"` // e.g. "send_email", "sync_records" - Payload string `gorm:"type:text"` // JSON or plain string, interpreted by the handler - Status QueueStatus `gorm:"not null;default:'pending';index"` - Priority int `gorm:"not null;default:0;index"` // higher = runs first within a tick - CreatedAt time.Time - UpdatedAt time.Time -} - -// TableName overrides the GORM default. -func (QueueItem) TableName() string { return "queue_items" } - -// ProcessedItem is an immutable record of a completed execution. -// Rows are never updated — only inserted. -type ProcessedItem struct { - ID uint `gorm:"primaryKey;autoIncrement"` - QueueItemID uint `gorm:"not null;index"` // FK → queue_items.id (soft ref) - TaskType string `gorm:"not null;index"` - Payload string `gorm:"type:text"` - Status ProcessedStatus `gorm:"not null;index"` - ErrorMsg string `gorm:"type:text"` // empty when Status == "ok" - StartedAt time.Time `gorm:"not null"` - FinishedAt time.Time `gorm:"not null"` - DurationMs int64 `gorm:"not null"` // FinishedAt - StartedAt in ms - CreatedAt time.Time -} - -// TableName overrides the GORM default. -func (ProcessedItem) TableName() string { return "processed_items" } - -// ─── Handler ───────────────────────────────────────────────────────────────── - -// HandlerFunc processes a single queue item. -// Return a non-nil error to mark the item as processed with error status. -type HandlerFunc func(ctx context.Context, item *QueueItem) error - -// Registry maps task types to their handlers. -// Register all handlers before calling Scheduler.Start. -type Registry struct { - handlers map[string]HandlerFunc -} - -// NewRegistry creates an empty handler registry. -func NewRegistry() *Registry { - return &Registry{handlers: make(map[string]HandlerFunc)} -} - -// Register associates a task type with a handler. -// Panics on duplicate registration to catch misconfiguration at startup. -func (r *Registry) Register(taskType string, h HandlerFunc) { - if _, exists := r.handlers[taskType]; exists { - panic(fmt.Sprintf("scheduler: handler already registered for task type %q", taskType)) - } - r.handlers[taskType] = h -} - -// get returns the handler for a task type, or a fallback error handler. -func (r *Registry) get(taskType string) HandlerFunc { - if h, ok := r.handlers[taskType]; ok { - return h - } - return func(_ context.Context, item *QueueItem) error { - return fmt.Errorf("no handler registered for task type %q", item.TaskType) - } -} - -// ─── Store ─────────────────────────────────────────────────────────────────── - -// Store handles all database operations for the scheduler. -type Store struct { - db *gorm.DB -} - -// NewStore creates a Store. -// Tables (queue_items, processed_items) -func NewStore(db *gorm.DB) (*Store, error) { - if db == nil { - return nil, fmt.Errorf("scheduler: db is nil") - } - return &Store{db: db}, nil -} - -// Enqueue inserts a new pending task. -func (s *Store) Enqueue(ctx context.Context, taskType, payload string, priority int) (*QueueItem, error) { - item := &QueueItem{ - TaskType: taskType, - Payload: payload, - Status: StatusPending, - Priority: priority, - } - if err := s.db.WithContext(ctx).Create(item).Error; err != nil { - return nil, fmt.Errorf("enqueue %s: %w", taskType, err) - } - return item, nil -} - -// ClaimBatch atomically claims up to `limit` pending tasks and marks them -// as "processing" so that concurrent scheduler instances never double-execute. -func (s *Store) ClaimBatch(ctx context.Context, limit int) ([]*QueueItem, error) { - var items []*QueueItem - - err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - query := tx. - Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}). - Where("status = ?", StatusPending). - Order("priority DESC, id ASC"). - Limit(limit). - Find(&items) - - if query.Error != nil { - return query.Error - } - if len(items) == 0 { - return nil - } - - ids := make([]uint, len(items)) - for i, it := range items { - ids[i] = it.ID - } - - return tx.Model(&QueueItem{}). - Where("id IN ?", ids). - Update("status", StatusProcessing). - Error - }) - - if err != nil { - return nil, fmt.Errorf("claim batch: %w", err) - } - return items, nil -} - -// RecordResult writes a ProcessedItem and deletes the original QueueItem -// in a single transaction. -func (s *Store) RecordResult(ctx context.Context, item *QueueItem, startedAt time.Time, execErr error) error { - finishedAt := time.Now() - - status := ProcessedOK - errMsg := "" - if execErr != nil { - status = ProcessedError - errMsg = execErr.Error() - } - - processed := &ProcessedItem{ - QueueItemID: item.ID, - TaskType: item.TaskType, - Payload: item.Payload, - Status: status, - ErrorMsg: errMsg, - StartedAt: startedAt, - FinishedAt: finishedAt, - DurationMs: finishedAt.Sub(startedAt).Milliseconds(), - } - - return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if err := tx.Create(processed).Error; err != nil { - return fmt.Errorf("insert processed_item: %w", err) - } - if err := tx.Delete(&QueueItem{}, item.ID).Error; err != nil { - return fmt.Errorf("delete queue_item %d: %w", item.ID, err) - } - return nil - }) -} - -// PendingCount returns the number of tasks currently in pending or processing state. -func (s *Store) PendingCount(ctx context.Context) (int64, error) { - var count int64 - err := s.db.WithContext(ctx). - Model(&QueueItem{}). - Where("status IN ?", []QueueStatus{StatusPending, StatusProcessing}). - Count(&count).Error - return count, err -} - -// RecentProcessed returns the last `n` processed items ordered by newest first. -func (s *Store) RecentProcessed(ctx context.Context, n int) ([]*ProcessedItem, error) { - var items []*ProcessedItem - err := s.db.WithContext(ctx). - Order("id DESC"). - Limit(n). - Find(&items).Error - return items, err -} - -// RecoverStuck resets any items stuck in "processing" state (e.g. after a -// crash) back to "pending" so they can be re-claimed on the next tick. -// Call this once at startup. -func (s *Store) RecoverStuck(ctx context.Context) (int64, error) { - result := s.db.WithContext(ctx). - Model(&QueueItem{}). - Where("status = ?", StatusProcessing). - Update("status", StatusPending) - return result.RowsAffected, result.Error -} - -// ─── Semaphore ─────────────────────────────────────────────────────────────── - -// Semaphore controls whether the scheduler is allowed to execute tasks. -// Green (true) means tasks can run; Red (false) means tasks are blocked. -type Semaphore struct { - green chan bool -} - -// NewSemaphore creates a Semaphore. Pass true to start green, false to start red. -func NewSemaphore(startGreen bool) *Semaphore { - s := &Semaphore{green: make(chan bool, 1)} - s.green <- startGreen - return s -} - -// SetGreen allows task execution to proceed. -func (s *Semaphore) SetGreen() { - <-s.green - s.green <- true -} - -// SetRed blocks task execution. -func (s *Semaphore) SetRed() { - <-s.green - s.green <- false -} - -// IsGreen reports whether execution is currently allowed. -func (s *Semaphore) IsGreen() bool { - val := <-s.green - s.green <- val - return val -} - -// ─── Scheduler ─────────────────────────────────────────────────────────────── - -// Scheduler polls the database every interval and executes pending tasks. -type Scheduler struct { - interval time.Duration - rateLimit int // max tasks per tick; 0 = unlimited - semaphore *Semaphore - store *Store - registry *Registry - - stop chan struct{} - done chan struct{} -} - -// SchedulerConfig holds constructor options. -type SchedulerConfig struct { - Interval time.Duration - RateLimit int // 0 = unlimited -} - -// NewScheduler creates a Scheduler. -func NewScheduler(cfg SchedulerConfig, sem *Semaphore, st *Store, reg *Registry) *Scheduler { - return &Scheduler{ - interval: cfg.Interval, - rateLimit: cfg.RateLimit, - semaphore: sem, - store: st, - registry: reg, - stop: make(chan struct{}), - done: make(chan struct{}), - } -} - -// Start launches the scheduler goroutine. -func (s *Scheduler) Start(ctx context.Context) { - go s.run(ctx) -} - -// Stop signals the scheduler and waits for it to exit cleanly. -func (s *Scheduler) Stop() { - close(s.stop) - <-s.done -} - -// run is the main event loop. -func (s *Scheduler) run(ctx context.Context) { - defer close(s.done) - - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - logger.ResolveLogger().Info("[scheduler] context cancelled, shutting down") - return - case <-s.stop: - logger.ResolveLogger().Info("[scheduler] stop requested, shutting down") - return - case t := <-ticker.C: - s.tick(ctx, t) - } - } -} - -// tick runs on every ticker event. -func (s *Scheduler) tick(ctx context.Context, t time.Time) { - ts := t.Format(time.TimeOnly) - - if !s.semaphore.IsGreen() { - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] tick %s — semaphore RED, skipping", ts)) - return - } - - limit := s.rateLimit - if limit == 0 { - limit = 1000 - } - - items, err := s.store.ClaimBatch(ctx, limit) - if err != nil { - logger.ResolveLogger().Error(fmt.Sprintf("[scheduler] tick %s — claim error: %v", ts, err)) - return - } - if len(items) == 0 { - return - } - - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] tick %s — executing %d task(s)", ts, len(items))) - - for i, item := range items { - s.execute(ctx, item, i+1, len(items)) - } - - if remaining, err := s.store.PendingCount(ctx); err == nil { - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] tick %s — done (%d task(s) still queued)", ts, remaining)) - } -} - -// execute runs one task and records the result regardless of outcome. -func (s *Scheduler) execute(ctx context.Context, item *QueueItem, pos, total int) { - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] [%d/%d] starting task id=%d type=%s", pos, total, item.ID, item.TaskType)) - - startedAt := time.Now() - handler := s.registry.get(item.TaskType) - execErr := handler(ctx, item) - - if execErr != nil { - logger.ResolveLogger().Error(fmt.Sprintf("[scheduler] [%d/%d] task id=%d FAILED: %v", pos, total, item.ID, execErr)) - } else { - logger.ResolveLogger().Info(fmt.Sprintf("[scheduler] [%d/%d] task id=%d OK (%.0fms)", - pos, total, item.ID, float64(time.Since(startedAt).Milliseconds()))) - } - - if err := s.store.RecordResult(ctx, item, startedAt, execErr); err != nil { - logger.ResolveLogger().Error(fmt.Sprintf("[scheduler] [%d/%d] failed to record result for id=%d: %v", pos, total, item.ID, err)) - } -}