From 7af7a1f70b8935b27e6b9e0da5127165a73c4026 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 16:54:48 +0600 Subject: [PATCH 1/6] feat: EnqueueOnce dedupes jobs by id while their row exists Queue.EnqueueOnce(item, id) and PriorityQueue.EnqueueOnce(item, priority, id) implement varmq's IDedupQueue and IDedupPriorityQueue. Opening a table adds a job_id TEXT column when it is missing and a unique partial index on it over the rows that have one. The insert ends in ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING, so a repeat stores nothing and returns true, while a NOT NULL failure such as Enqueue(nil) still fails. Enqueue and an empty id bind NULL. The index is __sqliteq_job_id rather than _job_id, for the reason the pending index has its suffix: tables and indexes share one namespace, and a queue named orders_job_id could not open beside orders. --- commit_test.go | 2 +- dedup_test.go | 381 ++++++++++++++++++++++++++++++++++++++++++++++ migration_test.go | 11 +- priority_queue.go | 6 + queue.go | 60 +++++++- 5 files changed, 446 insertions(+), 14 deletions(-) create mode 100644 dedup_test.go diff --git a/commit_test.go b/commit_test.go index b29a954..6b1a7e1 100644 --- a/commit_test.go +++ b/commit_test.go @@ -296,7 +296,7 @@ func TestCommitRoundFallsBackPerOp(t *testing.T) { defer qs.Close() add := func(tx *sql.Tx) (bool, error) { - _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now()) + _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now(), nil) return err == nil, err } bad := func(*sql.Tx) (bool, error) { return false, errors.New("bad op") } diff --git a/dedup_test.go b/dedup_test.go new file mode 100644 index 0000000..b2aee68 --- /dev/null +++ b/dedup_test.go @@ -0,0 +1,381 @@ +package sqliteq + +import ( + "fmt" + "path/filepath" + "sync" + "testing" +) + +// TestImplementsDedup checks at compile time that both kinds of queue +// implement varmq's IDedupQueue and IDedupPriorityQueue. +func TestImplementsDedup(t *testing.T) { + var _ interface{ EnqueueOnce(any, string) bool } = (*Queue)(nil) + var _ interface{ EnqueueOnce(any, int, string) bool } = (*PriorityQueue)(nil) +} + +// once adds item under id; on a priority queue it does so at priority 1 +type once func(item any, id string) bool + +// openOnce opens queue "q", or priority queue "q", in a fresh folder store, +// and returns its Queue with its EnqueueOnce. countRows reads table "q". +func openOnce(t *testing.T, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + qs := mustNew(t, t.TempDir()) + if priority { + pq, err := qs.NewPriorityQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create priority queue: %v", err) + } + return pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 1, id) }, qs + } + q, err := qs.NewQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create queue: %v", err) + } + return q, q.EnqueueOnce, qs +} + +// dequeueAll pops every pending item and returns them as strings, in order +func dequeueAll(t *testing.T, q *Queue) []string { + t.Helper() + var out []string + for { + item, ok := q.Dequeue() + if !ok { + return out + } + out = append(out, string(item.([]byte))) + } +} + +func TestEnqueueOnceRepeat(t *testing.T) { testOnceRepeat(t, false) } + +func testOnceRepeat(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("First EnqueueOnce failed") + } + if !add([]byte("b"), "j1") { + t.Error("A repeat returned false, want true") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a]", got) + } + + // Dequeue deleted the row, and the id with it + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after Dequeue returned false, want true") + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v after the re-add, want [c]", got) + } +} + +func TestEnqueueOnceWhileProcessing(t *testing.T) { testOnceWhileProcessing(t, false) } + +func testOnceWhileProcessing(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + t.Fatal("DequeueWithAckId failed") + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of a processing job returned false, want true") + } + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the processing one), got %d", n) + } +} + +func TestEnqueueOnceAfterAck(t *testing.T) { testOnceAfterAck(t, false) } + +func testOnceAfterAck(t *testing.T, priority bool) { + for _, remove := range []bool{true, false} { + t.Run(fmt.Sprintf("remove=%v", remove), func(t *testing.T) { + q, add, qs := openOnce(t, priority, WithRemoveOnComplete(remove)) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + if !q.Acknowledge(ackID) { + t.Fatal("Acknowledge failed") + } + + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after the ack returned false, want true") + } + if remove { + // The row is gone, and the id with it + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v, want [c]", got) + } + return + } + // The completed row keeps the id + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the completed one), got %d", n) + } + }) + } +} + +func TestEnqueueOncePriority(t *testing.T) { + t.Run("Repeat", func(t *testing.T) { testOnceRepeat(t, true) }) + t.Run("WhileProcessing", func(t *testing.T) { testOnceWhileProcessing(t, true) }) + t.Run("AfterAck", func(t *testing.T) { testOnceAfterAck(t, true) }) + + t.Run("Order", func(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + for _, c := range []struct { + item string + priority int + id string + }{ + {"low", 5, "a"}, + {"high", 1, "b"}, + {"mid", 3, "c"}, + {"high-again", 0, "b"}, // a repeat: neither stored nor moved up + {"low-again", 9, "a"}, + {"no-id", 2, ""}, + } { + if !pq.EnqueueOnce([]byte(c.item), c.priority, c.id) { + t.Fatalf("EnqueueOnce(%s, %d, %q) failed", c.item, c.priority, c.id) + } + } + + if got := dequeueAll(t, pq.Queue); fmt.Sprint(got) != "[high no-id mid low]" { + t.Errorf("Dequeued %v, want [high no-id mid low]", got) + } + }) +} + +// An empty id stores the item like Enqueue does, every time +func TestEnqueueOnceEmptyID(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + for i := 0; i < 3; i++ { + if !q.EnqueueOnce([]byte("x"), "") { + t.Fatalf("EnqueueOnce %d with no id failed", i) + } + if !q.Enqueue([]byte("y")) { + t.Fatalf("Enqueue %d failed", i) + } + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending items, got %d", n) + } + if n := countRows(t, q.f.db, "job_id IS NULL"); n != 6 { + t.Errorf("Expected 6 rows with a NULL job_id, got %d", n) + } +} + +// The conflict clause covers the job_id index only: a nil item still fails, +// and a failed add leaves its id free +func TestEnqueueOnceNil(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if add(nil, "j1") { + t.Error("EnqueueOnce(nil) succeeded") + } + if q.Enqueue(nil) { + t.Error("Enqueue(nil) succeeded") + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows, got %d", n) + } + if !add([]byte("a"), "j1") { + t.Error("EnqueueOnce after a failed one failed") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + }) + } +} + +// Two adds of one id in one grouped round: the second conflicts inside the +// round's transaction, both return true, and one row is stored. With a nil +// item in the round, the round falls back to one transaction per call and +// the same holds. +func TestEnqueueOnceOneRound(t *testing.T) { + for _, bad := range []bool{false, true} { + t.Run(fmt.Sprintf("bad=%v", bad), func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + first := func() bool { return q.Enqueue([]byte("x")) } + rest := []func() bool{ + func() bool { return q.EnqueueOnce([]byte("a"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("b"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("c"), "j2") }, + } + if bad { + rest = append(rest, + func() bool { return q.Enqueue(nil) }, + func() bool { return q.EnqueueOnce(nil, "j3") }, + ) + } + + got := oneRound(t, q.f, first, rest) + for i := 0; i < 4; i++ { + if !got[i] { + t.Errorf("Call %d failed", i) + } + } + for i := 4; i < len(got); i++ { + if got[i] { + t.Errorf("Call %d (a nil item) succeeded", i) + } + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 3 { + t.Errorf("Expected 3 rows (x, j1, j2), got %d", n) + } + }) + } +} + +// 32 concurrent adds of one id: every call returns true and one row is stored +func TestEnqueueOnceConcurrent(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + const n = 32 + var ( + wg sync.WaitGroup + start = make(chan struct{}) + oks [n]bool + ) + for g := 0; g < n; g++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + oks[g] = add([]byte(fmt.Sprint(g)), "j1") + }() + } + close(start) + wg.Wait() + + for g, ok := range oks { + if !ok { + t.Errorf("EnqueueOnce in goroutine %d returned false", g) + } + } + if l := q.Len(); l != 1 { + t.Errorf("Expected 1 pending item, got %d", l) + } + if c := countRows(t, q.f.db, "1"); c != 1 { + t.Errorf("Expected 1 row, got %d", c) + } + }) + } +} + +// A v0.2.3 file gains job_id and its unique partial index on open, keeps +// them on reopen, and its old rows still dequeue in order. +func TestEnqueueOnceOldFile(t *testing.T) { + for _, priority := range []bool{false, true} { + name := "jobs" + if priority { + name = "pjobs" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "v023.db") + writeV023File(t, path, name, priority) + + for round := 1; round <= 2; round++ { + qs := mustNew(t, path, WithSingleFile()) + var ( + q *Queue + add once + ) + if priority { + pq, err := qs.NewPriorityQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 priority queue: %v", round, err) + } + q, add = pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 0, id) } + } else { + var err error + if q, err = qs.NewQueue(name); err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 queue: %v", round, err) + } + add = q.EnqueueOnce + } + + if ok, err := hasColumn(q.f.db, name, "job_id"); err != nil || !ok { + t.Fatalf("Round %d: job_id column present = %v, %v", round, ok, err) + } + var unique, partial int + err := q.f.db.QueryRow(`SELECT "unique", partial FROM pragma_index_list(?) WHERE name = ?`, + name, name+jobIDIndexSuffix).Scan(&unique, &partial) + if err != nil || unique != 1 || partial != 1 { + t.Errorf("Round %d: job_id index unique=%d partial=%d err=%v, want a unique partial index", round, unique, partial, err) + } + if cols, err := indexColumns(q.f.db, name+jobIDIndexSuffix); err != nil || fmt.Sprint(cols) != "[job_id]" { + t.Errorf("Round %d: job_id index columns = %v, %v; want [job_id]", round, cols, err) + } + + if round == 1 { + if !add([]byte("new"), "j1") || !add([]byte("dup"), "j1") { + t.Error("EnqueueOnce on the old table failed") + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending jobs, got %d", n) + } + } else { + want := "[job-1 job-2 job-3 job-4 job-5 new]" + if got := dequeueAll(t, q); fmt.Sprint(got) != want { + t.Errorf("Dequeued %v, want %s", got, want) + } + } + + if err := qs.Close(); err != nil { + t.Fatalf("Round %d: failed to close: %v", round, err) + } + } + }) + } +} diff --git a/migration_test.go b/migration_test.go index affe32f..bc94204 100644 --- a/migration_test.go +++ b/migration_test.go @@ -125,7 +125,7 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} + want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } @@ -207,11 +207,12 @@ func TestPendingIndexFollowsKind(t *testing.T) { } } -// Tables and indexes share one namespace in SQLite, so the pending index -// must not take a name a queue could have: a file with queues "orders" and -// "orders_pending" opens in either order, and again on reopen. +// Tables and indexes share one namespace in SQLite, so the pending and +// job_id indexes must not take a name a queue could have: a file with queues +// "orders", "orders_pending" and "orders_job_id" opens in either order, and +// again on reopen. func TestPendingIndexNameFreeForQueues(t *testing.T) { - for _, names := range [][]string{{"orders", "orders_pending"}, {"orders_pending", "orders"}} { + for _, names := range [][]string{{"orders", "orders_pending", "orders_job_id"}, {"orders_job_id", "orders_pending", "orders"}} { t.Run(names[0]+"_first", func(t *testing.T) { path := filepath.Join(t.TempDir(), "shared.db") for round := 1; round <= 2; round++ { diff --git a/priority_queue.go b/priority_queue.go index bc2ff32..0aeae49 100644 --- a/priority_queue.go +++ b/priority_queue.go @@ -22,3 +22,9 @@ func newPriorityQueue(f *file, name string, opts ...Option) (*PriorityQueue, err func (pq *PriorityQueue) Enqueue(item any, priority int) bool { return pq.enqueue(item, priority, "") } + +// EnqueueOnce implements varmq.IDedupPriorityQueue: Queue.EnqueueOnce with a +// priority. A repeat leaves the stored item and its priority as they are. +func (pq *PriorityQueue) EnqueueOnce(item any, priority int, id string) bool { + return pq.enqueue(item, priority, id) +} diff --git a/queue.go b/queue.go index bdcfa9f..c2f62a3 100644 --- a/queue.go +++ b/queue.go @@ -80,6 +80,9 @@ func (q *Queue) initSchema() error { if err := q.migrate(); err != nil { return fmt.Errorf("failed to migrate indexes: %w", err) } + if err := q.addJobID(); err != nil { + return fmt.Errorf("failed to add job_id: %w", err) + } return nil } @@ -110,9 +113,13 @@ func (q *Queue) initTable() error { return err } -// pendingIndexSuffix names a queue's partial pending index. Tables and -// indexes share one namespace, so the suffix is one no queue name would take. -const pendingIndexSuffix = "__sqliteq_pending" +// pendingIndexSuffix and jobIDIndexSuffix name a queue's partial indexes. +// Tables and indexes share one namespace, so the suffixes are ones no queue +// name would take. +const ( + pendingIndexSuffix = "__sqliteq_pending" + jobIDIndexSuffix = "__sqliteq_job_id" +) // migrate drops the v0.2 indexes and makes sure the one partial index on // pending rows matches the queue order. On a migrated table it changes nothing. @@ -142,6 +149,24 @@ func (q *Queue) migrate() error { return err } +// addJobID adds the job_id column EnqueueOnce fills, when it is missing, and +// its unique index over the rows that have one. Rows without an id, old ones +// and those of Enqueue, are NULL and outside the index. +func (q *Queue) addJobID() error { + exists, err := hasColumn(q.f.db, q.name, "job_id") + if err != nil { + return err + } + if !exists { + if _, err := q.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN job_id TEXT", q.table)); err != nil { + return err + } + } + _, err = q.f.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (job_id) WHERE job_id IS NOT NULL", + quoteIdent(q.name+jobIDIndexSuffix), q.table)) + return err +} + // indexColumns lists an index's columns in order; none if it does not exist func indexColumns(db *sql.DB, index string) ([]string, error) { rows, err := db.Query("SELECT name FROM pragma_index_info(?) ORDER BY seqno", index) @@ -170,7 +195,7 @@ func (q *Queue) prepare() (err error) { } }() - cols, vals, returning := "data, status, ack, created_at, updated_at", "?, 'pending', 0, ?, ?", "id, data" + cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data" if q.priority { cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" } @@ -185,7 +210,9 @@ func (q *Queue) prepare() (err error) { stmt **sql.Stmt query string }{ - {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", q.table, cols, vals)}, + // The conflict target is the job_id index alone, so a NOT NULL or any + // other failure still errors; it must repeat the index's WHERE + {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, {&q.ack, ack}, @@ -266,8 +293,17 @@ func (q *Queue) Enqueue(item any) bool { return q.enqueue(item, 0, "") } +// EnqueueOnce implements varmq.IDedupQueue: it adds item under id unless a +// row with that id exists, pending, processing, or completed with +// WithRemoveOnComplete(false). A repeat stores nothing and returns true. An +// empty id behaves like Enqueue. +func (q *Queue) EnqueueOnce(item any, id string) bool { + return q.enqueue(item, 0, id) +} + // enqueue inserts one pending row in the file's next group commit round; -// priority is stored on priority queues only. id is unused for now. +// priority is stored on priority queues only. A row with a non-empty id is +// skipped, still with true, when a row with that id exists. func (q *Queue) enqueue(item any, priority int, id string) bool { if q.closed.Load() { return false @@ -280,18 +316,26 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { return false } + // No id is NULL, which the job_id index leaves out; "" would be one id + var jobID any + if id != "" { + jobID = id + } + now := time.Now().UTC() return q.f.b.do(func(tx *sql.Tx) (bool, error) { insert := q.insert if tx != nil { insert = tx.Stmt(insert) } + // One statement, so a round of one, run without a transaction, is atomic var err error if q.priority { - _, err = insert.Exec(data, now, now, priority) + _, err = insert.Exec(data, now, now, jobID, priority) } else { - _, err = insert.Exec(data, now, now) + _, err = insert.Exec(data, now, now, jobID) } + // A repeat id changes no row and is no error: it succeeds return err == nil, err }) } From 6a5b193100119d411a5cd8b068c65a91ddfdce97 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 16:54:48 +0600 Subject: [PATCH 2/6] docs: EnqueueOnce, and varmq v1.10.0 for batch pops and EnqueueOnce A short EnqueueOnce section: a repeat stores nothing and returns true, and the id is remembered while its row exists (pending, processing, or completed with WithRemoveOnComplete(false)). The Compatibility line now says batch pops and EnqueueOnce need varmq v1.10.0 or later, and the schema list names job_id. --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ff8415..4ec94c6 100644 --- a/README.md +++ b/README.md @@ -168,9 +168,20 @@ In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark] The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +## EnqueueOnce + +`EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. + +```go +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // stored +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stored +``` + +The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. + ## Compatibility -sqliteq does not import varmq. Queues and priority queues implement `DequeueBatchWithAckIds`, so with varmq ≥ v1.10.0 a worker claims several jobs in one statement. An older varmq never calls it and claims one job per call, which still works. `EnqueueOnce` comes in a later release. +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker claims several jobs in one statement through `DequeueBatchWithAckIds`, and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. ## How It Works @@ -180,6 +191,7 @@ SQLiteQ uses a SQLite database to store queue items with the following schema: - `data`: The serialized item data (stored as a JSON blob) - `status`: The status of the item ("pending", "processing", or "completed") - `ack_id`: Kept from v0.2 so old tables need no rebuild; no longer used +- `job_id`: The id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it - `created_at`: When the item was added to the queue - `updated_at`: When the item was last updated From e8c067e0a91704a6e6823de445b2a007080ee980 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 17:36:30 +0600 Subject: [PATCH 3/6] fix: reject nil options, pin EnqueueOnce across recovery, README wording A nil option panicked in New, in the registry's option loop and in newQueue. All three now return one error, so New, NewQueue and NewPriorityQueue never panic on it; a rejected call creates no folder, file or table. Two tests pin the EnqueueOnce id: a job claimed and never acked keeps its id through the requeue on the next open, so a repeat stores nothing and the job runs once; Purge frees the ids of pending and processing rows. README: any name SQLite accepts as a table name under WithSingleFile, the same handle only while it is open, the single-file layout does not create its folder, batch claims only for a worker whose sole source is the sqliteq queue, a Durability section for WAL with synchronous=NORMAL, and the Windows device names as queue names in the folder layout. --- README.md | 14 ++++++--- dedup_test.go | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- layout_test.go | 61 ++++++++++++++++++++++++++++++++++++++ option.go | 6 ++++ queue.go | 3 ++ queues.go | 3 ++ registry.go | 3 ++ 7 files changed, 165 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ec94c6..e82a0d8 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ func main() { } ``` -`New` returns an error instead of panicking: for a path it cannot use, an invalid option, or a path that is an existing file when `WithSingleFile()` is not given. +`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. -The same name always returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. +While a handle is open, the same name returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. ## Layout on disk @@ -138,11 +138,13 @@ if err != nil { defer queuesManager.Close() ``` +The file is created if it is missing, but its folder must already exist: `New` returns an error otherwise, unlike the folder layout, which creates its folder. + If you upgrade from v0.2, add `WithSingleFile()` to keep reading your existing file, or start with the folder layout for new data. ### Queue names -In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. With `WithSingleFile()` any name works, since it is only a table name. +In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. On Windows the folder layout also cannot use a reserved device name as a queue name (`CON`, `PRN`, `AUX`, `NUL`, `COM1` to `COM9`, `LPT1` to `LPT9`, in any letter case), because the name would become a file name such as `con.db`. sqliteq does not check for these. With `WithSingleFile()` the name is only a table name, so any name SQLite accepts as a table name works; SQLite reserves names that start with `sqlite_`. Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue, stored in `jobs.db` in the folder layout. @@ -168,6 +170,10 @@ In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark] The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +## Durability + +sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. A job whose `Enqueue` returned true survives a crash of the process. After a power loss or an operating system crash, the last commits before the failure may be lost, because SQLite rolls back what the WAL had not yet synced; the database itself stays consistent. + ## EnqueueOnce `EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. @@ -181,7 +187,7 @@ The id is remembered while its row exists: pending, processing, or completed wit ## Compatibility -sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker claims several jobs in one statement through `DequeueBatchWithAckIds`, and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. ## How It Works diff --git a/dedup_test.go b/dedup_test.go index b2aee68..35707c2 100644 --- a/dedup_test.go +++ b/dedup_test.go @@ -21,7 +21,13 @@ type once func(item any, id string) bool // and returns its Queue with its EnqueueOnce. countRows reads table "q". func openOnce(t *testing.T, priority bool, opts ...Option) (*Queue, once, Queues) { t.Helper() - qs := mustNew(t, t.TempDir()) + return openOnceAt(t, t.TempDir(), priority, opts...) +} + +// openOnceAt is openOnce in the folder store at dir, so a test can reopen it +func openOnceAt(t *testing.T, dir string, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + qs := mustNew(t, dir) if priority { pq, err := qs.NewPriorityQueue("q", opts...) if err != nil { @@ -146,10 +152,82 @@ func testOnceAfterAck(t *testing.T, priority bool) { } } +// A job claimed and never acked is requeued on the next open, and its id +// still blocks a repeat: the job is stored once and runs once. +func TestEnqueueOnceAcrossRecovery(t *testing.T) { testOnceAcrossRecovery(t, false) } + +func testOnceAcrossRecovery(t *testing.T, priority bool) { + dir := t.TempDir() + q, add, qs := openOnceAt(t, dir, priority) + if !add([]byte("a"), "j1") { + qs.Close() + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + qs.Close() + t.Fatal("DequeueWithAckId failed") + } + // The process "dies": the file closes with the row processing and no ack + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + + q, add, qs = openOnceAt(t, dir, priority) + defer qs.Close() + if n := q.Len(); n != 1 { + t.Fatalf("Len() = %d after reopen, want the claimed job back (1)", n) + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of the recovered job returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a] once", got) + } +} + +// Purge deletes the rows, so it frees their ids, pending and processing alike +func TestEnqueueOnceAfterPurge(t *testing.T) { testOnceAfterPurge(t, false) } + +func testOnceAfterPurge(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") || !add([]byte("b"), "j2") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { // j1 is processing, j2 pending + t.Fatal("DequeueWithAckId failed") + } + + q.Purge() + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Fatalf("Expected no rows after Purge, got %d", n) + } + + if !add([]byte("c"), "j1") || !add([]byte("d"), "j2") { + t.Error("EnqueueOnce of a purged id returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 2 { + t.Errorf("Expected 2 new rows, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c d]" { + t.Errorf("Dequeued %v, want [c d]", got) + } +} + func TestEnqueueOncePriority(t *testing.T) { t.Run("Repeat", func(t *testing.T) { testOnceRepeat(t, true) }) t.Run("WhileProcessing", func(t *testing.T) { testOnceWhileProcessing(t, true) }) t.Run("AfterAck", func(t *testing.T) { testOnceAfterAck(t, true) }) + t.Run("AcrossRecovery", func(t *testing.T) { testOnceAcrossRecovery(t, true) }) + t.Run("AfterPurge", func(t *testing.T) { testOnceAfterPurge(t, true) }) t.Run("Order", func(t *testing.T) { qs := mustNew(t, t.TempDir()) diff --git a/layout_test.go b/layout_test.go index 0ff30a7..1b3bd0e 100644 --- a/layout_test.go +++ b/layout_test.go @@ -3,6 +3,7 @@ package sqliteq import ( "context" "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -129,6 +130,66 @@ func TestNewBadPath(t *testing.T) { } } +// A nil option is an error from New, NewQueue and NewPriorityQueue, not a +// panic, and a rejected call leaves no folder, file or table behind +func TestNilOption(t *testing.T) { + t.Run("New", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "data") + for _, opts := range [][]StoreOption{{nil}, {WithSingleFile(), nil}} { + qs, err := New(dir, opts...) + if err == nil { + qs.Close() + t.Fatalf("Expected an error for a nil StoreOption in %d options", len(opts)) + } + if !errors.Is(err, errNilOption) { + t.Errorf("Got %v, want the nil option error", err) + } + } + if _, err := os.Stat(dir); err == nil { + t.Error("New created the folder before it rejected the option") + } + }) + + t.Run("NewQueue", func(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + defer qs.Close() + + if _, err := qs.NewQueue("q", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewQueue with a nil option: got %v, want the nil option error", err) + } + if _, err := qs.NewPriorityQueue("p", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewPriorityQueue with a nil option: got %v, want the nil option error", err) + } + if got := dbFiles(t, dir); len(got) != 0 { + t.Errorf("Expected no database file after the rejected calls, got %v", got) + } + + // The names are still free + if _, err := qs.NewQueue("q"); err != nil { + t.Errorf("Failed to create queue after the rejected call: %v", err) + } + if _, err := qs.NewPriorityQueue("p"); err != nil { + t.Errorf("Failed to create priority queue after the rejected call: %v", err) + } + }) + + // The registry rejects it before newQueue runs; newQueue's own check + // is the second line, and it must stop before it creates the table + t.Run("newQueue", func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + if _, err := newQueue(q.f, "other", false, nil); !errors.Is(err, errNilOption) { + t.Errorf("newQueue with a nil option: got %v, want the nil option error", err) + } + var n int + if err := q.f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = 'other'`).Scan(&n); err != nil || n != 0 { + t.Errorf("Expected no table \"other\", got count %d, err %v", n, err) + } + }) +} + // dbFiles lists the database files in dir, leaving out SQLite's -wal and -shm files. func dbFiles(t *testing.T, dir string) []string { t.Helper() diff --git a/option.go b/option.go index ad485f6..c56379f 100644 --- a/option.go +++ b/option.go @@ -1,5 +1,11 @@ package sqliteq +import "errors" + +// errNilOption is returned by New, NewQueue and NewPriorityQueue for a nil +// option, which they would otherwise call +var errNilOption = errors.New("sqliteq: option must not be nil") + // Option is a function type that can be used to configure a Queue type Option func(*Queue) diff --git a/queue.go b/queue.go index c2f62a3..512555e 100644 --- a/queue.go +++ b/queue.go @@ -55,6 +55,9 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro // Apply any provided options for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(q) } diff --git a/queues.go b/queues.go index 490f63e..14836db 100644 --- a/queues.go +++ b/queues.go @@ -29,6 +29,9 @@ type Queues interface { func New(path string, opts ...StoreOption) (Queues, error) { cfg := defaultStoreConfig() for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&cfg) } diff --git a/registry.go b/registry.go index 74d6151..24aeb0e 100644 --- a/registry.go +++ b/registry.go @@ -136,6 +136,9 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) want := Queue{removeOnComplete: true} for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&want) } From 3bb233bcb50c15f2235ac504429087baec35df08 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 18:39:46 +0600 Subject: [PATCH 4/6] docs: redesign README in the goptics house style, add a logo A text logo in light and dark variants, built like redisq's: three stacked queue bars and the wordmark, in SQLite blue. The README follows redisq's layout: logo, badges, tagline and section links, then features, installation (Go 1.24+ and cgo), quick start, usage, storage, configuration, delivery guarantees, upgrading from v0.2, how it works, development, contributing and license. New sections: a delivery guarantees table; an upgrade list (the path is a plain path, not a DSN; one shared handle per open name; recovery once per file open, which replaces RequeueNoAckRows); an option table. The CI badge now links the sqliteq.yml workflow. The column list gains ack and priority, and the single-file name rules note the index names. The measured WAL sizes are left out until the benchmark they come from is published; the computed sizes stay. --- README.md | 396 ++++++++++++++++++++++++-------------- assets/text-logo-dark.svg | 7 + assets/text-logo.svg | 7 + 3 files changed, 266 insertions(+), 144 deletions(-) create mode 100644 assets/text-logo-dark.svg create mode 100644 assets/text-logo.svg diff --git a/README.md b/README.md index e82a0d8..82bbfc0 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,49 @@ -# SQLiteQ: A SQLite-Based Persistent Queue for Go - -[![Go Reference](https://img.shields.io/badge/go-pkg-00ADD8.svg?logo=go)](https://pkg.go.dev/github.com/goptics/sqliteq) -[![Go Report Card](https://goreportcard.com/badge/github.com/goptics/sqliteq)](https://goreportcard.com/report/github.com/goptics/sqliteq) -[![Go Version](https://img.shields.io/badge/Go-1.24+-00ADD8?style=flat-square&logo=go)](https://golang.org/doc/devel/release.html) -[![CI](https://github.com/goptics/sqliteq/actions/workflows/sqliteq.yml/badge.svg)](https://github.com/goptics/sqliteq/actions/workflows/go.yml) -[![codecov](https://codecov.io/gh/goptics/sqliteq/branch/main/graph/badge.svg)](https://codecov.io/gh/goptics/sqliteq) - -SQLiteQ is a persistent queue implementation in Go using SQLite as the storage backend. It provides efficient enqueue and dequeue operations and maintains persistence across application restarts. - -## Features - -- Efficient enqueue and dequeue operations -- Persistence via SQLite storage -- Support for acknowledgment-based processing -- Simple and clean API following the Queue interface -- Built as a persistence adapter for [varmq](https://github.com/goptics/varmq) +
+ + + + sqliteq + + +

+ Go Reference + CI + Codecov + Go Report Card + Go Version + License +

+ +

SQLite-backed FIFO and priority queues for Go, with acknowledgments, batch pops, group commit and one file per queue.

+ +

+ Features · + Installation · + Quick Start · + Usage · + Storage · + Configuration · + Delivery Guarantees · + Upgrading · + Development · + Contributing +

+
+ +--- + +## ✨ Features + +- **FIFO and priority queues** stored in SQLite; a lower priority number is served first. +- **One file per queue** by default, so busy queues never wait on each other's write lock. `WithSingleFile()` keeps every queue in one file. +- **Acknowledgments**: items taken with an ack id that are never acknowledged go back to the queue when the file is next opened. +- **Batch dequeue** of many items in one SQL statement, each under its own ack id. +- **Group commit**: concurrent `Enqueue` and `Acknowledge` calls on one file share one transaction. +- **Deduplicated enqueue**: `EnqueueOnce` stores a job once per id while its row exists. +- **Automatic WAL checkpoints**, tunable with `WithCheckpointPages`. +- **Opens v0.2 files** and migrates their schema in place. +- **Thread-safe**: share one queue object across goroutines. +- **Persistence adapter for [varmq](https://github.com/goptics/varmq)**. ## Installation @@ -22,106 +51,160 @@ SQLiteQ is a persistent queue implementation in Go using SQLite as the storage b go get github.com/goptics/sqliteq ``` -## Usage +Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. + +## Quick Start ```go package main import ( - "fmt" - "log" + "fmt" + "log" - "github.com/goptics/sqliteq" + "github.com/goptics/sqliteq" ) func main() { - // The parameter is a folder, created if missing. Each queue gets its own - // database file in it: data/my_queue.db, data/my_priority_queue.db, ... - queuesManager, err := sqliteq.New("data") - if err != nil { - log.Fatalf("Failed to open queues: %v", err) - } - defer queuesManager.Close() - - // Create a new queue - // The parameter is the name of the queue; it becomes the table and file name - queue, err := queuesManager.NewQueue("my_queue") - if err != nil { - log.Fatalf("Failed to create queue: %v", err) - } - - // You can also create a queue with custom options - // For example, to keep acknowledged items in the database: - queueWithOptions, err := queuesManager.NewQueue("my_other_queue", - sqliteq.WithRemoveOnComplete(false)) // Set to false to keep acknowledged items - if err != nil { - log.Fatalf("Failed to create queue: %v", err) - } - _ = queueWithOptions - - // Create a priority queue - priorityQueue, err := queuesManager.NewPriorityQueue("my_priority_queue") - if err != nil { - log.Fatalf("Failed to create priority queue: %v", err) - } - _ = priorityQueue - - // Enqueue items - queue.Enqueue([]byte("item 1")) - queue.Enqueue([]byte("item 2")) - - // Get queue length - fmt.Printf("Queue length: %d\n", queue.Len()) - - // Get all pending items - items := queue.Values() - fmt.Printf("All items: %v\n", items) - - // Simple dequeue - item, success := queue.Dequeue() - if success { - fmt.Printf("Dequeued item: %v\n", string(item.([]byte))) - } - - // Dequeue with acknowledgment - item, success, ackID := queue.DequeueWithAckId() - if success { - fmt.Printf("Dequeued item: %v with ack ID: %s\n", string(item.([]byte)), ackID) - - // Process the item... - - // Acknowledge the item after processing - acknowledged := queue.Acknowledge(ackID) - fmt.Printf("Item acknowledged: %v\n", acknowledged) - - // Note: By default, acknowledged items are removed from the database - // With WithRemoveOnComplete(false), they would be marked as completed instead - } - - // Purge the queue - queue.Purge() + qs, err := sqliteq.New("data") // a folder, created if missing + if err != nil { + log.Fatal(err) + } + defer qs.Close() + + queue, err := qs.NewQueue("jobs") // stored in data/jobs.db + if err != nil { + log.Fatal(err) + } + + queue.Enqueue([]byte("hello world")) // any value database/sql accepts; false for others + + if data, ok := queue.Dequeue(); ok { + fmt.Println(string(data.([]byte))) // items come back as []byte + } } ``` -`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. +## Usage + +### Queue types + +| Type | Constructor | Order | +| --------------- | --------------------------- | ------------------------------ | +| `Queue` | `qs.NewQueue(name)` | FIFO | +| `PriorityQueue` | `qs.NewPriorityQueue(name)` | Lowest first, FIFO among equal | + +Both types support the acknowledgment, dedup and batch APIs below. Basic operations: + +```go +queue.Enqueue([]byte("message")) +queue.Len() // pending items +queue.Values() // pending items, in queue order +queue.Purge() // delete every item +``` While a handle is open, the same name returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. -## Layout on disk +### Priority queue + +```go +pq, err := qs.NewPriorityQueue("emails") +if err != nil { + log.Fatal(err) +} + +pq.Enqueue([]byte("newsletter"), 3) +pq.Enqueue([]byte("password reset"), 1) + +data, ok := pq.Dequeue() // "password reset" +``` + +### Acknowledgments + +```go +item, ok, ackID := queue.DequeueWithAckId() +if ok { + process(item) + queue.Acknowledge(ackID) +} +``` + +An item taken with `DequeueWithAckId` is marked as processing until it is acknowledged. The first time a queue is opened after its file is opened in the process, every item still marked as processing goes back to pending, so a job that was running when the process stopped runs again. There is no visibility timeout: unacknowledged items return only when the file is next opened, not when a closed queue is opened again on a file that is still open. + +By default an acknowledged item is deleted. With `WithRemoveOnComplete(false)` it is kept and marked as completed: + +```go +audit, err := qs.NewQueue("audit", sqliteq.WithRemoveOnComplete(false)) +``` + +### Enqueue once per job id + +`EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. + +```go +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // stored +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stored + +pq.EnqueueOnce([]byte("rebuild index"), 1, "rebuild-index") +``` + +The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. + +### Batch dequeue -By default `path` is a folder and every queue lives in its own SQLite file, named after the queue in lower case, with SQLite's `-wal` and `-shm` files beside it: +`DequeueBatchWithAckIds` takes up to `min(len(items), len(ackIDs))` items in one statement, each under its own ack id. `remaining` is the number of pending items left after it, counted up to 256; `Len` gives the exact count. + +```go +items := make([]any, 8) +ackIDs := make([]string, 8) + +for { + n, remaining, err := queue.DequeueBatchWithAckIds(items, ackIDs) + for i := range n { + process(items[i]) + queue.Acknowledge(ackIDs[i]) + } + if err != nil || remaining == 0 { + break + } +} +``` + +### With varmq + +sqliteq implements varmq's persistent queue interfaces: + +```go +q, err := qs.NewQueue("jobs") +if err != nil { + log.Fatal(err) +} + +worker := varmq.NewWorker(func(j varmq.Job[string]) { + fmt.Println(j.Data()) +}) +defer worker.WaitAndStop() + +queue := worker.WithPersistentQueue(q) +queue.Add("hello") +``` + +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. + +## Storage + +### Layout on disk + +By default `path` is a folder and every queue lives in its own SQLite file, named after the queue in lower case, with SQLite's `-wal` and `-shm` files beside it while the file is open (SQLite removes them when the last connection closes): ``` data/ -├── my_queue.db -├── my_queue.db-shm -├── my_queue.db-wal -├── my_other_queue.db -├── my_other_queue.db-shm -├── my_other_queue.db-wal -├── my_priority_queue.db -├── my_priority_queue.db-shm -└── my_priority_queue.db-wal +├── emails.db +├── emails.db-shm +├── emails.db-wal +├── jobs.db +├── jobs.db-shm +└── jobs.db-wal ``` Each file has its own connection pool, its own write lock and its own checkpoints, so busy queues do not wait on each other. `New` only creates the folder; a file is opened (and created) the first time you ask for its queue. @@ -131,89 +214,114 @@ Each file has its own connection pool, its own write lock and its own checkpoint `WithSingleFile()` makes `path` one database file that holds every queue as a table. This is the v0.2 layout, so a file written by v0.2 opens with this option and keeps its data: ```go -queuesManager, err := sqliteq.New("queue.db", sqliteq.WithSingleFile()) -if err != nil { - log.Fatalf("Failed to open queues: %v", err) -} -defer queuesManager.Close() +qs, err := sqliteq.New("queue.db", sqliteq.WithSingleFile()) ``` The file is created if it is missing, but its folder must already exist: `New` returns an error otherwise, unlike the folder layout, which creates its folder. -If you upgrade from v0.2, add `WithSingleFile()` to keep reading your existing file, or start with the folder layout for new data. - ### Queue names -In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. On Windows the folder layout also cannot use a reserved device name as a queue name (`CON`, `PRN`, `AUX`, `NUL`, `COM1` to `COM9`, `LPT1` to `LPT9`, in any letter case), because the name would become a file name such as `con.db`. sqliteq does not check for these. With `WithSingleFile()` the name is only a table name, so any name SQLite accepts as a table name works; SQLite reserves names that start with `sqlite_`. +In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. On Windows the folder layout also cannot use a reserved device name as a queue name (`CON`, `PRN`, `AUX`, `NUL`, `COM1` to `COM9`, `LPT1` to `LPT9`, in any letter case), because the name would become a file name such as `con.db`. sqliteq does not check for these. With `WithSingleFile()` the name is only a table name, so any name SQLite accepts as a table name works, with two exceptions: SQLite reserves names that start with `sqlite_`, and since a queue `` creates the indexes `__sqliteq_pending` and `__sqliteq_job_id` (SQLite keeps tables and indexes in one namespace), a queue named like one of those cannot open in the same file as ``; whichever opens second gets an error. Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue, stored in `jobs.db` in the folder layout. -## One process per file +### One process per file Run one process per database file. sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed. -Another process is not coordinated with this one. When a queue opens, it puts every item still marked as processing back to pending, so a second process opening a file that is in use would rerun the first process's running jobs. sqliteq does not take an OS lock to stop that. +Another process is not coordinated with this one. The first time a process opens a queue, it puts every item still marked as processing back to pending, so a second process opening a file that is in use would rerun the first process's running jobs. sqliteq does not take an OS lock to stop that. + +On macOS, spell a path the same way every time: the registry key keeps the letter case you pass, so `Data` and `data` would be two entries for one folder. On Windows, Go resolves existing folders and files to their on-disk letter case, so different spellings of them share one entry. + +## Configuration -On case-insensitive filesystems (macOS, Windows), spell a path the same way every time: the registry key is case-sensitive, so `Data` and `data` would be two entries for one folder. +| Option | Passed to | Default | Effect | +| ----------------------------- | ------------------------------ | ------------- | ---------------------------------------------------------------------- | +| `WithSingleFile()` | `New` | folder layout | `path` is one database file holding every queue | +| `WithCheckpointPages(n)` | `New` | `1000` | WAL size, in pages, at which SQLite checkpoints automatically | +| `WithRemoveOnComplete(false)` | `NewQueue`, `NewPriorityQueue` | `true` | Keep acknowledged items, marked as completed, instead of deleting them | -## Checkpoints +`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. + +### Checkpoints SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. A checkpoint runs inside the commit of the write that crosses that size, while that write holds the file's write lock. So with the default, any write on that file (an `Enqueue`, a claim, an ack or a purge) occasionally waits for a checkpoint's disk sync. ```go -queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) +qs, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) ``` Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark](https://github.com/goptics/varmq-benchmarks/tree/main/sqlite) (one queue in one file, 20,000 jobs), its `walMB` metric, the largest size the WAL file reached, was 4.1 MB with 1000 pages and 41 MB with 10000 pages, measured on one machine. +The WAL is a log, not extra stored data: the database holds the same items either way. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. + +## Delivery Guarantees -The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +| Call | Guarantee | If the process stops | +| -------------------------------------------- | --------------------------- | ---------------------------------------------------------- | +| `DequeueWithAckId`, `DequeueBatchWithAckIds` | At least once | Unacknowledged items go back to pending on the next open | +| `Dequeue` | At most once | The item is deleted when it is returned | +| `EnqueueOnce` | Once per id | Stored once while its row exists; the id survives recovery | +| `Enqueue` | Stored when it returns true | Survives a process crash | -## Durability +- Use `DequeueWithAckId` or `DequeueBatchWithAckIds` (varmq's persistent queue claims through one of them) for jobs you can't lose. +- Use globally unique ids with `EnqueueOnce` (UUID or nanoid, never a per-process counter). sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. A job whose `Enqueue` returned true survives a crash of the process. After a power loss or an operating system crash, the last commits before the failure may be lost, because SQLite rolls back what the WAL had not yet synced; the database itself stays consistent. -## EnqueueOnce +## Upgrading from v0.2 -`EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. +- `New` returns `(Queues, error)` instead of panicking, and takes store options. `NewQueue` and `NewPriorityQueue` keep their signatures. +- `New(path)` now treats `path` as a folder with one file per queue. Add `WithSingleFile()` to keep reading an existing v0.2 file. +- `path` is a file system path, not a DSN. A `file:` URI or `:memory:` is taken as a literal file name and opens a new empty database; a `?` or `#` in the path is an error. Pass the plain path of a v0.2 file. +- While a queue is open, the same name returns the same handle instead of a new one, so `Queue.Close` on it closes it for every holder. Asking for it with different options, or as the other kind of queue, is an error. +- A v0.2 table is migrated when it opens: its old indexes are replaced with a partial index on pending rows, and a `job_id` column is added with a unique index over the rows that have one. Its data is kept. +- Ack ids are row ids. An ack id from v0.2 no longer matches and returns false. +- Items come out in `id` order (a priority queue's in `priority`, then `id`), not `created_at`. +- `RequeueNoAckRows` is no longer exported, and nothing replaces it: unacknowledged items are requeued the first time a queue is opened after its file is opened in the process, not on every `NewQueue` call. -```go -queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // stored -queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stored -``` +## How It Works -The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. +Each queue is a table with these columns: -## Compatibility +- `id`: autoincrement primary key. Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal. +- `priority`: priority queues only; a lower number comes out first. Opening an older table as a priority queue adds it. +- `data`: the stored item. +- `status`: `pending`, `processing` or `completed`. +- `ack`: 0 when an item is added, 1 once it is acknowledged and kept with `WithRemoveOnComplete(false)`. On open, only `processing` rows with `ack` 0 go back to pending. +- `ack_id`: kept from v0.2 so old tables need no rebuild; no longer used. +- `job_id`: the id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it. +- `created_at`, `updated_at`: when the item was added and last changed. -sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. +On top of that table: -## How It Works +- Two partial indexes: one covers the pending items in queue order, and a unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. +- A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. +- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it. +- Concurrent `Enqueue` and `Acknowledge` calls on one file are committed together in one transaction. Each call still gets its own result, and if the shared transaction fails, each call is retried on its own so one bad call fails alone. + +## Development + +```bash +go test -race ./... # tests, with the race detector +go run ./example # a runnable example +gofmt -l . # formatting +``` + +Building and testing need cgo (`CGO_ENABLED=1`) and a C compiler. + +## API Reference -SQLiteQ uses a SQLite database to store queue items with the following schema: +See [GoDoc](https://pkg.go.dev/github.com/goptics/sqliteq). -- `id`: Unique identifier for each item (autoincrement primary key). Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal -- `data`: The serialized item data (stored as a JSON blob) -- `status`: The status of the item ("pending", "processing", or "completed") -- `ack_id`: Kept from v0.2 so old tables need no rebuild; no longer used -- `job_id`: The id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it -- `created_at`: When the item was added to the queue -- `updated_at`: When the item was last updated +## Star History -> NOTE: By default, when an item is acknowledged, it is removed from the database. However, you can configure the queue to keep acknowledged items by using the `WithRemoveOnComplete(false)` option when creating the queue. In this case, acknowledged items will be marked as "completed" but will remain in the database. +[![Star History Chart](https://api.star-history.com/svg?repos=goptics/sqliteq&type=Date)](https://www.star-history.com/#goptics/sqliteq&Date) -## Performance Considerations +## Contributing -- The queue is optimized for efficient enqueue and dequeue operations that scale well with queue size -- Operations leverage SQLite's indexing for logarithmic time complexity rather than true constant-time -- SQLite's WAL (Write-Ahead Logging) mode is enabled for better concurrent access -- One partial index covers the pending items in queue order; opening a v0.2 table replaces its old indexes with it -- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it -- A batch pop reports the items left after it, counting at most 256; `Len` gives the exact count +Open an issue to discuss a change, or send a pull request against `main` with tests (`go test -race ./...`). -## 👤 Author +## License -- GitHub: [@fahimfaisaal](https://github.com/fahimfaisaal) -- LinkedIn: [in/fahimfaisaal](https://www.linkedin.com/in/fahimfaisaal/) -- Twitter: [@FahimFaisaal](https://twitter.com/FahimFaisaal) +[MIT](LICENSE) diff --git a/assets/text-logo-dark.svg b/assets/text-logo-dark.svg new file mode 100644 index 0000000..f22b468 --- /dev/null +++ b/assets/text-logo-dark.svg @@ -0,0 +1,7 @@ + +sqliteq + + + +sqliteq + diff --git a/assets/text-logo.svg b/assets/text-logo.svg new file mode 100644 index 0000000..699c854 --- /dev/null +++ b/assets/text-logo.svg @@ -0,0 +1,7 @@ + +sqliteq + + + +sqliteq + From 5f0b9d31713694180849d10a7ad32432b4453a0b Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 18:43:11 +0600 Subject: [PATCH 5/6] docs: drop the v0.2 migration bullet from features --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 82bbfc0..8fc4b87 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ - **Group commit**: concurrent `Enqueue` and `Acknowledge` calls on one file share one transaction. - **Deduplicated enqueue**: `EnqueueOnce` stores a job once per id while its row exists. - **Automatic WAL checkpoints**, tunable with `WithCheckpointPages`. -- **Opens v0.2 files** and migrates their schema in place. - **Thread-safe**: share one queue object across goroutines. - **Persistence adapter for [varmq](https://github.com/goptics/varmq)**. From 6fda8e5c4c9c2a79bb041426c8feff1bb9d86faf Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 18:46:59 +0600 Subject: [PATCH 6/6] docs: drop the retired Go Report Card badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8fc4b87..6b4c501 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Go Reference CI Codecov - Go Report Card Go Version License