1342 lines
43 KiB
Go
1342 lines
43 KiB
Go
// Copyright 2026 Alibaba Group
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package personal
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
eventlock "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/lock"
|
|
)
|
|
|
|
type attemptTestClock struct {
|
|
mu sync.Mutex
|
|
now time.Time
|
|
}
|
|
|
|
func newAttemptTestClock() *attemptTestClock {
|
|
return &attemptTestClock{
|
|
now: time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC),
|
|
}
|
|
}
|
|
|
|
func (c *attemptTestClock) Now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.now
|
|
}
|
|
|
|
func (c *attemptTestClock) Advance(delta time.Duration) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.now = c.now.Add(delta)
|
|
}
|
|
|
|
func attemptTestFingerprint(label string) string {
|
|
return Fingerprint("https://mcp.example.test/dws", "idem-"+label, "profile-"+label)
|
|
}
|
|
|
|
func newAttemptTestStore(t *testing.T, clock *attemptTestClock) *AttemptStore {
|
|
t.Helper()
|
|
var sequence atomic.Int64
|
|
return NewAttemptStore(
|
|
t.TempDir(),
|
|
WithAttemptClock(clock.Now),
|
|
WithAttemptIDGenerator(func() (string, error) {
|
|
return fmt.Sprintf("attempt-%d", sequence.Add(1)), nil
|
|
}),
|
|
)
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptFingerprintIsCanonicalScopedAndOpaque(t *testing.T) {
|
|
base := Fingerprint(" HTTPS://MCP.Example.Test/dws/ ", " idem-1 ")
|
|
if base == "" || len(base) != 64 {
|
|
t.Fatalf("Fingerprint() = %q", base)
|
|
}
|
|
if got := Fingerprint("https://mcp.example.test/dws", "idem-1", ""); got != base {
|
|
t.Fatalf("empty profile changed fingerprint: %q != %q", got, base)
|
|
}
|
|
if got := Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-a"); got == base {
|
|
t.Fatal("profile selector did not scope fingerprint")
|
|
}
|
|
if left, right := Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-a"),
|
|
Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-b"); left == right {
|
|
t.Fatal("different profile selectors produced the same fingerprint")
|
|
}
|
|
if Fingerprint("", "idem") != "" || Fingerprint("https://mcp.example.test", "") != "" {
|
|
t.Fatal("blank required fingerprint input should return empty")
|
|
}
|
|
if strings.Contains(base, "idem") || strings.Contains(base, "mcp.example") {
|
|
t.Fatalf("fingerprint leaked source material: %q", base)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageRetryabilityValuePreservesUnknown(t *testing.T) {
|
|
if value, known := RetryabilityRetryable.Value(); !known || !value {
|
|
t.Fatalf("retryable Value() = %v, %v", value, known)
|
|
}
|
|
if value, known := RetryabilityNonRetryable.Value(); !known || value {
|
|
t.Fatalf("non-retryable Value() = %v, %v", value, known)
|
|
}
|
|
if _, known := RetryabilityUnknown.Value(); known {
|
|
t.Fatal("unknown retryability became known")
|
|
}
|
|
if _, known := Retryability("invalid").Value(); known {
|
|
t.Fatal("invalid retryability became known")
|
|
}
|
|
var nilBlocked *AttemptBlockedError
|
|
if nilBlocked.Error() != "" {
|
|
t.Fatalf("nil blocked error = %q", nilBlocked.Error())
|
|
}
|
|
blocked := &AttemptBlockedError{
|
|
State: AttemptStateCooldown,
|
|
NextAllowedAt: time.Date(2026, 7, 30, 10, 1, 0, 0, time.UTC),
|
|
}
|
|
if message := blocked.Error(); !strings.Contains(message, "cooldown") ||
|
|
!strings.Contains(message, "2026-07-30T10:01:00Z") {
|
|
t.Fatalf("blocked error = %q", message)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreClaimFailureBackoffAndRetryAfter(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("one")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if claim.AttemptID != "attempt-1" || len(claim.Fingerprints) != 1 {
|
|
t.Fatalf("claim = %#v", claim)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("second in-flight claim succeeded")
|
|
} else {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight ||
|
|
blocked.Retryability != RetryabilityUnknown || blocked.RetryAfter != time.Minute {
|
|
t.Fatalf("in-flight error = %#v, %v", blocked, err)
|
|
}
|
|
}
|
|
|
|
hold, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
ErrorCode: "TEMPORARY",
|
|
TraceID: "trace-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.State != AttemptStateCooldown || hold.FailureCount != 1 ||
|
|
hold.RetryAfter < 30*time.Second || hold.RetryAfter > 36*time.Second {
|
|
t.Fatalf("first hold = %#v", hold)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("cooldown claim succeeded")
|
|
} else {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.State != AttemptStateCooldown ||
|
|
blocked.Retryability != RetryabilityRetryable ||
|
|
blocked.ErrorCode != "TEMPORARY" || blocked.TraceID != "trace-1" {
|
|
t.Fatalf("cooldown error = %#v, %v", blocked, err)
|
|
}
|
|
}
|
|
|
|
clock.Advance(hold.RetryAfter + time.Second)
|
|
second, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondHold, err := store.CompleteFailure(second, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
RetryAfter: 5 * time.Minute,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if secondHold.FailureCount != 2 || secondHold.RetryAfter != 5*time.Minute ||
|
|
secondHold.Retryability != RetryabilityUnknown {
|
|
t.Fatalf("second hold = %#v", secondHold)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreBackoffSequenceAndTwentyFourHourReset(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("backoff")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
wantBases := []time.Duration{
|
|
30 * time.Second,
|
|
2 * time.Minute,
|
|
10 * time.Minute,
|
|
30 * time.Minute,
|
|
30 * time.Minute,
|
|
}
|
|
for index, base := range wantBases {
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatalf("claim %d: %v", index+1, err)
|
|
}
|
|
hold, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failure %d: %v", index+1, err)
|
|
}
|
|
if hold.FailureCount != index+1 ||
|
|
hold.RetryAfter < base ||
|
|
hold.RetryAfter > base+base/5 {
|
|
t.Fatalf("hold %d = %#v, base %s", index+1, hold, base)
|
|
}
|
|
if got := deterministicAttemptJitter(fingerprint, index+1, base); got != hold.RetryAfter-base {
|
|
t.Fatalf("jitter %d = %s, hold delta %s", index+1, got, hold.RetryAfter-base)
|
|
}
|
|
clock.Advance(hold.RetryAfter + time.Second)
|
|
}
|
|
|
|
clock.Advance(attemptFailureReset + time.Second)
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hold, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.FailureCount != 1 || hold.RetryAfter < 30*time.Second || hold.RetryAfter > 36*time.Second {
|
|
t.Fatalf("post-reset hold = %#v", hold)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreLongRetryAfterSurvivesFailureCountResetWindow(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("long-retry-after")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hold, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
RetryAfter: 48 * time.Hour,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.RetryAfter != 48*time.Hour {
|
|
t.Fatalf("hold Retry-After = %s, want 48h", hold.RetryAfter)
|
|
}
|
|
|
|
clock.Advance(24*time.Hour + time.Second)
|
|
if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("active 48h Retry-After was pruned at the 24h count reset boundary")
|
|
} else {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.NextAllowedAt != hold.NextAllowedAt {
|
|
t.Fatalf("long Retry-After block = %#v, %v", blocked, err)
|
|
}
|
|
}
|
|
|
|
clock.Advance(24 * time.Hour)
|
|
recovered, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
next, err := store.CompleteFailure(recovered, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if next.FailureCount != 1 {
|
|
t.Fatalf("failure count after an inactive 48h hold = %d, want 1", next.FailureCount)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreExpiredInFlightResetsOldFailureHistory(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("expired-inflight-reset")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
|
|
first, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
firstHold, err := store.CompleteFailure(first, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(firstHold.RetryAfter + time.Second)
|
|
crashed, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Simulate a process dying after Claim. The record remains in_flight with
|
|
// the old failure timestamp until a later process prunes it.
|
|
clock.Advance(attemptFailureReset + time.Second)
|
|
recovered, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovered.AttemptID == crashed.AttemptID {
|
|
t.Fatal("expired in-flight claim was not replaced")
|
|
}
|
|
hold, err := store.CompleteFailure(recovered, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.FailureCount != 1 {
|
|
t.Fatalf("failure count after expired in-flight reset = %d, want 1", hold.FailureCount)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreTerminalHoldAndSuccessClear(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("terminal")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventInChat}
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hold, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityNonRetryable,
|
|
ErrorCode: "GROUP_NOT_BELONG_TO_ORG",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.State != AttemptStateTerminalHold || hold.RetryAfter != time.Hour {
|
|
t.Fatalf("terminal hold = %#v", hold)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("terminal hold did not block")
|
|
} else {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.State != AttemptStateTerminalHold {
|
|
t.Fatalf("terminal block = %#v, %v", blocked, err)
|
|
}
|
|
if retryable, known := blocked.Retryability.Value(); !known || retryable {
|
|
t.Fatalf("terminal retryability = %v, %v", retryable, known)
|
|
}
|
|
}
|
|
clock.Advance(time.Hour + time.Second)
|
|
recovery, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.CompleteSuccess(recovery); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(store.workDir, AttemptStateFileName)); !os.IsNotExist(err) {
|
|
t.Fatalf("state file after success = %v, want missing", err)
|
|
}
|
|
if delay, state := attemptFailureDelay(fingerprint, 1, AttemptFailure{
|
|
Retryability: RetryabilityNonRetryable,
|
|
RetryAfter: 2 * time.Hour,
|
|
}); delay != time.Hour || state != AttemptStateTerminalHold {
|
|
t.Fatalf("terminal Retry-After = %s, %s", delay, state)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStorePartialFailureClearsSuccessAndReleasesUnexecuted(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprintA := attemptTestFingerprint("batch-a")
|
|
fingerprintB := attemptTestFingerprint("batch-b")
|
|
fingerprintC := attemptTestFingerprint("batch-c")
|
|
specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention}
|
|
specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat}
|
|
specC := AttemptSpec{Fingerprint: fingerprintC, EventKey: EventInChat}
|
|
|
|
seed, err := store.Claim([]AttemptSpec{specA}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seedHold, err := store.CompleteFailure(seed, nil, AttemptFailure{
|
|
Fingerprint: fingerprintA,
|
|
Retryability: RetryabilityRetryable,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(seedHold.RetryAfter + time.Second)
|
|
|
|
claim, err := store.Claim([]AttemptSpec{specA, specB, specC}, 3*time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hold, err := store.CompleteFailure(claim, []string{fingerprintA}, AttemptFailure{
|
|
Fingerprint: fingerprintB,
|
|
Retryability: RetryabilityUnknown,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hold.Fingerprint != fingerprintB || hold.State != AttemptStateCooldown {
|
|
t.Fatalf("batch hold = %#v", hold)
|
|
}
|
|
state, err := store.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(state.Records) != 1 {
|
|
t.Fatalf("records = %#v, want only failed item", state.Records)
|
|
}
|
|
if _, ok := state.Records[fingerprintB]; !ok {
|
|
t.Fatalf("failed fingerprint missing: %#v", state.Records)
|
|
}
|
|
if _, ok := state.Records[fingerprintA]; ok {
|
|
t.Fatal("successful fingerprint retained failure state")
|
|
}
|
|
if _, ok := state.Records[fingerprintC]; ok {
|
|
t.Fatal("unexecuted new fingerprint was not released")
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreReleaseRestoresPreviousState(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprintA := attemptTestFingerprint("release-a")
|
|
fingerprintB := attemptTestFingerprint("release-b")
|
|
specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention}
|
|
specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat}
|
|
|
|
seed, err := store.Claim([]AttemptSpec{specA}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seedHold, err := store.CompleteFailure(seed, nil, AttemptFailure{
|
|
Fingerprint: fingerprintA,
|
|
Retryability: RetryabilityUnknown,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(seedHold.RetryAfter + time.Second)
|
|
before, err := store.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previousA := before.Records[fingerprintA]
|
|
|
|
claim, err := store.Claim([]AttemptSpec{specA, specB}, 2*time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Release(claim); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, err := store.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(after.Records) != 1 || after.Records[fingerprintA] != previousA {
|
|
t.Fatalf("release state = %#v, want prior A %#v", after.Records, previousA)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreCASRejectsOldCompletion(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("cas")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
oldClaim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(time.Minute + time.Second)
|
|
newClaim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.CompleteSuccess(oldClaim); !errors.Is(err, ErrAttemptClaimStale) {
|
|
t.Fatalf("old completion error = %v, want stale", err)
|
|
}
|
|
if err := store.CompleteSuccess(newClaim); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreConcurrentClaimsOnlyOneWins(t *testing.T) {
|
|
workDir := t.TempDir()
|
|
fingerprint := attemptTestFingerprint("concurrent")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
const count = 100
|
|
var wg sync.WaitGroup
|
|
claims := make(chan *AttemptClaim, count)
|
|
errs := make(chan error, count)
|
|
for i := 0; i < count; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
claim, err := NewAttemptStore(workDir).Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
errs <- err
|
|
return
|
|
}
|
|
claims <- claim
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
close(claims)
|
|
close(errs)
|
|
|
|
var winners []*AttemptClaim
|
|
for claim := range claims {
|
|
winners = append(winners, claim)
|
|
}
|
|
if len(winners) != 1 {
|
|
t.Fatalf("winning claims = %d, want 1", len(winners))
|
|
}
|
|
for err := range errs {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight {
|
|
t.Fatalf("concurrent claim error = %v", err)
|
|
}
|
|
}
|
|
if err := NewAttemptStore(workDir).CompleteSuccess(winners[0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreBatchClaimIsAtomicWhenOneItemBlocked(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprintA := attemptTestFingerprint("atomic-a")
|
|
fingerprintB := attemptTestFingerprint("atomic-b")
|
|
specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention}
|
|
specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat}
|
|
first, err := store.Claim([]AttemptSpec{specA}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{specB, specA}, time.Minute); err == nil {
|
|
t.Fatal("batch with blocked item succeeded")
|
|
}
|
|
state, err := store.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(state.Records) != 1 {
|
|
t.Fatalf("atomic blocked claim wrote partial state: %#v", state.Records)
|
|
}
|
|
if _, ok := state.Records[fingerprintB]; ok {
|
|
t.Fatal("unblocked batch item was partially reserved")
|
|
}
|
|
if err := store.CompleteSuccess(first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStorePersistsNoRawFingerprintInputs(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
const (
|
|
endpoint = "https://sensitive.example.test/dws"
|
|
idem = "idem-sensitive-user-and-group"
|
|
profile = "private-profile-name"
|
|
)
|
|
fingerprint := Fingerprint(endpoint, idem, profile)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: fingerprint,
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, err := os.ReadFile(filepath.Join(store.workDir, AttemptStateFileName))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, raw := range []string{endpoint, idem, profile} {
|
|
if strings.Contains(string(data), raw) {
|
|
t.Fatalf("state leaked %q: %s", raw, data)
|
|
}
|
|
}
|
|
if err := store.CompleteSuccess(claim); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreFailsClosedOnCorruptStateAndWriteFailure(t *testing.T) {
|
|
t.Run("corrupt state", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
if err := os.WriteFile(
|
|
filepath.Join(store.workDir, AttemptStateFileName),
|
|
[]byte(`{"version":1,"records":`),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("corrupt"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "decode subscription attempt state") {
|
|
t.Fatalf("corrupt state error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unknown version", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
if err := os.WriteFile(
|
|
filepath.Join(store.workDir, AttemptStateFileName),
|
|
[]byte(`{"version":99,"records":{}}`),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("version"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
|
t.Fatalf("version error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("write failure", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
store.writeFile = func(string, []byte, os.FileMode) error {
|
|
return errors.New("disk full")
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("write"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "disk full") {
|
|
t.Fatalf("write error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("failure write keeps in-flight reservation", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("failure-write")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
claim, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originalWrite := store.writeFile
|
|
store.writeFile = func(string, []byte, os.FileMode) error {
|
|
return errors.New("disk full")
|
|
}
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
}); err == nil || !strings.Contains(err.Error(), "disk full") {
|
|
t.Fatalf("failure persistence error = %v", err)
|
|
}
|
|
store.writeFile = originalWrite
|
|
if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("failed persistence silently released in-flight reservation")
|
|
} else {
|
|
var blocked *AttemptBlockedError
|
|
if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight {
|
|
t.Fatalf("post-write-failure block = %#v, %v", blocked, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreFailsClosedOnInjectedIOErrors(t *testing.T) {
|
|
testClaim := func(store *AttemptStore, label string) error {
|
|
_, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint(label),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
return err
|
|
}
|
|
|
|
t.Run("read", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.readFile = func(string) ([]byte, error) {
|
|
return nil, errors.New("read denied")
|
|
}
|
|
if err := testClaim(store, "read-error"); err == nil || !strings.Contains(err.Error(), "read denied") {
|
|
t.Fatalf("read error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("mkdir", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.mkdirAll = func(string, os.FileMode) error {
|
|
return errors.New("mkdir denied")
|
|
}
|
|
if err := testClaim(store, "mkdir-error"); err == nil || !strings.Contains(err.Error(), "mkdir denied") {
|
|
t.Fatalf("mkdir error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("acquire", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.tryAcquire = func(string) (*eventlock.File, error) {
|
|
return nil, errors.New("lock denied")
|
|
}
|
|
if err := testClaim(store, "lock-error"); err == nil || !strings.Contains(err.Error(), "lock denied") {
|
|
t.Fatalf("lock error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("marshal", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.marshal = func(any, string, string) ([]byte, error) {
|
|
return nil, errors.New("marshal denied")
|
|
}
|
|
if err := testClaim(store, "marshal-error"); err == nil || !strings.Contains(err.Error(), "marshal denied") {
|
|
t.Fatalf("marshal error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("chmod", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.chmod = func(string, os.FileMode) error {
|
|
return errors.New("chmod denied")
|
|
}
|
|
if err := testClaim(store, "chmod-error"); err == nil ||
|
|
!strings.Contains(err.Error(), "chmod denied") {
|
|
t.Fatalf("chmod error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("rename and temporary cleanup", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.rename = func(string, string) error {
|
|
return errors.New("rename denied")
|
|
}
|
|
if err := testClaim(store, "rename-error"); err == nil || !strings.Contains(err.Error(), "rename denied") {
|
|
t.Fatalf("rename error = %v", err)
|
|
}
|
|
tmp := filepath.Join(store.workDir, AttemptStateFileName) + ".tmp"
|
|
if _, err := os.Stat(tmp); !os.IsNotExist(err) {
|
|
t.Fatalf("temporary file after rename error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("remove", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("remove-error"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.remove = func(string) error {
|
|
return errors.New("remove denied")
|
|
}
|
|
if err := store.CompleteSuccess(claim); err == nil || !strings.Contains(err.Error(), "remove denied") {
|
|
t.Fatalf("remove error = %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreTightensStateAndLockPermissions(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Windows does not expose Unix permission bits")
|
|
}
|
|
workDir := t.TempDir()
|
|
statePath := filepath.Join(workDir, AttemptStateFileName)
|
|
lockPath := filepath.Join(workDir, AttemptStateLockFileName)
|
|
tmpPath := statePath + ".tmp"
|
|
for _, path := range []string{lockPath, tmpPath} {
|
|
if err := os.WriteFile(path, []byte("stale"), 0o666); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chmod(path, 0o666); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
store := NewAttemptStore(workDir)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("permissions"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = store.Release(claim) }()
|
|
|
|
for _, path := range []string{statePath, lockPath} {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := info.Mode().Perm(); got != 0o600 {
|
|
t.Fatalf("%s mode = %o, want 600", filepath.Base(path), got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreRejectsInvalidPersistedRecords(t *testing.T) {
|
|
fingerprint := attemptTestFingerprint("invalid-record")
|
|
longValue := strings.Repeat("x", attemptMaxFieldLength+1)
|
|
tests := []struct {
|
|
name string
|
|
body string
|
|
}{
|
|
{"nil records", `{"version":1,"records":null}`},
|
|
{"key mismatch", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`,
|
|
fingerprint,
|
|
attemptTestFingerprint("other"),
|
|
)},
|
|
{"oversized field", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","event_key":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
longValue,
|
|
)},
|
|
{"negative failures", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","failure_count":-1,"next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
{"invalid retryability", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"sometimes"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
{"inflight incomplete", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"in_flight"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
{"cooldown nonretryable", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"non_retryable"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
{"terminal retryable", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"terminal_hold","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"retryable"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
{"invalid state", fmt.Sprintf(
|
|
`{"version":1,"records":{"%s":{"fingerprint":"%s","state":"open"}}}`,
|
|
fingerprint,
|
|
fingerprint,
|
|
)},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
workDir := t.TempDir()
|
|
if err := os.WriteFile(
|
|
filepath.Join(workDir, AttemptStateFileName),
|
|
[]byte(test.body),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := NewAttemptStore(workDir)
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("new"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil {
|
|
t.Fatalf("invalid persisted record was accepted: %s", test.body)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreLockTimeoutAndIDFailureAreFailClosed(t *testing.T) {
|
|
t.Run("lock timeout", func(t *testing.T) {
|
|
var tick atomic.Int64
|
|
base := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
store := NewAttemptStore(
|
|
t.TempDir(),
|
|
WithAttemptClock(func() time.Time {
|
|
return base.Add(time.Duration(tick.Add(1)) * 3 * time.Second)
|
|
}),
|
|
WithAttemptIDGenerator(func() (string, error) { return "attempt", nil }),
|
|
)
|
|
store.tryAcquire = func(string) (*eventlock.File, error) {
|
|
return nil, eventlock.ErrBusy
|
|
}
|
|
store.sleep = func(time.Duration) {}
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("lock"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "timed out") {
|
|
t.Fatalf("lock timeout error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("id failure", func(t *testing.T) {
|
|
store := NewAttemptStore(
|
|
t.TempDir(),
|
|
WithAttemptIDGenerator(func() (string, error) {
|
|
return "", errors.New("random source unavailable")
|
|
}),
|
|
)
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("id"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "random source unavailable") {
|
|
t.Fatalf("id error = %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreValidationAndFailureCAS(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("validation")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
if _, err := store.Claim(nil, time.Minute); err == nil {
|
|
t.Fatal("empty claim succeeded")
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{{Fingerprint: "raw-id"}}, time.Minute); err == nil {
|
|
t.Fatal("raw fingerprint claim succeeded")
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{spec}, 0); err == nil {
|
|
t.Fatal("zero lease claim succeeded")
|
|
}
|
|
var nilStore *AttemptStore
|
|
if _, err := nilStore.Claim([]AttemptSpec{spec}, time.Minute); err == nil {
|
|
t.Fatal("nil store claim succeeded")
|
|
}
|
|
uninitialized := &AttemptStore{workDir: t.TempDir()}
|
|
if _, err := uninitialized.Claim([]AttemptSpec{spec}, time.Minute); err == nil ||
|
|
!strings.Contains(err.Error(), "not initialized") {
|
|
t.Fatalf("uninitialized store error = %v", err)
|
|
}
|
|
|
|
claim, err := store.Claim([]AttemptSpec{spec, spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(claim.Fingerprints) != 1 {
|
|
t.Fatalf("duplicate specs were not deduplicated: %#v", claim.Fingerprints)
|
|
}
|
|
if _, err := store.CompleteFailure(claim, []string{fingerprint}, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityRetryable,
|
|
}); err == nil {
|
|
t.Fatal("same successful and failed fingerprint was accepted")
|
|
}
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: Retryability("invalid"),
|
|
}); err == nil {
|
|
t.Fatal("invalid retryability was accepted")
|
|
}
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: attemptTestFingerprint("not-in-claim"),
|
|
Retryability: RetryabilityUnknown,
|
|
}); err == nil {
|
|
t.Fatal("out-of-claim failure was accepted")
|
|
}
|
|
if _, err := store.CompleteFailure(claim, []string{attemptTestFingerprint("not-in-claim")}, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
}); err == nil {
|
|
t.Fatal("out-of-claim success was accepted")
|
|
}
|
|
if err := store.Release(claim); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.CompleteSuccess(nil); err == nil {
|
|
t.Fatal("nil success claim was accepted")
|
|
}
|
|
if err := store.Release(nil); err == nil {
|
|
t.Fatal("nil release claim was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreBoundsPersistedDiagnostics(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("bounded")
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: fingerprint,
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
longValue := strings.Repeat("x", attemptMaxFieldLength+100)
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
ErrorCode: longValue,
|
|
TraceID: longValue,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err := store.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record := state.Records[fingerprint]
|
|
if len(record.ErrorCode) != attemptMaxFieldLength ||
|
|
len(record.TraceID) != attemptMaxFieldLength {
|
|
t.Fatalf("diagnostic lengths = %d, %d", len(record.ErrorCode), len(record.TraceID))
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAttemptStoreChangedCoverageEdges(t *testing.T) {
|
|
t.Run("endpoint fallback", func(t *testing.T) {
|
|
const raw = "https://example.test/%zz"
|
|
if got := normalizeAttemptEndpoint(raw); got != raw {
|
|
t.Fatalf("normalizeAttemptEndpoint(%q) = %q", raw, got)
|
|
}
|
|
})
|
|
|
|
t.Run("invalid generated ids", func(t *testing.T) {
|
|
for _, attemptID := range []string{" ", strings.Repeat("x", attemptMaxFieldLength+1)} {
|
|
store := NewAttemptStore(
|
|
t.TempDir(),
|
|
WithAttemptIDGenerator(func() (string, error) { return attemptID, nil }),
|
|
)
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("invalid-id"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "attempt id is invalid") {
|
|
t.Fatalf("attempt ID %q error = %v", attemptID, err)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("complete success validates store", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("success-validate"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.now = nil
|
|
if err := store.CompleteSuccess(claim); err == nil || !strings.Contains(err.Error(), "not initialized") {
|
|
t.Fatalf("CompleteSuccess validation error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("complete success read failure", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("success-read"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.readFile = func(string) ([]byte, error) {
|
|
return nil, errors.New("success read denied")
|
|
}
|
|
if err := store.CompleteSuccess(claim); err == nil ||
|
|
!strings.Contains(err.Error(), "success read denied") {
|
|
t.Fatalf("CompleteSuccess read error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("complete failure rejects invalid claim", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
if _, err := store.CompleteFailure(nil, nil, AttemptFailure{}); err == nil {
|
|
t.Fatal("CompleteFailure accepted nil claim")
|
|
}
|
|
})
|
|
|
|
t.Run("complete failure validates store", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("failure-validate")
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: fingerprint,
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.now = nil
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
}); err == nil || !strings.Contains(err.Error(), "not initialized") {
|
|
t.Fatalf("CompleteFailure validation error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("complete failure read failure", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("failure-read")
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: fingerprint,
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.readFile = func(string) ([]byte, error) {
|
|
return nil, errors.New("failure read denied")
|
|
}
|
|
if _, err := store.CompleteFailure(claim, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
}); err == nil || !strings.Contains(err.Error(), "failure read denied") {
|
|
t.Fatalf("CompleteFailure read error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("complete failure stale claim", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("failure-stale")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
stale, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(time.Minute + time.Second)
|
|
current, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.CompleteFailure(stale, nil, AttemptFailure{
|
|
Fingerprint: fingerprint,
|
|
Retryability: RetryabilityUnknown,
|
|
}); !errors.Is(err, ErrAttemptClaimStale) {
|
|
t.Fatalf("CompleteFailure stale error = %v", err)
|
|
}
|
|
if err := store.CompleteSuccess(current); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("release validates store", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("release-validate"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.now = nil
|
|
if err := store.Release(claim); err == nil || !strings.Contains(err.Error(), "not initialized") {
|
|
t.Fatalf("Release validation error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("release read failure", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
claim, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("release-read"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.readFile = func(string) ([]byte, error) {
|
|
return nil, errors.New("release read denied")
|
|
}
|
|
if err := store.Release(claim); err == nil ||
|
|
!strings.Contains(err.Error(), "release read denied") {
|
|
t.Fatalf("Release read error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("release stale claim", func(t *testing.T) {
|
|
clock := newAttemptTestClock()
|
|
store := newAttemptTestStore(t, clock)
|
|
fingerprint := attemptTestFingerprint("release-stale")
|
|
spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention}
|
|
stale, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock.Advance(time.Minute + time.Second)
|
|
current, err := store.Claim([]AttemptSpec{spec}, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Release(stale); !errors.Is(err, ErrAttemptClaimStale) {
|
|
t.Fatalf("Release stale error = %v", err)
|
|
}
|
|
if err := store.CompleteSuccess(current); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("empty work directory", func(t *testing.T) {
|
|
if err := NewAttemptStore(" ").validate(); err == nil ||
|
|
!strings.Contains(err.Error(), "work directory is required") {
|
|
t.Fatalf("empty work directory error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("oversized event key", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("long-event"),
|
|
EventKey: strings.Repeat("e", attemptMaxFieldLength+1),
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "event key is too long") {
|
|
t.Fatalf("oversized event key error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("claim shape validation", func(t *testing.T) {
|
|
fingerprint := attemptTestFingerprint("claim-shape")
|
|
if err := validateAttemptClaim(&AttemptClaim{
|
|
AttemptID: "attempt",
|
|
Fingerprints: []string{"not-a-fingerprint"},
|
|
previous: map[string]attemptPrevious{"not-a-fingerprint": {}},
|
|
}); err == nil || !strings.Contains(err.Error(), "invalid fingerprint") {
|
|
t.Fatalf("invalid fingerprint claim error = %v", err)
|
|
}
|
|
if err := validateAttemptClaim(&AttemptClaim{
|
|
AttemptID: "attempt",
|
|
Fingerprints: []string{fingerprint},
|
|
previous: map[string]attemptPrevious{},
|
|
}); err == nil || !strings.Contains(err.Error(), "incomplete") {
|
|
t.Fatalf("incomplete claim error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unknown blocked state is ignored", func(t *testing.T) {
|
|
if blocked := blockedAttempt(attemptRecord{State: AttemptState("unknown")}, time.Now()); blocked != nil {
|
|
t.Fatalf("unknown state blocked = %#v", blocked)
|
|
}
|
|
})
|
|
|
|
t.Run("old abandoned first claim is pruned", func(t *testing.T) {
|
|
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
fingerprint := attemptTestFingerprint("prune-abandoned")
|
|
state := newAttemptStateFile()
|
|
state.Records[fingerprint] = attemptRecord{
|
|
Fingerprint: fingerprint,
|
|
State: AttemptStateInFlight,
|
|
AttemptID: "abandoned",
|
|
LeaseUntil: now.Add(-attemptFailureReset - time.Second),
|
|
}
|
|
pruneAttemptRecords(state, now)
|
|
if len(state.Records) != 0 {
|
|
t.Fatalf("abandoned first claim not pruned: %#v", state.Records)
|
|
}
|
|
})
|
|
|
|
t.Run("write validates state and directory", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
if err := store.write(nil); err == nil || !strings.Contains(err.Error(), "nil subscription attempt state") {
|
|
t.Fatalf("nil state write error = %v", err)
|
|
}
|
|
store.mkdirAll = func(string, os.FileMode) error {
|
|
return errors.New("write mkdir denied")
|
|
}
|
|
if err := store.write(newAttemptStateFile()); err == nil ||
|
|
!strings.Contains(err.Error(), "write mkdir denied") {
|
|
t.Fatalf("write mkdir error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("state chmod failure cleans temporary file", func(t *testing.T) {
|
|
store := NewAttemptStore(t.TempDir())
|
|
store.chmod = func(path string, mode os.FileMode) error {
|
|
if strings.HasSuffix(path, ".tmp") {
|
|
return errors.New("state chmod denied")
|
|
}
|
|
return os.Chmod(path, mode)
|
|
}
|
|
if _, err := store.Claim([]AttemptSpec{{
|
|
Fingerprint: attemptTestFingerprint("state-chmod"),
|
|
EventKey: EventMention,
|
|
}}, time.Minute); err == nil || !strings.Contains(err.Error(), "state chmod denied") {
|
|
t.Fatalf("state chmod error = %v", err)
|
|
}
|
|
tmp := filepath.Join(store.workDir, AttemptStateFileName) + ".tmp"
|
|
if _, err := os.Stat(tmp); !os.IsNotExist(err) {
|
|
t.Fatalf("temporary state after chmod error = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("lock wait clamps final sleep", func(t *testing.T) {
|
|
workDir := t.TempDir()
|
|
base := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
var nowCalls int
|
|
store := NewAttemptStore(workDir, WithAttemptClock(func() time.Time {
|
|
nowCalls++
|
|
if nowCalls == 1 {
|
|
return base
|
|
}
|
|
return base.Add(attemptLockWaitTimeout - 10*time.Millisecond)
|
|
}))
|
|
var acquireCalls int
|
|
store.tryAcquire = func(path string) (*eventlock.File, error) {
|
|
acquireCalls++
|
|
if acquireCalls == 1 {
|
|
return nil, eventlock.ErrBusy
|
|
}
|
|
return eventlock.TryAcquire(path)
|
|
}
|
|
var slept time.Duration
|
|
store.sleep = func(delay time.Duration) {
|
|
slept = delay
|
|
}
|
|
if err := store.withLock(func() error { return nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slept != 10*time.Millisecond {
|
|
t.Fatalf("final lock sleep = %s, want 10ms", slept)
|
|
}
|
|
})
|
|
|
|
t.Run("default attempt id", func(t *testing.T) {
|
|
attemptID, err := newAttemptID()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.TrimSpace(attemptID) == "" || len(attemptID) > attemptMaxFieldLength {
|
|
t.Fatalf("newAttemptID() = %q", attemptID)
|
|
}
|
|
})
|
|
}
|