From ec99654854540c103dd1bba04ea8abb1e65c5a9b Mon Sep 17 00:00:00 2001 From: wxianfeng Date: Thu, 30 Jul 2026 13:49:08 +0800 Subject: [PATCH] fix(event): prevent subscription retry storms --- docs/event-subprocess-contract.md | 38 +- internal/app/coverage_edges_test.go | 7 +- internal/app/event_command.go | 2 +- internal/app/event_personal_attempts.go | 529 +++++++ internal/app/event_personal_attempts_test.go | 1360 +++++++++++++++++ internal/app/event_personal_command.go | 342 +++-- .../app/event_personal_full_coverage_test.go | 9 +- internal/app/event_personal_many_test.go | 222 +++ internal/cli/schema_agent_metadata/index.json | 2 +- internal/cli/schema_agent_metadata_audit.json | 2 +- internal/cli/schema_catalog/catalog.json | 4 +- internal/errors/coverage_edges_test.go | 2 +- internal/errors/diagnostics.go | 1 + internal/errors/errors.go | 77 +- internal/errors/errors_test.go | 94 ++ internal/event/personal/attempt_store.go | 866 +++++++++++ internal/event/personal/attempt_store_test.go | 1341 ++++++++++++++++ internal/event/personal/client.go | 220 ++- internal/event/personal/client_test.go | 267 ++++ internal/helpers/devapp_test.go | 32 + internal/transport/client_test.go | 37 + skills/mono/references/products/event.md | 12 + skills/multi/dingtalk-event/SKILL.md | 12 + .../dingtalk-event/references/event-im.md | 12 + test/unit/skill_docs_policy_test.go | 41 + 25 files changed, 5357 insertions(+), 174 deletions(-) create mode 100644 internal/app/event_personal_attempts.go create mode 100644 internal/app/event_personal_attempts_test.go create mode 100644 internal/event/personal/attempt_store.go create mode 100644 internal/event/personal/attempt_store_test.go diff --git a/docs/event-subprocess-contract.md b/docs/event-subprocess-contract.md index 0b4b9c87..1fad80ab 100644 --- a/docs/event-subprocess-contract.md +++ b/docs/event-subprocess-contract.md @@ -4,7 +4,7 @@ Defines the stable `dws event consume` subprocess contract so an orchestrator can determine when the consumer is ready, stop it cleanly, and machine-read why it exited. -Scope of this branch: the four **contract** items below. Reconnect +Scope of this branch: the five **contract** items below. Reconnect resilience (keeping the stream alive across a transient upstream drop) is tracked separately and intentionally out of scope here. @@ -92,6 +92,42 @@ Ownership-based cleanup: - T4c (control): `kill -9` leaves subscribe_id lingering (documented risk; we only guarantee SIGTERM is clean, we do not fix kill -9 itself). +### 5. Subscription-create retry budget + +This policy covers all 16 public personal-event keys and every logical +subscription in a multi-event command. It applies only before the ready +marker; reconnecting an established Stream remains a separate mechanism. + +- ID resolution, `event consume`, and later `event status/stop` must use the + same `--profile`. A user or conversation ID resolved under another profile + must not be reused for the current subscription. +- A logical subscription is keyed by the current profile/identity, event key, + rule type, target, and filters. A new `subscribe_id`, `trace_id`, or process + does not reset its retry budget. +- `retryable=false` means `max_additional_attempts=0`. +- `retryable=true` means `max_additional_attempts=2`. A caller must honor + `retry_after_seconds` or `next_retry_at` when present and must not retry + early. +- An omitted retryable value (`retryable=unknown`) means + `max_additional_attempts=1`; a second unknown failure stops the operation. +- `in_flight` means the original logical request is still running. + `cooldown` and `terminal_hold` mean a guard is already delaying or blocking + it. These states must not recursively launch `event consume`, start a + parallel equivalent subscription, or reset the budget with a new subId or + trace. The caller waits for the original request/guard or stops. +- A multi-event command remains one original operation. A caller must not + split out a failed event, reorder events, or restart the command to bypass + a budget. Existing startup rollback cleans subscriptions created before a + later item fails. + +**Verification** +- T5a: non-retryable, retryable, and unknown failures allow respectively + 0, 2, and 1 additional attempts for the same logical subscription. +- T5b: a changed subId/trace or process restart does not increase the budget. +- T5c: `in_flight`/`cooldown` does not recursively issue another create. +- T5d: multi-event startup cannot be split or reordered to bypass the guard, + and a partial startup still rolls back earlier subscriptions. + ## Out of scope (next branch) **Reconnect resilience** — today `personal source` retries only diff --git a/internal/app/coverage_edges_test.go b/internal/app/coverage_edges_test.go index 75aa593c..1f69b573 100644 --- a/internal/app/coverage_edges_test.go +++ b/internal/app/coverage_edges_test.go @@ -1657,12 +1657,13 @@ func TestCrossPlatformCoveragePersonalEventCommandRuntimeCoverage(t *testing.T) CorpID: "corp", UserID: "user", ClientID: "client", }) t.Setenv("DWS_CONFIG_DIR", configDir) - var cancelCount int + var subscribeCount, cancelCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/event/sublist": _ = json.NewEncoder(w).Encode(map[string]any{"items": []map[string]any{{"subId": "sub", "eventKey": personal.EventMention, "ruleType": "at", "status": "active", "sourceId": "open"}}, "total": 1}) case "/subscription/user": + subscribeCount++ _ = json.NewEncoder(w).Encode(map[string]any{"success": true, "result": []string{"created"}}) case "/subscription/cancel": cancelCount++ @@ -1697,8 +1698,8 @@ func TestCrossPlatformCoveragePersonalEventCommandRuntimeCoverage(t *testing.T) if err := runPersonalEventConsume(cmd, personalConsumeOptions{Common: commonConsumeOptions{Foreground: true}, EventKey: personal.EventMention, ControlBaseURL: server.URL, StreamTicketMode: "invalid"}); err == nil { t.Fatal("invalid foreground consume succeeded") } - if cancelCount == 0 { - t.Fatal("failed foreground consume did not clean up subscription") + if subscribeCount != 0 || cancelCount != 0 { + t.Fatalf("invalid local configuration reached subscription control: subscribe=%d cancel=%d", subscribeCount, cancelCount) } if err := runPersonalEventStop(cmd, personalStopOptions{SubscribeID: "sub", All: true, ControlBaseURL: server.URL}); err == nil { diff --git a/internal/app/event_command.go b/internal/app/event_command.go index 89cbbf39..a6ac41f6 100644 --- a/internal/app/event_command.go +++ b/internal/app/event_command.go @@ -161,7 +161,7 @@ SIGTERM、关 stdin，或先用 dws event stop --dry-run 预览 "subscribe-id", "rule", "event-types", "filter", "foreground", "force", "debug-raw-events", ); err != nil { - return fmt.Errorf("event consume: %w", err) + return fmt.Errorf("event consume: %w", personalSubscriptionValidationError(err)) } } personalOpts.Common = commonConsumeOptions{ diff --git a/internal/app/event_personal_attempts.go b/internal/app/event_personal_attempts.go new file mode 100644 index 00000000..d88b9e2f --- /dev/null +++ b/internal/app/event_personal_attempts.go @@ -0,0 +1,529 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + apperrors "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/errors" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/personal" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/pkg/config" +) + +const personalSubscriptionAttemptOperation = "event.consume.personal.subscribe" + +type personalSubscriptionAttemptStore interface { + Claim([]personal.AttemptSpec, time.Duration) (*personal.AttemptClaim, error) + CompleteSuccess(*personal.AttemptClaim) error + CompleteFailure(*personal.AttemptClaim, []string, personal.AttemptFailure) (personal.AttemptHold, error) + Release(*personal.AttemptClaim) error +} + +var ( + personalNewSubscriptionAttemptStore = func(workDir string) personalSubscriptionAttemptStore { + return personal.NewAttemptStore(workDir) + } + personalSubscriptionAttemptNow = time.Now +) + +type personalSubscriptionAttemptItem struct { + eventKey string + fingerprint string +} + +type personalSubscriptionAttemptReservation struct { + store personalSubscriptionAttemptStore + claim *personal.AttemptClaim + items []personalSubscriptionAttemptItem +} + +type personalSubscriptionFailureClass struct { + retryability personal.Retryability + retryAfter time.Duration + code string + traceID string + reason string + auth bool +} + +func reservePersonalSubscriptionAttempts( + workDir string, + client *personal.Client, + identity personal.Identity, + profileSelector string, + plans []personalConsumeOptions, +) (*personalSubscriptionAttemptReservation, error) { + if len(plans) == 0 { + return nil, personalSubscriptionGuardError( + errors.New("personal event: no subscription attempts to reserve"), + ) + } + if client == nil { + return nil, personalSubscriptionGuardError( + errors.New("personal event: nil subscription control client"), + ) + } + if err := validatePersonalSubscriptionEndpoint(client.BaseURL); err != nil { + return nil, personalSubscriptionValidationError(err) + } + + items := make([]personalSubscriptionAttemptItem, 0, len(plans)) + specs := make([]personal.AttemptSpec, 0, len(plans)) + for _, plan := range plans { + prepared, err := preparePersonalSubscription(identity, plan) + if err != nil { + return nil, personalSubscriptionValidationError(err) + } + fingerprint := personal.Fingerprint( + client.BaseURL, + prepared.Request.IdempotencyKey, + profileSelector, + ) + items = append(items, personalSubscriptionAttemptItem{ + eventKey: prepared.EventKey, + fingerprint: fingerprint, + }) + specs = append(specs, personal.AttemptSpec{ + Fingerprint: fingerprint, + EventKey: prepared.EventKey, + }) + } + + store := personalNewSubscriptionAttemptStore(workDir) + if store == nil { + return nil, personalSubscriptionGuardError( + errors.New("personal event: subscription attempt store is unavailable"), + ) + } + claim, err := store.Claim(specs, personalSubscriptionAttemptLease(client, len(specs))) + if err != nil { + var blocked *personal.AttemptBlockedError + if errors.As(err, &blocked) { + return nil, personalSubscriptionBlockedError(blocked) + } + return nil, personalSubscriptionGuardError(err) + } + return &personalSubscriptionAttemptReservation{ + store: store, + claim: claim, + items: items, + }, nil +} + +func validatePersonalSubscriptionEndpoint(raw string) error { + raw = strings.TrimSpace(raw) + parsed, err := url.Parse(raw) + if err != nil || parsed.Host == "" || + (!strings.EqualFold(parsed.Scheme, "http") && + !strings.EqualFold(parsed.Scheme, "https")) { + if err == nil { + err = errors.New("an absolute http(s) URL is required") + } + return fmt.Errorf("personal event: invalid subscription control endpoint %q: %w", raw, err) + } + return nil +} + +func personalSubscriptionAttemptLease(client *personal.Client, batchSize int) time.Duration { + const ( + leaseOverhead = 30 * time.Second + minLease = time.Minute + maxLease = 10 * time.Minute + ) + if batchSize < 1 { + batchSize = 1 + } + timeout := config.HTTPTimeout + if client != nil && client.HTTPClient != nil && client.HTTPClient.Timeout > 0 { + timeout = client.HTTPClient.Timeout + } + maxRequestBudget := maxLease - leaseOverhead + if timeout <= 0 || timeout > maxRequestBudget/time.Duration(batchSize) { + return maxLease + } + lease := timeout*time.Duration(batchSize) + leaseOverhead + if lease < minLease { + return minLease + } + return lease +} + +func (r *personalSubscriptionAttemptReservation) completeSuccess() error { + if r == nil { + return nil + } + if r.store == nil || r.claim == nil { + return personalSubscriptionGuardError( + errors.New("personal event: subscription attempt reservation is incomplete"), + ) + } + if err := r.store.CompleteSuccess(r.claim); err != nil { + return personalSubscriptionGuardError(err) + } + return nil +} + +func (r *personalSubscriptionAttemptReservation) completeFailure( + ctx context.Context, + failedIndex int, + succeededCount int, + cause error, + override *personalSubscriptionFailureClass, +) error { + if r == nil { + return cause + } + if failedIndex < 0 || failedIndex >= len(r.items) || + succeededCount < 0 || succeededCount > failedIndex { + return personalSubscriptionGuardError(errors.Join( + cause, + errors.New("personal event: invalid subscription attempt completion indexes"), + )) + } + if personalSubscriptionCanceled(ctx, cause) { + // Cancellation is not a failed attempt. Restoring the claim normally + // completes immediately; if the lock cannot be acquired, leaving the + // finite lease behind is still safer than recording a false failure. + _ = r.store.Release(r.claim) + return cause + } + + classification := classifyPersonalSubscriptionFailure(cause, personalSubscriptionAttemptNow()) + if override != nil { + classification = *override + } + succeeded := make([]string, 0, succeededCount) + for i := 0; i < succeededCount; i++ { + succeeded = append(succeeded, r.items[i].fingerprint) + } + hold, err := r.store.CompleteFailure(r.claim, succeeded, personal.AttemptFailure{ + Fingerprint: r.items[failedIndex].fingerprint, + Retryability: classification.retryability, + RetryAfter: classification.retryAfter, + ErrorCode: classification.code, + TraceID: classification.traceID, + }) + if err != nil { + return personalSubscriptionGuardError(errors.Join(cause, err)) + } + return personalSubscriptionFailureError(cause, classification, hold) +} + +func personalSubscriptionCanceled(ctx context.Context, err error) bool { + if errors.Is(err, context.Canceled) { + return true + } + return ctx != nil && errors.Is(ctx.Err(), context.Canceled) +} + +func classifyPersonalSubscriptionFailure(err error, now time.Time) personalSubscriptionFailureClass { + classification := personalSubscriptionFailureClass{ + retryability: personal.RetryabilityUnknown, + reason: "personal_subscription_unknown", + } + + var apiErr *personal.APIError + if errors.As(err, &apiErr) { + classification.code = strings.TrimSpace(apiErr.Code) + classification.traceID = strings.TrimSpace(apiErr.TraceID) + classification.retryAfter = personalAPIRetryDelay(apiErr, now) + classification.auth = personalSubscriptionAuthFailure(apiErr.HTTPStatus, apiErr.Code) + switch { + case apiErr.Retryable != nil && *apiErr.Retryable: + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_server_retryable" + case apiErr.Retryable != nil: + classification.retryability = personal.RetryabilityNonRetryable + classification.reason = "personal_subscription_server_non_retryable" + case apiErr.HTTPStatus == http.StatusRequestTimeout || + apiErr.HTTPStatus == http.StatusTooEarly || + apiErr.HTTPStatus == http.StatusTooManyRequests || + apiErr.HTTPStatus >= http.StatusInternalServerError: + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_transient_http" + case personalSubscriptionTerminalBusinessCode(apiErr.Code): + classification.retryability = personal.RetryabilityNonRetryable + classification.reason = "personal_subscription_business_rejected" + case apiErr.HTTPStatus >= http.StatusBadRequest: + classification.retryability = personal.RetryabilityNonRetryable + classification.reason = "personal_subscription_http_rejected" + } + return classification + } + + if errors.Is(err, context.DeadlineExceeded) { + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_timeout" + return classification + } + var urlErr *url.Error + if errors.As(err, &urlErr) { + if strings.EqualFold(strings.TrimSpace(urlErr.Op), "parse") { + classification.retryability = personal.RetryabilityNonRetryable + classification.reason = "personal_subscription_invalid" + return classification + } + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_network" + return classification + } + var netErr net.Error + if errors.As(err, &netErr) { + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_network" + return classification + } + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + classification.retryability = personal.RetryabilityRetryable + classification.reason = "personal_subscription_network" + return classification + } + lower := strings.ToLower(err.Error()) + if strings.Contains(lower, "access token") || strings.Contains(lower, "oauth") { + classification.retryability = personal.RetryabilityNonRetryable + classification.reason = "personal_subscription_auth" + classification.auth = true + } + return classification +} + +func personalAPIRetryDelay(apiErr *personal.APIError, now time.Time) time.Duration { + if apiErr == nil { + return 0 + } + var delay time.Duration + if apiErr.RetryAfterSeconds != nil { + delay = maxPersonalRetryDelay(delay, personalRetrySeconds(*apiErr.RetryAfterSeconds)) + } + if apiErr.NextRetryAt != nil { + delay = maxPersonalRetryDelay(delay, apiErr.NextRetryAt.Sub(now)) + } + if raw, ok := apiErr.Details["retry_after"].(string); ok { + raw = strings.TrimSpace(raw) + if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil { + delay = maxPersonalRetryDelay(delay, personalRetrySeconds(seconds)) + } else if next, err := http.ParseTime(raw); err == nil { + delay = maxPersonalRetryDelay(delay, next.Sub(now)) + } + } + return delay +} + +func personalRetrySeconds(seconds int64) time.Duration { + if seconds <= 0 { + return 0 + } + if seconds > math.MaxInt64/int64(time.Second) { + return time.Duration(math.MaxInt64) + } + return time.Duration(seconds) * time.Second +} + +func maxPersonalRetryDelay(left, right time.Duration) time.Duration { + if right > left { + return right + } + return left +} + +func personalSubscriptionTerminalBusinessCode(raw string) bool { + code := strings.ToUpper(strings.TrimSpace(raw)) + replacer := strings.NewReplacer("-", "_", ".", "_", " ", "_") + code = replacer.Replace(code) + + // Keep this list deliberately conservative. Unknown server codes must stay + // unknown so a newly introduced transient condition cannot accidentally be + // converted into a one-hour terminal hold. + switch code { + case "INVALID_PARAM", "INVALID_PARAMS", "INVALID_PARAMETER", "INVALID_PARAMETERS", + "ILLEGAL_PARAM", "ILLEGAL_PARAMS", "ILLEGAL_PARAMETER", "ILLEGAL_PARAMETERS", + "PARAM_ERROR", "PARAMETER_ERROR", + "CLIENT_ID_REQUIRED", "SOURCE_ID_REQUIRED", "EVENT_KEY_REQUIRED", "RULE_TYPE_REQUIRED", + "NO_AUTH", "NO_PERMISSION", "PERMISSION_DENIED", "ACCESS_DENIED", + "FORBIDDEN", "UNAUTHORIZED", + "NOT_FOUND", "NOT_EXIST", "NOT_SUPPORTED", "UNSUPPORTED", + "UNIFIED_APP_ID_NOT_FOUND": + return true + } + + // Resource-qualified variants are stable business-rejection shapes. Avoid + // broad substring matching (for example, RETRY_REQUIRED must remain + // unknown). + for _, suffix := range []string{ + "_NOT_BELONG_TO_ORG", + "_DOES_NOT_BELONG_TO_ORG", + "_NOT_FOUND", + "_NOT_EXIST", + "_NOT_SUPPORTED", + "_UNSUPPORTED", + "_NO_PERMISSION", + "_PERMISSION_DENIED", + "_ACCESS_DENIED", + } { + if strings.HasSuffix(code, suffix) { + return true + } + } + return false +} + +func personalSubscriptionAuthFailure(status int, rawCode string) bool { + if status == http.StatusUnauthorized || status == http.StatusForbidden { + return true + } + code := strings.ToUpper(strings.TrimSpace(rawCode)) + for _, marker := range []string{ + "NO_AUTH", "UNAUTHORIZED", "FORBIDDEN", "PERMISSION", "ACCESS_DENIED", + } { + if strings.Contains(code, marker) { + return true + } + } + return false +} + +func personalSubscriptionFailureError( + cause error, + classification personalSubscriptionFailureClass, + hold personal.AttemptHold, +) error { + options := personalSubscriptionErrorOptions( + classification.retryability, + hold.RetryAfter, + hold.NextAllowedAt, + classification.code, + classification.traceID, + classification.reason, + cause, + ) + message := cause.Error() + if classification.retryability == personal.RetryabilityNonRetryable { + if classification.auth { + return apperrors.NewAuth(message, options...) + } + return apperrors.NewValidation(message, options...) + } + return apperrors.NewAPI(message, options...) +} + +func personalSubscriptionBlockedError(blocked *personal.AttemptBlockedError) error { + if blocked == nil { + return personalSubscriptionGuardError( + errors.New("personal event: nil blocked subscription attempt"), + ) + } + reason := "personal_subscription_" + string(blocked.State) + options := personalSubscriptionErrorOptions( + blocked.Retryability, + blocked.RetryAfter, + blocked.NextAllowedAt, + blocked.ErrorCode, + blocked.TraceID, + reason, + blocked, + ) + if blocked.Retryability == personal.RetryabilityNonRetryable { + if personalSubscriptionAuthFailure(0, blocked.ErrorCode) { + return apperrors.NewAuth(blocked.Error(), options...) + } + return apperrors.NewValidation(blocked.Error(), options...) + } + return apperrors.NewAPI(blocked.Error(), options...) +} + +func personalSubscriptionErrorOptions( + retryability personal.Retryability, + retryAfter time.Duration, + nextRetryAt time.Time, + code string, + traceID string, + reason string, + cause error, +) []apperrors.Option { + options := []apperrors.Option{ + apperrors.WithOperation(personalSubscriptionAttemptOperation), + apperrors.WithReason(reason), + apperrors.WithCause(cause), + } + if retryable, known := retryability.Value(); known { + options = append(options, apperrors.WithRetryable(retryable)) + } + if retryAfter > 0 { + options = append(options, apperrors.WithRetryAfterSeconds(ceilPersonalRetrySeconds(retryAfter))) + } + if !nextRetryAt.IsZero() { + options = append(options, apperrors.WithNextRetryAt(nextRetryAt)) + } + if code != "" || traceID != "" { + options = append(options, apperrors.WithServerDiag(apperrors.ServerDiagnostics{ + TraceID: strings.TrimSpace(traceID), + ServerErrorCode: strings.TrimSpace(code), + })) + } + return options +} + +func ceilPersonalRetrySeconds(delay time.Duration) int64 { + if delay <= 0 { + return 0 + } + seconds := int64(delay / time.Second) + if delay%time.Second != 0 { + seconds++ + } + return seconds +} + +func personalSubscriptionGuardError(cause error) error { + if cause == nil { + cause = errors.New("personal event: subscription attempt guard failed") + } + return apperrors.NewInternal( + fmt.Sprintf("personal subscription attempt guard failed: %v", cause), + apperrors.WithOperation(personalSubscriptionAttemptOperation), + apperrors.WithReason("personal_subscription_guard_failed"), + apperrors.WithRetryable(false), + apperrors.WithCause(cause), + ) +} + +func personalSubscriptionValidationError(cause error) error { + if cause == nil { + cause = errors.New("personal event: invalid subscription parameters") + } + return apperrors.NewValidation( + cause.Error(), + apperrors.WithOperation(personalSubscriptionAttemptOperation), + apperrors.WithReason("personal_subscription_invalid"), + apperrors.WithRetryable(false), + apperrors.WithCause(cause), + ) +} + +func personalSubscriptionLocalFailure(cause error) personalSubscriptionFailureClass { + return personalSubscriptionFailureClass{ + retryability: personal.RetryabilityNonRetryable, + reason: "personal_subscription_local_failure", + } +} diff --git a/internal/app/event_personal_attempts_test.go b/internal/app/event_personal_attempts_test.go new file mode 100644 index 00000000..c8f84d23 --- /dev/null +++ b/internal/app/event_personal_attempts_test.go @@ -0,0 +1,1360 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); + +package app + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + apperrors "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/errors" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/busctl" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/consume" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/personal" + eventtransport "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/transport" +) + +type personalRecordingAttemptStore struct { + mu sync.Mutex + specs []personal.AttemptSpec + lease time.Duration + claimCalls int + completeCalls int + failureCalls int + releaseCalls int + claimErr error + completeErr error + completeFailure error + failureHold personal.AttemptHold + lastFailure personal.AttemptFailure + lastSucceeded []string +} + +func (s *personalRecordingAttemptStore) Claim( + specs []personal.AttemptSpec, + lease time.Duration, +) (*personal.AttemptClaim, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.claimCalls++ + s.specs = append([]personal.AttemptSpec(nil), specs...) + s.lease = lease + if s.claimErr != nil { + return nil, s.claimErr + } + fingerprints := make([]string, 0, len(specs)) + for _, spec := range specs { + fingerprints = append(fingerprints, spec.Fingerprint) + } + return &personal.AttemptClaim{ + AttemptID: "recording-attempt", + Fingerprints: fingerprints, + }, nil +} + +func (s *personalRecordingAttemptStore) CompleteSuccess(*personal.AttemptClaim) error { + s.mu.Lock() + defer s.mu.Unlock() + s.completeCalls++ + return s.completeErr +} + +func (s *personalRecordingAttemptStore) CompleteFailure( + _ *personal.AttemptClaim, + succeeded []string, + failure personal.AttemptFailure, +) (personal.AttemptHold, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.failureCalls++ + s.lastFailure = failure + s.lastSucceeded = append([]string(nil), succeeded...) + if s.completeFailure != nil { + return personal.AttemptHold{}, s.completeFailure + } + hold := s.failureHold + hold.Fingerprint = failure.Fingerprint + hold.Retryability = failure.Retryability + return hold, nil +} + +func (s *personalRecordingAttemptStore) Release(*personal.AttemptClaim) error { + s.mu.Lock() + defer s.mu.Unlock() + s.releaseCalls++ + return nil +} + +func TestPersonalSubscriptionProtectionCoversAllPublicEvents(t *testing.T) { + oldFactory := personalNewSubscriptionAttemptStore + t.Cleanup(func() { personalNewSubscriptionAttemptStore = oldFactory }) + + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + client := personal.NewClient("https://mcp.example.test/dws", identity) + publicCount := 0 + ruleTypes := make(map[string]bool) + fingerprints := make(map[string]bool) + + for _, definition := range personal.Definitions() { + if !definition.Public { + continue + } + publicCount++ + ruleTypes[definition.RuleType] = true + opts := personalConsumeOptions{EventKey: definition.EventKey} + switch definition.RuleType { + case "singleChat", "sender": + opts.UserID = "target-user" + case "group": + opts.GroupID = "target-group" + } + + recording := &personalRecordingAttemptStore{} + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return recording + } + reservation, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + client, + identity, + "profile-a", + []personalConsumeOptions{opts}, + ) + if err != nil { + t.Fatalf("%s reservation failed: %v", definition.EventKey, err) + } + if recording.claimCalls != 1 || len(recording.specs) != 1 { + t.Fatalf("%s claim = %d, %#v", definition.EventKey, recording.claimCalls, recording.specs) + } + spec := recording.specs[0] + if spec.EventKey != definition.EventKey || len(spec.Fingerprint) != 64 { + t.Fatalf("%s spec = %#v", definition.EventKey, spec) + } + if fingerprints[spec.Fingerprint] { + t.Fatalf("%s reused another event fingerprint", definition.EventKey) + } + fingerprints[spec.Fingerprint] = true + if err := reservation.completeSuccess(); err != nil { + t.Fatalf("%s completion failed: %v", definition.EventKey, err) + } + } + + if publicCount != 16 { + t.Fatalf("public personal events = %d, want 16", publicCount) + } + for _, ruleType := range []string{"at", "all", "singleChat", "sender", "group"} { + if !ruleTypes[ruleType] { + t.Errorf("rule type %s was not protected", ruleType) + } + } +} + +func TestPersonalSubscriptionFingerprintChangesWithLogicalInputs(t *testing.T) { + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + fingerprint := func(endpoint, profile string, opts personalConsumeOptions) string { + t.Helper() + prepared, err := preparePersonalSubscription(identity, opts) + if err != nil { + t.Fatal(err) + } + return personal.Fingerprint(endpoint, prepared.Request.IdempotencyKey, profile) + } + baseOpts := personalConsumeOptions{ + EventKey: personal.EventSingleChat, + UserID: "target-a", + } + base := fingerprint("https://mcp.example.test/dws", "profile-a", baseOpts) + candidates := map[string]string{ + "endpoint": fingerprint("https://other.example.test/dws", "profile-a", baseOpts), + "profile": fingerprint("https://mcp.example.test/dws", "profile-b", baseOpts), + "event": fingerprint( + "https://mcp.example.test/dws", + "profile-a", + personalConsumeOptions{EventKey: personal.EventFromUser, UserID: "target-a"}, + ), + "target": fingerprint( + "https://mcp.example.test/dws", + "profile-a", + personalConsumeOptions{EventKey: personal.EventSingleChat, UserID: "target-b"}, + ), + "filter": fingerprint( + "https://mcp.example.test/dws", + "profile-a", + personalConsumeOptions{ + EventKey: personal.EventSingleChat, + UserID: "target-a", + QueryCSV: "urgent", + }, + ), + } + for dimension, candidate := range candidates { + if candidate == base { + t.Errorf("%s did not change subscription fingerprint", dimension) + } + } +} + +func TestPersonalSubscriptionRejectsMalformedEndpointBeforeClaim(t *testing.T) { + oldFactory := personalNewSubscriptionAttemptStore + t.Cleanup(func() { personalNewSubscriptionAttemptStore = oldFactory }) + factoryCalls := 0 + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + factoryCalls++ + return &personalRecordingAttemptStore{} + } + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + client := personal.NewClient("://malformed", identity) + _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + client, + identity, + "profile-a", + []personalConsumeOptions{{EventKey: personal.EventMention}}, + ) + if err == nil { + t.Fatal("malformed subscription endpoint was accepted") + } + var typed *apperrors.Error + if !errors.As(err, &typed) || typed.Category != apperrors.CategoryValidation || + !typed.RetryableSet || typed.Retryable { + t.Fatalf("malformed endpoint error = %#v, %v", typed, err) + } + if factoryCalls != 0 { + t.Fatalf("attempt-store factory calls = %d, want 0", factoryCalls) + } +} + +type personalRoundTripFunc func(*http.Request) (*http.Response, error) + +func (fn personalRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return fn(request) +} + +func TestPersonalSubscriptionProtectionConcurrentHundredAllowsOneHTTPRequest(t *testing.T) { + oldFactory := personalNewSubscriptionAttemptStore + t.Cleanup(func() { personalNewSubscriptionAttemptStore = oldFactory }) + personalNewSubscriptionAttemptStore = func(workDir string) personalSubscriptionAttemptStore { + return personal.NewAttemptStore(workDir) + } + + const concurrency = 100 + workDir := t.TempDir() + identity := personal.Identity{ + AccessToken: "token", + CorpID: "corp", + UserID: "self", + ClientID: "client", + SourceID: "source", + } + opts := personalConsumeOptions{EventKey: personal.EventMention} + var requests atomic.Int64 + requestStarted := make(chan struct{}, concurrency) + releaseRequest := make(chan struct{}) + transport := personalRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.Path != "/dws/subscription/user" { + t.Errorf("request path = %q", request.URL.Path) + } + requests.Add(1) + requestStarted <- struct{}{} + <-releaseRequest + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader( + `{"success":true,"result":["sub-one"]}`, + )), + Request: request, + }, nil + }) + + start := make(chan struct{}) + results := make(chan error, concurrency) + for i := 0; i < concurrency; i++ { + go func() { + <-start + client := personal.NewClient("https://mcp.example.test/dws", identity) + client.HTTPClient = &http.Client{Transport: transport, Timeout: 30 * time.Second} + reservation, err := reservePersonalSubscriptionAttempts( + workDir, + client, + identity, + "profile-a", + []personalConsumeOptions{opts}, + ) + if err != nil { + results <- err + return + } + _, _, _, err = ensurePersonalSubscription( + context.Background(), + client, + identity, + opts, + ) + if err == nil { + err = reservation.completeSuccess() + } + results <- err + }() + } + close(start) + + select { + case <-requestStarted: + case <-time.After(10 * time.Second): + t.Fatal("no subscription HTTP request started") + } + + for i := 0; i < concurrency-1; i++ { + select { + case err := <-results: + var blocked *personal.AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != personal.AttemptStateInFlight { + t.Fatalf("concurrent loser error = %v", err) + } + case <-time.After(10 * time.Second): + t.Fatalf("only %d concurrent losers completed", i) + } + } + close(releaseRequest) + select { + case err := <-results: + if err != nil { + t.Fatalf("winning request failed: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("winning request did not complete") + } + if got := requests.Load(); got != 1 { + t.Fatalf("subscription HTTP requests = %d, want 1", got) + } +} + +func TestPersonalSubscriptionFailureClassification(t *testing.T) { + retryable, nonRetryable := true, false + now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + networkErr := &url.Error{ + Op: "Post", + URL: "https://mcp.example.test", + Err: &net.DNSError{IsTimeout: true}, + } + tests := []struct { + name string + err error + want personal.Retryability + wantReason string + wantAuth bool + wantRetryWait time.Duration + }{ + { + name: "explicit false", + err: &personal.APIError{ + Code: "GROUP_NOT_BELONG_TO_ORG", Retryable: &nonRetryable, + }, + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_server_non_retryable", + }, + { + name: "explicit true", + err: &personal.APIError{ + Code: "BUSY", Retryable: &retryable, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_server_retryable", + }, + { + name: "terminal business code", + err: &personal.APIError{ + Code: "USER_NOT_FOUND", HTTPStatus: http.StatusOK, + }, + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_business_rejected", + }, + { + name: "permission", + err: &personal.APIError{ + Code: "PERMISSION_DENIED", HTTPStatus: http.StatusOK, + }, + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_business_rejected", + wantAuth: true, + }, + { + name: "429", + err: &personal.APIError{ + Code: "RATE_LIMIT", HTTPStatus: http.StatusTooManyRequests, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_transient_http", + }, + { + name: "408", + err: &personal.APIError{ + Code: "REQUEST_TIMEOUT", HTTPStatus: http.StatusRequestTimeout, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_transient_http", + }, + { + name: "425", + err: &personal.APIError{ + Code: "TOO_EARLY", HTTPStatus: http.StatusTooEarly, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_transient_http", + }, + { + name: "429 overrides terminal-looking business marker", + err: &personal.APIError{ + Code: "EVENT_KEY_NOT_SUPPORTED", HTTPStatus: http.StatusTooManyRequests, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_transient_http", + }, + { + name: "5xx", + err: &personal.APIError{ + Code: "SYSTEM_ERROR", HTTPStatus: http.StatusBadGateway, + }, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_transient_http", + }, + { + name: "http 200 system error remains unknown", + err: &personal.APIError{ + Code: "SYSTEM_ERROR", HTTPStatus: http.StatusOK, + }, + want: personal.RetryabilityUnknown, + wantReason: "personal_subscription_unknown", + }, + { + name: "unknown required code remains unknown", + err: &personal.APIError{ + Code: "RETRY_REQUIRED", HTTPStatus: http.StatusOK, + }, + want: personal.RetryabilityUnknown, + wantReason: "personal_subscription_unknown", + }, + { + name: "network", + err: networkErr, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_network", + }, + { + name: "malformed url is local validation", + err: &url.Error{ + Op: "parse", URL: "://malformed", Err: errors.New("missing protocol scheme"), + }, + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_invalid", + }, + { + name: "truncated response body", + err: fmt.Errorf("read response: %w", io.ErrUnexpectedEOF), + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_network", + }, + { + name: "deadline", + err: context.DeadlineExceeded, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_timeout", + }, + { + name: "longest retry after wins", + err: &personal.APIError{ + Code: "BUSY", + RetryAfterSeconds: int64Pointer(10), + NextRetryAt: timePointer(now.Add(45 * time.Second)), + Details: map[string]any{"retry_after": "60"}, + }, + want: personal.RetryabilityUnknown, + wantReason: "personal_subscription_unknown", + wantRetryWait: time.Minute, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := classifyPersonalSubscriptionFailure(test.err, now) + if got.retryability != test.want || got.reason != test.wantReason || + got.auth != test.wantAuth || got.retryAfter != test.wantRetryWait { + t.Fatalf("classification = %#v", got) + } + }) + } +} + +func TestPersonalSubscriptionFailureErrorPreservesTriStateAndDiagnostics(t *testing.T) { + now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + hold := personal.AttemptHold{ + State: personal.AttemptStateTerminalHold, + Retryability: personal.RetryabilityNonRetryable, + RetryAfter: time.Hour, + NextAllowedAt: now.Add(time.Hour), + } + cause := &personal.APIError{ + Code: "GROUP_NOT_BELONG_TO_ORG", + Message: "group does not belong to organization", + TraceID: "trace-1", + } + classification := personalSubscriptionFailureClass{ + retryability: personal.RetryabilityNonRetryable, + code: cause.Code, + traceID: cause.TraceID, + reason: "personal_subscription_business_rejected", + } + err := personalSubscriptionFailureError(cause, classification, hold) + var typed *apperrors.Error + if !errors.As(err, &typed) { + t.Fatalf("error type = %T", err) + } + if !typed.RetryableSet || typed.Retryable || + typed.RetryAfterSeconds == nil || *typed.RetryAfterSeconds != 3600 || + typed.NextRetryAt == nil || !typed.NextRetryAt.Equal(now.Add(time.Hour)) || + typed.ServerDiag.ServerErrorCode != cause.Code || + typed.ServerDiag.TraceID != cause.TraceID { + t.Fatalf("typed error = %#v", typed) + } + + unknown := personalSubscriptionFailureError( + errors.New("unknown"), + personalSubscriptionFailureClass{ + retryability: personal.RetryabilityUnknown, + reason: "personal_subscription_unknown", + }, + personal.AttemptHold{ + Retryability: personal.RetryabilityUnknown, + RetryAfter: 30 * time.Second, + NextAllowedAt: now.Add(30 * time.Second), + }, + ) + if !errors.As(unknown, &typed) || typed.RetryableSet { + t.Fatalf("unknown retryability was not preserved: %#v", typed) + } +} + +func TestPersonalSubscriptionBatchClaimFailureMakesZeroCreateCalls(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return identity, nil + } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + createCalls := 0 + personalEnsureSubscription = func( + context.Context, + *personal.Client, + personal.Identity, + personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + createCalls++ + return nil, "", "", errors.New("must not run") + } + + next := time.Now().UTC().Add(time.Minute) + recording := &personalRecordingAttemptStore{ + claimErr: &personal.AttemptBlockedError{ + State: personal.AttemptStateCooldown, + Retryability: personal.RetryabilityUnknown, + RetryAfter: time.Minute, + NextAllowedAt: next, + }, + } + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return recording + } + + err := runPersonalEventConsumeMany(newPersonalCoverageCommand(), personalConsumeOptions{ + EventKeys: []string{personal.EventMention, personal.EventAllSingleChat}, + }) + if err == nil { + t.Fatal("blocked batch succeeded") + } + if createCalls != 0 { + t.Fatalf("create calls = %d, want 0", createCalls) + } + if recording.claimCalls != 1 || len(recording.specs) != 2 { + t.Fatalf("batch claim = %d, %#v", recording.claimCalls, recording.specs) + } + var typed *apperrors.Error + if !errors.As(err, &typed) || typed.RetryableSet || + typed.NextRetryAt == nil || !typed.NextRetryAt.Equal(next) { + t.Fatalf("blocked error = %#v, %v", typed, err) + } +} + +func TestPersonalSubscriptionAttemptGuardBypassesNonCreatePaths(t *testing.T) { + oldFactory := personalNewSubscriptionAttemptStore + oldIdentity := personalResolveEventIdentity + oldEnsure := personalEnsureSubscription + oldGet := personalGetSubscription + oldList := personalListSubscriptions + oldUpsert := personalUpsertRunState + oldDelete := personalDeleteSubscription + oldRemove := personalRemoveRunStates + oldLoad := personalLoadRunStates + oldConsume := personalConsumeRun + oldRunMany := personalConsumeRunMany + oldValidate := personalValidateConsumeConfig + oldConflict := personalValidateNoOutputConflict + oldStopConsumers := personalStopConsumers + oldStopBus := personalStopBus + t.Cleanup(func() { + personalNewSubscriptionAttemptStore = oldFactory + personalResolveEventIdentity = oldIdentity + personalEnsureSubscription = oldEnsure + personalGetSubscription = oldGet + personalListSubscriptions = oldList + personalUpsertRunState = oldUpsert + personalDeleteSubscription = oldDelete + personalRemoveRunStates = oldRemove + personalLoadRunStates = oldLoad + personalConsumeRun = oldConsume + personalConsumeRunMany = oldRunMany + personalValidateConsumeConfig = oldValidate + personalValidateNoOutputConflict = oldConflict + personalStopConsumers = oldStopConsumers + personalStopBus = oldStopBus + }) + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + recording := &personalRecordingAttemptStore{} + factoryCalls := 0 + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + factoryCalls++ + return recording + } + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return identity, nil + } + personalEnsureSubscription = ensurePersonalSubscription + personalGetSubscription = func( + *personal.Client, + context.Context, + string, + ) (*personal.Subscription, error) { + return &personal.Subscription{ + SubscribeID: "sub-existing", + EventKey: personal.EventMention, + RuleType: "at", + Status: "active", + }, nil + } + personalListSubscriptions = func( + *personal.Client, + context.Context, + personal.ListOptions, + ) ([]personal.Subscription, error) { + return []personal.Subscription{{ + SubscribeID: "sub-existing", + EventKey: personal.EventMention, + RuleType: "at", + Status: "active", + }}, nil + } + personalUpsertRunState = func(string, personal.RunState) error { return nil } + personalDeleteSubscription = func(*personal.Client, context.Context, string) error { return nil } + personalRemoveRunStates = func(string, []string) error { return nil } + personalLoadRunStates = func(string) ([]personal.RunState, error) { return nil, nil } + personalConsumeRun = func(context.Context, consume.Config) error { return nil } + personalConsumeRunMany = func(context.Context, consume.Config, []consume.ConsumerSpec) error { + return nil + } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + personalValidateNoOutputConflict = func(consume.Config, string) error { return nil } + personalStopConsumers = func(string, []string) (eventtransport.ConsumerStopResp, error) { + return eventtransport.ConsumerStopResp{Stopped: []string{"sub-existing"}}, nil + } + personalStopBus = func(busctl.StopConfig) error { return busctl.ErrNotRunning } + + assertNoAttemptClaim := func(stage string) { + t.Helper() + recording.mu.Lock() + claimCalls := recording.claimCalls + recording.mu.Unlock() + if factoryCalls != 0 || claimCalls != 0 { + t.Fatalf( + "%s used subscription attempt guard: factory calls=%d claim calls=%d", + stage, + factoryCalls, + claimCalls, + ) + } + } + + if err := runPersonalEventConsumeSingle( + newPersonalCoverageCommand(), + personalConsumeOptions{ + EventKey: personal.EventMention, + Common: commonConsumeOptions{DryRun: true}, + }, + ); err != nil { + t.Fatalf("single dry-run error = %v", err) + } + assertNoAttemptClaim("single dry-run") + + if err := runPersonalEventConsumeMany( + newPersonalCoverageCommand(), + personalConsumeOptions{ + EventKeys: []string{personal.EventMention, personal.EventAllSingleChat}, + Common: commonConsumeOptions{DryRun: true}, + }, + ); err != nil { + t.Fatalf("multi dry-run error = %v", err) + } + assertNoAttemptClaim("multi dry-run") + + if err := runPersonalEventConsumeSingle( + newPersonalCoverageCommand(), + personalConsumeOptions{SubscribeID: "sub-existing"}, + ); err != nil { + t.Fatalf("subscribe-id reuse error = %v", err) + } + assertNoAttemptClaim("subscribe-id reuse") + + if err := runPersonalEventStatus( + newPersonalCoverageCommand(), + personalStatusOptions{Status: "all", Format: "json"}, + ); err != nil { + t.Fatalf("status error = %v", err) + } + assertNoAttemptClaim("status") + + if err := runPersonalEventStop( + newPersonalCoverageCommand(), + personalStopOptions{SubscribeID: "sub-existing"}, + ); err != nil { + t.Fatalf("stop error = %v", err) + } + assertNoAttemptClaim("stop") +} + +func TestPersonalSubscriptionLocalValidationRunsBeforeClaimAndCreate(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return identity, nil + } + wantErr := errors.New("invalid local output") + personalValidateConsumeConfig = func(consume.Config) error { return wantErr } + recording := &personalRecordingAttemptStore{} + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return recording + } + createCalls := 0 + personalEnsureSubscription = func( + context.Context, + *personal.Client, + personal.Identity, + personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + createCalls++ + return nil, "", "", errors.New("must not run") + } + + err := runPersonalEventConsumeSingle( + newPersonalCoverageCommand(), + personalConsumeOptions{EventKey: personal.EventMention}, + ) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want %v", err, wantErr) + } + if recording.claimCalls != 0 || createCalls != 0 { + t.Fatalf("claim calls = %d, create calls = %d", recording.claimCalls, createCalls) + } +} + +func TestPersonalSubscriptionSingleAttemptCompletionErrors(t *testing.T) { + tests := []struct { + name string + cancelContext bool + nilSub bool + upsertErr error + completeErr error + wantRelease int + wantFailure int + wantComplete int + wantDelete int + wantCanceledGC bool + }{ + { + name: "nil subscription records failure", + nilSub: true, + wantFailure: 1, + }, + { + name: "completion failure cleans subscription", + completeErr: errors.New("complete success failed"), + wantComplete: 1, + wantDelete: 1, + }, + { + name: "canceled local failure releases and uses canceled cleanup context", + cancelContext: true, + upsertErr: errors.New("save state failed"), + wantRelease: 1, + wantDelete: 1, + wantCanceledGC: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + identity := personal.Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + LocalSubject: "subject", + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return identity, nil + } + recording := &personalRecordingAttemptStore{completeErr: test.completeErr} + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return recording + } + personalEnsureSubscription = func( + context.Context, + *personal.Client, + personal.Identity, + personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + if test.nilSub { + return nil, "", "", nil + } + return &personal.Subscription{ + SubscribeID: "sub-one", + }, personal.EventMention, "at", nil + } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + personalUpsertRunState = func(string, personal.RunState) error { + return test.upsertErr + } + deleteCalls := 0 + personalDeleteSubscription = func( + _ *personal.Client, + cleanupCtx context.Context, + _ string, + ) error { + deleteCalls++ + if got := cleanupCtx.Err() != nil; got != test.wantCanceledGC { + t.Errorf("cleanup canceled = %t, want %t", got, test.wantCanceledGC) + } + return nil + } + personalRemoveRunStates = func(string, []string) error { return nil } + + cmd := newPersonalCoverageCommand() + if test.cancelContext { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cmd.SetContext(ctx) + } + err := runPersonalEventConsumeSingle(cmd, personalConsumeOptions{ + EventKey: personal.EventMention, + ControlBaseURL: "https://mcp.example.test/dws", + }) + if err == nil { + t.Fatal("single subscription error path succeeded") + } + if recording.releaseCalls != test.wantRelease || + recording.failureCalls != test.wantFailure || + recording.completeCalls != test.wantComplete || + deleteCalls != test.wantDelete { + t.Fatalf( + "release=%d failure=%d complete=%d delete=%d", + recording.releaseCalls, + recording.failureCalls, + recording.completeCalls, + deleteCalls, + ) + } + }) + } +} + +func TestPersonalSubscriptionManyCompletionFailureCleansBatch(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + identity := personal.Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + LocalSubject: "subject", + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return identity, nil + } + recording := &personalRecordingAttemptStore{ + completeErr: errors.New("complete success failed"), + } + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return recording + } + createCalls := 0 + personalEnsureSubscription = func( + _ context.Context, + _ *personal.Client, + _ personal.Identity, + opts personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + createCalls++ + return &personal.Subscription{ + SubscribeID: fmt.Sprintf("sub-%d", createCalls), + }, opts.EventKey, "all", nil + } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + personalUpsertRunState = func(string, personal.RunState) error { return nil } + deleteCalls := 0 + personalDeleteSubscription = func(*personal.Client, context.Context, string) error { + deleteCalls++ + return nil + } + personalRemoveRunStates = func(string, []string) error { return nil } + personalConsumeRunMany = func(context.Context, consume.Config, []consume.ConsumerSpec) error { + t.Fatal("consumer ran after attempt completion failure") + return nil + } + + err := runPersonalEventConsumeMany(newPersonalCoverageCommand(), personalConsumeOptions{ + EventKeys: []string{personal.EventMention, personal.EventAllSingleChat}, + ControlBaseURL: "https://mcp.example.test/dws", + }) + if err == nil { + t.Fatal("multi subscription completion failure succeeded") + } + if recording.completeCalls != 1 || createCalls != 2 || deleteCalls != 2 { + t.Fatalf( + "complete=%d create=%d delete=%d", + recording.completeCalls, + createCalls, + deleteCalls, + ) + } +} + +func TestPersonalSubscriptionRejectsNonPublicEventsOnChangedPaths(t *testing.T) { + const nonPublicEvent = personal.EventMention + oldLookup := personalLookupDefinition + oldGet := personalGetSubscription + t.Cleanup(func() { + personalLookupDefinition = oldLookup + personalGetSubscription = oldGet + }) + personalLookupDefinition = func(eventKey string) (personal.Definition, bool) { + definition, ok := personal.Lookup(eventKey) + if eventKey == nonPublicEvent { + definition.Public = false + } + return definition, ok + } + + err := runPersonalEventConsumeSingle( + newPersonalCoverageCommand(), + personalConsumeOptions{EventKey: nonPublicEvent}, + ) + var typed *apperrors.Error + if !errors.As(err, &typed) || typed.Category != apperrors.CategoryValidation || + !typed.RetryableSet || typed.Retryable { + t.Fatalf("single validation error = %#v, %v", typed, err) + } + + if _, err := preparePersonalSubscription( + personal.Identity{}, + personalConsumeOptions{EventKey: nonPublicEvent}, + ); err == nil { + t.Fatal("prepared a non-public personal event") + } + + personalGetSubscription = func( + *personal.Client, + context.Context, + string, + ) (*personal.Subscription, error) { + return &personal.Subscription{ + SubscribeID: "sub-existing", + EventKey: nonPublicEvent, + }, nil + } + if _, _, _, err := ensurePersonalSubscription( + context.Background(), + nil, + personal.Identity{}, + personalConsumeOptions{SubscribeID: "sub-existing"}, + ); err == nil { + t.Fatal("reused a subscription for a non-public personal event") + } +} + +func TestPersonalSubscriptionAttemptLeaseBounds(t *testing.T) { + client := personal.NewClient("https://mcp.example.test/dws", personal.Identity{}) + tests := []struct { + timeout time.Duration + batch int + want time.Duration + }{ + {timeout: 10 * time.Second, batch: 1, want: time.Minute}, + {timeout: 10 * time.Second, batch: 0, want: time.Minute}, + {timeout: 30 * time.Second, batch: 2, want: 90 * time.Second}, + {timeout: 30 * time.Second, batch: 16, want: 510 * time.Second}, + {timeout: time.Hour, batch: 16, want: 10 * time.Minute}, + } + for _, test := range tests { + client.HTTPClient.Timeout = test.timeout + if got := personalSubscriptionAttemptLease(client, test.batch); got != test.want { + t.Errorf("lease(%s, %d) = %s, want %s", test.timeout, test.batch, got, test.want) + } + } +} + +func TestPersonalSubscriptionAttemptGuardHelperEdges(t *testing.T) { + oldFactory := personalNewSubscriptionAttemptStore + t.Cleanup(func() { personalNewSubscriptionAttemptStore = oldFactory }) + + t.Run("default store factory", func(t *testing.T) { + if store := oldFactory(t.TempDir()); store == nil { + t.Fatal("default attempt-store factory returned nil") + } + }) + + identity := personal.Identity{ + CorpID: "corp", UserID: "self", ClientID: "client", SourceID: "source", + } + validClient := personal.NewClient("https://mcp.example.test/dws", identity) + validPlan := personalConsumeOptions{EventKey: personal.EventMention} + + t.Run("empty batch", func(t *testing.T) { + if _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + validClient, + identity, + "profile-a", + nil, + ); err == nil { + t.Fatal("empty attempt batch was accepted") + } + }) + + t.Run("nil client", func(t *testing.T) { + if _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + nil, + identity, + "profile-a", + []personalConsumeOptions{validPlan}, + ); err == nil { + t.Fatal("nil subscription client was accepted") + } + }) + + t.Run("relative endpoint", func(t *testing.T) { + client := personal.NewClient("relative/control", identity) + if _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + client, + identity, + "profile-a", + []personalConsumeOptions{validPlan}, + ); err == nil { + t.Fatal("relative subscription endpoint was accepted") + } + }) + + t.Run("nil store", func(t *testing.T) { + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return nil + } + defer func() { personalNewSubscriptionAttemptStore = oldFactory }() + if _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + validClient, + identity, + "profile-a", + []personalConsumeOptions{validPlan}, + ); err == nil { + t.Fatal("nil subscription attempt store was accepted") + } + }) + + t.Run("claim failure", func(t *testing.T) { + wantErr := errors.New("claim failed") + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return &personalRecordingAttemptStore{claimErr: wantErr} + } + defer func() { personalNewSubscriptionAttemptStore = oldFactory }() + _, err := reservePersonalSubscriptionAttempts( + t.TempDir(), + validClient, + identity, + "profile-a", + []personalConsumeOptions{validPlan}, + ) + if !errors.Is(err, wantErr) { + t.Fatalf("claim error = %v, want %v", err, wantErr) + } + }) + + t.Run("incomplete success reservation", func(t *testing.T) { + if err := (&personalSubscriptionAttemptReservation{}).completeSuccess(); err == nil { + t.Fatal("incomplete reservation succeeded") + } + }) + + t.Run("success completion failure", func(t *testing.T) { + wantErr := errors.New("complete success failed") + reservation := &personalSubscriptionAttemptReservation{ + store: &personalRecordingAttemptStore{completeErr: wantErr}, + claim: &personal.AttemptClaim{AttemptID: "attempt"}, + } + if err := reservation.completeSuccess(); !errors.Is(err, wantErr) { + t.Fatalf("complete-success error = %v, want %v", err, wantErr) + } + }) + + t.Run("nil failure reservation", func(t *testing.T) { + wantErr := errors.New("create failed") + var reservation *personalSubscriptionAttemptReservation + if err := reservation.completeFailure( + context.Background(), + 0, + 0, + wantErr, + nil, + ); !errors.Is(err, wantErr) { + t.Fatalf("nil reservation error = %v, want %v", err, wantErr) + } + }) + + t.Run("invalid failure indexes", func(t *testing.T) { + wantErr := errors.New("create failed") + reservation := &personalSubscriptionAttemptReservation{ + items: []personalSubscriptionAttemptItem{{fingerprint: "fingerprint"}}, + } + if err := reservation.completeFailure( + context.Background(), + 1, + 0, + wantErr, + nil, + ); !errors.Is(err, wantErr) { + t.Fatalf("invalid-index error = %v, want joined cause %v", err, wantErr) + } + }) + + t.Run("failure completion failure", func(t *testing.T) { + createErr := errors.New("create failed") + completeErr := errors.New("complete failure failed") + reservation := &personalSubscriptionAttemptReservation{ + store: &personalRecordingAttemptStore{completeFailure: completeErr}, + claim: &personal.AttemptClaim{AttemptID: "attempt"}, + items: []personalSubscriptionAttemptItem{{ + eventKey: personal.EventMention, + fingerprint: strings.Repeat("a", 64), + }}, + } + err := reservation.completeFailure( + context.Background(), + 0, + 0, + createErr, + nil, + ) + if !errors.Is(err, createErr) || !errors.Is(err, completeErr) { + t.Fatalf("completion error = %v, want joined errors", err) + } + }) +} + +func TestPersonalSubscriptionFailureClassificationEdges(t *testing.T) { + now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + + tests := []struct { + name string + err error + want personal.Retryability + wantReason string + wantAuth bool + }{ + { + name: "plain rejected http status", + err: &personal.APIError{ + Code: "BUSINESS_REJECTED", HTTPStatus: http.StatusBadRequest, + }, + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_http_rejected", + }, + { + name: "direct network error", + err: &net.DNSError{Err: "temporary resolver failure", Name: "mcp.example.test"}, + want: personal.RetryabilityRetryable, + wantReason: "personal_subscription_network", + }, + { + name: "token text", + err: errors.New("access token is missing"), + want: personal.RetryabilityNonRetryable, + wantReason: "personal_subscription_auth", + wantAuth: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := classifyPersonalSubscriptionFailure(test.err, now) + if got.retryability != test.want || got.reason != test.wantReason || + got.auth != test.wantAuth { + t.Fatalf("classification = %#v", got) + } + }) + } + + if delay := personalAPIRetryDelay(nil, now); delay != 0 { + t.Fatalf("nil API retry delay = %s", delay) + } + httpDate := now.Add(75 * time.Second).Format(http.TimeFormat) + if delay := personalAPIRetryDelay(&personal.APIError{ + Details: map[string]any{"retry_after": httpDate}, + }, now); delay != 75*time.Second { + t.Fatalf("HTTP-date retry delay = %s, want 75s", delay) + } + if delay := personalRetrySeconds(0); delay != 0 { + t.Fatalf("zero retry seconds = %s", delay) + } + overflowSeconds := int64(math.MaxInt64/int64(time.Second)) + 1 + if delay := personalRetrySeconds(overflowSeconds); delay != time.Duration(math.MaxInt64) { + t.Fatalf("overflow retry delay = %s", delay) + } + if delay := maxPersonalRetryDelay(time.Minute, time.Second); delay != time.Minute { + t.Fatalf("max retry delay = %s, want 1m", delay) + } + if !personalSubscriptionAuthFailure(http.StatusUnauthorized, "") { + t.Fatal("401 was not classified as an authentication failure") + } +} + +func TestPersonalSubscriptionErrorConstructionEdges(t *testing.T) { + nonRetryable := personalSubscriptionFailureClass{ + retryability: personal.RetryabilityNonRetryable, + reason: "personal_subscription_auth", + auth: true, + } + var typed *apperrors.Error + err := personalSubscriptionFailureError( + errors.New("authentication failed"), + nonRetryable, + personal.AttemptHold{}, + ) + if !errors.As(err, &typed) || typed.Category != apperrors.CategoryAuth { + t.Fatalf("authentication failure error = %#v, %v", typed, err) + } + + if err := personalSubscriptionBlockedError(nil); err == nil { + t.Fatal("nil blocked error was accepted") + } + for _, test := range []struct { + name string + code string + category apperrors.Category + }{ + {name: "blocked auth", code: "NO_PERMISSION", category: apperrors.CategoryAuth}, + {name: "blocked validation", code: "INVALID_PARAM", category: apperrors.CategoryValidation}, + } { + t.Run(test.name, func(t *testing.T) { + err := personalSubscriptionBlockedError(&personal.AttemptBlockedError{ + State: personal.AttemptStateTerminalHold, + Retryability: personal.RetryabilityNonRetryable, + ErrorCode: test.code, + }) + typed = nil + if !errors.As(err, &typed) || typed.Category != test.category { + t.Fatalf("blocked error = %#v, %v", typed, err) + } + }) + } + + if seconds := ceilPersonalRetrySeconds(0); seconds != 0 { + t.Fatalf("ceil zero retry seconds = %d", seconds) + } + if err := personalSubscriptionGuardError(nil); err == nil { + t.Fatal("nil guard cause did not produce an error") + } + if err := personalSubscriptionValidationError(nil); err == nil { + t.Fatal("nil validation cause did not produce an error") + } +} + +func TestNewPersonalEventControlClientSetsVersionHeaders(t *testing.T) { + oldVersion := version + t.Cleanup(func() { version = oldVersion }) + version = "1.2.3-test" + client := newPersonalEventControlClient( + t.TempDir(), + "https://mcp.example.test/dws", + personal.Identity{ClientID: "client", SourceID: "source"}, + ) + if client.ClientVersion != "1.2.3-test" { + t.Fatalf("ClientVersion = %q", client.ClientVersion) + } + if client.UserAgent != "dws-cli/1.2.3-test" { + t.Fatalf("UserAgent = %q", client.UserAgent) + } + + version = " " + client = newPersonalEventControlClient( + t.TempDir(), + "https://mcp.example.test/dws", + personal.Identity{ClientID: "client", SourceID: "source"}, + ) + if client.ClientVersion != "unknown" || client.UserAgent != "dws-cli/unknown" { + t.Fatalf( + "empty-version headers = %q, %q", + client.ClientVersion, + client.UserAgent, + ) + } +} + +func int64Pointer(value int64) *int64 { + return &value +} + +func timePointer(value time.Time) *time.Time { + return &value +} diff --git a/internal/app/event_personal_command.go b/internal/app/event_personal_command.go index 372c0604..99beaa9e 100644 --- a/internal/app/event_personal_command.go +++ b/internal/app/event_personal_command.go @@ -227,7 +227,7 @@ func runPersonalEventConsume(c *cobra.Command, opts personalConsumeOptions) erro func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions) error { ctx := c.Context() if err := ensurePublicPersonalEvent(opts.EventKey); err != nil { - return err + return personalSubscriptionValidationError(err) } rawFormat := "" if f := c.Flags().Lookup("format"); f != nil && f.Changed { @@ -238,7 +238,7 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions fmt.Fprintf(c.ErrOrStderr(), "WARN: --format %q has no meaning for event stream; using ndjson\n", rawFormat) } if err := validatePersonalEventOutputMode(opts.Flatten, opts.DebugRawEvents, normalised); err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } projector := personalEventProjector(opts.DebugRawEvents, opts.Flatten) @@ -255,12 +255,12 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions routes, err := consume.ParseRoutes(opts.Common.RoutesRaw) if err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } if opts.Common.DryRun { if strings.TrimSpace(opts.SubscribeID) == "" { if err := validatePersonalSubscriptionOptions(opts); err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } } cfg := consume.Config{ @@ -284,16 +284,100 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions DryRun: true, } applyPersonalConsumeFilters(&cfg, opts, strings.TrimSpace(opts.SubscribeID), opts.EventKey) - return personalConsumeRun(ctx, cfg) + if err := personalConsumeRun(ctx, cfg); err != nil { + return personalSubscriptionValidationError(err) + } + return nil + } + + cfg := consume.Config{ + WorkDir: workDir, + IPCEndpoint: ipcEndpoint, + ClientID: identity.ClientID, + SpawnExtraArgs: personalBusSpawnArgs(identity, opts.StreamTicketMode, opts.StreamTicketURL, spawnProfileSelector), + Compact: opts.Common.Compact, + MaxEvents: opts.Common.MaxEvents, + Duration: opts.Common.Duration, + EventKey: opts.EventKey, + Format: normalised, + Flatten: opts.Flatten, + OutputDir: opts.Common.OutputDir, + Routes: routes, + Projector: projector, + Stdout: c.OutOrStdout(), + Stderr: c.ErrOrStderr(), + Quiet: opts.Common.Quiet, + Foreground: opts.Common.Foreground, + Force: opts.Common.Force, + } + // Complete all local validation before creating a remote subscription. + // Otherwise an invalid output mode can repeatedly create and roll back a + // valid subscription when an outer agent relaunches the command. + applyEventConsumeStdin(&cfg, opts.Common.MaxEvents, opts.Common.Duration, c.InOrStdin()) + if err := personalValidateConsumeConfig(cfg); err != nil { + return personalSubscriptionValidationError(err) + } + if o := c.Flags().Lookup("output"); o != nil && o.Changed { + if err := personalValidateNoOutputConflict(cfg, o.Value.String()); err != nil { + return personalSubscriptionValidationError(err) + } + } + + var foregroundSource *source.PersonalSource + if opts.Common.Foreground { + foregroundSource, err = personalNewStreamSource(ctx, personalStreamSourceOptions{ + ConfigDir: configDir, + Identity: identity, + TicketMode: opts.StreamTicketMode, + TicketURL: opts.StreamTicketURL, + }) + if err != nil { + return personalSubscriptionValidationError(err) + } } client := newPersonalEventControlClient(configDir, personalEventControlBaseURL(opts.ControlBaseURL, configDir), identity) + var attempt *personalSubscriptionAttemptReservation + if strings.TrimSpace(opts.SubscribeID) == "" { + attempt, err = reservePersonalSubscriptionAttempts( + workDir, + client, + identity, + spawnProfileSelector, + []personalConsumeOptions{opts}, + ) + if err != nil { + return fmt.Errorf("event consume --as user: %w", err) + } + } sub, eventKey, ruleType, err := personalEnsureSubscription(ctx, client, identity, opts) if err != nil { + err = attempt.completeFailure(ctx, 0, 0, err, nil) return fmt.Errorf("event consume --as user: %w", err) } - if sub.SubscribeID == "" { - return fmt.Errorf("event consume --as user: server returned empty subscribe_id") + if sub == nil { + err = attempt.completeFailure( + ctx, + 0, + 0, + errors.New("personal event: server returned an empty subscription"), + nil, + ) + return fmt.Errorf("event consume --as user: %w", err) + } + if strings.TrimSpace(sub.SubscribeID) == "" { + err = attempt.completeFailure( + ctx, + 0, + 0, + errors.New("personal event: server returned empty subscribe_id"), + nil, + ) + return fmt.Errorf("event consume --as user: %w", err) + } + cleanup := func(cleanupCtx context.Context) { + _ = personalDeleteSubscription(client, cleanupCtx, sub.SubscribeID) + _ = personalRemoveRunStates(workDir, []string{sub.SubscribeID}) } if err := personalUpsertRunState(workDir, personal.RunState{ SubscribeID: sub.SubscribeID, @@ -303,11 +387,21 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions SourceID: identity.SourceID, IdentityHash: identityHash, }); err != nil { - return fmt.Errorf("event consume --as user: save run state: %w", err) + wrapped := fmt.Errorf("save run state: %w", err) + if attempt != nil { + cleanupCtx := context.Background() + if personalSubscriptionCanceled(ctx, wrapped) { + cleanupCtx = ctx + } + classification := personalSubscriptionLocalFailure(wrapped) + wrapped = attempt.completeFailure(ctx, 0, 0, wrapped, &classification) + cleanup(cleanupCtx) + } + return fmt.Errorf("event consume --as user: %w", wrapped) } - cleanup := func() { - _ = personalDeleteSubscription(client, context.Background(), sub.SubscribeID) - _ = personalRemoveRunStates(workDir, []string{sub.SubscribeID}) + if err := attempt.completeSuccess(); err != nil { + cleanup(context.Background()) + return fmt.Errorf("event consume --as user: %w", err) } // Ownership-based cleanup: a subscription this run CREATED is // unsubscribed on exit @@ -317,59 +411,17 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions // either way. selfCreated := strings.TrimSpace(opts.SubscribeID) == "" if opts.Ephemeral || selfCreated { - defer cleanup() + defer cleanup(context.Background()) } - cfg := consume.Config{ - WorkDir: workDir, - IPCEndpoint: ipcEndpoint, - ClientID: identity.ClientID, - SpawnExtraArgs: personalBusSpawnArgs(identity, opts.StreamTicketMode, opts.StreamTicketURL, spawnProfileSelector), - Compact: opts.Common.Compact, - MaxEvents: opts.Common.MaxEvents, - Duration: opts.Common.Duration, - EventKey: eventKey, - Format: normalised, - Flatten: opts.Flatten, - OutputDir: opts.Common.OutputDir, - Routes: routes, - Projector: projector, - ReadySubscribeID: sub.SubscribeID, - Stdout: c.OutOrStdout(), - Stderr: c.ErrOrStderr(), - Quiet: opts.Common.Quiet, - Foreground: opts.Common.Foreground, - Force: opts.Common.Force, - } - // Arm the stdin-EOF shutdown watcher only for a pipe-style, unbounded - // run (see shouldWatchStdinEOF). - applyEventConsumeStdin(&cfg, opts.Common.MaxEvents, opts.Common.Duration, c.InOrStdin()) + cfg.EventKey = eventKey + cfg.ReadySubscribeID = sub.SubscribeID applyPersonalConsumeFilters(&cfg, opts, sub.SubscribeID, eventKey) if opts.DebugRawEvents && !opts.Common.Quiet { fmt.Fprintf(c.ErrOrStderr(), "debug raw events enabled: local event filters disabled\nworkdir: %s\nbus_log: %s\n", workDir, filepath.Join(workDir, "bus.log")) } - if err := personalValidateConsumeConfig(cfg); err != nil { - return err - } - if o := c.Flags().Lookup("output"); o != nil && o.Changed { - if err := personalValidateNoOutputConflict(cfg, o.Value.String()); err != nil { - return err - } - } if opts.Common.Foreground { - src, err := personalNewStreamSource(ctx, personalStreamSourceOptions{ - ConfigDir: configDir, - Identity: identity, - TicketMode: opts.StreamTicketMode, - TicketURL: opts.StreamTicketURL, - }) - if err != nil { - if !opts.Ephemeral { - cleanup() - } - return err - } busCfg := bus.Config{ WorkDir: workDir, IPCEndpoint: ipcEndpoint, @@ -378,18 +430,18 @@ func runPersonalEventConsumeSingle(c *cobra.Command, opts personalConsumeOptions SourceKind: dwsevent.SourceKindPersonalStream, IdentityHash: identityHash, SourceID: identity.SourceID, - Source: src, + Source: foregroundSource, } bus.ApplyEnvTuning(&busCfg) err = personalBusRun(ctx, busCfg) if err != nil && !opts.Ephemeral { - cleanup() + cleanup(context.Background()) } return err } err = personalConsumeRun(ctx, cfg) if err != nil && !opts.Ephemeral { - cleanup() + cleanup(context.Background()) } return err } @@ -403,7 +455,7 @@ type personalMultiSubscription struct { func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) error { plans, err := preparePersonalMultiOptions(opts) if err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } rawFormat := "" if f := c.Flags().Lookup("format"); f != nil && f.Changed { @@ -414,7 +466,7 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) fmt.Fprintf(c.ErrOrStderr(), "WARN: --format %q has no meaning for event stream; using ndjson\n", rawFormat) } if err := validatePersonalEventOutputMode(opts.Flatten, opts.DebugRawEvents, normalised); err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } projector := personalEventProjector(false, opts.Flatten) @@ -428,15 +480,16 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) editionName := editionNameOrDefault() workDir := eventWorkDir(configDir, editionName, dwsevent.SourceKindPersonalStream, identityHash) ipcEndpoint := defaultIPCEndpoint(workDir, editionName, dwsevent.SourceKindPersonalStream, identityHash) + spawnProfileSelector := personalBusProfileSelector(configDir, identity) routes, err := consume.ParseRoutes(opts.Common.RoutesRaw) if err != nil { - return fmt.Errorf("event consume --as user: %w", err) + return fmt.Errorf("event consume --as user: %w", personalSubscriptionValidationError(err)) } baseCfg := consume.Config{ WorkDir: workDir, IPCEndpoint: ipcEndpoint, ClientID: identity.ClientID, - SpawnExtraArgs: personalBusSpawnArgs(identity, opts.StreamTicketMode, personalEventStreamTicketURL(opts.StreamTicketURL, configDir)), + SpawnExtraArgs: personalBusSpawnArgs(identity, opts.StreamTicketMode, personalEventStreamTicketURL(opts.StreamTicketURL, configDir), spawnProfileSelector), Compact: opts.Common.Compact, MaxEvents: opts.Common.MaxEvents, Duration: opts.Common.Duration, @@ -451,11 +504,11 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) } applyEventConsumeStdin(&baseCfg, opts.Common.MaxEvents, opts.Common.Duration, c.InOrStdin()) if err := personalValidateConsumeConfig(baseCfg); err != nil { - return err + return personalSubscriptionValidationError(err) } if o := c.Flags().Lookup("output"); o != nil && o.Changed { if err := personalValidateNoOutputConflict(baseCfg, o.Value.String()); err != nil { - return err + return personalSubscriptionValidationError(err) } } if opts.Common.DryRun { @@ -464,13 +517,23 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) } client := newPersonalEventControlClient(configDir, personalEventControlBaseURL(opts.ControlBaseURL, configDir), identity) + attempt, err := reservePersonalSubscriptionAttempts( + workDir, + client, + identity, + spawnProfileSelector, + plans, + ) + if err != nil { + return fmt.Errorf("event consume --as user: %w", err) + } created := make([]personalMultiSubscription, 0, len(plans)) - cleanup := func() { + cleanup := func(cleanupCtx context.Context) { ids := make([]string, 0, len(created)) for i := len(created) - 1; i >= 0; i-- { id := strings.TrimSpace(created[i].Sub.SubscribeID) ids = append(ids, id) - if err := personalDeleteSubscription(client, context.Background(), id); err != nil { + if err := personalDeleteSubscription(client, cleanupCtx, id); err != nil { fmt.Fprintf(c.ErrOrStderr(), "WARN: failed to clean personal subscription %s: %v\n", id, err) } } @@ -480,26 +543,45 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) } } } + failAndCleanup := func( + failedIndex int, + succeededCount int, + cause error, + override *personalSubscriptionFailureClass, + ) error { + cleanupCtx := context.Background() + if personalSubscriptionCanceled(ctx, cause) { + cleanupCtx = ctx + } + completed := attempt.completeFailure(ctx, failedIndex, succeededCount, cause, override) + // Persist the hold (or release a canceled claim) before any potentially + // slow remote rollback. Otherwise the attempt lease can expire while + // deleting earlier subscriptions and admit a duplicate create batch. + cleanup(cleanupCtx) + return completed + } seenSubscribeIDs := make(map[string]struct{}, len(plans)) - for _, plan := range plans { + for i, plan := range plans { sub, eventKey, ruleType, err := personalEnsureSubscription(ctx, client, identity, plan) if err != nil { - cleanup() + err = failAndCleanup(i, len(created), err, nil) return fmt.Errorf("event consume --as user: create subscription for %s: %w", plan.EventKey, err) } if sub == nil { - cleanup() - return fmt.Errorf("event consume --as user: server returned an empty subscription for %s", plan.EventKey) + cause := fmt.Errorf("personal event: server returned an empty subscription for %s", plan.EventKey) + cause = failAndCleanup(i, len(created), cause, nil) + return fmt.Errorf("event consume --as user: %w", cause) } id := strings.TrimSpace(sub.SubscribeID) if id == "" { - cleanup() - return fmt.Errorf("event consume --as user: server returned empty subscribe_id for %s", plan.EventKey) + cause := fmt.Errorf("personal event: server returned empty subscribe_id for %s", plan.EventKey) + cause = failAndCleanup(i, len(created), cause, nil) + return fmt.Errorf("event consume --as user: %w", cause) } if _, exists := seenSubscribeIDs[id]; exists { - _ = personalDeleteSubscription(client, context.Background(), id) - cleanup() - return fmt.Errorf("event consume --as user: server returned duplicate subscribe_id %s", id) + cause := fmt.Errorf("personal event: server returned duplicate subscribe_id %s", id) + cause = failAndCleanup(i, len(created), cause, nil) + return fmt.Errorf("event consume --as user: %w", cause) } seenSubscribeIDs[id] = struct{}{} item := personalMultiSubscription{Sub: sub, EventKey: eventKey, RuleType: ruleType} @@ -512,11 +594,17 @@ func runPersonalEventConsumeMany(c *cobra.Command, opts personalConsumeOptions) SourceID: identity.SourceID, IdentityHash: identityHash, }); err != nil { - cleanup() - return fmt.Errorf("event consume --as user: save run state for %s: %w", eventKey, err) + cause := fmt.Errorf("save run state for %s: %w", eventKey, err) + classification := personalSubscriptionLocalFailure(cause) + cause = failAndCleanup(i, len(created)-1, cause, &classification) + return fmt.Errorf("event consume --as user: %w", cause) } } - defer cleanup() + if err := attempt.completeSuccess(); err != nil { + cleanup(context.Background()) + return fmt.Errorf("event consume --as user: %w", err) + } + defer cleanup(context.Background()) specs := make([]consume.ConsumerSpec, 0, len(created)) for _, item := range created { @@ -705,6 +793,59 @@ func validatePersonalSubscriptionOptions(opts personalConsumeOptions) error { return err } +type personalPreparedSubscription struct { + EventKey string + RuleType string + Request personal.CreateSubscriptionRequest +} + +func preparePersonalSubscription(identity personal.Identity, opts personalConsumeOptions) (personalPreparedSubscription, error) { + if strings.TrimSpace(opts.EventKey) == "" { + return personalPreparedSubscription{}, fmt.Errorf("event_key is required unless --subscribe-id is provided") + } + if err := ensurePublicPersonalEvent(opts.EventKey); err != nil { + return personalPreparedSubscription{}, err + } + ruleType, ruleParam, err := personal.BuildRuleParam(opts.EventKey, personal.RuleOptions{ + RuleType: opts.Rule, + UserID: opts.UserID, + OpenDingTalkID: opts.OpenDingTalkID, + GroupID: opts.GroupID, + }) + if err != nil { + return personalPreparedSubscription{}, err + } + filter, filterCanonical, err := personal.BuildFilter(opts.FilterJSON, opts.QueryCSV) + if err != nil { + return personalPreparedSubscription{}, err + } + req := personal.CreateSubscriptionRequest{ + EventKey: opts.EventKey, + RuleType: ruleType, + Name: opts.Name, + RuleParam: ruleParam, + Filter: filter, + Delivery: map[string]any{"mode": "stream"}, + IdempotencyKey: personal.IdempotencyKey(identity, opts.EventKey, ruleType, ruleParam, filterCanonical), + } + if opts.TTL > 0 { + req.TTLSeconds = int64(opts.TTL.Seconds()) + } + return personalPreparedSubscription{ + EventKey: opts.EventKey, + RuleType: ruleType, + Request: req, + }, nil +} + +func createPreparedPersonalSubscription(ctx context.Context, client *personal.Client, plan personalPreparedSubscription) (*personal.Subscription, string, string, error) { + sub, err := personalCreateSubscription(client, ctx, plan.Request) + if err != nil { + return nil, "", "", err + } + return sub, plan.EventKey, plan.RuleType, nil +} + func ensurePersonalSubscription(ctx context.Context, client *personal.Client, identity personal.Identity, opts personalConsumeOptions) (*personal.Subscription, string, string, error) { if strings.TrimSpace(opts.SubscribeID) != "" { sub, err := personalGetSubscription(client, ctx, opts.SubscribeID) @@ -727,42 +868,11 @@ func ensurePersonalSubscription(ctx context.Context, client *personal.Client, id sub.SubscribeID = strings.TrimSpace(opts.SubscribeID) return sub, eventKey, ruleType, nil } - if strings.TrimSpace(opts.EventKey) == "" { - return nil, "", "", fmt.Errorf("event_key is required unless --subscribe-id is provided") - } - if err := ensurePublicPersonalEvent(opts.EventKey); err != nil { - return nil, "", "", err - } - ruleType, ruleParam, err := personal.BuildRuleParam(opts.EventKey, personal.RuleOptions{ - RuleType: opts.Rule, - UserID: opts.UserID, - OpenDingTalkID: opts.OpenDingTalkID, - GroupID: opts.GroupID, - }) + plan, err := preparePersonalSubscription(identity, opts) if err != nil { return nil, "", "", err } - filter, filterCanonical, err := personal.BuildFilter(opts.FilterJSON, opts.QueryCSV) - if err != nil { - return nil, "", "", err - } - req := personal.CreateSubscriptionRequest{ - EventKey: opts.EventKey, - RuleType: ruleType, - Name: opts.Name, - RuleParam: ruleParam, - Filter: filter, - Delivery: map[string]any{"mode": "stream"}, - IdempotencyKey: personal.IdempotencyKey(identity, opts.EventKey, ruleType, ruleParam, filterCanonical), - } - if opts.TTL > 0 { - req.TTLSeconds = int64(opts.TTL.Seconds()) - } - sub, err := personalCreateSubscription(client, ctx, req) - if err != nil { - return nil, "", "", err - } - return sub, opts.EventKey, ruleType, nil + return createPreparedPersonalSubscription(ctx, client, plan) } func runPersonalEventStatus(c *cobra.Command, opts personalStatusOptions) error { @@ -829,7 +939,7 @@ func ensurePublicPersonalEvent(eventKey string) error { if eventKey == "" { return nil } - if def, ok := personal.Lookup(eventKey); ok && !def.Public { + if def, ok := personalLookupDefinition(eventKey); ok && !def.Public { return personal.PublicAvailabilityError(eventKey) } return nil @@ -1091,6 +1201,12 @@ func resolvePersonalEventIdentity(ctx context.Context, configDir string, sourceI func newPersonalEventControlClient(configDir, baseURL string, identity personal.Identity) *personal.Client { identity.AccessToken = "" client := personal.NewClient(baseURL, identity) + version := strings.TrimSpace(RawVersion()) + if version == "" { + version = "unknown" + } + client.ClientVersion = version + client.UserAgent = "dws-cli/" + version client.AccessTokenProvider = func(ctx context.Context) (string, error) { return personalResolveAuxiliaryAccessToken(ctx, configDir, "") } diff --git a/internal/app/event_personal_full_coverage_test.go b/internal/app/event_personal_full_coverage_test.go index ba730dd4..a7cf1aa7 100644 --- a/internal/app/event_personal_full_coverage_test.go +++ b/internal/app/event_personal_full_coverage_test.go @@ -77,6 +77,7 @@ func TestCrossPlatformCoveragePersonalEventRemainingSchemaAndSubscriptionCoverag func TestCrossPlatformCoveragePersonalEventRemainingConsumeCoverage(t *testing.T) { oldIdentity := personalResolveEventIdentity oldEnsure := personalEnsureSubscription + oldAttemptStore := personalNewSubscriptionAttemptStore oldUpsert := personalUpsertRunState oldDelete := personalDeleteSubscription oldRemove := personalRemoveRunStates @@ -88,6 +89,7 @@ func TestCrossPlatformCoveragePersonalEventRemainingConsumeCoverage(t *testing.T t.Cleanup(func() { personalResolveEventIdentity = oldIdentity personalEnsureSubscription = oldEnsure + personalNewSubscriptionAttemptStore = oldAttemptStore personalUpsertRunState = oldUpsert personalDeleteSubscription = oldDelete personalRemoveRunStates = oldRemove @@ -97,6 +99,9 @@ func TestCrossPlatformCoveragePersonalEventRemainingConsumeCoverage(t *testing.T personalNewStreamSource = oldNewSource personalBusRun = oldBusRun }) + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return personalNoopAttemptStore{} + } wantErr := errors.New("consume") cmd := newPersonalCoverageCommand() @@ -154,11 +159,11 @@ func TestCrossPlatformCoveragePersonalEventRemainingConsumeCoverage(t *testing.T personalNewStreamSource = func(context.Context, personalStreamSourceOptions) (*source.PersonalSource, error) { return nil, wantErr } - if err := runPersonalEventConsume(cmd, personalConsumeOptions{EventKey: personal.EventMention, Common: commonConsumeOptions{Foreground: true}}); !errors.Is(err, wantErr) || deletes == 0 { + if err := runPersonalEventConsume(cmd, personalConsumeOptions{EventKey: personal.EventMention, Common: commonConsumeOptions{Foreground: true}}); !errors.Is(err, wantErr) || deletes != 0 { t.Fatalf("foreground source error = %v deletes=%d", err, deletes) } before := deletes - if err := runPersonalEventConsume(cmd, personalConsumeOptions{EventKey: personal.EventMention, Ephemeral: true, Common: commonConsumeOptions{Foreground: true}}); !errors.Is(err, wantErr) || deletes == before { + if err := runPersonalEventConsume(cmd, personalConsumeOptions{EventKey: personal.EventMention, Ephemeral: true, Common: commonConsumeOptions{Foreground: true}}); !errors.Is(err, wantErr) || deletes != before { t.Fatalf("ephemeral source error = %v deletes=%d", err, deletes) } personalNewStreamSource = func(context.Context, personalStreamSourceOptions) (*source.PersonalSource, error) { return nil, nil } diff --git a/internal/app/event_personal_many_test.go b/internal/app/event_personal_many_test.go index 4a546355..e3fb38d6 100644 --- a/internal/app/event_personal_many_test.go +++ b/internal/app/event_personal_many_test.go @@ -12,6 +12,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/busctl" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/consume" @@ -379,6 +380,157 @@ func TestRunPersonalEventConsumeManyRollsBackPartialCreation(t *testing.T) { } } +func TestRunPersonalEventConsumeManyPersistsFailureBeforeRollback(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + var order []string + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return &personalOrderingAttemptStore{order: &order} + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return personal.Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + LocalSubject: "subject", + }, nil + } + calls := 0 + personalEnsureSubscription = func( + _ context.Context, + _ *personal.Client, + _ personal.Identity, + opts personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + calls++ + if calls == 2 { + return nil, "", "", errors.New("second subscription failed") + } + return &personal.Subscription{SubscribeID: "sub-first"}, opts.EventKey, "all", nil + } + personalUpsertRunState = func(string, personal.RunState) error { return nil } + personalDeleteSubscription = func(_ *personal.Client, _ context.Context, _ string) error { + order = append(order, "delete") + return nil + } + personalRemoveRunStates = func(string, []string) error { return nil } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + + err := runPersonalEventConsume(newPersonalCoverageCommand(), personalConsumeOptions{ + EventKeys: []string{personal.EventMention, personal.EventAllSingleChat}, + }) + if err == nil { + t.Fatal("partial creation unexpectedly succeeded") + } + if !reflect.DeepEqual(order, []string{"complete_failure", "delete"}) { + t.Fatalf("failure/rollback order = %#v", order) + } +} + +func TestRunPersonalEventConsumeSinglePersistsLocalFailureBeforeRollback(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + var order []string + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return &personalOrderingAttemptStore{order: &order} + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return personal.Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + LocalSubject: "subject", + }, nil + } + personalEnsureSubscription = func( + _ context.Context, + _ *personal.Client, + _ personal.Identity, + opts personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + return &personal.Subscription{SubscribeID: "sub-one"}, opts.EventKey, "all", nil + } + personalUpsertRunState = func(string, personal.RunState) error { + return errors.New("state disk failed") + } + personalDeleteSubscription = func(_ *personal.Client, _ context.Context, _ string) error { + order = append(order, "delete") + return nil + } + personalRemoveRunStates = func(string, []string) error { return nil } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + + err := runPersonalEventConsume(newPersonalCoverageCommand(), personalConsumeOptions{ + EventKey: personal.EventMention, + }) + if err == nil { + t.Fatal("run-state failure unexpectedly succeeded") + } + if !reflect.DeepEqual(order, []string{"complete_failure", "delete"}) { + t.Fatalf("failure/rollback order = %#v", order) + } +} + +func TestRunPersonalEventConsumeManyCancellationReleasesBeforeCanceledCleanup(t *testing.T) { + restore := installPersonalManySeams(t) + defer restore() + t.Setenv("DWS_CONFIG_DIR", t.TempDir()) + + var order []string + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return &personalOrderingAttemptStore{order: &order} + } + personalResolveEventIdentity = func(context.Context, string, string) (personal.Identity, error) { + return personal.Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + LocalSubject: "subject", + }, nil + } + calls := 0 + personalEnsureSubscription = func( + _ context.Context, + _ *personal.Client, + _ personal.Identity, + opts personalConsumeOptions, + ) (*personal.Subscription, string, string, error) { + calls++ + if calls == 2 { + return nil, "", "", context.Canceled + } + return &personal.Subscription{SubscribeID: "sub-first"}, opts.EventKey, "all", nil + } + personalUpsertRunState = func(string, personal.RunState) error { return nil } + personalDeleteSubscription = func(_ *personal.Client, cleanupCtx context.Context, _ string) error { + if cleanupCtx.Err() == nil { + t.Fatal("cancellation cleanup received a live context") + } + order = append(order, "delete") + return cleanupCtx.Err() + } + personalRemoveRunStates = func(string, []string) error { return nil } + personalValidateConsumeConfig = func(consume.Config) error { return nil } + + cmd := newPersonalCoverageCommand() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cmd.SetContext(ctx) + err := runPersonalEventConsume(cmd, personalConsumeOptions{ + EventKeys: []string{personal.EventMention, personal.EventAllSingleChat}, + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation error = %v", err) + } + if !reflect.DeepEqual(order, []string{"release", "delete"}) { + t.Fatalf("release/canceled-cleanup order = %#v", order) + } +} + func TestRunPersonalEventConsumeManyRejectsInvalidSubscriptionResults(t *testing.T) { for _, test := range []struct { name string @@ -641,16 +793,21 @@ func installPersonalManySeams(t *testing.T) func() { oldIdentity := personalResolveEventIdentity oldLookup := personalLookupDefinition oldEnsure := personalEnsureSubscription + oldAttemptStore := personalNewSubscriptionAttemptStore oldUpsert := personalUpsertRunState oldDelete := personalDeleteSubscription oldRemove := personalRemoveRunStates oldRunMany := personalConsumeRunMany oldValidate := personalValidateConsumeConfig oldConflict := personalValidateNoOutputConflict + personalNewSubscriptionAttemptStore = func(string) personalSubscriptionAttemptStore { + return personalNoopAttemptStore{} + } return func() { personalResolveEventIdentity = oldIdentity personalLookupDefinition = oldLookup personalEnsureSubscription = oldEnsure + personalNewSubscriptionAttemptStore = oldAttemptStore personalUpsertRunState = oldUpsert personalDeleteSubscription = oldDelete personalRemoveRunStates = oldRemove @@ -659,3 +816,68 @@ func installPersonalManySeams(t *testing.T) func() { personalValidateNoOutputConflict = oldConflict } } + +type personalNoopAttemptStore struct{} + +func (personalNoopAttemptStore) Claim(specs []personal.AttemptSpec, _ time.Duration) (*personal.AttemptClaim, error) { + fingerprints := make([]string, 0, len(specs)) + for _, spec := range specs { + fingerprints = append(fingerprints, spec.Fingerprint) + } + return &personal.AttemptClaim{ + AttemptID: "test-attempt", + Fingerprints: fingerprints, + }, nil +} + +func (personalNoopAttemptStore) CompleteSuccess(*personal.AttemptClaim) error { + return nil +} + +func (personalNoopAttemptStore) CompleteFailure( + _ *personal.AttemptClaim, + _ []string, + failure personal.AttemptFailure, +) (personal.AttemptHold, error) { + return personal.AttemptHold{ + Fingerprint: failure.Fingerprint, + Retryability: failure.Retryability, + }, nil +} + +func (personalNoopAttemptStore) Release(*personal.AttemptClaim) error { + return nil +} + +type personalOrderingAttemptStore struct { + order *[]string +} + +func (s *personalOrderingAttemptStore) Claim( + specs []personal.AttemptSpec, + lease time.Duration, +) (*personal.AttemptClaim, error) { + return personalNoopAttemptStore{}.Claim(specs, lease) +} + +func (s *personalOrderingAttemptStore) CompleteSuccess(*personal.AttemptClaim) error { + *s.order = append(*s.order, "complete_success") + return nil +} + +func (s *personalOrderingAttemptStore) CompleteFailure( + _ *personal.AttemptClaim, + _ []string, + failure personal.AttemptFailure, +) (personal.AttemptHold, error) { + *s.order = append(*s.order, "complete_failure") + return personal.AttemptHold{ + Fingerprint: failure.Fingerprint, + Retryability: failure.Retryability, + }, nil +} + +func (s *personalOrderingAttemptStore) Release(*personal.AttemptClaim) error { + *s.order = append(*s.order, "release") + return nil +} diff --git a/internal/cli/schema_agent_metadata/index.json b/internal/cli/schema_agent_metadata/index.json index f80e1585..5ce455ff 100644 --- a/internal/cli/schema_agent_metadata/index.json +++ b/internal/cli/schema_agent_metadata/index.json @@ -1,6 +1,6 @@ { "version": 1, - "source_hash": "sha256:f8cb29bdbeaceae6f04cbac8a93ea77f2a84f93f8304d25878c827b959bc0ba0", + "source_hash": "sha256:a4998e1d96d1e816e6978125958b69002b5c9f82e6dfc44a0971ba8003fc6e9b", "surface_hash": "sha256:60eee8e2f37d6d9d60689efce85082798eb9ad38b7ba7c0b471c3de676a85a16", "coverage": { "surface_products": 26, diff --git a/internal/cli/schema_agent_metadata_audit.json b/internal/cli/schema_agent_metadata_audit.json index 01e0ac5f..9650b4ae 100644 --- a/internal/cli/schema_agent_metadata_audit.json +++ b/internal/cli/schema_agent_metadata_audit.json @@ -1,6 +1,6 @@ { "version": 1, - "source_hash": "sha256:f8cb29bdbeaceae6f04cbac8a93ea77f2a84f93f8304d25878c827b959bc0ba0", + "source_hash": "sha256:a4998e1d96d1e816e6978125958b69002b5c9f82e6dfc44a0971ba8003fc6e9b", "surface_hash": "sha256:60eee8e2f37d6d9d60689efce85082798eb9ad38b7ba7c0b471c3de676a85a16", "source_files": 160, "hint_files": 54, diff --git a/internal/cli/schema_catalog/catalog.json b/internal/cli/schema_catalog/catalog.json index 2a3ddf9a..b8a1aaa2 100644 --- a/internal/cli/schema_catalog/catalog.json +++ b/internal/cli/schema_catalog/catalog.json @@ -1,12 +1,12 @@ { "version": 1, "surface_hash": "sha256:60eee8e2f37d6d9d60689efce85082798eb9ad38b7ba7c0b471c3de676a85a16", - "source_hash": "sha256:735bbf8e04cfc44db741e9b14ffe1698f80aac5d2662d2c1c41b0aadcee9098d", + "source_hash": "sha256:d1d19f794effe155a1691c02315546f58aee3860fc694fc0b4456868b0dea3af", "catalog": { "agent_metadata": { "products_with_metadata": 26, "source": "embedded-skill-metadata", - "source_hash": "sha256:f8cb29bdbeaceae6f04cbac8a93ea77f2a84f93f8304d25878c827b959bc0ba0", + "source_hash": "sha256:a4998e1d96d1e816e6978125958b69002b5c9f82e6dfc44a0971ba8003fc6e9b", "surface_hash": "sha256:60eee8e2f37d6d9d60689efce85082798eb9ad38b7ba7c0b471c3de676a85a16", "surface_products": 26, "surface_tools": 845, diff --git a/internal/errors/coverage_edges_test.go b/internal/errors/coverage_edges_test.go index 6047a05f..dd3c425f 100644 --- a/internal/errors/coverage_edges_test.go +++ b/internal/errors/coverage_edges_test.go @@ -25,7 +25,7 @@ func TestCrossPlatformCoverageDiagnosticsAndErrorRenderingEdges(t *testing.T) { }), ) typed := err.(*Error) - if typed.Retryable || typed.ServerDiag.TraceID != "trace" { + if typed.Retryable || !typed.RetryableSet || typed.ServerDiag.TraceID != "trace" { t.Fatalf("diagnostics options = %#v", typed) } if (ServerDiagnostics{TraceID: "trace"}).IsEmpty() { diff --git a/internal/errors/diagnostics.go b/internal/errors/diagnostics.go index a1dedc0d..aea575a3 100644 --- a/internal/errors/diagnostics.go +++ b/internal/errors/diagnostics.go @@ -44,6 +44,7 @@ func WithServerDiag(diag ServerDiagnostics) Option { // Override retryable if server explicitly specified. if diag.ServerRetryable != nil { e.Retryable = *diag.ServerRetryable + e.RetryableSet = true } } } diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 137293f7..0514cac6 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "strings" + "time" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/jsonutil" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/tui" @@ -41,20 +42,23 @@ const ( // Error is the structured repository-local error model for the Go rewrite. type Error struct { - Category Category - Message string - Operation string - ServerKey string - Retryable bool - Reason string - Hint string - Actions []string - AvailableFlags []string - Snapshot string - RPCCode int `json:"rpc_code,omitempty"` - RPCData json.RawMessage `json:"rpc_data,omitempty"` - ServerDiag ServerDiagnostics `json:"-"` - Cause error `json:"-"` + Category Category + Message string + Operation string + ServerKey string + Retryable bool + RetryableSet bool + RetryAfterSeconds *int64 + NextRetryAt *time.Time + Reason string + Hint string + Actions []string + AvailableFlags []string + Snapshot string + RPCCode int `json:"rpc_code,omitempty"` + RPCData json.RawMessage `json:"rpc_data,omitempty"` + ServerDiag ServerDiagnostics `json:"-"` + Cause error `json:"-"` } func (e *Error) Error() string { @@ -107,6 +111,31 @@ func WithServerKey(serverKey string) Option { func WithRetryable(retryable bool) Option { return func(err *Error) { err.Retryable = retryable + err.RetryableSet = true + } +} + +// WithRetryAfterSeconds records the server-recommended delay before a retry. +// A zero delay is meaningful and is therefore preserved; negative values are +// ignored as invalid server guidance. +func WithRetryAfterSeconds(seconds int64) Option { + return func(err *Error) { + if seconds < 0 { + return + } + value := seconds + err.RetryAfterSeconds = &value + } +} + +// WithNextRetryAt records the absolute time at which a retry may be attempted. +func WithNextRetryAt(next time.Time) Option { + return func(err *Error) { + if next.IsZero() { + return + } + value := next.UTC() + err.NextRetryAt = &value } } @@ -266,8 +295,14 @@ func PrintJSON(w io.Writer, err error) error { if typed.ServerKey != "" { errorPayload["server_key"] = typed.ServerKey } - if typed.Retryable { - errorPayload["retryable"] = true + if typed.RetryableSet { + errorPayload["retryable"] = typed.Retryable + } + if typed.RetryAfterSeconds != nil { + errorPayload["retry_after_seconds"] = *typed.RetryAfterSeconds + } + if typed.NextRetryAt != nil { + errorPayload["next_retry_at"] = typed.NextRetryAt.UTC().Format(time.RFC3339) } if typed.Hint != "" { errorPayload["hint"] = typed.Hint @@ -383,8 +418,14 @@ func PrintHumanAt(w io.Writer, err error, v Verbosity) error { if line := formatAvailableFlagsHumanLine(typed.AvailableFlags); line != "" { lines = append(lines, tui.Dim(line)) } - if typed.Retryable { - lines = append(lines, tui.Warning("Retryable: true")) + if typed.RetryableSet { + lines = append(lines, tui.Warning(fmt.Sprintf("Retryable: %t", typed.Retryable))) + } + if typed.RetryAfterSeconds != nil { + lines = append(lines, tui.Warning(fmt.Sprintf("Retry After: %ds", *typed.RetryAfterSeconds))) + } + if typed.NextRetryAt != nil { + lines = append(lines, tui.Warning("Next Retry At: "+typed.NextRetryAt.UTC().Format(time.RFC3339))) } // Always shown when present: Trace ID, Server Code diff --git a/internal/errors/errors_test.go b/internal/errors/errors_test.go index 21a90930..880c0283 100644 --- a/internal/errors/errors_test.go +++ b/internal/errors/errors_test.go @@ -17,6 +17,7 @@ import ( stderrors "errors" "strings" "testing" + "time" ) func TestExitCodeByCategory(t *testing.T) { @@ -77,6 +78,99 @@ func TestPrintJSON(t *testing.T) { } } +func TestRetryabilityTriStateAndRetryTiming(t *testing.T) { + t.Parallel() + + next := time.Date(2026, time.July, 30, 4, 5, 6, 0, time.FixedZone("CST", 8*60*60)) + tests := []struct { + name string + err error + wantRetryable string + wantRetryAfter bool + wantNextRetryAt bool + }{ + { + name: "unknown is omitted", + err: NewAPI("unknown"), + }, + { + name: "explicit false is preserved", + err: NewValidation("terminal", WithRetryable(false)), + wantRetryable: `"retryable": false`, + }, + { + name: "explicit true with timing", + err: NewAPI("transient", WithRetryable(true), WithRetryAfterSeconds(30), WithNextRetryAt(next)), + wantRetryable: `"retryable": true`, + wantRetryAfter: true, + wantNextRetryAt: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var jsonOut strings.Builder + if err := PrintJSON(&jsonOut, tt.err); err != nil { + t.Fatalf("PrintJSON() error = %v", err) + } + gotJSON := jsonOut.String() + if tt.wantRetryable == "" { + if strings.Contains(gotJSON, `"retryable"`) { + t.Fatalf("unknown retryability must be omitted: %s", gotJSON) + } + } else if !strings.Contains(gotJSON, tt.wantRetryable) { + t.Fatalf("missing %s in %s", tt.wantRetryable, gotJSON) + } + if got := strings.Contains(gotJSON, `"retry_after_seconds": 30`); got != tt.wantRetryAfter { + t.Fatalf("retry_after_seconds presence = %v, want %v: %s", got, tt.wantRetryAfter, gotJSON) + } + if got := strings.Contains(gotJSON, `"next_retry_at": "2026-07-29T20:05:06Z"`); got != tt.wantNextRetryAt { + t.Fatalf("next_retry_at presence = %v, want %v: %s", got, tt.wantNextRetryAt, gotJSON) + } + + var humanOut strings.Builder + if err := PrintHuman(&humanOut, tt.err); err != nil { + t.Fatalf("PrintHuman() error = %v", err) + } + gotHuman := humanOut.String() + if tt.wantRetryable == "" { + if strings.Contains(gotHuman, "Retryable:") { + t.Fatalf("unknown retryability must be omitted: %s", gotHuman) + } + } else { + want := "Retryable: true" + if strings.Contains(tt.wantRetryable, "false") { + want = "Retryable: false" + } + if !strings.Contains(gotHuman, want) { + t.Fatalf("missing %q in %s", want, gotHuman) + } + } + if tt.wantRetryAfter && !strings.Contains(gotHuman, "Retry After: 30s") { + t.Fatalf("missing retry delay in %s", gotHuman) + } + if tt.wantNextRetryAt && !strings.Contains(gotHuman, "Next Retry At: 2026-07-29T20:05:06Z") { + t.Fatalf("missing next retry time in %s", gotHuman) + } + }) + } +} + +func TestRetryTimingOptionsIgnoreInvalidValues(t *testing.T) { + t.Parallel() + + err := NewAPI( + "invalid timing", + WithRetryAfterSeconds(-1), + WithNextRetryAt(time.Time{}), + ).(*Error) + if err.RetryAfterSeconds != nil || err.NextRetryAt != nil { + t.Fatalf("invalid retry timing was retained: %#v", err) + } +} + func TestPrintJSON_AvailableFlags(t *testing.T) { t.Parallel() diff --git a/internal/event/personal/attempt_store.go b/internal/event/personal/attempt_store.go new file mode 100644 index 00000000..efc7b36b --- /dev/null +++ b/internal/event/personal/attempt_store.go @@ -0,0 +1,866 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package personal + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + + eventlock "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/lock" + "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/pkg/config" +) + +const ( + // AttemptStateFileName stores retry suppression state for personal + // subscription creation. It intentionally does not share the successful + // subscription run-state file. + AttemptStateFileName = "personal_subscription_attempts.json" + // AttemptStateLockFileName serializes cross-process attempt-state changes. + AttemptStateLockFileName = "personal_subscription_attempts.lock" + + attemptStateVersion = 1 + + attemptLockWaitTimeout = 5 * time.Second + attemptLockRetryDelay = 25 * time.Millisecond + attemptFailureReset = 24 * time.Hour + attemptTerminalHold = time.Hour + attemptMaxFieldLength = 256 +) + +var ( + // ErrAttemptClaimStale means another process replaced at least one claimed + // fingerprint. Completion is rejected as a unit so an old process cannot + // overwrite newer state. + ErrAttemptClaimStale = errors.New("personal event: subscription attempt claim is stale") +) + +// AttemptState is the persisted lifecycle state of one subscription attempt. +type AttemptState string + +const ( + AttemptStateInFlight AttemptState = "in_flight" + AttemptStateCooldown AttemptState = "cooldown" + AttemptStateTerminalHold AttemptState = "terminal_hold" +) + +// Retryability preserves the distinction between an explicit server decision +// and an error for which the server did not provide retry guidance. +type Retryability string + +const ( + RetryabilityUnknown Retryability = "unknown" + RetryabilityRetryable Retryability = "retryable" + RetryabilityNonRetryable Retryability = "non_retryable" +) + +// Value converts retryability to a bool while preserving whether it is known. +func (r Retryability) Value() (value bool, known bool) { + switch r { + case RetryabilityRetryable: + return true, true + case RetryabilityNonRetryable: + return false, true + default: + return false, false + } +} + +// AttemptSpec identifies one normalized subscription request. Fingerprint must +// come from Fingerprint; only the digest and the non-sensitive event key are +// persisted. +type AttemptSpec struct { + Fingerprint string + EventKey string +} + +// AttemptClaim is the ownership token returned by Claim. Callers must finish +// it with CompleteSuccess, CompleteFailure, or Release. +type AttemptClaim struct { + AttemptID string + Fingerprints []string + + previous map[string]attemptPrevious +} + +type attemptPrevious struct { + record attemptRecord + present bool +} + +// AttemptBlockedError reports a local suppression decision. It never includes +// the raw endpoint, idempotency key, profile selector, rule parameter, or +// filter. +type AttemptBlockedError struct { + Fingerprint string + EventKey string + State AttemptState + Retryability Retryability + RetryAfter time.Duration + NextAllowedAt time.Time + FailureCount int + ErrorCode string + TraceID string +} + +func (e *AttemptBlockedError) Error() string { + if e == nil { + return "" + } + return fmt.Sprintf( + "personal event: subscription attempt %s until %s", + e.State, + e.NextAllowedAt.UTC().Format(time.RFC3339), + ) +} + +// AttemptFailure describes the one failed item in a claimed batch. +type AttemptFailure struct { + Fingerprint string + Retryability Retryability + RetryAfter time.Duration + ErrorCode string + TraceID string +} + +// AttemptHold is the persisted suppression decision after CompleteFailure. +type AttemptHold struct { + Fingerprint string + State AttemptState + Retryability Retryability + RetryAfter time.Duration + NextAllowedAt time.Time + FailureCount int +} + +// AttemptStoreOption customizes an AttemptStore. +type AttemptStoreOption func(*AttemptStore) + +// WithAttemptClock installs a clock, primarily for deterministic tests. +func WithAttemptClock(now func() time.Time) AttemptStoreOption { + return func(store *AttemptStore) { + if now != nil { + store.now = now + } + } +} + +// WithAttemptIDGenerator installs an attempt-ID generator. IDs are persisted +// only as CAS tokens and must not contain request data. +func WithAttemptIDGenerator(generate func() (string, error)) AttemptStoreOption { + return func(store *AttemptStore) { + if generate != nil { + store.newID = generate + } + } +} + +// AttemptStore coordinates personal subscription creation across CLI +// processes sharing one identity work directory. +type AttemptStore struct { + workDir string + now func() time.Time + newID func() (string, error) + + readFile func(string) ([]byte, error) + mkdirAll func(string, os.FileMode) error + tryAcquire func(string) (*eventlock.File, error) + remove func(string) error + marshal func(any, string, string) ([]byte, error) + writeFile func(string, []byte, os.FileMode) error + chmod func(string, os.FileMode) error + rename func(string, string) error + sleep func(time.Duration) +} + +// NewAttemptStore creates a fail-closed persistent attempt guard rooted in the +// identity-specific personal event work directory. +func NewAttemptStore(workDir string, options ...AttemptStoreOption) *AttemptStore { + store := &AttemptStore{ + workDir: strings.TrimSpace(workDir), + now: time.Now, + newID: newAttemptID, + readFile: os.ReadFile, + mkdirAll: os.MkdirAll, + tryAcquire: eventlock.TryAcquire, + remove: os.Remove, + marshal: json.MarshalIndent, + writeFile: os.WriteFile, + chmod: os.Chmod, + rename: os.Rename, + sleep: time.Sleep, + } + for _, option := range options { + if option != nil { + option(store) + } + } + return store +} + +// Fingerprint hashes an endpoint, the existing subscription idempotency key, +// and optional profile selectors into a stable non-reversible store key. +// Passing the active selector ensures two profiles resolving to the same +// corp/user can recover independently after a profile change. +func Fingerprint(endpoint, idempotencyKey string, profileSelector ...string) string { + endpoint = normalizeAttemptEndpoint(endpoint) + idempotencyKey = strings.TrimSpace(idempotencyKey) + if endpoint == "" || idempotencyKey == "" { + return "" + } + h := sha256.New() + writeFingerprintPart(h, "dws-personal-subscription-attempt-v1") + writeFingerprintPart(h, endpoint) + writeFingerprintPart(h, idempotencyKey) + for _, selector := range profileSelector { + if selector = strings.TrimSpace(selector); selector != "" { + writeFingerprintPart(h, selector) + } + } + return hex.EncodeToString(h.Sum(nil)) +} + +type fingerprintWriter interface { + Write([]byte) (int, error) +} + +func writeFingerprintPart(w fingerprintWriter, value string) { + var size [8]byte + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = w.Write(size[:]) + _, _ = w.Write([]byte(value)) +} + +func normalizeAttemptEndpoint(raw string) string { + raw = strings.TrimRight(strings.TrimSpace(raw), "/") + if raw == "" { + return "" + } + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return raw + } + parsed.Scheme = strings.ToLower(parsed.Scheme) + parsed.Host = strings.ToLower(parsed.Host) + parsed.Fragment = "" + parsed.Path = strings.TrimRight(parsed.Path, "/") + return parsed.String() +} + +// Claim atomically reserves every fingerprint in specs. If any fingerprint is +// still suppressed, none are reserved and an AttemptBlockedError is returned. +func (s *AttemptStore) Claim(specs []AttemptSpec, lease time.Duration) (*AttemptClaim, error) { + normalized, err := normalizeAttemptSpecs(specs) + if err != nil { + return nil, err + } + if lease <= 0 { + return nil, errors.New("personal event: subscription attempt lease must be positive") + } + if err := s.validate(); err != nil { + return nil, err + } + attemptID, err := s.newID() + if err != nil { + return nil, fmt.Errorf("personal event: generate subscription attempt id: %w", err) + } + attemptID = strings.TrimSpace(attemptID) + if attemptID == "" || len(attemptID) > attemptMaxFieldLength { + return nil, errors.New("personal event: generated subscription attempt id is invalid") + } + + now := s.now().UTC() + claim := &AttemptClaim{ + AttemptID: attemptID, + Fingerprints: make([]string, 0, len(normalized)), + previous: make(map[string]attemptPrevious, len(normalized)), + } + var blocked *AttemptBlockedError + err = s.withLock(func() error { + state, err := s.load() + if err != nil { + return err + } + pruneAttemptRecords(state, now) + + for _, spec := range normalized { + record, ok := state.Records[spec.Fingerprint] + if !ok { + continue + } + if candidate := blockedAttempt(record, now); candidate != nil { + blocked = candidate + return nil + } + } + for _, spec := range normalized { + previous, present := state.Records[spec.Fingerprint] + claim.previous[spec.Fingerprint] = attemptPrevious{ + record: previous, + present: present, + } + claim.Fingerprints = append(claim.Fingerprints, spec.Fingerprint) + record := previous + record.Fingerprint = spec.Fingerprint + record.EventKey = spec.EventKey + record.State = AttemptStateInFlight + record.AttemptID = attemptID + record.LeaseUntil = now.Add(lease) + record.NextAllowedAt = time.Time{} + record.Retryability = "" + record.ErrorCode = "" + record.TraceID = "" + state.Records[spec.Fingerprint] = record + } + return s.write(state) + }) + if err != nil { + return nil, err + } + if blocked != nil { + return nil, blocked + } + return claim, nil +} + +// CompleteSuccess clears failure history for every item in a successful +// claimed batch. +func (s *AttemptStore) CompleteSuccess(claim *AttemptClaim) error { + if err := validateAttemptClaim(claim); err != nil { + return err + } + if err := s.validate(); err != nil { + return err + } + return s.withLock(func() error { + state, err := s.load() + if err != nil { + return err + } + if !claimOwnsAll(state, claim) { + return ErrAttemptClaimStale + } + for _, fingerprint := range claim.Fingerprints { + delete(state.Records, fingerprint) + } + return s.write(state) + }) +} + +// CompleteFailure records the failed item, clears successfully completed +// items, and restores every unexecuted item to its exact pre-claim state. +func (s *AttemptStore) CompleteFailure( + claim *AttemptClaim, + succeeded []string, + failure AttemptFailure, +) (AttemptHold, error) { + var zero AttemptHold + if err := validateAttemptClaim(claim); err != nil { + return zero, err + } + failure.Fingerprint = strings.TrimSpace(failure.Fingerprint) + if !containsFingerprint(claim.Fingerprints, failure.Fingerprint) { + return zero, errors.New("personal event: failed fingerprint is not part of the attempt claim") + } + if !validRetryability(failure.Retryability) { + return zero, fmt.Errorf("personal event: invalid retryability %q", failure.Retryability) + } + succeededSet, err := normalizeSucceededFingerprints(claim, succeeded, failure.Fingerprint) + if err != nil { + return zero, err + } + if err := s.validate(); err != nil { + return zero, err + } + + now := s.now().UTC() + var hold AttemptHold + err = s.withLock(func() error { + state, err := s.load() + if err != nil { + return err + } + if !claimOwnsAll(state, claim) { + return ErrAttemptClaimStale + } + + for _, fingerprint := range claim.Fingerprints { + switch { + case fingerprint == failure.Fingerprint: + previous := claim.previous[fingerprint] + record := previous.record + count := record.FailureCount + if record.LastFailureAt.IsZero() || now.Sub(record.LastFailureAt) >= attemptFailureReset { + count = 0 + } + count++ + delay, stateName := attemptFailureDelay(fingerprint, count, failure) + record.Fingerprint = fingerprint + record.EventKey = state.Records[fingerprint].EventKey + record.State = stateName + record.AttemptID = "" + record.LeaseUntil = time.Time{} + record.FailureCount = count + record.LastFailureAt = now + record.NextAllowedAt = now.Add(delay) + record.Retryability = failure.Retryability + record.ErrorCode = boundedAttemptField(failure.ErrorCode) + record.TraceID = boundedAttemptField(failure.TraceID) + state.Records[fingerprint] = record + hold = AttemptHold{ + Fingerprint: fingerprint, + State: stateName, + Retryability: failure.Retryability, + RetryAfter: delay, + NextAllowedAt: record.NextAllowedAt, + FailureCount: count, + } + case succeededSet[fingerprint]: + delete(state.Records, fingerprint) + default: + restoreAttemptPrevious(state, fingerprint, claim.previous[fingerprint]) + } + } + return s.write(state) + }) + if err != nil { + return zero, err + } + return hold, nil +} + +// Release restores every claimed item without recording a remote failure. It +// is intended for local failures before a subscription request is sent. +func (s *AttemptStore) Release(claim *AttemptClaim) error { + if err := validateAttemptClaim(claim); err != nil { + return err + } + if err := s.validate(); err != nil { + return err + } + return s.withLock(func() error { + state, err := s.load() + if err != nil { + return err + } + if !claimOwnsAll(state, claim) { + return ErrAttemptClaimStale + } + for _, fingerprint := range claim.Fingerprints { + restoreAttemptPrevious(state, fingerprint, claim.previous[fingerprint]) + } + return s.write(state) + }) +} + +func (s *AttemptStore) validate() error { + if s == nil { + return errors.New("personal event: nil subscription attempt store") + } + if s.workDir == "" { + return errors.New("personal event: subscription attempt work directory is required") + } + if s.now == nil || s.newID == nil || s.readFile == nil || s.mkdirAll == nil || + s.tryAcquire == nil || s.remove == nil || s.marshal == nil || + s.writeFile == nil || s.chmod == nil || s.rename == nil || s.sleep == nil { + return errors.New("personal event: subscription attempt store is not initialized") + } + return nil +} + +func normalizeAttemptSpecs(specs []AttemptSpec) ([]AttemptSpec, error) { + if len(specs) == 0 { + return nil, errors.New("personal event: at least one subscription attempt is required") + } + out := make([]AttemptSpec, 0, len(specs)) + seen := make(map[string]struct{}, len(specs)) + for _, spec := range specs { + spec.Fingerprint = strings.TrimSpace(spec.Fingerprint) + spec.EventKey = strings.TrimSpace(spec.EventKey) + if !validAttemptFingerprint(spec.Fingerprint) { + return nil, errors.New("personal event: subscription attempt fingerprint is invalid") + } + if len(spec.EventKey) > attemptMaxFieldLength { + return nil, errors.New("personal event: subscription attempt event key is too long") + } + if _, ok := seen[spec.Fingerprint]; ok { + continue + } + seen[spec.Fingerprint] = struct{}{} + out = append(out, spec) + } + return out, nil +} + +func normalizeSucceededFingerprints( + claim *AttemptClaim, + succeeded []string, + failed string, +) (map[string]bool, error) { + out := make(map[string]bool, len(succeeded)) + for _, fingerprint := range succeeded { + fingerprint = strings.TrimSpace(fingerprint) + if fingerprint == failed { + return nil, errors.New("personal event: failed fingerprint cannot also be successful") + } + if !containsFingerprint(claim.Fingerprints, fingerprint) { + return nil, errors.New("personal event: successful fingerprint is not part of the attempt claim") + } + out[fingerprint] = true + } + return out, nil +} + +func validateAttemptClaim(claim *AttemptClaim) error { + if claim == nil || strings.TrimSpace(claim.AttemptID) == "" || + len(claim.Fingerprints) == 0 || claim.previous == nil { + return errors.New("personal event: invalid subscription attempt claim") + } + for _, fingerprint := range claim.Fingerprints { + if !validAttemptFingerprint(fingerprint) { + return errors.New("personal event: subscription attempt claim contains an invalid fingerprint") + } + if _, ok := claim.previous[fingerprint]; !ok { + return errors.New("personal event: subscription attempt claim is incomplete") + } + } + return nil +} + +func containsFingerprint(fingerprints []string, target string) bool { + for _, fingerprint := range fingerprints { + if fingerprint == target { + return true + } + } + return false +} + +func validAttemptFingerprint(value string) bool { + if len(value) != sha256.Size*2 { + return false + } + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} + +func validRetryability(value Retryability) bool { + switch value { + case RetryabilityUnknown, RetryabilityRetryable, RetryabilityNonRetryable: + return true + default: + return false + } +} + +func attemptFailureDelay( + fingerprint string, + count int, + failure AttemptFailure, +) (time.Duration, AttemptState) { + if failure.Retryability == RetryabilityNonRetryable { + return attemptTerminalHold, AttemptStateTerminalHold + } + base := attemptBackoff(count) + delay := base + deterministicAttemptJitter(fingerprint, count, base) + if failure.RetryAfter > delay { + delay = failure.RetryAfter + } + return delay, AttemptStateCooldown +} + +func attemptBackoff(count int) time.Duration { + switch count { + case 1: + return 30 * time.Second + case 2: + return 2 * time.Minute + case 3: + return 10 * time.Minute + default: + return 30 * time.Minute + } +} + +func deterministicAttemptJitter(fingerprint string, count int, base time.Duration) time.Duration { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", fingerprint, count))) + // 0..2000 basis points, inclusive (0%..20%). + basisPoints := int(binary.BigEndian.Uint16(sum[:2])) % 2001 + return time.Duration(int64(base) * int64(basisPoints) / 10000) +} + +func boundedAttemptField(value string) string { + value = strings.TrimSpace(value) + if len(value) > attemptMaxFieldLength { + return value[:attemptMaxFieldLength] + } + return value +} + +func restoreAttemptPrevious(state *attemptStateFile, fingerprint string, previous attemptPrevious) { + if previous.present { + state.Records[fingerprint] = previous.record + return + } + delete(state.Records, fingerprint) +} + +func claimOwnsAll(state *attemptStateFile, claim *AttemptClaim) bool { + for _, fingerprint := range claim.Fingerprints { + record, ok := state.Records[fingerprint] + if !ok || record.State != AttemptStateInFlight || record.AttemptID != claim.AttemptID { + return false + } + } + return true +} + +func blockedAttempt(record attemptRecord, now time.Time) *AttemptBlockedError { + var next time.Time + retryability := record.Retryability + switch record.State { + case AttemptStateInFlight: + if !record.LeaseUntil.After(now) { + return nil + } + next = record.LeaseUntil + retryability = RetryabilityUnknown + case AttemptStateCooldown, AttemptStateTerminalHold: + if !record.NextAllowedAt.After(now) { + return nil + } + next = record.NextAllowedAt + default: + return nil + } + return &AttemptBlockedError{ + Fingerprint: record.Fingerprint, + EventKey: record.EventKey, + State: record.State, + Retryability: retryability, + RetryAfter: next.Sub(now), + NextAllowedAt: next, + FailureCount: record.FailureCount, + ErrorCode: record.ErrorCode, + TraceID: record.TraceID, + } +} + +type attemptStateFile struct { + Version int `json:"version"` + Records map[string]attemptRecord `json:"records"` +} + +type attemptRecord struct { + Fingerprint string `json:"fingerprint"` + EventKey string `json:"event_key,omitempty"` + State AttemptState `json:"state"` + AttemptID string `json:"attempt_id,omitempty"` + LeaseUntil time.Time `json:"lease_until,omitempty"` + FailureCount int `json:"failure_count,omitempty"` + LastFailureAt time.Time `json:"last_failure_at,omitempty"` + NextAllowedAt time.Time `json:"next_allowed_at,omitempty"` + Retryability Retryability `json:"retryability,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + TraceID string `json:"trace_id,omitempty"` +} + +func (s *AttemptStore) load() (*attemptStateFile, error) { + path := filepath.Join(s.workDir, AttemptStateFileName) + data, err := s.readFile(path) + if os.IsNotExist(err) { + return newAttemptStateFile(), nil + } + if err != nil { + return nil, fmt.Errorf("personal event: read subscription attempt state: %w", err) + } + var state attemptStateFile + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("personal event: decode subscription attempt state: %w", err) + } + if state.Version != attemptStateVersion { + return nil, fmt.Errorf( + "personal event: unsupported subscription attempt state version %d", + state.Version, + ) + } + if state.Records == nil { + return nil, errors.New("personal event: subscription attempt state has no records map") + } + for fingerprint, record := range state.Records { + if err := validateAttemptRecord(fingerprint, record); err != nil { + return nil, err + } + } + return &state, nil +} + +func newAttemptStateFile() *attemptStateFile { + return &attemptStateFile{ + Version: attemptStateVersion, + Records: make(map[string]attemptRecord), + } +} + +func validateAttemptRecord(fingerprint string, record attemptRecord) error { + if !validAttemptFingerprint(fingerprint) || fingerprint != record.Fingerprint { + return errors.New("personal event: subscription attempt state contains an invalid fingerprint") + } + if len(record.EventKey) > attemptMaxFieldLength || + len(record.AttemptID) > attemptMaxFieldLength || + len(record.ErrorCode) > attemptMaxFieldLength || + len(record.TraceID) > attemptMaxFieldLength { + return errors.New("personal event: subscription attempt state contains an oversized field") + } + if record.FailureCount < 0 { + return errors.New("personal event: subscription attempt state contains a negative failure count") + } + if record.Retryability != "" && !validRetryability(record.Retryability) { + return errors.New("personal event: subscription attempt state contains invalid retryability") + } + switch record.State { + case AttemptStateInFlight: + if record.AttemptID == "" || record.LeaseUntil.IsZero() { + return errors.New("personal event: in-flight subscription attempt state is incomplete") + } + case AttemptStateCooldown: + if record.NextAllowedAt.IsZero() || + (record.Retryability != RetryabilityUnknown && + record.Retryability != RetryabilityRetryable) { + return errors.New("personal event: cooldown subscription attempt state is invalid") + } + case AttemptStateTerminalHold: + if record.NextAllowedAt.IsZero() || + record.Retryability != RetryabilityNonRetryable { + return errors.New("personal event: terminal subscription attempt state is invalid") + } + default: + return errors.New("personal event: subscription attempt state contains an invalid state") + } + return nil +} + +func pruneAttemptRecords(state *attemptStateFile, now time.Time) { + for fingerprint, record := range state.Records { + switch { + case !record.LastFailureAt.IsZero() && + !now.Before(record.LastFailureAt) && + now.Sub(record.LastFailureAt) >= attemptFailureReset && + ((record.State == AttemptStateInFlight && !record.LeaseUntil.After(now)) || + ((record.State == AttemptStateCooldown || + record.State == AttemptStateTerminalHold) && + !record.NextAllowedAt.After(now))): + delete(state.Records, fingerprint) + case record.State == AttemptStateInFlight && + record.LastFailureAt.IsZero() && + !record.LeaseUntil.IsZero() && + !now.Before(record.LeaseUntil) && + now.Sub(record.LeaseUntil) >= attemptFailureReset: + delete(state.Records, fingerprint) + } + } +} + +func (s *AttemptStore) write(state *attemptStateFile) error { + if state == nil { + return errors.New("personal event: nil subscription attempt state") + } + if err := s.mkdirAll(s.workDir, config.DirPerm); err != nil { + return fmt.Errorf("personal event: create subscription attempt directory: %w", err) + } + path := filepath.Join(s.workDir, AttemptStateFileName) + if len(state.Records) == 0 { + if err := s.remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("personal event: remove empty subscription attempt state: %w", err) + } + return nil + } + + keys := make([]string, 0, len(state.Records)) + for fingerprint := range state.Records { + keys = append(keys, fingerprint) + } + sort.Strings(keys) + ordered := make(map[string]attemptRecord, len(keys)) + for _, fingerprint := range keys { + ordered[fingerprint] = state.Records[fingerprint] + } + payload := attemptStateFile{Version: attemptStateVersion, Records: ordered} + data, err := s.marshal(payload, "", " ") + if err != nil { + return fmt.Errorf("personal event: encode subscription attempt state: %w", err) + } + data = append(data, '\n') + tmp := path + ".tmp" + if err := s.writeFile(tmp, data, config.FilePerm); err != nil { + return fmt.Errorf("personal event: write subscription attempt state: %w", err) + } + if err := s.chmod(tmp, config.FilePerm); err != nil { + _ = s.remove(tmp) + return fmt.Errorf("personal event: secure subscription attempt state: %w", err) + } + if err := s.rename(tmp, path); err != nil { + _ = s.remove(tmp) + return fmt.Errorf("personal event: replace subscription attempt state: %w", err) + } + return nil +} + +func (s *AttemptStore) withLock(fn func() error) error { + if err := s.mkdirAll(s.workDir, config.DirPerm); err != nil { + return fmt.Errorf("personal event: create subscription attempt directory: %w", err) + } + lockPath := filepath.Join(s.workDir, AttemptStateLockFileName) + deadline := s.now().Add(attemptLockWaitTimeout) + for { + held, err := s.tryAcquire(lockPath) + if err == nil { + if err := s.chmod(lockPath, config.FilePerm); err != nil { + _ = held.Close() + return fmt.Errorf("personal event: secure subscription attempt lock: %w", err) + } + defer held.Close() + return fn() + } + if !errors.Is(err, eventlock.ErrBusy) { + return fmt.Errorf("personal event: acquire subscription attempt lock: %w", err) + } + remaining := deadline.Sub(s.now()) + if remaining <= 0 { + return fmt.Errorf( + "personal event: timed out waiting for subscription attempt lock after %s", + attemptLockWaitTimeout, + ) + } + delay := attemptLockRetryDelay + if remaining < delay { + delay = remaining + } + s.sleep(delay) + } +} + +func newAttemptID() (string, error) { + return rand.Text(), nil +} diff --git a/internal/event/personal/attempt_store_test.go b/internal/event/personal/attempt_store_test.go new file mode 100644 index 00000000..2280e7e1 --- /dev/null +++ b/internal/event/personal/attempt_store_test.go @@ -0,0 +1,1341 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package personal + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + eventlock "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/event/lock" +) + +type attemptTestClock struct { + mu sync.Mutex + now time.Time +} + +func newAttemptTestClock() *attemptTestClock { + return &attemptTestClock{ + now: time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC), + } +} + +func (c *attemptTestClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *attemptTestClock) Advance(delta time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(delta) +} + +func attemptTestFingerprint(label string) string { + return Fingerprint("https://mcp.example.test/dws", "idem-"+label, "profile-"+label) +} + +func newAttemptTestStore(t *testing.T, clock *attemptTestClock) *AttemptStore { + t.Helper() + var sequence atomic.Int64 + return NewAttemptStore( + t.TempDir(), + WithAttemptClock(clock.Now), + WithAttemptIDGenerator(func() (string, error) { + return fmt.Sprintf("attempt-%d", sequence.Add(1)), nil + }), + ) +} + +func TestAttemptFingerprintIsCanonicalScopedAndOpaque(t *testing.T) { + base := Fingerprint(" HTTPS://MCP.Example.Test/dws/ ", " idem-1 ") + if base == "" || len(base) != 64 { + t.Fatalf("Fingerprint() = %q", base) + } + if got := Fingerprint("https://mcp.example.test/dws", "idem-1", ""); got != base { + t.Fatalf("empty profile changed fingerprint: %q != %q", got, base) + } + if got := Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-a"); got == base { + t.Fatal("profile selector did not scope fingerprint") + } + if left, right := Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-a"), + Fingerprint("https://mcp.example.test/dws", "idem-1", "profile-b"); left == right { + t.Fatal("different profile selectors produced the same fingerprint") + } + if Fingerprint("", "idem") != "" || Fingerprint("https://mcp.example.test", "") != "" { + t.Fatal("blank required fingerprint input should return empty") + } + if strings.Contains(base, "idem") || strings.Contains(base, "mcp.example") { + t.Fatalf("fingerprint leaked source material: %q", base) + } +} + +func TestRetryabilityValuePreservesUnknown(t *testing.T) { + if value, known := RetryabilityRetryable.Value(); !known || !value { + t.Fatalf("retryable Value() = %v, %v", value, known) + } + if value, known := RetryabilityNonRetryable.Value(); !known || value { + t.Fatalf("non-retryable Value() = %v, %v", value, known) + } + if _, known := RetryabilityUnknown.Value(); known { + t.Fatal("unknown retryability became known") + } + if _, known := Retryability("invalid").Value(); known { + t.Fatal("invalid retryability became known") + } + var nilBlocked *AttemptBlockedError + if nilBlocked.Error() != "" { + t.Fatalf("nil blocked error = %q", nilBlocked.Error()) + } + blocked := &AttemptBlockedError{ + State: AttemptStateCooldown, + NextAllowedAt: time.Date(2026, 7, 30, 10, 1, 0, 0, time.UTC), + } + if message := blocked.Error(); !strings.Contains(message, "cooldown") || + !strings.Contains(message, "2026-07-30T10:01:00Z") { + t.Fatalf("blocked error = %q", message) + } +} + +func TestAttemptStoreClaimFailureBackoffAndRetryAfter(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("one") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if claim.AttemptID != "attempt-1" || len(claim.Fingerprints) != 1 { + t.Fatalf("claim = %#v", claim) + } + if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("second in-flight claim succeeded") + } else { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight || + blocked.Retryability != RetryabilityUnknown || blocked.RetryAfter != time.Minute { + t.Fatalf("in-flight error = %#v, %v", blocked, err) + } + } + + hold, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + ErrorCode: "TEMPORARY", + TraceID: "trace-1", + }) + if err != nil { + t.Fatal(err) + } + if hold.State != AttemptStateCooldown || hold.FailureCount != 1 || + hold.RetryAfter < 30*time.Second || hold.RetryAfter > 36*time.Second { + t.Fatalf("first hold = %#v", hold) + } + if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("cooldown claim succeeded") + } else { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != AttemptStateCooldown || + blocked.Retryability != RetryabilityRetryable || + blocked.ErrorCode != "TEMPORARY" || blocked.TraceID != "trace-1" { + t.Fatalf("cooldown error = %#v, %v", blocked, err) + } + } + + clock.Advance(hold.RetryAfter + time.Second) + second, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + secondHold, err := store.CompleteFailure(second, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + RetryAfter: 5 * time.Minute, + }) + if err != nil { + t.Fatal(err) + } + if secondHold.FailureCount != 2 || secondHold.RetryAfter != 5*time.Minute || + secondHold.Retryability != RetryabilityUnknown { + t.Fatalf("second hold = %#v", secondHold) + } +} + +func TestAttemptStoreBackoffSequenceAndTwentyFourHourReset(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("backoff") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + wantBases := []time.Duration{ + 30 * time.Second, + 2 * time.Minute, + 10 * time.Minute, + 30 * time.Minute, + 30 * time.Minute, + } + for index, base := range wantBases { + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatalf("claim %d: %v", index+1, err) + } + hold, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatalf("failure %d: %v", index+1, err) + } + if hold.FailureCount != index+1 || + hold.RetryAfter < base || + hold.RetryAfter > base+base/5 { + t.Fatalf("hold %d = %#v, base %s", index+1, hold, base) + } + if got := deterministicAttemptJitter(fingerprint, index+1, base); got != hold.RetryAfter-base { + t.Fatalf("jitter %d = %s, hold delta %s", index+1, got, hold.RetryAfter-base) + } + clock.Advance(hold.RetryAfter + time.Second) + } + + clock.Advance(attemptFailureReset + time.Second) + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + hold, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatal(err) + } + if hold.FailureCount != 1 || hold.RetryAfter < 30*time.Second || hold.RetryAfter > 36*time.Second { + t.Fatalf("post-reset hold = %#v", hold) + } +} + +func TestAttemptStoreLongRetryAfterSurvivesFailureCountResetWindow(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("long-retry-after") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + hold, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + RetryAfter: 48 * time.Hour, + }) + if err != nil { + t.Fatal(err) + } + if hold.RetryAfter != 48*time.Hour { + t.Fatalf("hold Retry-After = %s, want 48h", hold.RetryAfter) + } + + clock.Advance(24*time.Hour + time.Second) + if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("active 48h Retry-After was pruned at the 24h count reset boundary") + } else { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.NextAllowedAt != hold.NextAllowedAt { + t.Fatalf("long Retry-After block = %#v, %v", blocked, err) + } + } + + clock.Advance(24 * time.Hour) + recovered, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + next, err := store.CompleteFailure(recovered, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatal(err) + } + if next.FailureCount != 1 { + t.Fatalf("failure count after an inactive 48h hold = %d, want 1", next.FailureCount) + } +} + +func TestAttemptStoreExpiredInFlightResetsOldFailureHistory(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("expired-inflight-reset") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + + first, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + firstHold, err := store.CompleteFailure(first, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatal(err) + } + clock.Advance(firstHold.RetryAfter + time.Second) + crashed, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + // Simulate a process dying after Claim. The record remains in_flight with + // the old failure timestamp until a later process prunes it. + clock.Advance(attemptFailureReset + time.Second) + recovered, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if recovered.AttemptID == crashed.AttemptID { + t.Fatal("expired in-flight claim was not replaced") + } + hold, err := store.CompleteFailure(recovered, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatal(err) + } + if hold.FailureCount != 1 { + t.Fatalf("failure count after expired in-flight reset = %d, want 1", hold.FailureCount) + } +} + +func TestAttemptStoreTerminalHoldAndSuccessClear(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("terminal") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventInChat} + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + hold, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityNonRetryable, + ErrorCode: "GROUP_NOT_BELONG_TO_ORG", + }) + if err != nil { + t.Fatal(err) + } + if hold.State != AttemptStateTerminalHold || hold.RetryAfter != time.Hour { + t.Fatalf("terminal hold = %#v", hold) + } + if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("terminal hold did not block") + } else { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != AttemptStateTerminalHold { + t.Fatalf("terminal block = %#v, %v", blocked, err) + } + if retryable, known := blocked.Retryability.Value(); !known || retryable { + t.Fatalf("terminal retryability = %v, %v", retryable, known) + } + } + clock.Advance(time.Hour + time.Second) + recovery, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if err := store.CompleteSuccess(recovery); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(store.workDir, AttemptStateFileName)); !os.IsNotExist(err) { + t.Fatalf("state file after success = %v, want missing", err) + } + if delay, state := attemptFailureDelay(fingerprint, 1, AttemptFailure{ + Retryability: RetryabilityNonRetryable, + RetryAfter: 2 * time.Hour, + }); delay != time.Hour || state != AttemptStateTerminalHold { + t.Fatalf("terminal Retry-After = %s, %s", delay, state) + } +} + +func TestAttemptStorePartialFailureClearsSuccessAndReleasesUnexecuted(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprintA := attemptTestFingerprint("batch-a") + fingerprintB := attemptTestFingerprint("batch-b") + fingerprintC := attemptTestFingerprint("batch-c") + specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention} + specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat} + specC := AttemptSpec{Fingerprint: fingerprintC, EventKey: EventInChat} + + seed, err := store.Claim([]AttemptSpec{specA}, time.Minute) + if err != nil { + t.Fatal(err) + } + seedHold, err := store.CompleteFailure(seed, nil, AttemptFailure{ + Fingerprint: fingerprintA, + Retryability: RetryabilityRetryable, + }) + if err != nil { + t.Fatal(err) + } + clock.Advance(seedHold.RetryAfter + time.Second) + + claim, err := store.Claim([]AttemptSpec{specA, specB, specC}, 3*time.Minute) + if err != nil { + t.Fatal(err) + } + hold, err := store.CompleteFailure(claim, []string{fingerprintA}, AttemptFailure{ + Fingerprint: fingerprintB, + Retryability: RetryabilityUnknown, + }) + if err != nil { + t.Fatal(err) + } + if hold.Fingerprint != fingerprintB || hold.State != AttemptStateCooldown { + t.Fatalf("batch hold = %#v", hold) + } + state, err := store.load() + if err != nil { + t.Fatal(err) + } + if len(state.Records) != 1 { + t.Fatalf("records = %#v, want only failed item", state.Records) + } + if _, ok := state.Records[fingerprintB]; !ok { + t.Fatalf("failed fingerprint missing: %#v", state.Records) + } + if _, ok := state.Records[fingerprintA]; ok { + t.Fatal("successful fingerprint retained failure state") + } + if _, ok := state.Records[fingerprintC]; ok { + t.Fatal("unexecuted new fingerprint was not released") + } +} + +func TestAttemptStoreReleaseRestoresPreviousState(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprintA := attemptTestFingerprint("release-a") + fingerprintB := attemptTestFingerprint("release-b") + specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention} + specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat} + + seed, err := store.Claim([]AttemptSpec{specA}, time.Minute) + if err != nil { + t.Fatal(err) + } + seedHold, err := store.CompleteFailure(seed, nil, AttemptFailure{ + Fingerprint: fingerprintA, + Retryability: RetryabilityUnknown, + }) + if err != nil { + t.Fatal(err) + } + clock.Advance(seedHold.RetryAfter + time.Second) + before, err := store.load() + if err != nil { + t.Fatal(err) + } + previousA := before.Records[fingerprintA] + + claim, err := store.Claim([]AttemptSpec{specA, specB}, 2*time.Minute) + if err != nil { + t.Fatal(err) + } + if err := store.Release(claim); err != nil { + t.Fatal(err) + } + after, err := store.load() + if err != nil { + t.Fatal(err) + } + if len(after.Records) != 1 || after.Records[fingerprintA] != previousA { + t.Fatalf("release state = %#v, want prior A %#v", after.Records, previousA) + } +} + +func TestAttemptStoreCASRejectsOldCompletion(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("cas") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + oldClaim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + clock.Advance(time.Minute + time.Second) + newClaim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if err := store.CompleteSuccess(oldClaim); !errors.Is(err, ErrAttemptClaimStale) { + t.Fatalf("old completion error = %v, want stale", err) + } + if err := store.CompleteSuccess(newClaim); err != nil { + t.Fatal(err) + } +} + +func TestAttemptStoreConcurrentClaimsOnlyOneWins(t *testing.T) { + workDir := t.TempDir() + fingerprint := attemptTestFingerprint("concurrent") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + const count = 100 + var wg sync.WaitGroup + claims := make(chan *AttemptClaim, count) + errs := make(chan error, count) + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + claim, err := NewAttemptStore(workDir).Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + errs <- err + return + } + claims <- claim + }() + } + wg.Wait() + close(claims) + close(errs) + + var winners []*AttemptClaim + for claim := range claims { + winners = append(winners, claim) + } + if len(winners) != 1 { + t.Fatalf("winning claims = %d, want 1", len(winners)) + } + for err := range errs { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight { + t.Fatalf("concurrent claim error = %v", err) + } + } + if err := NewAttemptStore(workDir).CompleteSuccess(winners[0]); err != nil { + t.Fatal(err) + } +} + +func TestAttemptStoreBatchClaimIsAtomicWhenOneItemBlocked(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprintA := attemptTestFingerprint("atomic-a") + fingerprintB := attemptTestFingerprint("atomic-b") + specA := AttemptSpec{Fingerprint: fingerprintA, EventKey: EventMention} + specB := AttemptSpec{Fingerprint: fingerprintB, EventKey: EventSingleChat} + first, err := store.Claim([]AttemptSpec{specA}, time.Minute) + if err != nil { + t.Fatal(err) + } + if _, err := store.Claim([]AttemptSpec{specB, specA}, time.Minute); err == nil { + t.Fatal("batch with blocked item succeeded") + } + state, err := store.load() + if err != nil { + t.Fatal(err) + } + if len(state.Records) != 1 { + t.Fatalf("atomic blocked claim wrote partial state: %#v", state.Records) + } + if _, ok := state.Records[fingerprintB]; ok { + t.Fatal("unblocked batch item was partially reserved") + } + if err := store.CompleteSuccess(first); err != nil { + t.Fatal(err) + } +} + +func TestAttemptStorePersistsNoRawFingerprintInputs(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + const ( + endpoint = "https://sensitive.example.test/dws" + idem = "idem-sensitive-user-and-group" + profile = "private-profile-name" + ) + fingerprint := Fingerprint(endpoint, idem, profile) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: fingerprint, + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(store.workDir, AttemptStateFileName)) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{endpoint, idem, profile} { + if strings.Contains(string(data), raw) { + t.Fatalf("state leaked %q: %s", raw, data) + } + } + if err := store.CompleteSuccess(claim); err != nil { + t.Fatal(err) + } +} + +func TestAttemptStoreFailsClosedOnCorruptStateAndWriteFailure(t *testing.T) { + t.Run("corrupt state", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + if err := os.WriteFile( + filepath.Join(store.workDir, AttemptStateFileName), + []byte(`{"version":1,"records":`), + 0o600, + ); err != nil { + t.Fatal(err) + } + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("corrupt"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "decode subscription attempt state") { + t.Fatalf("corrupt state error = %v", err) + } + }) + + t.Run("unknown version", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + if err := os.WriteFile( + filepath.Join(store.workDir, AttemptStateFileName), + []byte(`{"version":99,"records":{}}`), + 0o600, + ); err != nil { + t.Fatal(err) + } + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("version"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("version error = %v", err) + } + }) + + t.Run("write failure", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + store.writeFile = func(string, []byte, os.FileMode) error { + return errors.New("disk full") + } + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("write"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("write error = %v", err) + } + }) + + t.Run("failure write keeps in-flight reservation", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("failure-write") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + claim, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + originalWrite := store.writeFile + store.writeFile = func(string, []byte, os.FileMode) error { + return errors.New("disk full") + } + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + }); err == nil || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("failure persistence error = %v", err) + } + store.writeFile = originalWrite + if _, err := store.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("failed persistence silently released in-flight reservation") + } else { + var blocked *AttemptBlockedError + if !errors.As(err, &blocked) || blocked.State != AttemptStateInFlight { + t.Fatalf("post-write-failure block = %#v, %v", blocked, err) + } + } + }) +} + +func TestAttemptStoreFailsClosedOnInjectedIOErrors(t *testing.T) { + testClaim := func(store *AttemptStore, label string) error { + _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint(label), + EventKey: EventMention, + }}, time.Minute) + return err + } + + t.Run("read", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.readFile = func(string) ([]byte, error) { + return nil, errors.New("read denied") + } + if err := testClaim(store, "read-error"); err == nil || !strings.Contains(err.Error(), "read denied") { + t.Fatalf("read error = %v", err) + } + }) + + t.Run("mkdir", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.mkdirAll = func(string, os.FileMode) error { + return errors.New("mkdir denied") + } + if err := testClaim(store, "mkdir-error"); err == nil || !strings.Contains(err.Error(), "mkdir denied") { + t.Fatalf("mkdir error = %v", err) + } + }) + + t.Run("acquire", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.tryAcquire = func(string) (*eventlock.File, error) { + return nil, errors.New("lock denied") + } + if err := testClaim(store, "lock-error"); err == nil || !strings.Contains(err.Error(), "lock denied") { + t.Fatalf("lock error = %v", err) + } + }) + + t.Run("marshal", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.marshal = func(any, string, string) ([]byte, error) { + return nil, errors.New("marshal denied") + } + if err := testClaim(store, "marshal-error"); err == nil || !strings.Contains(err.Error(), "marshal denied") { + t.Fatalf("marshal error = %v", err) + } + }) + + t.Run("chmod", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.chmod = func(string, os.FileMode) error { + return errors.New("chmod denied") + } + if err := testClaim(store, "chmod-error"); err == nil || + !strings.Contains(err.Error(), "chmod denied") { + t.Fatalf("chmod error = %v", err) + } + }) + + t.Run("rename and temporary cleanup", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.rename = func(string, string) error { + return errors.New("rename denied") + } + if err := testClaim(store, "rename-error"); err == nil || !strings.Contains(err.Error(), "rename denied") { + t.Fatalf("rename error = %v", err) + } + tmp := filepath.Join(store.workDir, AttemptStateFileName) + ".tmp" + if _, err := os.Stat(tmp); !os.IsNotExist(err) { + t.Fatalf("temporary file after rename error = %v", err) + } + }) + + t.Run("remove", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("remove-error"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.remove = func(string) error { + return errors.New("remove denied") + } + if err := store.CompleteSuccess(claim); err == nil || !strings.Contains(err.Error(), "remove denied") { + t.Fatalf("remove error = %v", err) + } + }) +} + +func TestAttemptStoreTightensStateAndLockPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not expose Unix permission bits") + } + workDir := t.TempDir() + statePath := filepath.Join(workDir, AttemptStateFileName) + lockPath := filepath.Join(workDir, AttemptStateLockFileName) + tmpPath := statePath + ".tmp" + for _, path := range []string{lockPath, tmpPath} { + if err := os.WriteFile(path, []byte("stale"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o666); err != nil { + t.Fatal(err) + } + } + + store := NewAttemptStore(workDir) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("permissions"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + defer func() { _ = store.Release(claim) }() + + for _, path := range []string{statePath, lockPath} { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("%s mode = %o, want 600", filepath.Base(path), got) + } + } +} + +func TestAttemptStoreRejectsInvalidPersistedRecords(t *testing.T) { + fingerprint := attemptTestFingerprint("invalid-record") + longValue := strings.Repeat("x", attemptMaxFieldLength+1) + tests := []struct { + name string + body string + }{ + {"nil records", `{"version":1,"records":null}`}, + {"key mismatch", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`, + fingerprint, + attemptTestFingerprint("other"), + )}, + {"oversized field", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","event_key":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`, + fingerprint, + fingerprint, + longValue, + )}, + {"negative failures", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","failure_count":-1,"next_allowed_at":"2026-07-30T11:00:00Z","retryability":"unknown"}}}`, + fingerprint, + fingerprint, + )}, + {"invalid retryability", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"sometimes"}}}`, + fingerprint, + fingerprint, + )}, + {"inflight incomplete", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"in_flight"}}}`, + fingerprint, + fingerprint, + )}, + {"cooldown nonretryable", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"cooldown","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"non_retryable"}}}`, + fingerprint, + fingerprint, + )}, + {"terminal retryable", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"terminal_hold","next_allowed_at":"2026-07-30T11:00:00Z","retryability":"retryable"}}}`, + fingerprint, + fingerprint, + )}, + {"invalid state", fmt.Sprintf( + `{"version":1,"records":{"%s":{"fingerprint":"%s","state":"open"}}}`, + fingerprint, + fingerprint, + )}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + workDir := t.TempDir() + if err := os.WriteFile( + filepath.Join(workDir, AttemptStateFileName), + []byte(test.body), + 0o600, + ); err != nil { + t.Fatal(err) + } + store := NewAttemptStore(workDir) + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("new"), + EventKey: EventMention, + }}, time.Minute); err == nil { + t.Fatalf("invalid persisted record was accepted: %s", test.body) + } + }) + } +} + +func TestAttemptStoreLockTimeoutAndIDFailureAreFailClosed(t *testing.T) { + t.Run("lock timeout", func(t *testing.T) { + var tick atomic.Int64 + base := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + store := NewAttemptStore( + t.TempDir(), + WithAttemptClock(func() time.Time { + return base.Add(time.Duration(tick.Add(1)) * 3 * time.Second) + }), + WithAttemptIDGenerator(func() (string, error) { return "attempt", nil }), + ) + store.tryAcquire = func(string) (*eventlock.File, error) { + return nil, eventlock.ErrBusy + } + store.sleep = func(time.Duration) {} + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("lock"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("lock timeout error = %v", err) + } + }) + + t.Run("id failure", func(t *testing.T) { + store := NewAttemptStore( + t.TempDir(), + WithAttemptIDGenerator(func() (string, error) { + return "", errors.New("random source unavailable") + }), + ) + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("id"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "random source unavailable") { + t.Fatalf("id error = %v", err) + } + }) +} + +func TestAttemptStoreValidationAndFailureCAS(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("validation") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + if _, err := store.Claim(nil, time.Minute); err == nil { + t.Fatal("empty claim succeeded") + } + if _, err := store.Claim([]AttemptSpec{{Fingerprint: "raw-id"}}, time.Minute); err == nil { + t.Fatal("raw fingerprint claim succeeded") + } + if _, err := store.Claim([]AttemptSpec{spec}, 0); err == nil { + t.Fatal("zero lease claim succeeded") + } + var nilStore *AttemptStore + if _, err := nilStore.Claim([]AttemptSpec{spec}, time.Minute); err == nil { + t.Fatal("nil store claim succeeded") + } + uninitialized := &AttemptStore{workDir: t.TempDir()} + if _, err := uninitialized.Claim([]AttemptSpec{spec}, time.Minute); err == nil || + !strings.Contains(err.Error(), "not initialized") { + t.Fatalf("uninitialized store error = %v", err) + } + + claim, err := store.Claim([]AttemptSpec{spec, spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if len(claim.Fingerprints) != 1 { + t.Fatalf("duplicate specs were not deduplicated: %#v", claim.Fingerprints) + } + if _, err := store.CompleteFailure(claim, []string{fingerprint}, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityRetryable, + }); err == nil { + t.Fatal("same successful and failed fingerprint was accepted") + } + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: Retryability("invalid"), + }); err == nil { + t.Fatal("invalid retryability was accepted") + } + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: attemptTestFingerprint("not-in-claim"), + Retryability: RetryabilityUnknown, + }); err == nil { + t.Fatal("out-of-claim failure was accepted") + } + if _, err := store.CompleteFailure(claim, []string{attemptTestFingerprint("not-in-claim")}, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + }); err == nil { + t.Fatal("out-of-claim success was accepted") + } + if err := store.Release(claim); err != nil { + t.Fatal(err) + } + if err := store.CompleteSuccess(nil); err == nil { + t.Fatal("nil success claim was accepted") + } + if err := store.Release(nil); err == nil { + t.Fatal("nil release claim was accepted") + } +} + +func TestAttemptStoreBoundsPersistedDiagnostics(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("bounded") + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: fingerprint, + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + longValue := strings.Repeat("x", attemptMaxFieldLength+100) + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + ErrorCode: longValue, + TraceID: longValue, + }); err != nil { + t.Fatal(err) + } + state, err := store.load() + if err != nil { + t.Fatal(err) + } + record := state.Records[fingerprint] + if len(record.ErrorCode) != attemptMaxFieldLength || + len(record.TraceID) != attemptMaxFieldLength { + t.Fatalf("diagnostic lengths = %d, %d", len(record.ErrorCode), len(record.TraceID)) + } +} + +func TestAttemptStoreChangedCoverageEdges(t *testing.T) { + t.Run("endpoint fallback", func(t *testing.T) { + const raw = "https://example.test/%zz" + if got := normalizeAttemptEndpoint(raw); got != raw { + t.Fatalf("normalizeAttemptEndpoint(%q) = %q", raw, got) + } + }) + + t.Run("invalid generated ids", func(t *testing.T) { + for _, attemptID := range []string{" ", strings.Repeat("x", attemptMaxFieldLength+1)} { + store := NewAttemptStore( + t.TempDir(), + WithAttemptIDGenerator(func() (string, error) { return attemptID, nil }), + ) + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("invalid-id"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "attempt id is invalid") { + t.Fatalf("attempt ID %q error = %v", attemptID, err) + } + } + }) + + t.Run("complete success validates store", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("success-validate"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.now = nil + if err := store.CompleteSuccess(claim); err == nil || !strings.Contains(err.Error(), "not initialized") { + t.Fatalf("CompleteSuccess validation error = %v", err) + } + }) + + t.Run("complete success read failure", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("success-read"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.readFile = func(string) ([]byte, error) { + return nil, errors.New("success read denied") + } + if err := store.CompleteSuccess(claim); err == nil || + !strings.Contains(err.Error(), "success read denied") { + t.Fatalf("CompleteSuccess read error = %v", err) + } + }) + + t.Run("complete failure rejects invalid claim", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + if _, err := store.CompleteFailure(nil, nil, AttemptFailure{}); err == nil { + t.Fatal("CompleteFailure accepted nil claim") + } + }) + + t.Run("complete failure validates store", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("failure-validate") + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: fingerprint, + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.now = nil + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + }); err == nil || !strings.Contains(err.Error(), "not initialized") { + t.Fatalf("CompleteFailure validation error = %v", err) + } + }) + + t.Run("complete failure read failure", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("failure-read") + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: fingerprint, + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.readFile = func(string) ([]byte, error) { + return nil, errors.New("failure read denied") + } + if _, err := store.CompleteFailure(claim, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + }); err == nil || !strings.Contains(err.Error(), "failure read denied") { + t.Fatalf("CompleteFailure read error = %v", err) + } + }) + + t.Run("complete failure stale claim", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("failure-stale") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + stale, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + clock.Advance(time.Minute + time.Second) + current, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if _, err := store.CompleteFailure(stale, nil, AttemptFailure{ + Fingerprint: fingerprint, + Retryability: RetryabilityUnknown, + }); !errors.Is(err, ErrAttemptClaimStale) { + t.Fatalf("CompleteFailure stale error = %v", err) + } + if err := store.CompleteSuccess(current); err != nil { + t.Fatal(err) + } + }) + + t.Run("release validates store", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("release-validate"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.now = nil + if err := store.Release(claim); err == nil || !strings.Contains(err.Error(), "not initialized") { + t.Fatalf("Release validation error = %v", err) + } + }) + + t.Run("release read failure", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + claim, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("release-read"), + EventKey: EventMention, + }}, time.Minute) + if err != nil { + t.Fatal(err) + } + store.readFile = func(string) ([]byte, error) { + return nil, errors.New("release read denied") + } + if err := store.Release(claim); err == nil || + !strings.Contains(err.Error(), "release read denied") { + t.Fatalf("Release read error = %v", err) + } + }) + + t.Run("release stale claim", func(t *testing.T) { + clock := newAttemptTestClock() + store := newAttemptTestStore(t, clock) + fingerprint := attemptTestFingerprint("release-stale") + spec := AttemptSpec{Fingerprint: fingerprint, EventKey: EventMention} + stale, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + clock.Advance(time.Minute + time.Second) + current, err := store.Claim([]AttemptSpec{spec}, time.Minute) + if err != nil { + t.Fatal(err) + } + if err := store.Release(stale); !errors.Is(err, ErrAttemptClaimStale) { + t.Fatalf("Release stale error = %v", err) + } + if err := store.CompleteSuccess(current); err != nil { + t.Fatal(err) + } + }) + + t.Run("empty work directory", func(t *testing.T) { + if err := NewAttemptStore(" ").validate(); err == nil || + !strings.Contains(err.Error(), "work directory is required") { + t.Fatalf("empty work directory error = %v", err) + } + }) + + t.Run("oversized event key", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("long-event"), + EventKey: strings.Repeat("e", attemptMaxFieldLength+1), + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "event key is too long") { + t.Fatalf("oversized event key error = %v", err) + } + }) + + t.Run("claim shape validation", func(t *testing.T) { + fingerprint := attemptTestFingerprint("claim-shape") + if err := validateAttemptClaim(&AttemptClaim{ + AttemptID: "attempt", + Fingerprints: []string{"not-a-fingerprint"}, + previous: map[string]attemptPrevious{"not-a-fingerprint": {}}, + }); err == nil || !strings.Contains(err.Error(), "invalid fingerprint") { + t.Fatalf("invalid fingerprint claim error = %v", err) + } + if err := validateAttemptClaim(&AttemptClaim{ + AttemptID: "attempt", + Fingerprints: []string{fingerprint}, + previous: map[string]attemptPrevious{}, + }); err == nil || !strings.Contains(err.Error(), "incomplete") { + t.Fatalf("incomplete claim error = %v", err) + } + }) + + t.Run("unknown blocked state is ignored", func(t *testing.T) { + if blocked := blockedAttempt(attemptRecord{State: AttemptState("unknown")}, time.Now()); blocked != nil { + t.Fatalf("unknown state blocked = %#v", blocked) + } + }) + + t.Run("old abandoned first claim is pruned", func(t *testing.T) { + now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + fingerprint := attemptTestFingerprint("prune-abandoned") + state := newAttemptStateFile() + state.Records[fingerprint] = attemptRecord{ + Fingerprint: fingerprint, + State: AttemptStateInFlight, + AttemptID: "abandoned", + LeaseUntil: now.Add(-attemptFailureReset - time.Second), + } + pruneAttemptRecords(state, now) + if len(state.Records) != 0 { + t.Fatalf("abandoned first claim not pruned: %#v", state.Records) + } + }) + + t.Run("write validates state and directory", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + if err := store.write(nil); err == nil || !strings.Contains(err.Error(), "nil subscription attempt state") { + t.Fatalf("nil state write error = %v", err) + } + store.mkdirAll = func(string, os.FileMode) error { + return errors.New("write mkdir denied") + } + if err := store.write(newAttemptStateFile()); err == nil || + !strings.Contains(err.Error(), "write mkdir denied") { + t.Fatalf("write mkdir error = %v", err) + } + }) + + t.Run("state chmod failure cleans temporary file", func(t *testing.T) { + store := NewAttemptStore(t.TempDir()) + store.chmod = func(path string, mode os.FileMode) error { + if strings.HasSuffix(path, ".tmp") { + return errors.New("state chmod denied") + } + return os.Chmod(path, mode) + } + if _, err := store.Claim([]AttemptSpec{{ + Fingerprint: attemptTestFingerprint("state-chmod"), + EventKey: EventMention, + }}, time.Minute); err == nil || !strings.Contains(err.Error(), "state chmod denied") { + t.Fatalf("state chmod error = %v", err) + } + tmp := filepath.Join(store.workDir, AttemptStateFileName) + ".tmp" + if _, err := os.Stat(tmp); !os.IsNotExist(err) { + t.Fatalf("temporary state after chmod error = %v", err) + } + }) + + t.Run("lock wait clamps final sleep", func(t *testing.T) { + workDir := t.TempDir() + base := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC) + var nowCalls int + store := NewAttemptStore(workDir, WithAttemptClock(func() time.Time { + nowCalls++ + if nowCalls == 1 { + return base + } + return base.Add(attemptLockWaitTimeout - 10*time.Millisecond) + })) + var acquireCalls int + store.tryAcquire = func(path string) (*eventlock.File, error) { + acquireCalls++ + if acquireCalls == 1 { + return nil, eventlock.ErrBusy + } + return eventlock.TryAcquire(path) + } + var slept time.Duration + store.sleep = func(delay time.Duration) { + slept = delay + } + if err := store.withLock(func() error { return nil }); err != nil { + t.Fatal(err) + } + if slept != 10*time.Millisecond { + t.Fatalf("final lock sleep = %s, want 10ms", slept) + } + }) + + t.Run("default attempt id", func(t *testing.T) { + attemptID, err := newAttemptID() + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(attemptID) == "" || len(attemptID) > attemptMaxFieldLength { + t.Fatalf("newAttemptID() = %q", attemptID) + } + }) +} diff --git a/internal/event/personal/client.go b/internal/event/personal/client.go index 5c7ab042..d9738d03 100644 --- a/internal/event/personal/client.go +++ b/internal/event/personal/client.go @@ -23,6 +23,7 @@ import ( "log/slog" "net/http" "net/url" + "strconv" "strings" "time" @@ -67,6 +68,8 @@ type Client struct { HTTPClient *http.Client Identity Identity AccessTokenProvider func(context.Context) (string, error) + ClientVersion string + UserAgent string } type CreateSubscriptionRequest struct { @@ -141,10 +144,14 @@ func (s dwsSubscription) toSubscription() Subscription { } type APIError struct { - Code string `json:"code"` - Message string `json:"message"` - Retryable bool `json:"retryable,omitempty"` - Details map[string]any `json:"details,omitempty"` + Code string `json:"code"` + Message string `json:"message"` + Retryable *bool `json:"retryable,omitempty"` + RetryAfterSeconds *int64 `json:"retry_after_seconds,omitempty"` + NextRetryAt *time.Time `json:"next_retry_at,omitempty"` + TraceID string `json:"trace_id,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` + Details map[string]any `json:"details,omitempty"` } func (e *APIError) Error() string { @@ -179,7 +186,7 @@ func (c *Client) CreateSubscription(ctx context.Context, req CreateSubscriptionR var sub Subscription if err := c.do(ctx, http.MethodPost, "/subscription/user", nil, c.buildCreateRequest(req), &sub); err != nil { var apiErr *APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) && isDuplicateSubscriptionCode(apiErr.Code) { if subID, ok := apiErr.Details["subscribe_id"].(string); ok && subID != "" { return &Subscription{ SubscribeID: subID, @@ -378,14 +385,20 @@ func (c *Client) do(ctx context.Context, method, path string, q url.Values, body return fmt.Errorf("personal event: read response: %w", err) } responseLog := sanitizeLogPayload(data) + responseID := firstNonEmpty(responseRequestID(data), responseHeaderRequestID(resp.Header)) + responseTrace := firstNonEmpty(responseBodyTraceID(data), responseTraceID(resp.Header)) if resp.StatusCode < 200 || resp.StatusCode >= 300 { if apiErr := decodeAPIError(data); apiErr != nil { - apiErr = withRequestDetails(apiErr, method, path, resp.StatusCode, responseRequestID(data)) - logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, responseRequestID(data), apiErr) + apiErr = withHTTPResponseDetails(apiErr, method, path, resp, responseID, responseTrace) + logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, responseID, apiErr) return apiErr } - logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, responseRequestID(data), nil) - return fmt.Errorf("personal event: HTTP %d", resp.StatusCode) + apiErr := withHTTPResponseDetails(&APIError{ + Code: fmt.Sprintf("HTTP_%d", resp.StatusCode), + Message: firstNonEmpty(http.StatusText(resp.StatusCode), "HTTP request failed"), + }, method, path, resp, responseID, responseTrace) + logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, responseID, apiErr) + return apiErr } if len(bytes.TrimSpace(data)) == 0 { logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, "", nil) @@ -393,23 +406,25 @@ func (c *Client) do(ctx context.Context, method, path string, q url.Values, body } var env responseEnvelope if err := json.Unmarshal(data, &env); err == nil && (env.Success != nil || env.Error != nil || env.Result != nil || env.ErrorCode != "" || env.ErrorMsg != "") { + envID := firstNonEmpty(env.requestID(), responseID) + envTrace := firstNonEmpty(env.traceID(), responseTrace) if env.Success == nil { if apiErr := env.apiError(); apiErr != nil { - apiErr = withRequestDetails(apiErr, method, path, resp.StatusCode, env.requestID()) - logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, env.requestID(), apiErr) + apiErr = withHTTPResponseDetails(apiErr, method, path, resp, envID, envTrace) + logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, envID, apiErr) return apiErr } } if env.Success != nil && !*env.Success { if apiErr := env.apiError(); apiErr != nil { - apiErr = withRequestDetails(apiErr, method, path, resp.StatusCode, env.requestID()) - logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, env.requestID(), apiErr) + apiErr = withHTTPResponseDetails(apiErr, method, path, resp, envID, envTrace) + logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, envID, apiErr) return apiErr } - logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, env.requestID(), nil) + logControlRequest("personal event control request failed", method, path, q, resp.StatusCode, requestLog, responseLog, envID, nil) return errors.New("personal event: request failed") } - logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, env.requestID(), nil) + logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, envID, nil) if env.Result == nil { return nil } @@ -419,10 +434,10 @@ func (c *Client) do(ctx context.Context, method, path string, q url.Values, body return decodeResult(env.Result, out) } if out == nil { - logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, responseRequestID(data), nil) + logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, responseID, nil) return nil } - logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, responseRequestID(data), nil) + logControlRequest("personal event control request", method, path, q, resp.StatusCode, requestLog, responseLog, responseID, nil) return json.Unmarshal(data, out) } @@ -451,25 +466,58 @@ func (c *Client) decorate(req *http.Request, accessToken string) { if c.Identity.CorpID != "" { req.Header.Set("X-DWS-Corp-Id", c.Identity.CorpID) } + if version := strings.TrimSpace(c.ClientVersion); version != "" { + req.Header.Set("X-Cli-Version", version) + } + if userAgent := strings.TrimSpace(c.UserAgent); userAgent != "" { + req.Header.Set("User-Agent", userAgent) + } req.Header.Set("Accept", "application/json") } type responseEnvelope struct { - Success *bool `json:"success"` - RequestID string `json:"request_id,omitempty"` - RequestID2 string `json:"requestId,omitempty"` - Result json.RawMessage `json:"result"` - Error *APIError `json:"error"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorMsg string `json:"errorMsg,omitempty"` + Success *bool `json:"success"` + RequestID string `json:"request_id,omitempty"` + RequestID2 string `json:"requestId,omitempty"` + TraceID string `json:"trace_id,omitempty"` + TraceID2 string `json:"traceId,omitempty"` + Arguments []json.RawMessage `json:"arguments,omitempty"` + Result json.RawMessage `json:"result"` + Error *APIError `json:"error"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMsg string `json:"errorMsg,omitempty"` + Retryable *bool `json:"retryable,omitempty"` + RetryAfterSeconds *int64 `json:"retry_after_seconds,omitempty"` + RetryAfterSecondsCamel *int64 `json:"retryAfterSeconds,omitempty"` + NextRetryAt *time.Time `json:"next_retry_at,omitempty"` + NextRetryAtCamel *time.Time `json:"nextRetryAt,omitempty"` } func (e responseEnvelope) apiError() *APIError { if e.Error != nil { + if e.Error.Retryable == nil { + e.Error.Retryable = e.Retryable + } + if e.Error.RetryAfterSeconds == nil { + e.Error.RetryAfterSeconds = firstInt64Pointer(e.RetryAfterSeconds, e.RetryAfterSecondsCamel) + } + if e.Error.NextRetryAt == nil { + e.Error.NextRetryAt = firstTimePointer(e.NextRetryAt, e.NextRetryAtCamel) + } + if e.Error.TraceID == "" { + e.Error.TraceID = e.traceID() + } return e.Error } if e.ErrorCode != "" || e.ErrorMsg != "" { - return &APIError{Code: e.ErrorCode, Message: e.ErrorMsg} + return &APIError{ + Code: e.ErrorCode, + Message: e.ErrorMsg, + Retryable: e.Retryable, + RetryAfterSeconds: firstInt64Pointer(e.RetryAfterSeconds, e.RetryAfterSecondsCamel), + NextRetryAt: firstTimePointer(e.NextRetryAt, e.NextRetryAtCamel), + TraceID: e.traceID(), + } } return nil } @@ -478,6 +526,10 @@ func (e responseEnvelope) requestID() string { return firstNonEmpty(e.RequestID, e.RequestID2) } +func (e responseEnvelope) traceID() string { + return firstNonEmpty(e.TraceID, e.TraceID2, firstArgumentString(e.Arguments)) +} + func decodeResult(raw json.RawMessage, out any) error { if len(raw) == 0 || string(raw) == "null" { return nil @@ -511,11 +563,8 @@ func decodeSubscriptionResult(raw json.RawMessage) (Subscription, bool) { func decodeAPIError(data []byte) *APIError { var env responseEnvelope if err := json.Unmarshal(data, &env); err == nil { - if env.Error != nil { - return env.Error - } - if env.ErrorCode != "" || env.ErrorMsg != "" { - return &APIError{Code: env.ErrorCode, Message: env.ErrorMsg} + if apiErr := env.apiError(); apiErr != nil { + return apiErr } } var apiErr APIError @@ -533,12 +582,24 @@ func responseRequestID(data []byte) string { return "" } +func responseBodyTraceID(data []byte) string { + var env responseEnvelope + if err := json.Unmarshal(data, &env); err == nil { + if env.Error != nil && strings.TrimSpace(env.Error.TraceID) != "" { + return strings.TrimSpace(env.Error.TraceID) + } + return env.traceID() + } + return "" +} + func withRequestDetails(apiErr *APIError, method, path string, status int, requestID string) *APIError { if apiErr == nil { return nil } + apiErr.HTTPStatus = status if apiErr.Details == nil { - apiErr.Details = make(map[string]any, 4) + apiErr.Details = make(map[string]any, 8) } apiErr.Details["method"] = method apiErr.Details["path"] = path @@ -549,6 +610,91 @@ func withRequestDetails(apiErr *APIError, method, path string, status int, reque return apiErr } +func withHTTPResponseDetails(apiErr *APIError, method, path string, resp *http.Response, requestID, traceID string) *APIError { + if apiErr == nil { + return nil + } + status := 0 + if resp != nil { + status = resp.StatusCode + traceID = firstNonEmpty(apiErr.TraceID, traceID, responseTraceID(resp.Header)) + } + apiErr = withRequestDetails(apiErr, method, path, status, requestID) + if apiErr.TraceID == "" { + apiErr.TraceID = strings.TrimSpace(traceID) + } + if resp == nil { + return apiErr + } + + retryAfter := strings.TrimSpace(resp.Header.Get("Retry-After")) + if retryAfter == "" { + return apiErr + } + apiErr.Details["retry_after"] = retryAfter + if apiErr.RetryAfterSeconds == nil && apiErr.NextRetryAt == nil { + if seconds, err := strconv.ParseInt(retryAfter, 10, 64); err == nil && seconds >= 0 { + apiErr.RetryAfterSeconds = &seconds + } else if next, err := http.ParseTime(retryAfter); err == nil { + next = next.UTC() + apiErr.NextRetryAt = &next + } + } + if apiErr.RetryAfterSeconds != nil { + apiErr.Details["retry_after_seconds"] = *apiErr.RetryAfterSeconds + } + if apiErr.NextRetryAt != nil { + apiErr.Details["next_retry_at"] = apiErr.NextRetryAt.UTC().Format(time.RFC3339) + } + return apiErr +} + +func responseTraceID(headers http.Header) string { + for _, key := range []string{"X-Trace-Id", "X-Dingtalk-Trace-Id"} { + if value := strings.TrimSpace(headers.Get(key)); value != "" { + return value + } + } + return "" +} + +func responseHeaderRequestID(headers http.Header) string { + return strings.TrimSpace(headers.Get("X-Request-Id")) +} + +func firstArgumentString(arguments []json.RawMessage) string { + if len(arguments) == 0 { + return "" + } + var value string + if err := json.Unmarshal(arguments[0], &value); err != nil { + return "" + } + return strings.TrimSpace(value) +} + +func firstInt64Pointer(values ...*int64) *int64 { + for _, value := range values { + if value == nil { + continue + } + copy := *value + return © + } + return nil +} + +func firstTimePointer(values ...*time.Time) *time.Time { + for _, value := range values { + if value == nil || value.IsZero() { + continue + } + copy := value.UTC() + return © + } + return nil +} + func logControlRequest(message, method, path string, q url.Values, status int, requestPayload, responsePayload, requestID string, apiErr *APIError) { attrs := []any{ "method", method, @@ -568,6 +714,9 @@ func logControlRequest(message, method, path string, q url.Values, status int, r attrs = append(attrs, "request_id", requestID) } if apiErr != nil { + if apiErr.TraceID != "" { + attrs = append(attrs, "trace_id", apiErr.TraceID) + } if apiErr.Code != "" { attrs = append(attrs, "error_code", apiErr.Code) } @@ -660,6 +809,15 @@ func isNotFound(err error) bool { return errors.As(err, &apiErr) && (apiErr.Code == "PERSONAL_EVENT_NOT_FOUND" || apiErr.Code == "NOT_FOUND") } +func isDuplicateSubscriptionCode(code string) bool { + switch strings.ToUpper(strings.TrimSpace(code)) { + case "DUP", "DUPLICATE_SUBSCRIPTION", "SUBSCRIPTION_ALREADY_EXISTS": + return true + default: + return false + } +} + func dwsStatusString(raw json.RawMessage) string { raw = bytes.TrimSpace(raw) if len(raw) == 0 || string(raw) == "null" { diff --git a/internal/event/personal/client_test.go b/internal/event/personal/client_test.go index 6ac034b6..a2abd3c8 100644 --- a/internal/event/personal/client_test.go +++ b/internal/event/personal/client_test.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "testing" + "time" ) func TestClientCreateSubscriptionDWSRequestAndArrayResponse(t *testing.T) { @@ -282,6 +283,12 @@ func TestClientBusinessErrorHTTP200(t *testing.T) { apiErr.Details["http_status"] != http.StatusOK || apiErr.Details["request_id"] != "req-1" { t.Fatalf("details = %#v", apiErr.Details) } + if apiErr.Retryable != nil { + t.Fatalf("retryable = %v, want unknown", *apiErr.Retryable) + } + if apiErr.HTTPStatus != http.StatusOK || apiErr.TraceID != "" { + t.Fatalf("HTTP diagnostics = status %d trace %q", apiErr.HTTPStatus, apiErr.TraceID) + } out := logs.String() for _, want := range []string{"/subscription/user", "INVALID_PARAM", "clientId is empty", "req-1", "request", "response"} { if !strings.Contains(out, want) { @@ -290,6 +297,266 @@ func TestClientBusinessErrorHTTP200(t *testing.T) { } } +func TestClientBusinessErrorPreservesRetryContractAndClientHeaders(t *testing.T) { + nextRetryAt := "2026-07-30T04:05:06Z" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Cli-Version"); got != "1.2.3" { + t.Fatalf("X-Cli-Version = %q", got) + } + if got := r.Header.Get("User-Agent"); got != "dws-cli/1.2.3" { + t.Fatalf("User-Agent = %q", got) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "success": false, + "errorCode": "RATE_LIMITED", + "errorMsg": "slow down", + "retryable": false, + "retryAfterSeconds": 30, + "nextRetryAt": nextRetryAt, + "arguments": []string{"portal-trace-1"}, + }) + })) + defer srv.Close() + + c := NewClient(srv.URL, Identity{AccessToken: "token", ClientID: "client", SourceID: "open"}) + c.ClientVersion = "1.2.3" + c.UserAgent = "dws-cli/1.2.3" + _, err := c.CreateSubscription(t.Context(), CreateSubscriptionRequest{ + EventKey: EventMention, + RuleType: "at", + RuleParam: map[string]any{}, + }) + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error = %v, want *APIError", err) + } + if apiErr.Retryable == nil || *apiErr.Retryable { + t.Fatalf("retryable = %v, want explicit false", apiErr.Retryable) + } + if apiErr.RetryAfterSeconds == nil || *apiErr.RetryAfterSeconds != 30 { + t.Fatalf("retry_after_seconds = %v", apiErr.RetryAfterSeconds) + } + if apiErr.NextRetryAt == nil || apiErr.NextRetryAt.Format(time.RFC3339) != nextRetryAt { + t.Fatalf("next_retry_at = %v", apiErr.NextRetryAt) + } + if apiErr.TraceID != "portal-trace-1" || apiErr.HTTPStatus != http.StatusOK { + t.Fatalf("HTTP diagnostics = trace %q status %d", apiErr.TraceID, apiErr.HTTPStatus) + } + if _, exists := apiErr.Details["request_id"]; exists { + t.Fatalf("portal trace was mislabeled as request_id: %#v", apiErr.Details) + } +} + +func TestClientErrorDiagnosticIdentityPrecedence(t *testing.T) { + tests := []struct { + name string + body map[string]any + headers http.Header + wantTraceID string + wantRequestID string + }{ + { + name: "nested trace wins", + body: map[string]any{ + "success": false, + "requestId": "body-request", + "traceId": "top-trace", + "arguments": []string{"portal-trace"}, + "error": map[string]any{ + "code": "SYSTEM_ERROR", + "message": "failed", + "trace_id": "nested-trace", + }, + }, + headers: http.Header{ + "X-Trace-Id": []string{"header-trace"}, + "X-Request-Id": []string{"header-request"}, + }, + wantTraceID: "nested-trace", + wantRequestID: "body-request", + }, + { + name: "top-level trace wins over arguments and header", + body: map[string]any{ + "success": false, + "requestId": "body-request", + "traceId": "top-trace", + "arguments": []string{"portal-trace"}, + "error": map[string]any{"code": "SYSTEM_ERROR", "message": "failed"}, + }, + headers: http.Header{"X-Trace-Id": []string{"header-trace"}}, + wantTraceID: "top-trace", + wantRequestID: "body-request", + }, + { + name: "portal argument wins over header", + body: map[string]any{ + "success": false, + "requestId": "body-request", + "arguments": []string{"portal-trace"}, + "error": map[string]any{"code": "SYSTEM_ERROR", "message": "failed"}, + }, + headers: http.Header{"X-Trace-Id": []string{"header-trace"}}, + wantTraceID: "portal-trace", + wantRequestID: "body-request", + }, + { + name: "headers are independent fallbacks", + body: map[string]any{ + "success": false, + "error": map[string]any{"code": "SYSTEM_ERROR", "message": "failed"}, + }, + headers: http.Header{ + "X-Trace-Id": []string{"header-trace"}, + "X-Request-Id": []string{"header-request"}, + }, + wantTraceID: "header-trace", + wantRequestID: "header-request", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + for key, values := range test.headers { + for _, value := range values { + w.Header().Add(key, value) + } + } + _ = json.NewEncoder(w).Encode(test.body) + })) + defer srv.Close() + + client := NewClient(srv.URL, Identity{ + AccessToken: "token", + ClientID: "client", + SourceID: "open", + }) + _, err := client.CreateSubscription(t.Context(), CreateSubscriptionRequest{ + EventKey: EventMention, + RuleType: "at", + RuleParam: map[string]any{}, + }) + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error = %v, want *APIError", err) + } + if apiErr.TraceID != test.wantTraceID { + t.Fatalf("trace_id = %q, want %q", apiErr.TraceID, test.wantTraceID) + } + if got, _ := apiErr.Details["request_id"].(string); got != test.wantRequestID { + t.Fatalf("request_id = %q, want %q; details=%#v", got, test.wantRequestID, apiErr.Details) + } + }) + } +} + +func TestClientHTTPErrorPreservesRetryAfterAndHeaderTrace(t *testing.T) { + tests := []struct { + name string + retryAfter string + wantSeconds int64 + wantNextRetryAt string + }{ + {name: "delta seconds", retryAfter: "45", wantSeconds: 45}, + {name: "http date", retryAfter: "Thu, 30 Jul 2026 04:05:06 GMT", wantNextRetryAt: "2026-07-30T04:05:06Z"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Retry-After", tt.retryAfter) + w.Header().Set("X-Trace-Id", "header-trace") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + c := NewClient(srv.URL, Identity{AccessToken: "token", ClientID: "client", SourceID: "open"}) + _, err := c.CreateSubscription(t.Context(), CreateSubscriptionRequest{ + EventKey: EventMention, + RuleType: "at", + RuleParam: map[string]any{}, + }) + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error = %v, want *APIError", err) + } + if calls != 1 { + t.Fatalf("HTTP calls = %d, want one-shot client", calls) + } + if apiErr.Code != "HTTP_429" || apiErr.HTTPStatus != http.StatusTooManyRequests || apiErr.TraceID != "header-trace" { + t.Fatalf("API error = %#v", apiErr) + } + if apiErr.Retryable != nil { + t.Fatalf("retryable = %v, want unknown", apiErr.Retryable) + } + if tt.wantSeconds > 0 { + if apiErr.RetryAfterSeconds == nil || *apiErr.RetryAfterSeconds != tt.wantSeconds { + t.Fatalf("retry_after_seconds = %v", apiErr.RetryAfterSeconds) + } + } + if tt.wantNextRetryAt != "" { + if apiErr.NextRetryAt == nil || apiErr.NextRetryAt.Format(time.RFC3339) != tt.wantNextRetryAt { + t.Fatalf("next_retry_at = %v", apiErr.NextRetryAt) + } + } + }) + } +} + +func TestClientErrorDiagnosticHelpersHandleNilAndMalformedInputs(t *testing.T) { + if got := withHTTPResponseDetails(nil, http.MethodPost, "/subscription/user", nil, "request-1", "trace-1"); got != nil { + t.Fatalf("withHTTPResponseDetails(nil) = %#v, want nil", got) + } + + apiErr := &APIError{Code: "SYSTEM_ERROR", Message: "failed"} + got := withHTTPResponseDetails(apiErr, http.MethodPost, "/subscription/user", nil, "request-1", " trace-1 ") + if got != apiErr { + t.Fatalf("withHTTPResponseDetails() returned a different error: got %#v, want %#v", got, apiErr) + } + if got.HTTPStatus != 0 || got.TraceID != "trace-1" { + t.Fatalf("diagnostics = status %d trace %q", got.HTTPStatus, got.TraceID) + } + if got.Details["request_id"] != "request-1" { + t.Fatalf("details = %#v", got.Details) + } + + if got := firstArgumentString([]json.RawMessage{json.RawMessage(`{`)}); got != "" { + t.Fatalf("firstArgumentString(malformed) = %q, want empty", got) + } +} + +func TestClientCreateSubscriptionDoesNotTreatArbitraryErrorSubscribeIDAsSuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": map[string]any{ + "code": "SYSTEM_ERROR", + "message": "registration failed", + "details": map[string]any{"subscribe_id": "pending-sub"}, + }, + }) + })) + defer srv.Close() + + c := NewClient(srv.URL, Identity{AccessToken: "token", ClientID: "client", SourceID: "open"}) + sub, err := c.CreateSubscription(t.Context(), CreateSubscriptionRequest{ + EventKey: EventMention, + RuleType: "at", + RuleParam: map[string]any{}, + }) + if err == nil || sub != nil { + t.Fatalf("subscription = %#v, error = %v; arbitrary errors must stay failures", sub, err) + } + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Code != "SYSTEM_ERROR" { + t.Fatalf("error = %#v", err) + } +} + func TestClientOmitsCorpHeaderWhenUnknown(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("X-DWS-Corp-Id"); got != "" { diff --git a/internal/helpers/devapp_test.go b/internal/helpers/devapp_test.go index f3e911b6..ad79c0f0 100644 --- a/internal/helpers/devapp_test.go +++ b/internal/helpers/devapp_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" stderrors "errors" + "io" "reflect" "strings" "testing" @@ -651,6 +652,37 @@ func TestDevAppEventSubscribeUsesEventCodes(t *testing.T) { } } +type devAppFailingCountRunner struct { + calls int + err error +} + +func (r *devAppFailingCountRunner) Run(_ context.Context, invocation executor.Invocation) (executor.Result, error) { + r.calls++ + return executor.Result{Invocation: invocation}, r.err +} + +func TestDevAppEventSubscribeRunnerFailureIsNotRetried(t *testing.T) { + wantErr := stderrors.New("event subscription failed") + runner := &devAppFailingCountRunner{err: wantErr} + root := newDevAppTestRoot(runner) + root.SetOut(io.Discard) + root.SetErr(io.Discard) + root.SetArgs([]string{ + "dev", "app", "event", "subscribe", + "--unified-app-id", "u-1", + "--event-codes", "a,b", + "--yes", + }) + + if err := root.Execute(); !stderrors.Is(err, wantErr) { + t.Fatalf("Execute() error = %v, want %v", err, wantErr) + } + if runner.calls != 1 { + t.Fatalf("runner calls = %d, want 1", runner.calls) + } +} + func TestDevAppEventSubscribeRequiresEventCodes(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/transport/client_test.go b/internal/transport/client_test.go index 969a8796..d6f9fc90 100644 --- a/internal/transport/client_test.go +++ b/internal/transport/client_test.go @@ -292,6 +292,43 @@ func TestCallToolUsesJSONRPCMethod(t *testing.T) { } } +func TestCallToolDevAppEventSubscribeRetriesAreBounded(t *testing.T) { + attempts := 0 + httpClient := &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return &http.Response{ + StatusCode: http.StatusServiceUnavailable, + Header: make(http.Header), + Body: http.NoBody, + Request: req, + }, nil + }), + } + client := NewClient(httpClient) + client.sleep = func(context.Context, time.Duration) error { return nil } + + _, err := client.CallTool( + context.Background(), + "https://mcp.example.test/dws", + "subscribe_dev_app_events", + map[string]any{ + "unifiedAppId": "u-1", + "eventCodes": []string{"event-a", "event-b"}, + }, + ) + if err == nil { + t.Fatal("CallTool() succeeded after repeated HTTP 503 responses") + } + wantAttempts := client.MaxRetries + 1 + if attempts != wantAttempts { + t.Fatalf("HTTP attempts = %d, want configured bound %d", attempts, wantAttempts) + } + if attempts > 2 { + t.Fatalf("dev app event subscribe HTTP attempts = %d, want at most 2 by default", attempts) + } +} + func TestCallToolInjectsAuthHeaders(t *testing.T) { t.Parallel() diff --git a/skills/mono/references/products/event.md b/skills/mono/references/products/event.md index a00ecdfd..c8054d6a 100644 --- a/skills/mono/references/products/event.md +++ b/skills/mono/references/products/event.md @@ -171,6 +171,18 @@ dws event stop --all --yes - 订阅清理：本次新建的订阅任意退出即自动退订；`--subscribe-id` 复用的保留；`--ephemeral` 强制退订。优雅停用 SIGTERM、关 stdin，或外部先预览 `dws event stop --dry-run`、确认后加 `--yes`。不要 `kill -9`（跳过退订、泄漏服务端订阅）。 - 一个 consume 可监听多个兼容 event key，每个事件仍有独立订阅和 consumer，共用一个 bus、远程连接及输出。`event stop ` 只移除目标事件，最后一个被移除后进程退出。 +## 订阅创建失败与重试预算 + +以下约束适用于上表全部 16 个事件以及多事件命令中的每一项，只治理 `[event] ready` 之前的订阅创建；ready 之后的 Stream 断线由长连接重连机制处理。 + +- 解析人名或群名、执行 `event consume` 以及后续 `event status/stop` 必须使用同一个 `--profile`。不得把其它 profile 下解析出的 userId、openDingtalkId 或 openConversationId 直接带入当前 profile 的订阅。 +- 同一逻辑订阅由当前 profile / 身份、event key、rule type、目标和过滤条件共同确定。`subscribe_id`、`trace_id` 以及重新启动进程都只是诊断或执行信息，不会生成新预算。 +- `retryable=false`：`max_additional_attempts=0`，立即停止，不得自动重跑。 +- `retryable=true`：`max_additional_attempts=2`，初次失败后最多再尝试 2 次；错误若给出 `retry_after_seconds` 或 `next_retry_at`，不得提前重试。 +- 未返回 `retryable`（即 `retryable=unknown`）：`max_additional_attempts=1`，最多补偿尝试 1 次；仍无法确认时停止并上报错误与 trace。 +- `in_flight` 表示同一逻辑订阅已有请求执行中；`cooldown` 或 `terminal_hold` 表示当前被退避或终态保护。遇到这些状态不得递归调用 `event consume`、并行启动相同订阅或通过新 subId / trace 绕过；等待原请求或保护时间结束，并继续消耗原预算。 +- 多事件命令必须作为同一次原始操作治理。不得把失败事件拆成新的单事件命令、调整顺序或反复重启来重置预算；启动中任一项失败时，由 CLI 回滚本次已经创建的订阅。 + ## Output parsing - 推荐 `--flatten -f ndjson`：顶层业务字段，一行一个事件 JSON，适合 Agent 管道读取。 diff --git a/skills/multi/dingtalk-event/SKILL.md b/skills/multi/dingtalk-event/SKILL.md index 512c8ffb..c4c1d302 100644 --- a/skills/multi/dingtalk-event/SKILL.md +++ b/skills/multi/dingtalk-event/SKILL.md @@ -74,6 +74,18 @@ description: 钉钉个人 IM 事件长连接监听、订阅与消费，覆盖消 - `--debug-raw-events` 只用于联调确认服务端推送是否到达本地连接；正常任务不要使用。它和 `--flatten` 互斥，`-f raw` 也不能与 `--flatten` 同时使用。 - 排查：consume 报 bus 启动失败 → 报错已带真实原因，先查 `dws --profile auth status`（非默认组织带对 `--profile`）；本地日志见 `~/.dws/events//personal_stream//bus.log`（`hash` 见 `dws event status` 的 Workdir）；有残留先用 `dws event stop --all --dry-run` 预览，确认后加 `--yes` 清理。看着"挂住"无输出多是误加了 `--foreground`（那是跑 bus、不打印事件），去掉即可。 +## 订阅创建失败与重试预算 + +以下约束适用于上表全部 16 个事件以及多事件命令中的每一项，只治理 `[event] ready` 之前的订阅创建；ready 之后的 Stream 断线由长连接重连机制处理。 + +- 解析人名或群名、执行 `event consume` 以及后续 `event status/stop` 必须使用同一个 `--profile`。不得把其它 profile 下解析出的 userId、openDingtalkId 或 openConversationId 直接带入当前 profile 的订阅。 +- 同一逻辑订阅由当前 profile / 身份、event key、rule type、目标和过滤条件共同确定。`subscribe_id`、`trace_id` 以及重新启动进程都只是诊断或执行信息，不会生成新预算。 +- `retryable=false`：`max_additional_attempts=0`，立即停止，不得自动重跑。 +- `retryable=true`：`max_additional_attempts=2`，初次失败后最多再尝试 2 次；错误若给出 `retry_after_seconds` 或 `next_retry_at`，不得提前重试。 +- 未返回 `retryable`（即 `retryable=unknown`）：`max_additional_attempts=1`，最多补偿尝试 1 次；仍无法确认时停止并上报错误与 trace。 +- `in_flight` 表示同一逻辑订阅已有请求执行中；`cooldown` 或 `terminal_hold` 表示当前被退避或终态保护。遇到这些状态不得递归调用 `event consume`、并行启动相同订阅或通过新 subId / trace 绕过；等待原请求或保护时间结束，并继续消耗原预算。 +- 多事件命令必须作为同一次原始操作治理。不得把失败事件拆成新的单事件命令、调整顺序或反复重启来重置预算；启动中任一项失败时，由 CLI 回滚本次已经创建的订阅。 + ## Call flow 1. 从用户意图选择事件码；人名或群名先解析成必填 ID。 diff --git a/skills/multi/dingtalk-event/references/event-im.md b/skills/multi/dingtalk-event/references/event-im.md index 03dc4c82..5615a6ab 100644 --- a/skills/multi/dingtalk-event/references/event-im.md +++ b/skills/multi/dingtalk-event/references/event-im.md @@ -369,6 +369,18 @@ dws event stop --all --yes 裸 `dws event stop` 不会取消订阅。本次 consume 新建的订阅会在 SIGTERM、Ctrl+C、stdin EOF、duration 或 max-events 等干净退出时自动取消；通过 `--subscribe-id` 复用的订阅默认保留。多事件进程中，停止一个 `subscribe_id` 只移除对应逻辑 consumer，其余事件继续监听；最后一个被移除时进程退出。需要从外部取消时，使用事件输出或 `status` 里的 `subscribe_id`，先执行 `dws event stop --dry-run`，确认预览后再加 `--yes`。不要使用 `kill -9`，它会跳过清理。 +## 订阅创建失败与重试预算 + +以下约束适用于上表全部 16 个事件以及多事件命令中的每一项，只治理 `[event] ready` 之前的订阅创建；ready 之后的 Stream 断线由长连接重连机制处理。 + +- 解析人名或群名、执行 `event consume` 以及后续 `event status/stop` 必须使用同一个 `--profile`。不得把其它 profile 下解析出的 userId、openDingtalkId 或 openConversationId 直接带入当前 profile 的订阅。 +- 同一逻辑订阅由当前 profile / 身份、event key、rule type、目标和过滤条件共同确定。`subscribe_id`、`trace_id` 以及重新启动进程都只是诊断或执行信息，不会生成新预算。 +- `retryable=false`：`max_additional_attempts=0`，立即停止，不得自动重跑。 +- `retryable=true`：`max_additional_attempts=2`，初次失败后最多再尝试 2 次；错误若给出 `retry_after_seconds` 或 `next_retry_at`，不得提前重试。 +- 未返回 `retryable`（即 `retryable=unknown`）：`max_additional_attempts=1`，最多补偿尝试 1 次；仍无法确认时停止并上报错误与 trace。 +- `in_flight` 表示同一逻辑订阅已有请求执行中；`cooldown` 或 `terminal_hold` 表示当前被退避或终态保护。遇到这些状态不得递归调用 `event consume`、并行启动相同订阅或通过新 subId / trace 绕过；等待原请求或保护时间结束，并继续消耗原预算。 +- 多事件命令必须作为同一次原始操作治理。不得把失败事件拆成新的单事件命令、调整顺序或反复重启来重置预算；启动中任一项失败时，由 CLI 回滚本次已经创建的订阅。 + ## Troubleshooting - 没有输出：单事件确认 stderr 已出现 `ready event_key=...`；多事件确认已出现 `ready event_count=...`，不要把某条 `subscription` 行误认为整体就绪。 diff --git a/test/unit/skill_docs_policy_test.go b/test/unit/skill_docs_policy_test.go index 7d54bc71..1bf00297 100644 --- a/test/unit/skill_docs_policy_test.go +++ b/test/unit/skill_docs_policy_test.go @@ -130,6 +130,47 @@ func TestEventSkillUsesFlatOutputContract(t *testing.T) { } } +func TestEventSkillPinsSubscriptionRetryBudget(t *testing.T) { + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller(0) failed") + } + root := filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..")) + paths := []string{ + filepath.Join(root, "skills", "multi", "dingtalk-event", "SKILL.md"), + filepath.Join(root, "skills", "multi", "dingtalk-event", "references", "event-im.md"), + filepath.Join(root, "skills", "mono", "references", "products", "event.md"), + filepath.Join(root, "docs", "event-subprocess-contract.md"), + } + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + text := string(content) + for _, required := range []string{ + "16", + "--profile", + "retryable=false", + "max_additional_attempts=0", + "retryable=true", + "max_additional_attempts=2", + "retryable=unknown", + "max_additional_attempts=1", + "retry_after_seconds", + "next_retry_at", + "in_flight", + "cooldown", + "subscribe_id", + "trace_id", + } { + if !strings.Contains(text, required) { + t.Errorf("%s missing subscription retry contract %q", path, required) + } + } + } +} + func TestEventSkillFrontmatterAdvertisesGroupMemberLifecycle(t *testing.T) { _, filename, _, ok := runtime.Caller(0) if !ok {