diff --git a/internal/runtime/executor/codex_executor_html_rejection_test.go b/internal/runtime/executor/codex_executor_html_rejection_test.go new file mode 100644 index 00000000000..4a45d089e7d --- /dev/null +++ b/internal/runtime/executor/codex_executor_html_rejection_test.go @@ -0,0 +1,131 @@ +package executor + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator" +) + +func TestCodexHTMLForbiddenClassification(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + want int + }{ + {"observed edge page", 403, "\n", 502}, + {"doctype and case", 403, " \nForbidden", 502}, + {"byte order mark", 403, "\xef\xbb\xbfForbidden", 502}, + {"structured permission denial", 403, `{"error":{"type":"permission_error","message":"Access denied"}}`, 403}, + {"html inside structured denial", 403, `{"error":{"message":"Access denied"}}`, 403}, + {"plain denial", 403, "Forbidden", 403}, + {"similar prefix", 403, "Forbidden", 403}, + {"unauthorized unchanged", 401, "Unauthorized", 401}, + {"quota unchanged", 429, `{"error":{"type":"usage_limit_reached","resets_in_seconds":3600}}`, 429}, + {"gateway error unchanged", 502, "Bad gateway", 502}, + } { + t.Run(tt.name, func(t *testing.T) { + err := newCodexStatusErr(tt.status, []byte(tt.body)) + if err.StatusCode() != tt.want { + t.Fatalf("status = %d, want %d", err.StatusCode(), tt.want) + } + if tt.status == 403 && tt.want == 502 { + if err.RetryAfter() == nil || *err.RetryAfter() != 5*time.Second || err.IsCredentialScoped() { + t.Fatalf("HTML rejection must have short, non-quota retry: %#v", err) + } + if strings.Contains(err.Error(), "Forbidden")) + if err.RetryAfter() == nil || *err.RetryAfter() != 45*time.Second { + t.Fatalf("upstream retry hint was not honoured: %#v", err) + } +} + +func TestCodexHTMLForbiddenRecovery(t *testing.T) { + for _, stream := range []bool{false, true} { + name := "nonstream" + if stream { + name = "stream" + } + t.Run(name, func(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) == 1 { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, "Unavailable") + return + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_recovered\",\"object\":\"response\",\"status\":\"completed\",\"model\":\"gpt-6.1-sol\",\"output\":[{\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]}}\n\n") + })) + t.Cleanup(server.Close) + executor := NewCodexExecutor(&config.Config{}) + auth := codexOAuthTestAuth(server.URL) + auth.ID = "html-recovery-" + name + model := "gpt-6.1-sol" + reg := registry.GetGlobalRegistry() + reg.RegisterClient(auth.ID, "codex", []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { reg.UnregisterClient(auth.ID) }) + manager := cliproxyauth.NewManager(nil, nil, nil) + manager.SetRetryConfig(0, 0, 0) + manager.RegisterExecutor(executor) + if _, err := manager.Register(context.Background(), auth); err != nil { + t.Fatal(err) + } + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"model":"gpt-6.1-sol","messages":[{"role":"user","content":"Reply OK"}],"max_tokens":256}`)} + opts := cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude, Stream: stream} + before := time.Now() + var firstErr error + if stream { + _, firstErr = manager.ExecuteStream(context.Background(), []string{"codex"}, req, opts) + } else { + _, firstErr = manager.Execute(context.Background(), []string{"codex"}, req, opts) + } + if firstErr == nil || !strings.Contains(firstErr.Error(), "upstream_html_rejection") { + t.Fatalf("first request should report transient HTML rejection: %v", firstErr) + } + updated, _ := manager.GetByID(auth.ID) + state := updated.ModelStates[model] + if state == nil || !state.NextRetryAfter.After(before) || state.NextRetryAfter.After(time.Now().Add(6*time.Second)) { + t.Fatalf("expected short retry instead of thirty-minute auth cooldown: %#v", state) + } + if updated.Quota.Exceeded || state.Quota.Exceeded { + t.Fatal("HTML rejection was incorrectly recorded as quota exhaustion") + } + // Exercise the recovered upstream with the exact same credential and + // executor, without sleeps, token refresh, or a process restart. + if stream { + result, err := executor.ExecuteStream(context.Background(), auth, req, opts) + if err != nil { + t.Fatal(err) + } + for chunk := range result.Chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + } + } else if _, err := executor.Execute(context.Background(), auth, req, opts); err != nil { + t.Fatal(err) + } + if attempts.Load() != 2 { + t.Fatalf("unexpected upstream attempts: %d", attempts.Load()) + } + }) + } +} diff --git a/internal/runtime/executor/codex_executor_terminal.go b/internal/runtime/executor/codex_executor_terminal.go index 82ca00bfaee..495aa7daf71 100644 --- a/internal/runtime/executor/codex_executor_terminal.go +++ b/internal/runtime/executor/codex_executor_terminal.go @@ -323,6 +323,21 @@ func newCodexStatusErrWithCooling(statusCode int, body []byte, modelLevelCooling } func newCodexStatusErrWithHeadersAndCooling(statusCode int, headers http.Header, body []byte, modelLevelCooling bool) statusErr { + if helps.IsCodexHTMLForbidden(statusCode, body) { + // An HTML edge rejection is not evidence of an invalid credential. + // Keep retrying bounded by the existing manager policy and honour any + // upstream Retry-After without retaining the HTML page in client history. + retryAfter := 5 * time.Second + if hint := parseRetryAfterHeader(headers, time.Now()); hint != nil && *hint > retryAfter { + retryAfter = *hint + } + return statusErr{ + code: http.StatusBadGateway, + msg: `{"error":{"type":"server_error","code":"upstream_html_rejection","message":"Codex upstream returned an HTML HTTP 403 page. Retry after network connectivity is restored."}}`, + retryAfter: &retryAfter, + diagnostic: "Codex edge HTML rejection; original HTTP status 403", + } + } errCode := statusCode isUsageLimit := isCodexUsageLimitError(body) credentialScoped := isUsageLimit && !modelLevelCooling diff --git a/internal/runtime/executor/helps/codex_html_rejection.go b/internal/runtime/executor/helps/codex_html_rejection.go new file mode 100644 index 00000000000..cf1eaf46185 --- /dev/null +++ b/internal/runtime/executor/helps/codex_html_rejection.go @@ -0,0 +1,29 @@ +package helps + +import ( + "bytes" + "net/http" +) + +// IsCodexHTMLForbidden identifies an edge HTML page, not a structured API +// permission error. A transient network rejection must not disable OAuth for +// the ordinary thirty-minute forbidden-credential cooldown. +func IsCodexHTMLForbidden(status int, body []byte) bool { + if status != http.StatusForbidden { + return false + } + prefix := bytes.TrimSpace(bytes.TrimPrefix(body, []byte("\xef\xbb\xbf"))) + if len(prefix) > 64 { + prefix = prefix[:64] + } + prefix = bytes.ToLower(prefix) + for _, tag := range [][]byte{[]byte(" len(tag) { + switch prefix[len(tag)] { + case '>', ' ', '\t', '\r', '\n': + return true + } + } + } + return false +} diff --git a/sdk/cliproxy/auth/conductor_cooldown.go b/sdk/cliproxy/auth/conductor_cooldown.go index 388ac914f67..653882638de 100644 --- a/sdk/cliproxy/auth/conductor_cooldown.go +++ b/sdk/cliproxy/auth/conductor_cooldown.go @@ -8,6 +8,7 @@ import ( "io" "net" "net/http" + "net/url" "regexp" "sort" "strings" @@ -22,6 +23,7 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" "github.com/router-for-me/CLIProxyAPI/v8/internal/thinking" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + "golang.org/x/net/http2" ) var quotaCooldownDisabled atomic.Bool @@ -2123,6 +2125,15 @@ func isTransientTransportError(err error) bool { if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return true } + // A POST rejected by HTTP/2 before response headers is a transport + // failure, not a bad credential. Body/stream errors have no url.Error + // wrapper and must not gain a replay after output has been delivered. + var requestErr *url.Error + var streamErr http2.StreamError + if errors.As(err, &requestErr) && requestErr != nil && strings.EqualFold(requestErr.Op, "Post") && + errors.As(requestErr.Err, &streamErr) && streamErr.StreamID != 0 && streamErr.Code == http2.ErrCodeProtocol { + return true + } var dnsErr *net.DNSError if errors.As(err, &dnsErr) && dnsErr != nil && (dnsErr.IsTimeout || dnsErr.IsTemporary || dnsErr.Timeout()) { return true diff --git a/sdk/cliproxy/auth/conductor_http2_protocol_test.go b/sdk/cliproxy/auth/conductor_http2_protocol_test.go new file mode 100644 index 00000000000..159ce7b3a21 --- /dev/null +++ b/sdk/cliproxy/auth/conductor_http2_protocol_test.go @@ -0,0 +1,97 @@ +package auth + +import ( + "context" + "errors" + "net/http" + "net/url" + "testing" + + "github.com/google/uuid" + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + "golang.org/x/net/http2" +) + +func protocolRoundTripError() error { + return &url.Error{Op: "Post", URL: "https://chatgpt.com/backend-api/codex/responses", Err: http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol, Cause: errors.New("received from peer")}} +} + +func TestHTTP2ProtocolErrorOnlyRetriesBeforeResponse(t *testing.T) { + for _, tt := range []struct { + name string + err error + want bool + }{ + {"POST round trip", protocolRoundTripError(), true}, + {"body read after headers", http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol}, false}, + {"plain message after headers", errors.New("stream error: stream ID 1; PROTOCOL_ERROR; received from peer"), false}, + {"canceled request", errors.Join(protocolRoundTripError(), context.Canceled), false}, + {"caller deadline", errors.Join(protocolRoundTripError(), context.DeadlineExceeded), false}, + {"real unauthorized", &Error{HTTPStatus: http.StatusUnauthorized, Message: protocolRoundTripError().Error()}, false}, + {"real quota", &Error{HTTPStatus: http.StatusTooManyRequests, Message: protocolRoundTripError().Error()}, false}, + {"real upstream status", &Error{HTTPStatus: http.StatusServiceUnavailable, Message: protocolRoundTripError().Error()}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := isTransientTransportError(tt.err); got != tt.want { + t.Fatalf("transient = %t, want %t", got, tt.want) + } + }) + } +} + +func TestHTTP2ProtocolRoundTripRecoveryWithoutCooldown(t *testing.T) { + for _, stream := range []bool{false, true} { + name := "nonstream" + if stream { + name = "stream" + } + t.Run(name, func(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(1, 0, 0) + executor := &transportThenSuccessExecutor{identifier: "codex", fail: protocolRoundTripError()} + m.RegisterExecutor(executor) + model, authID := "protocol-model-"+uuid.NewString(), "protocol-auth-"+uuid.NewString() + registry.GetGlobalRegistry().RegisterClient(authID, "codex", []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(authID) }) + if _, err := m.Register(context.Background(), &Auth{ID: authID, Provider: "codex"}); err != nil { + t.Fatal(err) + } + if stream { + result, err := m.ExecuteStream(context.Background(), []string{"codex"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}) + if err != nil { + t.Fatal(err) + } + for chunk := range result.Chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + } + } else if _, err := m.Execute(context.Background(), []string{"codex"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}); err != nil { + t.Fatal(err) + } + if executor.callCount() != 2 { + t.Fatalf("calls = %d, want 2", executor.callCount()) + } + assertNoCooldown(t, m, authID, model) + }) + } +} + +func TestHTTP2ProtocolRoundTripRetryBound(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(3, 0, 0) + executor := &alwaysFailExecutor{identifier: "codex", fail: protocolRoundTripError()} + m.RegisterExecutor(executor) + model, authID := "protocol-model-"+uuid.NewString(), "protocol-auth-"+uuid.NewString() + registry.GetGlobalRegistry().RegisterClient(authID, "codex", []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(authID) }) + if _, err := m.Register(context.Background(), &Auth{ID: authID, Provider: "codex"}); err != nil { + t.Fatal(err) + } + _, err := m.Execute(context.Background(), []string{"codex"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}) + if !errors.Is(err, executor.fail) || executor.callCount() != 4 { + t.Fatalf("error = %v, calls = %d; want original failure after four attempts", err, executor.callCount()) + } + assertNoCooldown(t, m, authID, model) +}