fix(providers): stop reading "no local credentials" as "model not found"
The diagnostic logging just added (cf42152) immediately paid off: the
second laptop's real error, for all five fallback models, was google-auth's
own "Your default credentials were not found" — raised client-side before
any request reaches Google. This laptop has no service account JSON and
has never run `gcloud auth application-default login`; it was never about
Vertex AI Model Garden rollout at all.
The bug was mine: that message contains the literal substring "not
found", so _is_model_not_found_error's naive check misread "you have no
Google credentials on this machine" as "this specific model doesn't
exist" — retrying all five models for an identical, unfixable-by-retrying
failure, then still reporting the wrong final message ("model does not
exist or is not available") because _gemini_error_message made the same
substring mistake.
_is_missing_credentials_error checks for this specific error first, in
both places: the fallback loop now fails fast on the first model instead
of wasting four more identical round trips, and the final message tells
the operator exactly what is actually true — no Google Cloud credentials
exist on this machine, it is per-machine state, and there are two ways to
fix it (paste a service account JSON, or run the gcloud login command
locally) — not that a model is unavailable, which was never the case.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -46,14 +46,37 @@ def _is_native_gemini_provider(provider_id: str, data: ProvidersData) -> bool:
|
||||
return get_provider_chat_model(provider_id, data) == "GeminiChatModel"
|
||||
|
||||
|
||||
def _is_missing_credentials_error(exc: Exception) -> bool:
|
||||
"""True when the SDK never reached Google at all — no local credentials.
|
||||
|
||||
Raised client-side by google-auth when Vertex mode has no service
|
||||
account JSON and this machine has no cached `gcloud auth
|
||||
application-default login`. Checked before `_is_model_not_found_error`
|
||||
because its own message is "Your default **credentials were not
|
||||
found**" — containing the literal substring "not found", which would
|
||||
otherwise be misread as "this one model doesn't exist" and retried
|
||||
against all five models for an identical failure every time, arriving
|
||||
at the same wrong "model does not exist" message after five wasted
|
||||
round trips instead of the real, fixable cause on the first one.
|
||||
"""
|
||||
lowered = str(exc).lower()
|
||||
return (
|
||||
"default credentials were not found" in lowered
|
||||
or "defaultcredentialserror" in lowered
|
||||
)
|
||||
|
||||
|
||||
def _is_model_not_found_error(exc: Exception) -> bool:
|
||||
"""True when Google rejected the request because of the model id only.
|
||||
|
||||
Distinguishes "this one model isn't enabled for this project/region yet"
|
||||
from a genuine problem with the key, billing or quota — the former
|
||||
should try the next candidate model, not report the whole connection
|
||||
as broken.
|
||||
as broken. Callers must rule out `_is_missing_credentials_error` first:
|
||||
that error also contains "not found" but is not about any model.
|
||||
"""
|
||||
if _is_missing_credentials_error(exc):
|
||||
return False
|
||||
error_msg = str(exc)
|
||||
return "404" in error_msg or "not found" in error_msg.lower()
|
||||
|
||||
@@ -85,6 +108,15 @@ def _gemini_error_message(defn_name: str, exc: Exception) -> str:
|
||||
logger.warning("Gemini/Vertex request failed for %s: %s", defn_name, exc)
|
||||
error_msg = str(exc)
|
||||
lowered = error_msg.lower()
|
||||
if _is_missing_credentials_error(exc):
|
||||
return (
|
||||
f"{defn_name} (Vertex AI) has no local Google Cloud "
|
||||
"credentials on this machine. Either paste a service account "
|
||||
"JSON above, or run `gcloud auth application-default login` "
|
||||
"in a terminal on this machine, then try again. This is "
|
||||
"per-machine: a working setup on another computer does not "
|
||||
"carry over."
|
||||
)
|
||||
invalid_markers = (
|
||||
"api key not valid",
|
||||
"api_key_invalid",
|
||||
|
||||
@@ -14,10 +14,23 @@ the whole connection as broken over exactly one model.
|
||||
from adclaw.providers.registry import GEMINI_MODELS
|
||||
from adclaw.providers.store import (
|
||||
_GEMINI_PROBE_MODEL_IDS,
|
||||
_is_missing_credentials_error,
|
||||
_is_model_not_found_error,
|
||||
_test_native_gemini_provider_connection,
|
||||
)
|
||||
|
||||
# Verbatim from a real failure: a machine with Vertex enabled but no
|
||||
# service account JSON and no `gcloud auth application-default login`
|
||||
# ever run. Raised client-side by google-auth before any network call is
|
||||
# made — every model would fail identically, since none of them are the
|
||||
# actual problem.
|
||||
_MISSING_ADC_ERROR = (
|
||||
"Your default credentials were not found. To set up Application "
|
||||
"Default Credentials, see "
|
||||
"https://cloud.google.com/docs/authentication/external/set-up-adc "
|
||||
"for more information."
|
||||
)
|
||||
|
||||
|
||||
class _FakeModels:
|
||||
"""Stands in for `client.aio.models`. Queued outcomes, one per call."""
|
||||
@@ -155,3 +168,73 @@ async def test_client_construction_failure_is_reported_gracefully(monkeypatch):
|
||||
def test_is_model_not_found_error_recognises_graph_style_404s():
|
||||
assert _is_model_not_found_error(_not_found("gemini-3.6-flash")) is True
|
||||
assert _is_model_not_found_error(Exception("403 PERMISSION_DENIED")) is False
|
||||
|
||||
|
||||
# --- the real incident: missing local Google credentials, not a model problem ---
|
||||
#
|
||||
# Reported live: a second laptop, same API key, same real Project ID,
|
||||
# failed on every one of the five fallback models with the exact same
|
||||
# "model does not exist" message. The raw error, logged only after this
|
||||
# fix, was google-auth's own "Your default credentials were not found" —
|
||||
# a client-side failure before any request reaches Google, misread as a
|
||||
# per-model 404 because its own text contains "not found".
|
||||
|
||||
|
||||
def test_missing_credentials_error_is_not_mistaken_for_a_missing_model():
|
||||
"""The bug: this error's own text contains "not found", so the naive
|
||||
substring check in _is_model_not_found_error treated "you have no
|
||||
Google credentials on this machine" as "this one model is missing" —
|
||||
and retried four more times for an identical, unfixable-by-retrying
|
||||
failure."""
|
||||
exc = Exception(_MISSING_ADC_ERROR)
|
||||
|
||||
assert _is_missing_credentials_error(exc) is True
|
||||
assert _is_model_not_found_error(exc) is False
|
||||
|
||||
|
||||
async def test_missing_credentials_fails_fast_without_trying_every_model(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Retrying five models for a machine-wide auth problem wastes five
|
||||
round trips and still arrives at a wrong final message. It must stop
|
||||
on the first attempt."""
|
||||
client = _FakeClient([Exception(_MISSING_ADC_ERROR)])
|
||||
monkeypatch.setattr(
|
||||
"adclaw.providers.store._gemini_vertex_enabled", lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"adclaw.providers.store._create_gemini_vertex_client",
|
||||
lambda genai: client,
|
||||
)
|
||||
|
||||
result = await _test_native_gemini_provider_connection(
|
||||
"Gemini (Google)", "AIzafake",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert len(client.aio.models.calls) == 1
|
||||
assert "credentials" in result["message"].lower()
|
||||
# The old, wrong message must not appear — that was the entire bug.
|
||||
assert "does not exist or is not available" not in result["message"]
|
||||
|
||||
|
||||
async def test_missing_credentials_message_says_what_to_do_and_that_its_per_machine(
|
||||
monkeypatch,
|
||||
):
|
||||
client = _FakeClient([Exception(_MISSING_ADC_ERROR)])
|
||||
monkeypatch.setattr(
|
||||
"adclaw.providers.store._gemini_vertex_enabled", lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"adclaw.providers.store._create_gemini_vertex_client",
|
||||
lambda genai: client,
|
||||
)
|
||||
|
||||
result = await _test_native_gemini_provider_connection(
|
||||
"Gemini (Google)", "AIzafake",
|
||||
)
|
||||
|
||||
message = result["message"].lower()
|
||||
assert "gcloud auth application-default login" in message
|
||||
assert "service account" in message
|
||||
assert "per-machine" in message or "another computer" in message
|
||||
|
||||
Reference in New Issue
Block a user