Fail live model probe on API errors

.github/workflows/tests.yml

@@ -0,0 +1,44 @@
+name: Run tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Unit tests
+            run_live: "0"
+            command: pytest
+            api_base_url: "https://k050506koch-gpt3-dev-api.hf.space"
+          - name: Live model probe
+            run_live: "1"
+            command: pytest -k live_more_models
+            api_base_url: "https://k050506koch-gpt3-dev-api.hf.space"
+
+    runs-on: ubuntu-latest
+    name: ${{ matrix.name }}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-test.txt
+
+      - name: Run tests
+        env:
+          RUN_LIVE_API_TESTS: ${{ matrix.run_live }}
+          API_BASE_URL: ${{ matrix.api_base_url }}
+        run: ${{ matrix.command }}

README.md

@@ -125,7 +125,7 @@ export RUN_LIVE_API_TESTS=0 #or 1
 ```
 ```bash
 pytest
-pytest -k pytest -k live_more_models
+pytest -k live_more_models
 ```
 
 ## Project Structure

requirements-test.txt

@@ -0,0 +1,9 @@
+fastapi>=0.110.0
+httpx>=0.27.0
+pytest>=7.4.0
+pytest-asyncio>=0.23.0
+pydantic>=2.6.0
+pydantic-settings>=2.1.0
+sse-starlette>=1.6.5
+python-multipart>=0.0.9
+tiktoken>=0.5.2

tests/test_live_more_models.py

@@ -1,9 +1,8 @@
 """Additional live API tests to exercise multiple models.
 
 Skipped by default; set RUN_LIVE_API_TESTS=1 to enable.
-This test is lenient: if a model is listed by /v1/models but is not
-actually usable (e.g., missing HF auth or unavailable backend), the
-test skips that case instead of failing the suite.
+The suite skips candidates that are missing from /v1/models but now
+fails whenever the live API returns an error so issues surface in CI.
 """
 from __future__ import annotations
 
@@ -57,9 +56,12 @@ def test_completion_for_models(model: str) -> None:
     with httpx.Client(timeout=timeout) as client:
         resp = client.post(f"{BASE_URL}/v1/completions", json=payload)
 
-    if resp.status_code != 200:
-        pytest.skip(
-            f"model {model} not usable on server (status={resp.status_code}): {resp.text[:500]}"
+    try:
+        resp.raise_for_status()
+    except httpx.HTTPStatusError as exc:  # pragma: no cover - only hit when live API misbehaves
+        pytest.fail(
+            f"model {model} returned an error response: "
+            f"{exc.response.status_code} {exc.response.text[:500]}"
         )
 
     body = resp.json()