Repository navigation
Regression suite #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs the source-level regression suite against a given Open WebUI ref. | |
| # Callable from open-webui/open-webui so a release can be gated on it. | |
| name: Regression suite | |
| on: | |
| workflow_call: | |
| inputs: | |
| open-webui-ref: | |
| description: Ref of open-webui/open-webui to test (tag, branch or SHA) | |
| required: true | |
| type: string | |
| tests-ref: | |
| description: Ref of open-webui/tests to run | |
| required: false | |
| type: string | |
| default: main | |
| suite: | |
| description: Test paths to run | |
| required: false | |
| type: string | |
| default: unit | |
| workflow_dispatch: | |
| inputs: | |
| open-webui-ref: | |
| description: Ref of open-webui/open-webui to test (tag, branch or SHA) | |
| required: true | |
| type: string | |
| default: dev | |
| tests-ref: | |
| description: Ref of open-webui/tests to run | |
| required: false | |
| type: string | |
| default: main | |
| suite: | |
| description: Test paths to run | |
| required: false | |
| type: string | |
| default: unit | |
| # Queued rather than cancelled: a caller gating a release on this workflow would | |
| # otherwise see its required check go red because another run took the slot. | |
| concurrency: | |
| group: regression-${{ inputs.open-webui-ref }}-${{ inputs.tests-ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| run: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Check out the test suite | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/tests | |
| ref: ${{ inputs.tests-ref }} | |
| path: tests | |
| - name: Check out Open WebUI | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/open-webui | |
| ref: ${{ inputs.open-webui-ref }} | |
| path: open-webui | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| open-webui/backend/requirements.txt | |
| tests/pyproject.toml | |
| # torch comes in transitively via sentence-transformers and defaults to the CUDA | |
| # build, which is several GB of nvidia-* wheels this suite never runs. Pinning the | |
| # CPU build first, as the Dockerfile does, keeps the install to a few minutes. | |
| - name: Install dependencies | |
| run: | | |
| uv pip install --system 'torch<=2.9.1' --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system -r open-webui/backend/requirements.txt | |
| uv pip install --system -e "./tests[postgres]" | |
| # Importing open_webui.config creates DATA_DIR and clears STATIC_DIR, so each job | |
| # gets its own scratch rather than the shared temp dir the suite falls back to. | |
| - name: Run the suite | |
| working-directory: tests | |
| env: | |
| OPEN_WEBUI_SOURCE_DIR: ${{ github.workspace }}/open-webui/backend | |
| DATA_DIR: ${{ runner.temp }}/owui-data | |
| STATIC_DIR: ${{ runner.temp }}/owui-static | |
| WEBUI_SECRET_KEY: ci-regression-key | |
| run: | | |
| mkdir -p "$DATA_DIR" "$STATIC_DIR" | |
| pytest ${{ inputs.suite }} -q --html=report.html --self-contained-html | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: regression-report-${{ inputs.open-webui-ref }} | |
| path: tests/report.html | |
| if-no-files-found: ignore | |
| # The integration suite talks to a running instance over HTTP, so it boots one | |
| # from the same checkout rather than reading the source. | |
| integration: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| OPEN_WEBUI_URL: http://127.0.0.1:8080 | |
| ADMIN_USER_EMAIL: admin@example.com | |
| ADMIN_USER_PASSWORD: adminpassword123 | |
| TEST_USER_EMAIL: test@example.com | |
| TEST_USER_PASSWORD: testpassword123 | |
| steps: | |
| - name: Check out the test suite | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/tests | |
| ref: ${{ inputs.tests-ref }} | |
| path: tests | |
| - name: Check out Open WebUI | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/open-webui | |
| ref: ${{ inputs.open-webui-ref }} | |
| path: open-webui | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| open-webui/backend/requirements.txt | |
| tests/pyproject.toml | |
| - name: Install dependencies | |
| run: | | |
| uv pip install --system 'torch<=2.9.1' --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system -r open-webui/backend/requirements.txt | |
| uv pip install --system -e "./tests[postgres]" | |
| # Ollama probing and the local embedding model download are both startup | |
| # costs this suite never uses. | |
| - name: Start Open WebUI | |
| working-directory: open-webui/backend | |
| env: | |
| DATA_DIR: ${{ runner.temp }}/owui-data | |
| WEBUI_SECRET_KEY: ci-regression-key | |
| ENABLE_OLLAMA_API: 'false' | |
| RAG_EMBEDDING_ENGINE: openai | |
| run: | | |
| mkdir -p "$DATA_DIR" | |
| nohup python -m uvicorn open_webui.main:app --host 127.0.0.1 --port 8080 > "$RUNNER_TEMP/open-webui.log" 2>&1 & | |
| for _ in $(seq 1 120); do | |
| if curl -fsS "$OPEN_WEBUI_URL/health" >/dev/null; then | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Open WebUI did not become healthy" | |
| tail -100 "$RUNNER_TEMP/open-webui.log" | |
| exit 1 | |
| # The first signup becomes the admin and turns signup off, so the second | |
| # account has to come from the admin API. | |
| - name: Create the accounts the suite signs in with | |
| run: | | |
| admin_token="$(curl -fsS "$OPEN_WEBUI_URL/api/v1/auths/signup" \ | |
| -H 'Content-Type: application/json' \ | |
| -d "{\"name\":\"Admin\",\"email\":\"$ADMIN_USER_EMAIL\",\"password\":\"$ADMIN_USER_PASSWORD\"}" \ | |
| | python -c 'import json,sys; print(json.load(sys.stdin)["token"])')" | |
| curl -fsS "$OPEN_WEBUI_URL/api/v1/auths/add" \ | |
| -H 'Content-Type: application/json' \ | |
| -H "Authorization: Bearer $admin_token" \ | |
| -d "{\"name\":\"Test\",\"email\":\"$TEST_USER_EMAIL\",\"password\":\"$TEST_USER_PASSWORD\",\"role\":\"user\"}" \ | |
| >/dev/null | |
| - name: Run the integration suite | |
| working-directory: tests | |
| run: pytest integration -q --html=integration-report.html --self-contained-html | |
| - name: Show the backend log | |
| if: failure() | |
| run: tail -200 "$RUNNER_TEMP/open-webui.log" | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-report-${{ inputs.open-webui-ref }} | |
| path: tests/integration-report.html | |
| if-no-files-found: ignore | |
| # The vitest suite drives the frontend source, so it needs the checkout's node_modules. | |
| frontend: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Check out the test suite | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/tests | |
| ref: ${{ inputs.tests-ref }} | |
| path: tests | |
| - name: Check out Open WebUI | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: open-webui/open-webui | |
| ref: ${{ inputs.open-webui-ref }} | |
| path: open-webui | |
| - uses: actions/setup-node@v5 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: | | |
| open-webui/package-lock.json | |
| tests/frontend/package-lock.json | |
| # The checkout's tsconfig extends the one svelte-kit sync generates. | |
| - name: Install Open WebUI dependencies | |
| working-directory: open-webui | |
| run: | | |
| npm ci --force | |
| npx svelte-kit sync | |
| - name: Install the suite | |
| working-directory: tests/frontend | |
| run: npm ci | |
| - name: Run the frontend suite | |
| working-directory: tests/frontend | |
| env: | |
| OPEN_WEBUI_SOURCE_DIR: ${{ github.workspace }}/open-webui/backend | |
| run: npx vitest run |