Skip to content

tests: refactor oriole pg_regress and add Docker regress coverage #2011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
samrose merged 7 commits into from
Jan 22, 2026
Merged

tests: refactor oriole pg_regress and add Docker regress coverage #2011

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5ceacca
tests: refactor oriole pg_regress and add Docker regress coverage
samrose Jan 22, 2026
6fe3cfc
tests: don't cover index_advisor for orioledb
samrose Jan 22, 2026
e7ca5b9
tests: align test with pr to correct docs
samrose Jan 22, 2026
ebc0ec6
Update test-docker-image.sh
samrose Jan 22, 2026
dc47b0f
Update test-docker-image.sh
samrose Jan 22, 2026
f35e6a3
fix: replace basename with _basename to avoid collision
samrose Jan 22, 2026
7793d19
Update nix/checks.nix
samrose Jan 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Dockerfile-orioledb-17
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,9 @@ RUN sed -i \
chown -R postgres:postgres /etc/postgresql-custom

# Remove items from postgresql.conf
RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
#as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present
RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
# As of pg 16.4+ db_user_namespace is totally deprecated and will break the server if setting is present
RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf"
RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf"
RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf"
RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf"
Expand Down
53 changes: 48 additions & 5 deletions nix/checks.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,32 +113,54 @@
let
name = pkg.version;
in
if builtins.match "15.*" name != null then
# Check orioledb first since "17_15" would match "17.*" pattern
if builtins.match "17_[0-9]+" name != null then
"orioledb-17"
else if builtins.match "15.*" name != null then
"15"
else if builtins.match "17.*" name != null then
"17"
else if builtins.match "orioledb-17.*" name != null then
"orioledb-17"
else
throw "Unsupported PostgreSQL version: ${name}";

# Tests to skip for OrioleDB (not compatible with OrioleDB storage)
orioledbSkipTests = [
"index_advisor" # index_advisor doesn't support OrioleDB tables
];

# Helper function to filter SQL files based on version
filterTestFiles =
version: dir:
let
files = builtins.readDir dir;
# Get list of OrioleDB-specific test basenames , then strip the orioledb prefix from them
orioledbVariants = pkgs.lib.pipe files [
builtins.attrNames
(builtins.filter (n: builtins.match "z_orioledb-17_.*\\.sql" n != null))
(map (n: builtins.substring 14 (pkgs.lib.stringLength n - 18) n)) # Remove "z_orioledb-17_" prefix (14 chars) and ".sql" suffix (4 chars)
];
hasOrioledbVariant = basename: builtins.elem basename orioledbVariants;
isValidFile =
name:
let
isVersionSpecific = builtins.match "z_.*" name != null;
basename = builtins.substring 0 (pkgs.lib.stringLength name - 4) name; # Remove .sql
# Skip tests that don't work with OrioleDB
isSkippedForOrioledb = version == "orioledb-17" && builtins.elem basename orioledbSkipTests;
matchesVersion =
if isVersionSpecific then
if isSkippedForOrioledb then
false
else if isVersionSpecific then
if version == "orioledb-17" then
builtins.match "z_orioledb-17_.*" name != null
else if version == "17" then
builtins.match "z_17_.*" name != null
else
builtins.match "z_15_.*" name != null
else
# For common tests: exclude if OrioleDB variant exists and we're running OrioleDB
if version == "orioledb-17" && hasOrioledbVariant basename then
false
else
true;
in
Expand Down Expand Up @@ -275,7 +297,14 @@
if [[ "${pgpkg.version}" == *"17"* ]]; then
perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf"
fi
#NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole
# Configure OrioleDB if running orioledb-17 check
#shellcheck disable=SC2193
if [[ "${pgpkg.version}" == *"_"* ]]; then
log info "Configuring OrioleDB..."
# Add orioledb to shared_preload_libraries
perl -pi -e "s/(shared_preload_libraries = ')/\$1orioledb, /" "$PGTAP_CLUSTER/postgresql.conf"
log info "OrioleDB added to shared_preload_libraries"
fi

# Check if postgresql.conf exists
if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then
Expand All @@ -302,6 +331,13 @@
log info "Creating test database"
log_cmd createdb -p ${pgPort} -h localhost --username=supabase_admin testing

# Create orioledb extension if running orioledb-17 check (before prime.sql)
#shellcheck disable=SC2193
if [[ "${pgpkg.version}" == *"_"* ]]; then
log info "Creating orioledb extension..."
log_cmd psql -p ${pgPort} -h localhost --username=supabase_admin -d testing -c "CREATE EXTENSION IF NOT EXISTS orioledb;"
fi

log info "Loading prime SQL file"
if ! log_cmd psql -p ${pgPort} -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then
log error "Error executing SQL file. PostgreSQL log content:"
Expand Down Expand Up @@ -341,6 +377,13 @@

check_postgres_ready

# Create orioledb extension if running orioledb-17 check (before prime.sql)
#shellcheck disable=SC2193
if [[ "${pgpkg.version}" == *"_"* ]]; then
log info "Creating orioledb extension for pg_regress tests..."
log_cmd psql -p ${pgPort} -h localhost --no-password --username=supabase_admin -d postgres -c "CREATE EXTENSION IF NOT EXISTS orioledb;"
fi

log info "Loading prime SQL file"
if ! log_cmd psql -p ${pgPort} -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql} 2>&1; then
log error "Error executing SQL file"
Expand Down
160 changes: 160 additions & 0 deletions nix/docs/docker-testing.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
# Docker Image Testing

This document describes how to test PostgreSQL Docker images against the pg_regress test suite.

## Overview

The `test-docker-image.sh` script builds a Docker image from one of the project's Dockerfiles and runs the existing `nix/tests/` test suite against it. This validates that Docker images work correctly before deployment.

## Quick Start

```bash
# Test PostgreSQL 17 image
./test-docker-image.sh Dockerfile-17

# Test PostgreSQL 15 image
./test-docker-image.sh Dockerfile-15

# Test OrioleDB 17 image
./test-docker-image.sh Dockerfile-orioledb-17
```

## Requirements

- Docker
- Nix (provides psql and pg_regress from the flake)

## Usage

```
Usage: ./test-docker-image.sh [OPTIONS] DOCKERFILE

Test a PostgreSQL Docker image against the pg_regress test suite.

Arguments:
DOCKERFILE The Dockerfile to build and test (e.g., Dockerfile-17)

Options:
-h, --help Show this help message
--no-build Skip building the image (use existing)
--keep Keep the container running after tests (for debugging)
```

### Examples

```bash
# Build and test
./test-docker-image.sh Dockerfile-17

# Test without rebuilding (faster iteration)
./test-docker-image.sh --no-build Dockerfile-17

# Keep container running for debugging
./test-docker-image.sh --keep Dockerfile-17
# Then connect with:
# psql -h localhost -p 5435 -U supabase_admin postgres
```

## How It Works

1. **Build** - Builds Docker image from the specified Dockerfile
2. **Start** - Runs container with PostgreSQL exposed on a test port
3. **Wait** - Waits for PostgreSQL to be ready (pg_isready)
4. **HTTP Mock** - Starts the HTTP mock server inside the container for `http` extension tests
5. **Setup** - Runs `prime.sql` to enable all extensions, creates `test_config` table
6. **Patch** - Copies test files to temp dir, patches expected outputs for Docker-specific differences
7. **Test** - Runs pg_regress with version-filtered test files
8. **Compare** - Checks output against patched expected files
9. **Report** - Shows pass/fail, prints diffs on failure
10. **Cleanup** - Removes container (unless `--keep`)

## Version Mapping

| Dockerfile | Version | Test Port | Test Filter | Tests |
|------------|---------|-----------|-------------|-------|
| Dockerfile-15 | 15 | 5436 | `z_15_*.sql` + common | 53 |
| Dockerfile-17 | 17 | 5435 | `z_17_*.sql` + common | 49 |
| Dockerfile-orioledb-17 | orioledb-17 | 5437 | `z_orioledb-17_*.sql` + common | 47 |

## Test Filtering

Tests in `nix/tests/sql/` are filtered by PostgreSQL version:

- Files without `z_` prefix run for **all versions**
- Files starting with `z_15_` run only for PostgreSQL 15
- Files starting with `z_17_` run only for PostgreSQL 17
- Files starting with `z_orioledb-17_` run only for OrioleDB 17

## CI Integration

### GitHub Actions

```yaml
jobs:
test-docker:
runs-on: ubuntu-latest
strategy:
matrix:
dockerfile:
- Dockerfile-15
- Dockerfile-17
- Dockerfile-orioledb-17
steps:
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v24
- name: Test Docker image
run: ./test-docker-image.sh ${{ matrix.dockerfile }}
```

## Debugging Failed Tests

When tests fail, the script outputs `regression.diffs` showing the differences between expected and actual output.

To investigate further:

```bash
# Run with --keep to preserve container
./test-docker-image.sh --keep Dockerfile-17

# Connect to the running database
psql -h localhost -p 5435 -U supabase_admin postgres

# Run individual test manually
psql -h localhost -p 5435 -U supabase_admin postgres -f nix/tests/sql/pgroonga.sql
```

## Relationship to Nix Checks

This script complements `nix flake check` which tests the Nix-built PostgreSQL packages directly. The Docker tests validate that:

1. Docker image builds correctly
2. Extensions are properly installed in the container
3. Configuration files are correctly applied
4. The containerized PostgreSQL behaves the same as the Nix-built version

## Known Differences (Auto-Patched)

The script automatically patches expected outputs at runtime to handle Docker-specific differences:

| Difference | Affected Tests | Cause |
|------------|----------------|-------|
| `$user` → `\$user` in search_path | pgmq, vault, roles | Docker image configuration escapes `$` in search_path |

These patches are applied to a temporary copy of the test files - the original files are never modified.

### OrioleDB-Specific Test Files

For tests that produce different output under OrioleDB (due to the orioledb extension being loaded or different storage behavior), create OrioleDB-specific versions:

- `nix/tests/sql/z_orioledb-17_<testname>.sql` - OrioleDB version of the test
- `nix/tests/expected/z_orioledb-17_<testname>.out` - Expected output for OrioleDB

When an OrioleDB variant exists, the common test is automatically skipped for OrioleDB runs. This approach is used by both the Docker test script and Nix flake checks.

## Adding New Tests

1. Add SQL file to `nix/tests/sql/`
2. Add expected output to `nix/tests/expected/`
3. For version-specific tests, prefix with `z_15_`, `z_17_`, or `z_orioledb-17_`

See existing tests for examples.
10 changes: 6 additions & 4 deletions nix/tests/expected/docs-functions.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,15 @@ language plpgsql
security definer set search_path = ''
as $$
begin
select 'hello world';
return 'hello world';
end;
$$;
select hello_world_definer();
ERROR: query has no destination for result data
HINT: If you want to discard the results of a SELECT, use PERFORM instead.
CONTEXT: PL/pgSQL function public.hello_world_definer() line 3 at SQL statement
hello_world_definer
---------------------
hello world
(1 row)

revoke execute on function public.hello_world from public;
revoke execute on function public.hello_world from anon;
grant execute on function public.hello_world to authenticated;
Expand Down
Loading
Loading