diff --git a/.clangd b/.clangd new file mode 100644 index 000000000..dd2584a0a --- /dev/null +++ b/.clangd @@ -0,0 +1,6 @@ +CompileFlags: + CompilationDatabase: .cache/clangd + Add: -Wno-unqualified-std-cast-call +Diagnostics: + Includes: + IgnoreHeader: duckdb\.hpp diff --git a/.github/actions/setup-postgres/action.yml b/.github/actions/setup-postgres/action.yml new file mode 100644 index 000000000..c52515d0e --- /dev/null +++ b/.github/actions/setup-postgres/action.yml @@ -0,0 +1,40 @@ +name: Setup Postgres +description: Start Postgres (optionally with SSL enabled) in a Docker container on localhost:5432 +inputs: + postgres-version: + description: Postgres major version (Docker image tag) + required: true + ssl: + description: Whether to enable SSL on the Postgres server + required: false + default: 'false' +runs: + using: composite + steps: + - name: Start Postgres container + shell: bash + run: | + SSL_ARGS=() + if [ "${{ inputs.ssl }}" == "true" ]; then + SSL_ARGS=( + -c ssl=on + -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem + -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key + ) + fi + docker run -d --name postgres --network host \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=postgres \ + -v /tmp:/tmp \ + postgres:${{ inputs.postgres-version }} \ + -c port=5432 \ + "${SSL_ARGS[@]}" + for i in $(seq 1 60); do + if docker exec postgres pg_isready -h localhost -p 5432 -U postgres; then + break + fi + sleep 1 + done + docker exec postgres pg_isready -h localhost -p 5432 -U postgres + PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c "SELECT version()" diff --git a/.github/workflows/IntegrationTests.yml b/.github/workflows/IntegrationTests.yml index 420905328..a67b42f6b 100644 --- a/.github/workflows/IntegrationTests.yml +++ b/.github/workflows/IntegrationTests.yml @@ -16,37 +16,33 @@ jobs: format-check: name: Format Check - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'duckdb' && 'namespace-profile-linux-slim' || 'ubuntu-latest' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: submodules: 'true' - - - name: Install - run: | - pip3 install \ + - run: | + python -m pip install --user --break-system-packages \ black==25.1.0 \ clang_format==11.0.1 \ cmake-format==0.6.13 - - - name: Format Check - run: | + export PATH=~/.local/bin:$PATH python -m pip show black python -m pip show cmake_format python -m pip show clang_format + which clang-format clang-format --version - clang-format --dump-config make format-check linux-tests: name: Linux (pg-17) needs: format-check - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'duckdb' && 'namespace-profile-linux-x64' || 'ubuntu-latest' }} env: GEN: ninja - CC: 'ccache gcc' - CXX: 'ccache g++' + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -61,6 +57,8 @@ jobs: - name: Install Dependencies run: | + echo "set man-db/auto-update false" | sudo debconf-communicate || true + sudo dpkg-reconfigure man-db || true sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ build-essential \ @@ -85,7 +83,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension run: | @@ -98,14 +96,10 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: 17 - username: 'postgres' - password: 'postgres' - database: 'postgres' - port: '5432' - ssl: true + ssl: false - name: Run tests env: @@ -114,6 +108,7 @@ jobs: PGUSER: postgres PGPASSWORD: postgres POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' run: | source ./create-postgres-tables.sh @@ -123,20 +118,20 @@ jobs: linux-pgbouncer: name: Linux PgBouncer (${{ matrix.postgres.name }}) needs: linux-tests - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'duckdb' && 'namespace-profile-linux-x64' || 'ubuntu-latest' }} strategy: matrix: postgres: - - name: pg-14 - version: 14 + - name: pg-15 + version: 15 - name: pg-18 version: 18 env: GEN: ninja - CC: 'ccache gcc' - CXX: 'ccache g++' + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -151,13 +146,14 @@ jobs: - name: Install Dependencies run: | + echo "set man-db/auto-update false" | sudo debconf-communicate || true + sudo dpkg-reconfigure man-db || true sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ build-essential \ ccache \ cmake \ - ninja-build \ - pgbouncer + ninja-build - name: Cache Key id: cache_key @@ -176,20 +172,16 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension run: | make release - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: ${{ matrix.postgres.version }} - username: 'postgres' - password: 'postgres' - database: 'postgres' - port: '5432' ssl: true - name: Setup PgBouncer @@ -198,18 +190,43 @@ jobs: openssl req -new -x509 -days 365 -nodes -text -subj "/CN=localhost" \ -out ./pgbouncer/pgbouncer.crt \ -keyout ./pgbouncer/pgbouncer.key - sudo chown -R postgres:postgres ./pgbouncer - sudo cp -p ./pgbouncer/* /etc/pgbouncer/ - sudo ls -la /etc/pgbouncer - sudo sed -i 's|^;bardb = host=localhost dbname=bazdb|postgresscanner = host=localhost dbname=postgresscanner|' /etc/pgbouncer/pgbouncer.ini - sudo sed -i 's|^;pool_mode = session|pool_mode = transaction|' /etc/pgbouncer/pgbouncer.ini - sudo sed -i 's|^;max_prepared_statements = 0|max_prepared_statements = 1024|' /etc/pgbouncer/pgbouncer.ini - sudo sed -i 's|^;client_tls_sslmode = disable|client_tls_sslmode = require|' /etc/pgbouncer/pgbouncer.ini - sudo sed -i 's|^;client_tls_key_file =|client_tls_key_file = /etc/pgbouncer/pgbouncer.key|' /etc/pgbouncer/pgbouncer.ini - sudo sed -i 's|^;client_tls_cert_file =|client_tls_cert_file = /etc/pgbouncer/pgbouncer.crt|' /etc/pgbouncer/pgbouncer.ini - echo '"postgres" "postgres"' | sudo tee /etc/pgbouncer/userlist.txt - sudo cat /etc/pgbouncer/pgbouncer.ini - sudo systemctl restart pgbouncer + cat > ./pgbouncer/pgbouncer.ini <<'EOF' + [databases] + postgresscanner = host=localhost port=5432 dbname=postgresscanner + + [pgbouncer] + logfile = /var/log/postgresql/pgbouncer.log + listen_addr = localhost + listen_port = 6432 + unix_socket_dir = + auth_type = md5 + auth_file = /etc/pgbouncer/userlist.txt + pool_mode = transaction + max_prepared_statements = 1024 + client_tls_sslmode = require + client_tls_key_file = /etc/pgbouncer/pgbouncer.key + client_tls_cert_file = /etc/pgbouncer/pgbouncer.crt + EOF + echo '"postgres" "postgres"' > ./pgbouncer/userlist.txt + chmod 755 ./pgbouncer + chmod 644 ./pgbouncer/* + ls -la ./pgbouncer + cat ./pgbouncer/pgbouncer.ini + sudo mkdir -p /var/log/postgresql + sudo chmod 777 /var/log/postgresql + docker run -d --name pgbouncer --network host \ + -v ${{ github.workspace }}/pgbouncer:/etc/pgbouncer:ro \ + -v /var/log/postgresql:/var/log/postgresql \ + edoburu/pgbouncer:latest \ + /usr/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini + for i in $(seq 1 30); do + if pg_isready -h localhost -p 6432; then + break + fi + sleep 1 + done + docker logs pgbouncer + pg_isready -h localhost -p 6432 - name: Setup test data env: @@ -257,7 +274,7 @@ jobs: linux-sanitized: name: Linux ${{ matrix.sanitizer.name }} (pg-17) needs: linux-tests - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'duckdb' && 'namespace-profile-linux-x64' || 'ubuntu-latest' }} strategy: matrix: @@ -268,9 +285,9 @@ jobs: threadsan_enabled: 1 env: - CMAKE_BUILD_PARALLEL_LEVEL: 2 - CC: 'ccache gcc' - CXX: 'ccache g++' + GEN: ninja + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -285,6 +302,8 @@ jobs: - name: Install Dependencies run: | + echo "set man-db/auto-update false" | sudo debconf-communicate || true + sudo dpkg-reconfigure man-db || true sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ build-essential \ @@ -310,14 +329,13 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension env: THREADSAN: ${{ matrix.sanitizer.threadsan_enabled }} run: | make relassert - cat ./build/relassert/src/CMakeFiles/duckdb.dir/flags.make cd build ln -s relassert release @@ -328,13 +346,9 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: 17 - username: 'postgres' - password: 'postgres' - database: 'postgres' - port: '5432' ssl: true - name: Run tests @@ -345,6 +359,8 @@ jobs: PGPASSWORD: postgres PGSSLMODE: require POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 + SANITIZER_BUILD: 1 LOCAL_EXTENSION_REPO: 'build/relassert/repository' run: | source ./create-postgres-tables.sh @@ -420,7 +436,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Prepare build tree shell: bash @@ -442,7 +458,7 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: '17' username: 'postgres' @@ -470,6 +486,7 @@ jobs: PGPASSWORD: postgres PGSSLMODE: require POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' shell: bash run: | diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 560257565..337a49287 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,7 +14,6 @@ jobs: name: Build extension binaries uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: - duckdb_version: main ci_tools_version: main extension_name: postgres_scanner exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw' @@ -27,9 +26,9 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: - duckdb_version: main ci_tools_version: main extension_name: postgres_scanner exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw' opt_in_archs: 'linux_amd64_musl;linux_arm64_musl;windows_amd64_arm' deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }} + deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }} diff --git a/.gitignore b/.gitignore index 76c2cc0a0..87a16b92d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,12 @@ build cmake-build-debug -.idea +/.idea .DS_Store -.vscode -.cache +/.vscode +/.cache +/.local duckdb_unittest_tempdir +.envrc +docker-compose.yml +/AGENTS.md +/CLAUDE.md diff --git a/.gitmodules b/.gitmodules index ac125b1d2..b58f97c4c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "duckdb"] path = duckdb - url = https://github.com/duckdb/duckdb.git + url = https://github.com/altertable-ai/duckdb + branch = altertable-v2.0-cyanoptera [submodule "extension-ci-tools"] path = extension-ci-tools url = https://github.com/duckdb/extension-ci-tools.git diff --git a/CMakeLists.txt b/CMakeLists.txt index b30ca04a7..101e1640d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,33 +2,22 @@ cmake_minimum_required(VERSION 2.8.12...3.29) set(TARGET_NAME postgres_scanner) project(${TARGET_NAME}) -add_definitions( - -DFRONTEND=1 - -D_GNU_SOURCE=1 - -DUSE_OPENSSL=1 - -DHAVE_BIO_GET_DATA=1 - -DHAVE_BIO_METH_NEW=1) +add_definitions(-DFRONTEND=1 -D_GNU_SOURCE=1 -DUSE_OPENSSL=1 + -DHAVE_BIO_GET_DATA=1 -DHAVE_BIO_METH_NEW=1) find_package(OpenSSL REQUIRED) find_package(PostgreSQL REQUIRED) if(NOT MSVC) - add_compile_options( - -Wno-pedantic - -Wno-sign-compare - -Wno-unused-variable) + add_compile_options(-Wno-pedantic -Wno-sign-compare -Wno-unused-variable) endif() -include_directories( - include - database-connector/src/include - ${OPENSSL_INCLUDE_DIR} - ${PostgreSQL_INCLUDE_DIRS}) +include_directories(include database-connector/src/include + ${OPENSSL_INCLUDE_DIR} ${PostgreSQL_INCLUDE_DIRS}) if(WIN32) - include_directories( - vcpkg_ports/libpq/include/port/win32 - vcpkg_ports/libpq/include/port/win32_msvc) + include_directories(vcpkg_ports/libpq/include/port/win32 + vcpkg_ports/libpq/include/port/win32_msvc) endif() add_subdirectory(src) @@ -40,35 +29,27 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) build_static_extension(${TARGET_NAME} ${ALL_OBJECT_FILES}) build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${ALL_OBJECT_FILES}) -# Fix for ELF symbol conflict when loaded inside PostgreSQL backend -# Use -Bsymbolic to force the extension to prefer its own symbols over global ones -# This fixes the pg_link_canary_is_frontend() check failure where calls from within -# this extension resolve to the backend's version (returns false) instead of ours (returns true) +# Fix for ELF symbol conflict when loaded inside PostgreSQL backend Use +# -Bsymbolic to force the extension to prefer its own symbols over global ones +# This fixes the pg_link_canary_is_frontend() check failure where calls from +# within this extension resolve to the backend's version (returns false) instead +# of ours (returns true) if(NOT WIN32 AND NOT APPLE) - target_link_options(${LOADABLE_EXTENSION_NAME} PRIVATE - "-Wl,-Bsymbolic" - ) + target_link_options(${LOADABLE_EXTENSION_NAME} PRIVATE "-Wl,-Bsymbolic") endif() -target_link_libraries(${LOADABLE_EXTENSION_NAME} - OpenSSL::SSL - OpenSSL::Crypto - PostgreSQL::PostgreSQL - ${CURL_LIBRARIES}) +target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto + PostgreSQL::PostgreSQL ${CURL_LIBRARIES}) set_property(TARGET ${EXTENSION_NAME} PROPERTY C_STANDARD 99) set_property(TARGET ${LOADABLE_EXTENSION_NAME} PROPERTY C_STANDARD 99) if(WIN32) - target_link_libraries(${LOADABLE_EXTENSION_NAME} - wsock32 - ws2_32 - wldap32 - secur32 - crypt32) + target_link_libraries(${LOADABLE_EXTENSION_NAME} wsock32 ws2_32 wldap32 + secur32 crypt32) endif() install( - TARGETS ${EXTENSION_NAME} - EXPORT "${DUCKDB_EXPORT_SET}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") + TARGETS ${EXTENSION_NAME} + EXPORT "${DUCKDB_EXPORT_SET}" + LIBRARY DESTINATION "${INSTALL_LIB_DIR}" + ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") diff --git a/README.md b/README.md index e1b7f6be9..0447b130d 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,68 @@ D SELECT * FROM postgres_db.uuids; For more information on how to use the connector, refer to the [Postgres documentation on the website](https://duckdb.org/docs/extensions/postgres). +### Remote Pushdown + +Queries and statements that only reference tables of a single attached Postgres database are executed in Postgres as a whole (instead of scanning the tables and executing the query in DuckDB), when every construct they use can be translated to Postgres SQL with the same semantics. This covers e.g. joins, aggregates, window functions, CTEs, set operations, subqueries, many scalar functions, `UPDATE` / `DELETE` / `INSERT` (including `RETURNING` and `ON CONFLICT`), `CREATE TABLE ... AS`, `CREATE VIEW` and most `DROP` / `ALTER TABLE` statements. `EXPLAIN` shows the generated SQL (`Postgres Query`), and `SET pg_debug_show_queries = true` also prints why a query is not pushed down. + +```sql +D EXPLAIN SELECT customer, sum(amount) FROM postgres_db.orders WHERE status = 'open' GROUP BY customer; +╭─ Postgres Query ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ SQL: │ +│ SELECT "customer", sum("amount") AS "sum(amount)" FROM "public"."orders" WHERE ("status" = 'open') GROUP BY "customer" │ +│ Projections: c0, c1 │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +A pushed down query is executed as a single Postgres query over a single connection. Queries that are plain scans of a single table are instead executed through the regular table scan, which reads the table in parallel over multiple connections (by ranges of the `ctid`). A query is a plain scan when it reads a single table, selects only columns (or `*`), filters only with comparisons of columns with constants, `IN` lists and `IS [NOT] NULL` combined with `AND` / `OR`, and has no `ORDER BY` or `OFFSET` (a `LIMIT` is fine). For example, this query is still executed with a parallel scan: + +```sql +SELECT id, customer, amount FROM postgres_db.orders WHERE status = 'open' AND amount > 100 LIMIT 10; +``` + +Any other construct makes the query a candidate for pushdown, including seemingly simple ones such as a computed column, a `LIKE` filter or an `ORDER BY`. These queries are executed by Postgres over a single connection, which can be slower than a parallel scan for queries that read most of a large table: + +```sql +-- pushed down: the computed column makes it more than a plain scan +SELECT id, amount * 1.2 AS amount_with_tax FROM postgres_db.orders WHERE status = 'open'; +-- pushed down as well +SELECT * FROM postgres_db.orders WHERE customer LIKE 'c1%'; +SELECT * FROM postgres_db.orders ORDER BY id; +``` + +Plain scans are pushed down as well when the settings at attach time do not allow parallel scans (`pg_use_ctid_scan = false`, the text protocol through `pg_query_mode`, `pg_use_text_protocol` or the `USE_TEXT_PROTOCOL` attach option, or Postgres versions before 14). + +Remote pushdown is enabled by default. It can be disabled for newly attached databases: + +```sql +SET pg_enable_remote_pushdown = false; +``` + +Known differences to executing the query in DuckDB: + +* Floating point results of statistical aggregates (e.g. `corr`, `regr_*`, `quantile_cont`) can differ in the last digits. +* Postgres raises errors for floating point overflows, where DuckDB produces `inf`. +* `HUGEINT` results are transferred as `NUMERIC(38, 0)` - values of 10^38 or more raise an error. +* Duplicate result column names are made unique (e.g. `i` and `i_1`), like for the result of a subquery. +* `CURRENT_DATE` / `CURRENT_TIMESTAMP` (without parentheses) are evaluated by Postgres, at the start of the Postgres transaction. +* DuckDB orders strings byte-wise. When the database collation orders them differently (e.g. `en_US.UTF-8`), string ordering (`ORDER BY`, `<`, `min` / `max`, ...) is pushed down with the `C` collation (`COLLATE "C"`), which cannot use indexes built with the database collation. `SELECT DISTINCT` / set operations / `DISTINCT` aggregates ordered by strings are then executed locally, and `CREATE TABLE ... AS` / `CREATE VIEW` statements that order strings are not pushed down as a whole. Column-level collations are not taken into account. +* `lower` / `ILIKE` are only pushed down when Postgres maps Unicode characters to lower case. +* Casts of string columns to dates, timestamps or booleans use the Postgres parsing rules, which accept some inputs that DuckDB rejects (and vice versa). +* Unqualified tables are only pushed down when the Postgres database has no schema named `main` (or when it is attached with the `SCHEMA` option). + +### Connection Pooling + +By default, the extension opens a new connection whenever none are idle. Behind a transaction-mode pooler (e.g. PgBouncer `pool_mode=transaction`), this can exceed the pooler's own pool size under load and hang. If that applies to you, cap and wait instead: + +```sql +SET pg_pool_acquire_mode = 'wait'; +SET pg_pool_max_connections = 16; -- ~80-90% of your pooler's pool size + +ATTACH 'host=pgbouncer-host port=6432 dbname=mydb' AS postgres_db (TYPE postgres); +``` + +Note that `SET` statements are global and only apply pre-`ATTACH`. To change pooling on an already-attached database, use `postgres_configure_pool(catalog_name=...)` instead. Leave `force` mode if you connect directly to Postgres or through a session-mode pooler. + ### AWS RDS IAM Authentication The extension supports AWS RDS IAM-based authentication, which allows you to connect to RDS/Aurora PostgreSQL instances using IAM database authentication instead of static passwords. This feature automatically generates temporary authentication tokens using the AWS SDK. @@ -129,27 +191,20 @@ Requires a running Postgres instance reachable via libpq defaults (e.g. `psql` w PGHOST=localhost PGPORT=5432 PGUSER= PGPASSWORD= ./create-postgres-tables.sh ``` -The extension is loaded from `~/.duckdb/extensions///`, copied there on first load and not overwritten on reruns. Create a symlink so the latest build is always used: +Run the tests with `scripts/run-tests.sh`, which also links the freshly built extension into `~/.duckdb/extensions///` (the extension is copied there on first load and not overwritten on reruns otherwise): ```bash -DUCKDB_VERSION=$(ls build/reldebug/repository/) -DUCKDB_PLATFORM=$(ls build/reldebug/repository/${DUCKDB_VERSION}/) -ln -sf "$(pwd)/build/reldebug/extension/postgres_scanner/postgres_scanner.duckdb_extension" \ - ~/.duckdb/extensions/${DUCKDB_VERSION}/${DUCKDB_PLATFORM}/postgres_scanner.duckdb_extension +POSTGRES_TEST_DATABASE_AVAILABLE=1 scripts/run-tests.sh build/release ``` -Run all tests: +Run a single test: ```bash -POSTGRES_TEST_DATABASE_AVAILABLE=1 LOCAL_EXTENSION_REPO=./build/release/repository \ - ./build/release/test/unittest --autoloading available +POSTGRES_TEST_DATABASE_AVAILABLE=1 scripts/run-tests.sh build/release test/sql/storage/attach_simple.test ``` -Run a single test: +The script runs `unittest --keep-home --order lex`. Do not set `LOCAL_EXTENSION_REPO` (or pass `--autoloading`) - the test runner then autoloads extensions instead of loading them with `require`, and settings of the extension are unknown until it is autoloaded. -```bash -POSTGRES_TEST_DATABASE_AVAILABLE=1 LOCAL_EXTENSION_REPO=./build/release/repository \ - ./build/release/test/unittest --autoloading available test/sql/storage/attach_simple.test -``` +The remote pushdown tests pass with both byte-wise (e.g. `C.UTF-8`) and other (e.g. `en_US.UTF-8`, as in the Postgres Docker images used by the CI) database collations. > **Note:** debug and release launchers and extensions should not be mixed — use `build/reldebug` consistently if building with `make reldebug`. diff --git a/create-postgres-tables.sh b/create-postgres-tables.sh index 441605476..3d2c9f72d 100755 --- a/create-postgres-tables.sh +++ b/create-postgres-tables.sh @@ -24,13 +24,14 @@ CALL dbgen(sf=0.01, schema='tpch'); CALL dsdgen(sf=0.01, schema='tpcds'); EXPORT DATABASE '${ABS_DIR_PREFIX}/postgresscannertmp'; " | \ -$DUCKDB_PATH +$DUCKDB_PATH -unsigned dropdb --if-exists postgresscanner createdb postgresscanner psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/schema.sql -psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/load.sql +sed -E "s/,[[:space:]]*allow_quoted_nulls[[:space:]]+[^,)]+//g" ${ABS_DIR_PREFIX}/postgresscannertmp/load.sql > ${ABS_DIR_PREFIX}/postgresscannertmp/load_filtered.sql +psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/load_filtered.sql rm -rf ${ABS_DIR_PREFIX}/postgresscannertmp psql -d postgresscanner < test/all_pg_types.sql diff --git a/database-connector b/database-connector index 7777dac88..c305670b5 160000 --- a/database-connector +++ b/database-connector @@ -1 +1 @@ -Subproject commit 7777dac887a48769684900a627ecaaf6feb3f4c5 +Subproject commit c305670b5729cc6dbda8d395eeb63b4d1487c064 diff --git a/duckdb b/duckdb index f0af2e1b4..03bbf5adc 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit f0af2e1b4462143158289d5bb8028d52ee1f3342 +Subproject commit 03bbf5adcb196cf05f4f46b15ba3daf655a9b5b2 diff --git a/extension-ci-tools b/extension-ci-tools index a00c74e02..78bd1e4ab 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit a00c74e02aefbfa3fd31a4c74c4b6bc260af0de7 +Subproject commit 78bd1e4abbd543822166d5b07644751ba1fc6e8d diff --git a/extension_config.cmake b/extension_config.cmake index 04f1fa899..b19867de3 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -3,7 +3,6 @@ # Extension from this repo duckdb_extension_load(postgres_scanner SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} - DONT_LINK ) duckdb_extension_load(tpch) diff --git a/scripts/win_resources.patch b/scripts/win_resources.patch index ca35ead6a..f2a65b72e 100644 --- a/scripts/win_resources.patch +++ b/scripts/win_resources.patch @@ -1,8 +1,8 @@ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 24afa3db42..d3f04aed2d 100644 +index 9dd2ea45d2..a583df2d9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt -@@ -130,7 +130,6 @@ if(WIN32 AND NOT MINGW) +@@ -132,7 +132,6 @@ if(WIN32 AND NOT MINGW) set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -D DUCKDB_DEV_ITERATION=\"${RC_DEV_ITERATION}\"") @@ -11,14 +11,14 @@ index 24afa3db42..d3f04aed2d 100644 install( diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt -index 0d26359008..04c5c7f01f 100644 +index 7c376be703..956a2df514 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt -@@ -77,7 +77,6 @@ if(WIN32 AND NOT MINGW) +@@ -74,7 +74,6 @@ if(WIN32 AND NOT MINGW) "${CMAKE_RC_FLAGS} -D DUCKDB_DEV_ITERATION=\"${RC_DEV_ITERATION}\"") set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -D DUCKDB_COPYRIGHT_YEAR=\"${DUCKDB_COPYRIGHT_YEAR}\"") - target_sources(shell PRIVATE rc/duckdb.rc) - endif() - - set_target_properties(shell PROPERTIES OUTPUT_NAME duckdb) + # Opt into the Windows Segment Heap to avoid the default NT heap's lock + # contention under multi-threaded allocation (see #24027). CMake merges + # .manifest sources into the linker manifest via mt.exe. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 63d27c0b2..e5113651c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,21 +5,24 @@ add_subdirectory(storage) add_library( postgres_ext_library OBJECT - postgres_attach.cpp postgres_aws.cpp postgres_binary_copy.cpp postgres_binary_file_reader.cpp postgres_binary_parser.cpp postgres_binary_reader.cpp + postgres_connection_string.cpp postgres_connection.cpp postgres_copy_from.cpp postgres_copy_to.cpp - postgres_execute.cpp postgres_extension.cpp postgres_filter_pushdown.cpp postgres_hstore.cpp postgres_parameters.cpp + postgres_pin_connection.cpp + postgres_prepared_statement.cpp postgres_query.cpp + postgres_sql_writer.cpp + postgres_result_reader.cpp postgres_scanner.cpp postgres_secrets.cpp postgres_storage.cpp diff --git a/src/dbconnector/CMakeLists.txt b/src/dbconnector/CMakeLists.txt index 09ece0687..bbb3a8599 100644 --- a/src/dbconnector/CMakeLists.txt +++ b/src/dbconnector/CMakeLists.txt @@ -3,12 +3,13 @@ set(POSTGRES_DBCONNECTOR_PATH add_library( postgres_ext_dbconnector OBJECT + ${POSTGRES_DBCONNECTOR_PATH}/src/attached/attached_catalog.cpp + ${POSTGRES_DBCONNECTOR_PATH}/src/attached/attached_table.cpp + ${POSTGRES_DBCONNECTOR_PATH}/src/functions/configure_pool.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/query/query_writer.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_pushdown.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_util.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/aggregate_optimizer.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/order_by_and_limit_optimizer.cpp) + ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/include/postgres_aws.hpp b/src/include/postgres_aws.hpp index cd14a2bb1..728c0fd4f 100644 --- a/src/include/postgres_aws.hpp +++ b/src/include/postgres_aws.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include "duckdb/main/attached_database.hpp" #include #include "duckdb.hpp" @@ -21,6 +22,12 @@ struct PostgresAwsRdsTokenConfig { std::string rds_secret_name; std::string base_connection_string; + //! Options of the RDS template secret, resolved once at ATTACH time. The template is not looked up + //! by name when the token is re-generated, because such lookup searches all secret storages, + //! including the Postgres secret storage of the attached DB, that requires a new connection. + identifier_tree_t template_options; + //! Overrides the max token age, set from 'aws_rds_token_max_age_seconds' secret option + int64_t max_age_seconds_override = -1; bool Enabled(); int64_t MaxAgeSeconds(); @@ -30,7 +37,8 @@ struct PostgresAws { static std::string GenerateRdsAuthToken(AttachedDatabase &attached_db, const PostgresAwsRdsTokenConfig &token_config); - static PostgresAwsRdsTokenConfig ExtractTokenConfigFromSecret(optional_ptr secret_entry); + static PostgresAwsRdsTokenConfig ExtractTokenConfigFromSecret(ClientContext &context, + optional_ptr secret_entry); }; } // namespace duckdb diff --git a/src/include/postgres_binary_copy.hpp b/src/include/postgres_binary_copy.hpp index fe873fc1a..2e8fb9d53 100644 --- a/src/include/postgres_binary_copy.hpp +++ b/src/include/postgres_binary_copy.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_utils.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_result.hpp" #include "duckdb/function/copy_function.hpp" @@ -34,7 +35,7 @@ class PostgresBinaryCopyFunction : public CopyFunction { GlobalFunctionData &gstate); static unique_ptr PostgresBinaryReadBind(ClientContext &context, CopyFromFunctionBindInput &info, - vector &expected_names, + vector &expected_names, vector &expected_types); }; diff --git a/src/include/postgres_binary_file_reader.hpp b/src/include/postgres_binary_file_reader.hpp index de73ad268..1aff03604 100644 --- a/src/include/postgres_binary_file_reader.hpp +++ b/src/include/postgres_binary_file_reader.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_binary_parser.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/file_system.hpp" namespace duckdb { @@ -16,9 +17,13 @@ namespace duckdb { class PostgresBinaryFileReader { public: static constexpr idx_t DEFAULT_BUFFER_SIZE = 32 * 1024 * 1024; + //! Size of the binary COPY file header: the signature, followed by the flags field and the header + //! extension area length (4 bytes each). Must match the header consumed by PostgresBinaryParser::CheckHeader. + static constexpr idx_t COPY_FILE_HEADER_SIZE = PostgresConversion::COPY_HEADER_LENGTH + 8; PostgresBinaryFileReader(ClientContext &context, const string &file_path, vector types, - vector postgres_types, idx_t buffer_size = DEFAULT_BUFFER_SIZE); + vector postgres_types, PostgresTypeConfig type_config, + idx_t buffer_size = DEFAULT_BUFFER_SIZE); bool ReadChunk(DataChunk &output); @@ -37,6 +42,7 @@ class PostgresBinaryFileReader { idx_t leftover; idx_t leftover_offset; bool finished; + bool header_scanned; }; } // namespace duckdb diff --git a/src/include/postgres_binary_parser.hpp b/src/include/postgres_binary_parser.hpp index 0a1e386b6..e477bd44f 100644 --- a/src/include/postgres_binary_parser.hpp +++ b/src/include/postgres_binary_parser.hpp @@ -9,6 +9,8 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/common/exception/conversion_exception.hpp" +#include "duckdb/common/operator/add.hpp" #include "duckdb/common/types/interval.hpp" #include "postgres_conversion.hpp" #include "postgres_utils.hpp" @@ -17,11 +19,18 @@ namespace duckdb { class PostgresBinaryParser { public: - PostgresBinaryParser(vector types, vector postgres_types); + PostgresBinaryParser(vector types, vector postgres_types, + PostgresTypeConfig type_config); void SetBuffer(data_ptr_t buf, idx_t len); + //! Decode rows from the buffer into output until it is full or the buffer runs out; + //! returns true if output is full. Only the chunk's count is updated, not its vectors' + //! sizes: the caller must call output.SetChildCardinality(output.size()) once it is + //! done filling the chunk. bool ReadChunk(DataChunk &output, const vector &column_ids); void CheckHeader(); + void ParseValue(const LogicalType &type, const PostgresType &postgres_type, int32_t value_len, Vector &out_vec, + idx_t output_offset); private: bool Ready() { @@ -38,6 +47,7 @@ class PostgresBinaryParser { vector types; vector postgres_types; + PostgresTypeConfig type_config; private: template @@ -58,10 +68,13 @@ class PostgresBinaryParser { return val; } + //! Defined out of line so the bounds checks that call it stay small enough to inline + [[noreturn]] static void ThrowOutOfBuffer(const char *function_name); + template inline T ReadInteger() { if (buffer_ptr + sizeof(T) > end) { - throw IOException("Postgres binary reader - out of buffer in ReadInteger"); + ThrowOutOfBuffer("ReadInteger"); } return ReadIntegerUnchecked(); } @@ -110,7 +123,16 @@ class PostgresBinaryParser { if (usec == POSTGRES_NINFINITY) { return timestamp_t::ninfinity(); } - return timestamp_t(usec + (POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS)); + int64_t timestamp_usec; + if (!TryAddOperator::Operation(static_cast(usec), + static_cast(POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS), timestamp_usec)) { + throw ConversionException("timestamp out of range"); + } + auto timestamp = timestamp_t(timestamp_usec); + if (!timestamp.IsFinite()) { + throw ConversionException("timestamp out of range"); + } + return timestamp; } inline interval_t ReadInterval() { @@ -131,7 +153,7 @@ class PostgresBinaryParser { const char *ReadString(idx_t string_length) { if (buffer_ptr + string_length > end) { - throw IOException("Postgres binary reader - out of buffer in ReadString"); + ThrowOutOfBuffer("ReadString"); } auto result = const_char_ptr_cast(buffer_ptr); buffer_ptr += string_length; @@ -139,15 +161,27 @@ class PostgresBinaryParser { } PostgresDecimalConfig ReadDecimalConfig(); + PostgresDecimal ReadDecimalAsDouble(); + + static PostgresDecimalKind NonFiniteDecimalKindFromSign(uint16_t dec_sign); + + static string NonFiniteDecimalKindToString(PostgresDecimalKind kind); template - T ReadDecimal() { + PostgresDecimal ReadDecimal() { // this is wild auto config = ReadDecimalConfig(); + + // we do not support non-finite numerics, need to return NULL or throw + if (config.sign == NUMERIC_NAN || config.sign == NUMERIC_PINF || config.sign == NUMERIC_NINF) { + PostgresDecimalKind kind = NonFiniteDecimalKindFromSign(config.sign); + return PostgresDecimal(kind); + } + auto scale_POWER = OP::GetPowerOfTen(config.scale); if (config.ndigits == 0) { - return 0; + return PostgresDecimal(static_cast(0)); } T integral_part = 0, fractional_part = 0; @@ -199,7 +233,8 @@ class PostgresBinaryParser { // finally auto base_res = OP::Finalize(config, integral_part + fractional_part); - return (config.is_negative ? -base_res : base_res); + auto val = (config.is_negative ? -base_res : base_res); + return PostgresDecimal(val); } void ReadGeometry(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset); @@ -208,6 +243,8 @@ class PostgresBinaryParser { uint32_t current_count, uint32_t dimensions[], uint32_t ndim); void ReadValue(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset); + + bool CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind); }; } // namespace duckdb diff --git a/src/include/postgres_binary_reader.hpp b/src/include/postgres_binary_reader.hpp index 26f89fbe3..200e5b583 100644 --- a/src/include/postgres_binary_reader.hpp +++ b/src/include/postgres_binary_reader.hpp @@ -9,14 +9,18 @@ #pragma once #include "postgres_binary_parser.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_result_reader.hpp" #include "postgres_connection.hpp" namespace duckdb { +enum class PostgresBinaryReaderMode { UNINITIALIZED, COPY, PLAIN_QUERY }; + struct PostgresBinaryReader : public PostgresResultReader { explicit PostgresBinaryReader(PostgresConnection &con, const vector &column_ids, - const PostgresBindData &bind_data); + const PostgresBindData &bind_data, + optional_ptr prepared_statement = nullptr); ~PostgresBinaryReader() override; public: @@ -27,9 +31,17 @@ struct PostgresBinaryReader : public PostgresResultReader { bool FetchNextBuffer(); void FreeBuffer(); + PostgresReadResult ReadCopy(DataChunk &result); + PostgresReadResult ReadPlain(DataChunk &result); + private: + PostgresBinaryReaderMode mode = PostgresBinaryReaderMode::UNINITIALIZED; PostgresBinaryParser parser; + // copy mode data_ptr_t buffer = nullptr; + // plain mode + unique_ptr result; + idx_t row_offset = 0; }; } // namespace duckdb diff --git a/src/include/postgres_binary_writer.hpp b/src/include/postgres_binary_writer.hpp index 1d951afa7..77d2acb7e 100644 --- a/src/include/postgres_binary_writer.hpp +++ b/src/include/postgres_binary_writer.hpp @@ -9,7 +9,9 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/common/operator/subtract.hpp" #include "duckdb/common/types/interval.hpp" +#include "duckdb/common/types/geometry_crs.hpp" #include "duckdb/common/serializer/memory_stream.hpp" #include "duckdb/common/vector/list_vector.hpp" #include "duckdb/common/vector/map_vector.hpp" @@ -105,12 +107,12 @@ class PostgresBinaryWriter { } void WriteTime(dtime_t value) { - WriteInteger(value.micros); + WriteInteger(value.value); } void WriteTimeTZ(dtime_tz_t value) { WriteRawInteger(sizeof(uint64_t) + sizeof(int32_t)); - WriteRawInteger(value.time().micros); + WriteRawInteger(value.time().value); WriteRawInteger(-value.offset()); } @@ -121,7 +123,14 @@ class PostgresBinaryWriter { if (value == timestamp_t::ninfinity()) { return POSTGRES_NINFINITY; } - return uint64_t(value.value - (POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS)); + int64_t postgres_usec; + if (!TrySubtractOperator::Operation(value.value, static_cast(POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS), + postgres_usec) || + postgres_usec < POSTGRES_MIN_TIMESTAMP || postgres_usec >= POSTGRES_END_TIMESTAMP) { + throw InvalidInputException("TIMESTAMP \"%s\" is out of range for Postgres' TIMESTAMP field", + Timestamp::ToString(value)); + } + return static_cast(postgres_usec); } void WriteTimestamp(timestamp_t value) { @@ -210,6 +219,44 @@ class PostgresBinaryWriter { stream.WriteData(const_data_ptr_cast(str_data), str_size); } + //! Write GEOMETRY to PostGIS. If the type carries CRS metadata, writes EWKB + //! (WKB with SRID) so PostGIS receives the correct SRID. + void WriteGeometry(string_t value, const LogicalType &type) { + if (!GeoType::HasCRS(type)) { + WriteRawBlob(value); + return; + } + auto &crs = GeoType::GetCRS(type); + + // Extract SRID from CRS identifier (e.g., "EPSG:4326" → 4326) + auto &id = crs.GetIdentifier(); + auto colon = id.find(':'); + if (colon == string::npos) { + WriteRawBlob(value); + return; + } + int32_t srid; + try { + srid = std::stoi(id.substr(colon + 1)); + } catch (...) { + WriteRawBlob(value); + return; + } + + // Write EWKB: WKB with SRID flag set on the type field + 4-byte SRID inserted. + // [byte_order:1] [type|0x20000000:4] [srid:4] [coordinates...] + auto wkb_size = value.GetSize(); + auto wkb_data = const_data_ptr_cast(value.GetData()); + WriteRawInteger(NumericCast(wkb_size + 4)); + stream.WriteData(wkb_data, 1); // byte order + uint32_t wkb_type; + memcpy(&wkb_type, wkb_data + 1, 4); + wkb_type |= 0x20000000; + stream.WriteData(const_data_ptr_cast(reinterpret_cast(&wkb_type)), 4); // type + SRID flag + stream.WriteData(const_data_ptr_cast(reinterpret_cast(&srid)), 4); // SRID (LE, matching WKB) + stream.WriteData(wkb_data + 5, wkb_size - 5); // rest of payload + } + void WriteVarchar(string_t value) { auto str_size = value.GetSize(); auto str_data = value.GetData(); @@ -357,7 +404,7 @@ class PostgresBinaryWriter { } case LogicalTypeId::GEOMETRY: { auto data = FlatVector::GetData(col)[r]; - WriteRawBlob(data); + WriteGeometry(data, type); break; } case LogicalTypeId::ENUM: { diff --git a/src/include/postgres_connection.hpp b/src/include/postgres_connection.hpp index 8d5cc7b72..c1ae76c38 100644 --- a/src/include/postgres_connection.hpp +++ b/src/include/postgres_connection.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_utils.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_parameters.hpp" #include "postgres_result.hpp" #include "duckdb/common/shared_ptr.hpp" @@ -21,21 +22,12 @@ class PostgresSchemaEntry; class PostgresTableEntry; class PostgresStatement; class PostgresResult; +class PostgresPreparedStatement; struct IndexInfo; -struct OwnedPostgresConnection { - explicit OwnedPostgresConnection(PGconn *conn = nullptr); - OwnedPostgresConnection(const OwnedPostgresConnection &) = delete; - OwnedPostgresConnection &operator=(const OwnedPostgresConnection &) = delete; - ~OwnedPostgresConnection(); - - PGconn *connection; - mutex connection_lock; -}; - class PostgresConnection { public: - explicit PostgresConnection(shared_ptr connection = nullptr); + explicit PostgresConnection(PGconn *connection = nullptr); ~PostgresConnection(); // disable copy constructors PostgresConnection(const PostgresConnection &other) = delete; @@ -47,12 +39,15 @@ class PostgresConnection { public: static PostgresConnection Open(const string &dsn, const string &attach_path); void Execute(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); unique_ptr TryQuery(optional_ptr context, const string &query, optional_ptr error_message = nullptr, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); unique_ptr Query(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); + //! Executes a statement that was prepared on this connection + unique_ptr QueryStmt(optional_ptr context, const PostgresPreparedStatement &stmt, + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); //! Submits a set of queries to be executed in the connection. vector> ExecuteQueries(ClientContext &context, const string &queries); @@ -76,28 +71,33 @@ class PostgresConnection { bool PingServer(const std::string &health_check_query); void Reset(const std::string &health_check_query); - shared_ptr GetConnection() { - return connection; - } string GetDSN() { return dsn; } + //! Locks the connection, must be held when calling libpq functions on the result of GetConn() + unique_lock Lock() { + return unique_lock(connection_lock); + } PGconn *GetConn() { - if (!connection || !connection->connection) { + if (!connection) { throw InternalException("PostgresConnection::GetConn - no connection available"); } - return connection->connection; + return connection; } static void DebugSetPrintQueries(bool print); static bool DebugPrintQueries(); private: + //! Requires connection_lock to be held PGresult *PQExecute(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); + //! Requires connection_lock to be held + bool PingServerInternal(const std::string &health_check_query); - shared_ptr connection; + PGconn *connection = nullptr; + mutex connection_lock; string dsn; }; diff --git a/src/include/postgres_connection_string.hpp b/src/include/postgres_connection_string.hpp new file mode 100644 index 000000000..d65fb9592 --- /dev/null +++ b/src/include/postgres_connection_string.hpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// postgres_binary_writer.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "duckdb.hpp" + +namespace duckdb { + +enum class PostgresConnectionStringFormat { KEY_VALUE, URI }; + +struct PostgresConnectionString { + static string Escape(const string &input); + + static string ExtractOption(const KeyValueSecret &kv_secret, const string &name); + + static bool ContainsApplicationName(const std::string &conn_str); + + //! Whether libpq reads the connection string as a URI rather than key/value pairs + static bool IsURI(const string &conn_string); + + static void AppendDefaultApplicationName(string &conn_string, PostgresConnectionStringFormat format); +}; + +} // namespace duckdb diff --git a/src/include/postgres_conversion.hpp b/src/include/postgres_conversion.hpp index 391429e50..1d1241c9c 100644 --- a/src/include/postgres_conversion.hpp +++ b/src/include/postgres_conversion.hpp @@ -29,8 +29,11 @@ #define POSTGRES_DATE_NINF 2147483648U #define POSTGRES_EPOCH_TS 211813488000000000ULL #define DUCKDB_EPOCH_TS 210866803200000000ULL -#define POSTGRES_INFINITY 9223372036854775807ULL -#define POSTGRES_NINFINITY 9223372036854775808ULL +/* Values from PostgreSQL's datatype/timestamp.h. The end is exclusive. */ +#define POSTGRES_MIN_TIMESTAMP (-211813488000000000LL) +#define POSTGRES_END_TIMESTAMP 9223371331200000000LL +#define POSTGRES_INFINITY 9223372036854775807ULL +#define POSTGRES_NINFINITY 9223372036854775808ULL #define NBASE 10000 #define DEC_DIGITS 4 /* decimal digits per NBASE digit */ @@ -96,6 +99,21 @@ struct PostgresDecimalConfig { uint16_t ndigits; int16_t weight; bool is_negative; + uint16_t sign; +}; + +enum class PostgresDecimalKind { ORDINARY, NOT_A_NUMBER, POSITIVE_INFINITY, NEGATIVE_INFINITY }; + +template +struct PostgresDecimal { + T value; + PostgresDecimalKind kind; + + PostgresDecimal(T val) : value(val), kind(PostgresDecimalKind::ORDINARY) { + } + + PostgresDecimal(PostgresDecimalKind kind_p) : value(0), kind(kind_p) { + } }; struct PostgresConversion { @@ -190,14 +208,4 @@ struct DecimalConversionHugeint { } }; -struct DecimalConversionDouble { - static double GetPowerOfTen(idx_t index) { - return pow(10, double(index)); - } - - static double Finalize(const PostgresDecimalConfig &config, double result) { - return result / GetPowerOfTen(config.scale); - } -}; - } // namespace duckdb diff --git a/src/include/postgres_filter_pushdown.hpp b/src/include/postgres_filter_pushdown.hpp index 3b1a02988..d91983dec 100644 --- a/src/include/postgres_filter_pushdown.hpp +++ b/src/include/postgres_filter_pushdown.hpp @@ -8,14 +8,19 @@ #pragma once +#include "duckdb/planner/filter/expression_filter.hpp" +#include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/table_filter_set.hpp" +#include "postgres_version.hpp" namespace duckdb { class PostgresFilterPushdown { public: + static bool CanPushExpressionDown(ClientContext &, const LogicalGet &, Expression &expr); + static string TransformFilters(const vector &column_ids, optional_ptr filters, - const vector &names); + const vector &names, PostgresInstanceType instance_type); private: // TODO diff --git a/src/include/postgres_logging.hpp b/src/include/postgres_logging.hpp index 1fc9650b9..ef4c840a8 100644 --- a/src/include/postgres_logging.hpp +++ b/src/include/postgres_logging.hpp @@ -2,6 +2,7 @@ #pragma once #include "duckdb/logging/logging.hpp" +#include "duckdb/logging/log_type.hpp" #include "duckdb/common/string_util.hpp" namespace duckdb { diff --git a/src/include/postgres_oauth.hpp b/src/include/postgres_oauth.hpp index 2e36feede..6e1a62eae 100644 --- a/src/include/postgres_oauth.hpp +++ b/src/include/postgres_oauth.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" namespace duckdb { diff --git a/src/include/postgres_prepared_statement.hpp b/src/include/postgres_prepared_statement.hpp new file mode 100644 index 000000000..8ce5c9bc7 --- /dev/null +++ b/src/include/postgres_prepared_statement.hpp @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// postgres_prepared_statement.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "postgres_result.hpp" + +namespace duckdb { +class PostgresConnection; + +//! Owns a prepared statement created with PQprepare: holds the statement name, the query text, the ID of the +//! pooled connection the statement was prepared on and the PQprepare result. The constructor throws if the +//! preparation failed. +//! TODO: unless Destroy() is called, the statement is not closed on the server when this object is destroyed, it +//! is leaked until the connection is closed. +class PostgresPreparedStatement { +public: + PostgresPreparedStatement(PGconn *conn, uint64_t pool_connection_id_p, string name_p, string query_p, + PGresult *prepare_result_p); + + PostgresPreparedStatement(const PostgresPreparedStatement &) = delete; + PostgresPreparedStatement &operator=(const PostgresPreparedStatement &) = delete; + +public: + //! Generates a prepared statement name that is unique within the process + static string GenerateName(); + + uint64_t GetPoolConnectionId() const; + const string &GetName() const; + const string &GetQuery() const; + PGresult *GetPrepareResult() const; + + //! Closes the statement on the server, con must be the connection the statement was prepared on + void Destroy(PostgresConnection &con); + bool IsDestroyed() const; + +private: + uint64_t pool_connection_id; + string name; + string query; + PostgresResult prepare_result; + bool destroyed = false; +}; + +} // namespace duckdb diff --git a/src/include/postgres_result.hpp b/src/include/postgres_result.hpp index 262adfef5..2ba2e1f67 100644 --- a/src/include/postgres_result.hpp +++ b/src/include/postgres_result.hpp @@ -48,10 +48,14 @@ class PostgresResult { D_ASSERT(res); return PQntuples(res); } - idx_t AffectedRows() { - auto affected = PQcmdTuples(res); - if (!affected) { - throw InternalException("Postgres scanner - AffectedRows called but none were available"); + idx_t ColumnCount() { + D_ASSERT(res); + return PQnfields(res); + } + int64_t AffectedRows() { + char *affected = PQcmdTuples(res); + if (!affected || strlen(affected) == 0) { + return -1; } return atoll(affected); } diff --git a/src/include/postgres_result_reader.hpp b/src/include/postgres_result_reader.hpp index 1915c3b53..a668a1f36 100644 --- a/src/include/postgres_result_reader.hpp +++ b/src/include/postgres_result_reader.hpp @@ -9,20 +9,24 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/types/interval.hpp" #include "postgres_conversion.hpp" #include "postgres_utils.hpp" +#include "postgres_result.hpp" namespace duckdb { class PostgresConnection; +class PostgresPreparedStatement; struct PostgresBindData; enum class PostgresReadResult { FINISHED, HAVE_MORE_TUPLES }; struct PostgresResultReader { explicit PostgresResultReader(PostgresConnection &con_p, const vector &column_ids, - const PostgresBindData &bind_data) - : con(con_p), column_ids(column_ids), bind_data(bind_data) { + const PostgresBindData &bind_data, + optional_ptr prepared_statement_p = nullptr) + : con(con_p), column_ids(column_ids), bind_data(bind_data), prepared_statement(prepared_statement_p) { } virtual ~PostgresResultReader() = default; @@ -34,10 +38,18 @@ struct PostgresResultReader { virtual void BeginCopy(ClientContext &context, const string &sql) = 0; virtual PostgresReadResult Read(DataChunk &result) = 0; +protected: + //! Runs the query, executing the statement prepared in bind instead if it was not used yet and the query matches + unique_ptr Query(ClientContext &context, const string &sql, int format); + //! Closes the statement prepared in bind if this reader has it and it was not closed yet + void DestroyPreparedStatement(); + protected: PostgresConnection &con; const vector &column_ids; const PostgresBindData &bind_data; + //! Statement prepared on con, executed instead of the query if the query matches it + optional_ptr prepared_statement; }; } // namespace duckdb diff --git a/src/include/postgres_scanner.hpp b/src/include/postgres_scanner.hpp index 9e1fb6d3c..699c11650 100644 --- a/src/include/postgres_scanner.hpp +++ b/src/include/postgres_scanner.hpp @@ -9,12 +9,12 @@ #pragma once #include "duckdb.hpp" - -#include "dbconnector/bind_data.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_utils.hpp" #include "postgres_connection.hpp" #include "postgres_parameters.hpp" +#include "postgres_prepared_statement.hpp" #include "storage/postgres_connection_pool.hpp" namespace duckdb { @@ -23,15 +23,25 @@ struct PostgresLocalState; struct PostgresGlobalState; class PostgresTransaction; -struct PostgresBindData : public dbconnector::BindData { +struct PostgresBindData : public FunctionData { static constexpr const idx_t DEFAULT_PAGES_PER_TASK = 1000; public: - PostgresBindData(ClientContext &context); + PostgresBindData(ClientContext &context, const Identifier &catalog_name); - PostgresVersion version; string schema_name; string table_name; + + Identifier catalog_name; + QualifiedName qualified_table_name; + + PostgresQueryMode query_mode = PostgresQueryMode::AUTO; + + // required for get_bind_info and only used there + weak_ptr context_ptr; + + //! Set by postgres_query, executed (without parameters) on the scan connection before the main query + vector preliminary_queries; string sql; PostgresParameters params; idx_t pages_approx = 0; @@ -50,49 +60,35 @@ struct PostgresBindData : public dbconnector::BindData { bool emit_ctid = false; bool use_transaction = true; bool use_text_protocol = false; + //! Plain queries with binary results are used, but a scan falls back to the text protocol when it reads a column + //! that can only be read using the text protocol + bool text_protocol_fallback = false; //! Set by postgres_query's bind when the statement returns no columns (a command like DDL, or //! DML without RETURNING). InitGlobalState executes it and returns a single-row Success result. bool command_only = false; + //! Set by postgres_query, clears the attached catalog cache after the query is executed + bool clear_catalog_cache = false; idx_t max_threads = 1; + PostgresTypeConfig type_config; + //! The server the scan reads from - set in PrepareBind + PostgresInstanceType instance_type = PostgresInstanceType::POSTGRES; + + string limit_clause; + + uint64_t pinned_connection_id = 0; - dbconnector::optimizer::OrderByAndLimitBindData order_by_and_limit_bind_data; - dbconnector::optimizer::AggregateBindData aggregate_bind_data; + //! Statement prepared by postgres_query's bind to describe the query result + unique_ptr prepared_statement; public: void SetTablePages(idx_t approx_num_pages); - void SetCatalog(PostgresCatalog &catalog); - void SetTable(PostgresTableEntry &table); - optional_ptr GetCatalog() const { - return pg_catalog; - } - optional_ptr GetTable() const { - return pg_table; - } - unique_ptr Copy() const override { throw NotImplementedException(""); } bool Equals(const FunctionData &other_p) const override { return false; } - - dbconnector::optimizer::OrderByAndLimitBindData &GetOrderByAndLimitBindData() override { - return order_by_and_limit_bind_data; - } - - dbconnector::optimizer::AggregateBindData &GetAggregateBindData() override { - return aggregate_bind_data; - } - -private: - optional_ptr pg_catalog; - optional_ptr pg_table; -}; - -class PostgresAttachFunction : public TableFunction { -public: - PostgresAttachFunction(); }; class PostgresScanFunction : public TableFunction { @@ -100,12 +96,7 @@ class PostgresScanFunction : public TableFunction { PostgresScanFunction(); static void PrepareBind(PostgresVersion version, ClientContext &context, PostgresBindData &bind, - int64_t approx_num_pages); -}; - -class PostgresScanFunctionFilterPushdown : public TableFunction { -public: - PostgresScanFunctionFilterPushdown(); + int64_t approx_num_pages, optional_ptr pg_catalog); }; class PostgresClearCacheFunction : public TableFunction { @@ -126,4 +117,14 @@ class PostgresExecuteFunction : public TableFunction { PostgresExecuteFunction(); }; +class PostgresPinConnectionFunction : public ScalarFunction { +public: + PostgresPinConnectionFunction(); +}; + +class PostgresClosePinnedConnectionFunction : public ScalarFunction { +public: + PostgresClosePinnedConnectionFunction(); +}; + } // namespace duckdb diff --git a/src/include/postgres_secrets.hpp b/src/include/postgres_secrets.hpp index a4713abfe..45e4b5dc1 100644 --- a/src/include/postgres_secrets.hpp +++ b/src/include/postgres_secrets.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/main/secret/secret_manager.hpp" namespace duckdb { diff --git a/src/include/postgres_sql_writer.hpp b/src/include/postgres_sql_writer.hpp new file mode 100644 index 000000000..1e9315435 --- /dev/null +++ b/src/include/postgres_sql_writer.hpp @@ -0,0 +1,256 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// postgres_sql_writer.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/optional_ptr.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/parser/parsed_expression.hpp" +#include "duckdb/parser/query_node.hpp" +#include "duckdb/parser/result_modifier.hpp" +#include "duckdb/parser/sql_statement.hpp" +#include "duckdb/parser/tableref.hpp" + +namespace duckdb { +class BaseTableRef; +class AlterInfo; +class ColumnDefinition; +class CreateStatement; +struct CreateViewInfo; +struct DropInfo; +class DeleteQueryNode; +class ExpressionListRef; +class InsertQueryNode; +class MergeQueryNode; +class OnConflictInfo; +class UpdateQueryNode; +class UpdateSetInfo; +class CaseExpression; +class FunctionArgument; +class ColumnRefExpression; +class FunctionExpression; +class OperatorExpression; +class PostgresCatalog; +class RecursiveCTENode; +class SelectNode; +class SetOperationNode; +class SubqueryExpression; +class TableCatalogEntry; +class WindowExpression; + +//! Thrown by the writer when a construct turns out not to be translatable once its types are known - the +//! query is then executed locally instead +class PostgresPushdownNotSupported : public std::exception { +public: + explicit PostgresPushdownNotSupported(string reason_p) : reason(std::move(reason_p)) { + } + const char *what() const noexcept override { + return reason.c_str(); + } + +private: + string reason; +}; + +//! A catalog entry that is modified by a statement, and has to be removed from the cache +struct PostgresCacheInvalidation { + CatalogType type; + string schema; + string name; + //! Remove all tables of the schema from the cache + bool all_tables = false; +}; + +//! The result of serializing a query or statement for Postgres +struct PostgresWriterResult { + //! Statements executed before the main statement (e.g. setting the session time zone) + vector preliminary_queries; + //! The main statement, its result is returned to DuckDB + string sql; + //! Whether the main statement is a DML statement whose number of changed rows is the result + bool changed_rows = false; + //! Whether the result of the statement depends on the session time zone + bool requires_time_zone = false; + //! The cached catalog entries that are modified by the statement + vector invalidations; +}; + +//! Serializes a parsed query tree into Postgres SQL. +//! This mirrors the structure of the DuckDB ToString() methods, with Postgres-specific differences +//! applied during serialization: +//! * identifiers are quoted using the spelling of the corresponding Postgres catalog entries +//! (DuckDB identifiers are case-insensitive, quoted Postgres identifiers are not) +//! * base tables are always qualified with their Postgres schema +//! * functions and operators are remapped where semantics are identical under a different +//! name or form (e.g. count_star() -> count(*), year(x) -> EXTRACT(YEAR FROM x), a / b -> float8 division) +//! * ORDER BY entries get an explicit NULLS FIRST / NULLS LAST +//! Constructs that cannot be serialized are rejected up-front by PostgresCapabilities - encountering +//! them here is an internal error. +class PostgresSQLWriter { +public: + PostgresSQLWriter(ClientContext &context, PostgresCatalog &catalog); + + //! Serialize a query node - result_types are the types DuckDB itself binds the query to (if known) + static PostgresWriterResult WriteQuery(ClientContext &context, PostgresCatalog &catalog, QueryNode &node, + optional_ptr> result_types); + + //! Serialize a (DDL) statement + static PostgresWriterResult WriteStatement(ClientContext &context, PostgresCatalog &catalog, + SQLStatement &statement); + + //! Collect the names of all CTEs defined anywhere in the query + static void CollectCTENames(QueryNode &node, identifier_set_t &cte_names); + //! Qualify all base table references with the catalog name again (the optimizer strips it before + //! pushdown), so that the query can be bound by DuckDB itself + static void QualifyTables(QueryNode &node, const Identifier &catalog_name, const identifier_set_t &cte_names); + +private: + //! Collect the Postgres spelling of all identifiers referenced by the query + void CollectNames(QueryNode &node); + void CollectNames(TableRef &ref); + void CollectNames(ParsedExpression &expr); + void AddCatalogName(const string &name); + void AddCatalogColumnName(const string &name); + void AddQueryName(const Identifier &name); + optional_ptr LookupTable(const BaseTableRef &ref); + optional_ptr LookupTable(const QualifiedName &qualified_name); + void AddTableNames(TableCatalogEntry &table); + bool IsCTEReference(const BaseTableRef &ref); + + void CollectResultNodes(const QueryNode &node); + string WriteResultCast(const string &expr, idx_t column_index); + string WriteResultCast(const string &expr, const LogicalType &type); + //! The type DuckDB binds an expression at the current position in the query to - INVALID if unknown + LogicalType InferType(const ParsedExpression &expr); + LogicalType InferTypeInternal(const ParsedExpression &expr); + //! The type of a column of a base table of an enclosing query (e.g. a correlated or a LATERAL reference) + LogicalType InferOuterColumnType(const ColumnRefExpression &column_ref); + //! The result types of a query (empty if they cannot be determined) + vector InferQueryTypes(const QueryNode &node); + void CollectScopeTables(const TableRef &ref, vector> &tables, bool &only_base_tables); + //! The table a (base table) binding refers to at the current position in the query + optional_ptr FindBindingTable(const Identifier &binding); + string WriteOrderingKey(const ParsedExpression &expr); + //! "ordering" - whether the operands are ordered (e.g. <, BETWEEN), as opposed to compared for equality + vector WriteComparisonOperands(const vector> &operands, + bool ordering = false); + //! Write an expression that is ordered (ORDER BY key, operand of <, min, ...) - DuckDB orders strings + //! byte-wise, so the C collation is pinned when the database collation orders them differently + string WriteOrderedOperand(const string &expr_str, const LogicalType &type); + string WriteOrderedExpression(const ParsedExpression &expr); + string WriteStringOperand(const ParsedExpression &expr); + string WriteCast(const ParsedExpression &child, const LogicalType &target_type); + string WriteCast(const string &child_str, const LogicalType &source_type, const LogicalType &target_type); + string WriteLength(const string &name, const ParsedExpression &child); + + string WriteQueryNode(const QueryNode &node); + void WriteCreateTable(CreateStatement &statement, PostgresWriterResult &result); + void WriteCreateView(CreateViewInfo &info, PostgresWriterResult &result); + string WriteDrop(const DropInfo &info, PostgresWriterResult &result); + void VerifyEntryExists(CatalogType type, const QualifiedName &name); + string WriteAlter(const AlterInfo &info, PostgresWriterResult &result); + string WriteQualifiedTableName(const QualifiedName &name); + //! The Postgres schema and name of an entry + pair ResolveEntryName(const QualifiedName &name); + string WriteSelectNode(const SelectNode &node); + string WriteSetOperationNode(const SetOperationNode &node); + string WriteRecursiveCTENode(const RecursiveCTENode &node); + string WriteUpdateNode(const UpdateQueryNode &node); + string WriteDeleteNode(const DeleteQueryNode &node); + string WriteInsertNode(const InsertQueryNode &node); + string WriteMergeNode(const MergeQueryNode &node); + string WriteOnConflict(const OnConflictInfo &info, TableCatalogEntry &table, const Identifier &target_binding, + const vector> &columns, + optional_ptr values_list, + const vector> &value_columns); + string WriteSetClause(const UpdateSetInfo &set_info, TableCatalogEntry &table, bool on_conflict); + string WriteAssignment(const ParsedExpression &expr, TableCatalogEntry &table, const ColumnDefinition &column, + optional_ptr source_type_override = nullptr); + string WriteAssignmentCast(const string &expr_str, const LogicalType &source_type, TableCatalogEntry &table, + const ColumnDefinition &column); + string WriteColumnName(TableCatalogEntry &table, const ColumnDefinition &column); + string WriteReturningList(const vector> &returning_list); + TableCatalogEntry &GetTargetTable(const TableRef &ref); + string WriteCTEMap(const CommonTableExpressionMap &cte_map); + string WriteResultModifiers(const QueryNode &node, + optional_ptr>> select_list = nullptr); + //! "node" - the query whose result is ordered, nullptr for the ORDER BY of a window / aggregate + string WriteOrderList(const vector &orders, optional_ptr node = nullptr); + + string WriteTableRef(const TableRef &ref, bool lateral = false); + string WriteTableAlias(const TableRef &ref); + + string WriteExpression(const ParsedExpression &expr); + string WriteExpressionList(const vector> &expressions, const string &separator = ", "); + string WriteArgumentList(const vector &arguments); + string WriteFunction(const FunctionExpression &func); + string WriteFunctionCall(const string &function_name, const vector &children, bool distinct, + const string &order_by); + string WriteAggregate(const FunctionExpression &func, const string &name); + bool TryWriteSpecialFunction(const string &name, const vector &children, string &result); + string WriteDatePart(const string &part, const ParsedExpression &arg); + string WriteDateDiff(const string &part, const ParsedExpression &start, const ParsedExpression &end); + string WriteRound(const vector &children, bool half_even); + string WriteDivision(const ParsedExpression &left, const ParsedExpression &right); + bool HasDefaultDivisionSemantics(); + string WriteFrameOffset(const WindowExpression &window, const ParsedExpression &offset, const string &mode); + string WriteWindow(const WindowExpression &window); + string WriteOperator(const OperatorExpression &op); + string WriteCase(const CaseExpression &case_expr); + string WriteSubquery(const SubqueryExpression &subquery); + string WriteColumnRef(const ColumnRefExpression &column_ref); + string WriteConstant(const Value &value); + string WriteType(const LogicalType &type); + string WriteIdentifier(const Identifier &identifier); + string WriteIdentifier(const string &identifier); + +private: + ClientContext &context; + PostgresCatalog &catalog; + //! The Postgres spelling of identifiers, keyed case-insensitively + case_insensitive_map_t identifier_spelling; + //! The names of the columns of the tables referenced by the query + case_insensitive_set_t catalog_columns; + //! The types columns are read as by DuckDB, if they differ from their Postgres type ("?" if ambiguous) + case_insensitive_map_t column_read_types; + //! The aliases of single-column table functions (without column aliases), and the names of the functions + case_insensitive_map_t table_function_aliases; + //! The tables base table references are bound to + case_insensitive_map_t> binding_tables; + //! The names tables are bound to in the query (table names, aliases, CTE names) + case_insensitive_set_t table_bindings; + //! Identifiers which have multiple spellings in the Postgres catalog - written as specified in the query + case_insensitive_set_t ambiguous_identifiers; + //! The CTE names visible at the current position in the query + vector> cte_scopes; + //! The tables in scope at the current position in the query (e.g. the FROM clause of the enclosing SELECT, + //! or the target and FROM clause of an UPDATE) + vector>>> table_scopes; + //! Whether the query formats dates / times as text, which requires the ISO DateStyle + bool requires_iso_date_style = false; + //! Whether the result of the query depends on the session time zone + bool requires_time_zone = false; + //! Cache of inferred expression types + unordered_map type_cache; + //! The SELECT nodes producing the query result (e.g. the children of a top-level UNION), and the + //! types DuckDB binds the result to + unordered_set result_nodes; + optional_ptr> result_types; + //! The nesting depth of the query node that is being written + idx_t query_depth = 0; + //! The target table of the ON CONFLICT DO UPDATE clause that is being written (if any) + optional_ptr on_conflict_target; + idx_t on_conflict_depth = 0; + string on_conflict_binding; + //! Counter for generating aliases for unnamed subqueries + idx_t unnamed_subquery_index = 0; +}; + +} // namespace duckdb diff --git a/src/include/postgres_text_reader.hpp b/src/include/postgres_text_reader.hpp index cbd9491af..cf449b911 100644 --- a/src/include/postgres_text_reader.hpp +++ b/src/include/postgres_text_reader.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_result_reader.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_connection.hpp" #include "postgres_result.hpp" @@ -16,7 +17,8 @@ namespace duckdb { struct PostgresTextReader : public PostgresResultReader { explicit PostgresTextReader(ClientContext &context, PostgresConnection &con, const vector &column_ids, - const PostgresBindData &bind_data); + const PostgresBindData &bind_data, + optional_ptr prepared_statement = nullptr); ~PostgresTextReader() override; public: diff --git a/src/include/postgres_utils.hpp b/src/include/postgres_utils.hpp index 84c07968b..c095ebd5d 100644 --- a/src/include/postgres_utils.hpp +++ b/src/include/postgres_utils.hpp @@ -8,20 +8,36 @@ #pragma once +#include + #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include #include "postgres_version.hpp" #include "duckdb/main/secret/secret_manager.hpp" namespace duckdb { +class PostgresCatalog; class PostgresSchemaEntry; class PostgresTransaction; +enum class PostgresQueryMode { AUTO, COPY_BINARY, PLAIN_QUERY_BINARY, PLAIN_QUERY_TEXT }; + struct PostgresTypeData { int64_t type_modifier = 0; string type_name; string type_schema; idx_t array_dimensions = 0; + //! The server the type comes from - some type names (e.g. "geometry") have server-specific semantics + PostgresInstanceType instance_type = PostgresInstanceType::POSTGRES; +}; + +struct PostgresTypeConfig { + bool array_as_varchar = false; + bool numeric_as_varchar = false; + bool numeric_nan_as_null = true; + + static PostgresTypeConfig FromContext(optional_ptr context); }; enum class PostgresTypeAnnotation { @@ -37,7 +53,15 @@ enum class PostgresTypeAnnotation { GEOM_BOX, GEOM_PATH, GEOM_POLYGON, - GEOM_CIRCLE + GEOM_CIRCLE, + //! hex encoded binary data without the "\\x" prefix (Redshift VARBYTE) + HEX_BLOB, + //! Redshift INTERVAL YEAR TO MONTH - the binary format is the number of months (int32) + INTERVAL_YEAR_TO_MONTH, + //! Redshift INTERVAL DAY TO SECOND - the binary format is the number of microseconds (int64) + INTERVAL_DAY_TO_SECOND, + //! Redshift HLLSKETCH - can only be read using the text protocol + TEXT_PROTOCOL_ONLY }; struct PostgresType { @@ -58,16 +82,27 @@ struct PostgresCopyState { enum class PostgresIsolationLevel { READ_COMMITTED, REPEATABLE_READ, SERIALIZABLE }; +enum class PostgresConnectCheck { THROW_ON_CONNECTION_BAD, RETURN_ON_CONNECTION_BAD }; + +enum class PostgresUseCtidScan : uint8_t { DEFAULT, ENABLED, DISABLED }; + class PostgresUtils { public: static PGconn *PGConnect(const string &dsn, const string &attach_path); + static bool UseInformationSchemaIntrospection(optional_ptr context); + static bool StalenessQueryEnabled(ClientContext &context); + static string GetCustomStalenessQuery(ClientContext &context); + static string DataTypeToTypeName(const string &data_type); static LogicalType ToPostgresType(const LogicalType &input); static LogicalType TypeToLogicalType(optional_ptr transaction, - optional_ptr schema, const PostgresTypeData &input, + optional_ptr schema, + const PostgresTypeConfig &type_config, const PostgresTypeData &input, PostgresType &postgres_type); static string TypeToString(const LogicalType &input); static string PostgresOidToName(uint32_t oid); + //! Maps the OIDs of the Redshift-specific types to the type names, returns an empty string for other OIDs + static string RedshiftOidToName(uint32_t oid); static uint32_t ToPostgresOid(const LogicalType &input); static uint32_t TypeNameToPostgresOid(const string &type_name); static bool SupportedPostgresOid(const LogicalType &input); @@ -77,12 +112,22 @@ class PostgresUtils { static PostgresVersion ExtractPostgresVersion(const string &version); - static string EscapeConnectionString(const string &input); - static string ExtractConnectionOption(const KeyValueSecret &kv_secret, const string &name); static string WriteLiteral(const string &identifier); + static string WriteLiteralsCommaSeparated(const vector &literals); static string WriteIdentifier(const string &identifier); + static PostgresQueryMode ExtractQueryMode(ClientContext &ctx); + static PostgresQueryMode ExtractQueryMode(Value &value); + static PostgresCatalog &GetCatalogByName(ClientContext &context, const std::string &name); + + static void DeduplicateColumns(vector &names); + private: + static mutex libpq_init_lock; + static bool libpq_connect_succeeded_at_least_once; + + static PGconn *PGConnectInternal(const string &dsn, const string &attach_path, PostgresConnectCheck check); + static string EscapeQuotes(const string &text, char quote); static string WriteQuoted(const string &text, char quote); }; diff --git a/src/include/postgres_version.hpp b/src/include/postgres_version.hpp index d26b0526a..aa1db5090 100644 --- a/src/include/postgres_version.hpp +++ b/src/include/postgres_version.hpp @@ -14,6 +14,10 @@ namespace duckdb { enum class PostgresInstanceType { UNKNOWN, POSTGRES, AURORA, REDSHIFT }; +//! How scans decide between the binary COPY protocol and the text protocol. +//! AUTO derives it from the server (the binary COPY protocol is not supported by Redshift). +enum class PostgresTextProtocolMode { AUTO, TEXT, BINARY }; + struct PostgresVersion { PostgresVersion() { } diff --git a/src/include/storage/postgres_capabilities.hpp b/src/include/storage/postgres_capabilities.hpp new file mode 100644 index 000000000..bc10a0578 --- /dev/null +++ b/src/include/storage/postgres_capabilities.hpp @@ -0,0 +1,95 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/postgres_capabilities.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/type_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_expression.hpp" +#include "duckdb/parser/query_node.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/sql_statement.hpp" +#include "duckdb/parser/tableref.hpp" + +#include "postgres_version.hpp" + +namespace duckdb { + +//! Server properties that influence which constructs can be pushed down into Postgres +struct PostgresPushdownInfo { + PostgresVersion version; + //! Whether the database default collation orders strings byte-wise (C / POSIX / C.UTF-8 / builtin), + //! which matches DuckDB's (binary) string ordering + bool binary_collation = false; + //! Whether lower() / ILIKE apply Unicode case mapping (as DuckDB does) instead of only mapping ASCII characters + bool unicode_case_mapping = false; + //! Whether the database was attached read-only - DML and DDL are never pushed down then + bool read_only = false; +}; + +//! Decides which parsed constructs can be serialized by PostgresSQLWriter and executed in Postgres +//! with the same semantics as in DuckDB +class PostgresCapabilities { +public: + PostgresCapabilities() { + } + explicit PostgresCapabilities(PostgresPushdownInfo info_p) : info(std::move(info_p)) { + } + + bool Supports(RemoteCapability capability) const; + bool SupportsPushdown(const ParsedExpression &expression); + bool SupportsPushdown(const TableRef &ref); + bool SupportsPushdown(const QueryNode &node); + bool SupportsPushdown(const SQLStatement &statement); + + const PostgresPushdownInfo &GetInfo() const { + return info; + } + + //! Resolve a cast target / column type expression against the built-in DuckDB types, + //! returns LogicalTypeId::INVALID for types that are not built-in + static LogicalType ResolveType(const TypeExpression &type_expr); + //! Constant INTERVAL values are represented as add(add(to_months(m), to_days(d)), to_microseconds(us)) - + //! recognize this representation + static bool TryGetIntervalLiteral(const ParsedExpression &expr, interval_t &result); + //! Whether a constant value can be serialized as a Postgres literal + static bool SupportsValue(const Value &value); + //! Get the value of a constant expression (or of a cast of a constant) + static bool TryGetConstantValue(const ParsedExpression &expr, Value &result); + +private: + bool SupportsFunction(const FunctionExpression &func); + bool SupportsWindow(const WindowExpression &window); + bool SupportsCTEMap(const CommonTableExpressionMap &cte_map); + bool SupportsResultModifiers(const QueryNode &node, + optional_ptr>> select_list = nullptr); + bool SupportsOrderEntries(const vector &orders, + optional_ptr>> select_list = nullptr); + bool SupportsSelectAliasUsage(const SelectNode &select); + bool SupportsCastType(const LogicalType &type); + //! Whether an expression / query can be serialized without depending on the types it operates on + bool SupportsTypeIndependent(ParsedExpression &expr); + bool SupportsQueryTypeIndependent(QueryNode &node); + //! Whether string ordering (ORDER BY, <, min, ...) has to be blocked because it differs from DuckDB's + //! The checks for statements - these cannot fall back to local execution when the writer cannot serialize them + PostgresCapabilities StatementCapabilities() const; + bool StringOrderingDiffers() const { + return !info.binary_collation && !writer_pins_collation; + } + +private: + PostgresPushdownInfo info; + //! Whether the writer pins the C collation for string ordering (and falls back to local execution where it + //! cannot) - not possible for statements, which cannot be executed locally instead + bool writer_pins_collation = true; +}; + +} // namespace duckdb diff --git a/src/include/storage/postgres_catalog.hpp b/src/include/storage/postgres_catalog.hpp index 9161e46fd..4e0617ea7 100644 --- a/src/include/storage/postgres_catalog.hpp +++ b/src/include/storage/postgres_catalog.hpp @@ -8,13 +8,22 @@ #pragma once +#include "dbconnector/attached.hpp" + #include +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include #include "duckdb/catalog/catalog.hpp" #include "duckdb/common/enums/access_mode.hpp" #include "postgres_aws.hpp" #include "postgres_connection.hpp" +#include "storage/postgres_capabilities.hpp" #include "storage/postgres_schema_set.hpp" #include "storage/postgres_connection_pool.hpp" #include "storage/postgres_secret_storage.hpp" @@ -22,25 +31,29 @@ namespace duckdb { class PostgresCatalog; class PostgresSchemaEntry; +struct PostgresCacheInvalidation; class PostgresCatalog : public Catalog { public: explicit PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, string attach_path, AccessMode access_mode, - string schema_to_load, PostgresIsolationLevel isolation_level, const string &secret_name, - SecretStorageTable secret_storage_table_p); + vector schemas_to_load, PostgresIsolationLevel isolation_level, + const string &secret_name, SecretStorageTable secret_storage_table_p, + PostgresTextProtocolMode text_protocol_mode, string connect_display = string()); ~PostgresCatalog(); string attach_path; AccessMode access_mode; PostgresIsolationLevel isolation_level; + PostgresTextProtocolMode text_protocol_mode; + PostgresUseCtidScan use_ctid_scan = PostgresUseCtidScan::DEFAULT; public: void Initialize(bool load_builtin) override; string GetCatalogType() override { return "postgres"; } - string GetDefaultSchema() const override { - return default_schema.empty() ? "public" : default_schema; + optional GetDefaultSchema() const override { + return default_schema.empty() ? Identifier("public") : default_schema; } string GetConnectionString(); @@ -68,10 +81,20 @@ class PostgresCatalog : public Catalog { DatabaseSize GetDatabaseSize(ClientContext &context) override; + //! Whether the database default collation orders strings byte-wise, like DuckDB + bool HasBinaryCollation() const { + return capabilities.GetInfo().binary_collation; + } PostgresVersion GetPostgresVersion() const { return version; } + bool UseTextProtocol(PostgresQueryMode query_mode, bool use_text_protocol_setting) const; + //! Whether scans should use plain queries with binary results, falling back to the text protocol for queries + //! that read columns that can only be read using the text protocol. Used for Redshift, which does not support + //! the binary COPY protocol. + bool UseBinaryPlainQueryWithTextFallback(PostgresQueryMode query_mode, bool use_text_protocol_setting) const; + //! Label all postgres scans in the sub-tree as requiring materialization //! This is used for e.g. insert queries that have both (1) a scan from a postgres table, and (2) a sink into one static void MaterializePostgresScans(PhysicalOperator &op); @@ -82,8 +105,17 @@ class PostgresCatalog : public Catalog { string GetDBPath() override; bool Supports(RemoteCapability capability) const override { - return capability == RemoteCapability::IS_REMOTE || capability == RemoteCapability::CONNECT; + if (!remote_pushdown_enabled) { + return capability == RemoteCapability::IS_REMOTE || capability == RemoteCapability::CONNECT; + } + return capabilities.Supports(capability); } + bool SupportsPushdown(const ParsedExpression &expression) override; + bool SupportsPushdown(const TableRef &ref) override; + bool SupportsPushdown(const QueryNode &node) override; + bool SupportsPushdown(const SQLStatement &statement) override; + unique_ptr RemoteExecute(ClientContext &context, unique_ptr node) override; + unique_ptr RemoteExecute(ClientContext &context, unique_ptr statement) override; unique_ptr RemoteExecute(ClientContext &context, const string &sql) override; string GetConnectDisplay() override; @@ -95,7 +127,10 @@ class PostgresCatalog : public Catalog { return connection_pool; } + static dbconnector::attached::AttachedCatalog Lookup(ClientContext &ctx, const Identifier &name); + void ClearCache(); + void InvalidateCacheEntry(ClientContext &context, const PostgresCacheInvalidation &invalidation); //! Whether or not this catalog should search a specific type with the standard priority CatalogLookupBehavior CatalogTypeLookupRule(CatalogType type) const override { @@ -115,16 +150,28 @@ class PostgresCatalog : public Catalog { void DropSchema(ClientContext &context, DropInfo &info) override; void RegisterSecretStorage(); + void InitializeRemotePushdown(ClientContext &context, PostgresConnection &connection); + //! Statements to run before a pushed down query (e.g. aligning the session time zone with DuckDB's) + vector GetRemotePushdownPreliminaryQueries(ClientContext &context, bool requires_time_zone); static unique_ptr GetSecretEntry(ClientContext &ctx, const std::string &secret_name); static string CreateConnectionString(optional_ptr secret_entry, const string &attach_path); private: PostgresVersion version; + PostgresCapabilities capabilities; + bool remote_pushdown_enabled = false; + //! Whether table scans can be parallelized with ctid ranges (pg_use_ctid_scan, the protocol and the version at + //! attach time) - otherwise plain table scans are pushed down as well + bool parallel_scans_supported = false; + //! The Postgres session time zone at attach time + string remote_time_zone; PostgresSchemaSet schemas; shared_ptr connection_pool; - string default_schema; + Identifier default_schema; SecretStorageTable secret_storage_table; + //! Overrides what the shell prompt displays for this catalog - see GetConnectDisplay + string connect_display; PostgresAwsRdsTokenConfig rds_token_config; std::mutex rds_token_mutex; diff --git a/src/include/storage/postgres_catalog_set.hpp b/src/include/storage/postgres_catalog_set.hpp index d454c591d..c8eb7ad8d 100644 --- a/src/include/storage/postgres_catalog_set.hpp +++ b/src/include/storage/postgres_catalog_set.hpp @@ -9,9 +9,12 @@ #pragma once #include "duckdb/transaction/transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/common/shared_ptr.hpp" +#include "duckdb/common/thread.hpp" namespace duckdb { struct DropInfo; @@ -29,6 +32,8 @@ class PostgresCatalogSet { const std::function &callback); virtual optional_ptr CreateEntry(PostgresTransaction &transaction, shared_ptr entry); void ClearEntries(); + //! Remove an entry from the cache (without dropping it) - it is reloaded when it is looked up the next time + void InvalidateEntry(const string &name); virtual bool SupportReload() const { return false; } @@ -41,7 +46,24 @@ class PostgresCatalogSet { virtual bool HasInternalDependencies() const { return false; } + //! Empty (default) means staleness is never checked once loaded + virtual string GetStalenessQuery(ClientContext &context) const { + return string(); + } void TryLoadEntries(ClientContext &context, PostgresTransaction &transaction); + void RefreshStalenessSignature(PostgresTransaction &transaction, bool use_transaction_connection); + +private: + //! Runs LoadEntries + RefreshStalenessSignature, must be called with load_lock held + void LoadEntriesLocked(ClientContext &context, PostgresTransaction &transaction); + //! Must be called with load_lock held + void ClearEntriesLocked(); + //! Returns with entry_lock held + unique_lock LoadEntriesForRead(ClientContext &context, PostgresTransaction &transaction); + +protected: +public: + void PromoteStalenessSignature(string signature); protected: Catalog &catalog; @@ -52,6 +74,8 @@ class PostgresCatalogSet { unordered_map> entries; case_insensitive_map_t entry_map; atomic is_loaded; + atomic loading_thread; + string staleness_signature; }; class PostgresInSchemaSet : public PostgresCatalogSet { diff --git a/src/include/storage/postgres_connection_pool.hpp b/src/include/storage/postgres_connection_pool.hpp index b366423ec..982350705 100644 --- a/src/include/storage/postgres_connection_pool.hpp +++ b/src/include/storage/postgres_connection_pool.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include "duckdb/main/client_context.hpp" #include "duckdb/common/common.hpp" #include "duckdb/common/mutex.hpp" @@ -30,14 +31,7 @@ class PostgresConnectionPool : public dbconnector::pool::ConnectionPool index_result = nullptr); public: - static string GetInitializeQuery(const string &schema = string()); + static string GetInitializeQuery(const vector &schemas); optional_ptr CreateIndex(PostgresTransaction &transaction, CreateIndexInfo &info, TableCatalogEntry &table); diff --git a/src/include/storage/postgres_insert.hpp b/src/include/storage/postgres_insert.hpp index 3d20e97f5..b3626b39b 100644 --- a/src/include/storage/postgres_insert.hpp +++ b/src/include/storage/postgres_insert.hpp @@ -9,18 +9,23 @@ #pragma once #include "duckdb/execution/physical_operator.hpp" -#include "duckdb/common/index_vector.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "storage/postgres_catalog.hpp" namespace duckdb { class PostgresInsert : public PhysicalOperator { public: //! INSERT INTO - PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, - physical_index_vector_t column_index_map); + PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, TableCatalogEntry &table); //! CREATE TABLE AS - PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, - unique_ptr info); + PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); //! The table to insert into optional_ptr table; @@ -28,10 +33,10 @@ class PostgresInsert : public PhysicalOperator { optional_ptr schema; //! Create table info, in case of CREATE TABLE AS unique_ptr info; - //! column_index_map - physical_index_vector_t column_index_map; //! Whether or not we can keep the copy alive during Sink calls bool keep_copy_alive = true; + //! Whether to use plain INSERTs instead of COPY + bool use_plain_inserts = false; public: // Source interface @@ -44,10 +49,30 @@ class PostgresInsert : public PhysicalOperator { public: // Sink interface - unique_ptr GetGlobalSinkState(ClientContext &context) const override; - SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; + unique_ptr GetGlobalSinkState(ClientContext &context) const override { + if (use_plain_inserts) { + return GetGlobalSinkStatePlain(context); + } else { + return GetGlobalSinkStateCopy(context); + } + } + + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override { + if (use_plain_inserts) { + return SinkPlain(context, chunk, input); + } else { + return SinkCopy(context, chunk, input); + } + } + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const override; + OperatorSinkFinalizeInput &input) const override { + if (use_plain_inserts) { + return FinalizePlain(pipeline, event, context, input); + } else { + return FinalizeCopy(pipeline, event, context, input); + } + } bool IsSink() const override { return true; @@ -59,6 +84,17 @@ class PostgresInsert : public PhysicalOperator { string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; + +private: + static bool UsePlainInserts(PostgresCatalog &pg_catalog, ClientContext &context); + unique_ptr GetGlobalSinkStateCopy(ClientContext &context) const; + unique_ptr GetGlobalSinkStatePlain(ClientContext &context) const; + SinkResultType SinkCopy(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const; + SinkResultType SinkPlain(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const; + SinkFinalizeType FinalizeCopy(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const; + SinkFinalizeType FinalizePlain(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const; }; } // namespace duckdb diff --git a/src/include/storage/postgres_optimizer.hpp b/src/include/storage/postgres_optimizer.hpp index 832a5d86c..c6d47513e 100644 --- a/src/include/storage/postgres_optimizer.hpp +++ b/src/include/storage/postgres_optimizer.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/main/config.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "duckdb/optimizer/optimizer_extension.hpp" namespace duckdb { diff --git a/src/include/storage/postgres_schema_entry.hpp b/src/include/storage/postgres_schema_entry.hpp index 75e23643a..d853ebe85 100644 --- a/src/include/storage/postgres_schema_entry.hpp +++ b/src/include/storage/postgres_schema_entry.hpp @@ -9,6 +9,10 @@ #pragma once #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "storage/postgres_table_set.hpp" #include "storage/postgres_index_set.hpp" #include "storage/postgres_type_set.hpp" @@ -46,6 +50,11 @@ class PostgresSchemaEntry : public SchemaCatalogEntry { static bool SchemaIsInternal(const string &name); + //! Remove an entry from the cache - after it was modified by a statement that was executed in Postgres + void InvalidateEntry(CatalogType type, const string &name); + //! Remove all tables from the cache + void InvalidateTables(); + private: void AlterTable(PostgresTransaction &transaction, RenameTableInfo &info); void AlterTable(PostgresTransaction &transaction, RenameColumnInfo &info); diff --git a/src/include/storage/postgres_schema_set.hpp b/src/include/storage/postgres_schema_set.hpp index 979a545e5..9ac1a3cb1 100644 --- a/src/include/storage/postgres_schema_set.hpp +++ b/src/include/storage/postgres_schema_set.hpp @@ -9,6 +9,8 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "storage/postgres_schema_entry.hpp" namespace duckdb { @@ -16,19 +18,21 @@ struct CreateSchemaInfo; class PostgresSchemaSet : public PostgresCatalogSet { public: - explicit PostgresSchemaSet(Catalog &catalog, string schema_to_load); + explicit PostgresSchemaSet(Catalog &catalog, vector schemas_to_load); public: optional_ptr CreateSchema(PostgresTransaction &transaction, CreateSchemaInfo &info); - static string GetInitializeQuery(const string &schema = string()); + static string GetInitializeQuery(const vector &schemas); + static string GetInitializeQueryInformationSchema(const vector &schemas); protected: void LoadEntries(ClientContext &context, PostgresTransaction &transaction) override; + void LoadEntriesInformationSchema(ClientContext &context, PostgresTransaction &transaction); protected: //! Schema to load - if empty loads all schemas (default behavior) - string schema_to_load; + vector schemas_to_load; }; } // namespace duckdb diff --git a/src/include/storage/postgres_secret_storage.hpp b/src/include/storage/postgres_secret_storage.hpp index 3504fe208..68a77e4f6 100644 --- a/src/include/storage/postgres_secret_storage.hpp +++ b/src/include/storage/postgres_secret_storage.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/main/secret/secret_storage.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/main/secret/secret_manager.hpp" #include "storage/postgres_connection_pool.hpp" diff --git a/src/include/storage/postgres_table_entry.hpp b/src/include/storage/postgres_table_entry.hpp index f21f8c3fb..ca0d6aac8 100644 --- a/src/include/storage/postgres_table_entry.hpp +++ b/src/include/storage/postgres_table_entry.hpp @@ -8,19 +8,35 @@ #pragma once +#include "dbconnector/attached.hpp" + #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/common/unordered_map.hpp" #include "postgres_utils.hpp" namespace duckdb { +//! How Postgres fills in a column that is not specified in an INSERT +enum class PostgresColumnDefault : uint8_t { + //! No default - the column is set to NULL + NONE, + //! DEFAULT expression, SERIAL or identity column - NULL values are replaced with the Postgres-side default + DEFAULT_VALUE, + //! Generated column - values cannot be inserted + GENERATED +}; + struct PostgresTableInfo { PostgresTableInfo() { create_info = make_uniq(); create_info->columns.SetAllowDuplicates(true); } PostgresTableInfo(const string &schema, const string &table) { - create_info = make_uniq(Identifier(), Identifier(schema), Identifier(table)); + create_info = make_uniq(QualifiedName(Identifier(), Identifier(schema), Identifier(table))); create_info->columns.SetAllowDuplicates(true); } PostgresTableInfo(const SchemaCatalogEntry &schema, const string &table) { @@ -29,13 +45,16 @@ struct PostgresTableInfo { } const string &GetTableName() const { - return create_info->table.GetIdentifierName(); + return create_info->GetTableName().GetIdentifierName(); } unique_ptr create_info; vector postgres_types; vector postgres_names; + vector postgres_type_names; + vector column_defaults; int64_t approx_num_pages = 0; + unordered_map attnum_to_logical; }; class PostgresTableEntry : public TableCatalogEntry { @@ -44,6 +63,8 @@ class PostgresTableEntry : public TableCatalogEntry { PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, PostgresTableInfo &info); public: + const ColumnList &GetColumns() const override; + unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; @@ -56,6 +77,14 @@ class PostgresTableEntry : public TableCatalogEntry { //! Get the copy format (text or binary) that should be used when writing data to this table PostgresCopyFormat GetCopyFormat(ClientContext &context); + static dbconnector::attached::AttachedTable Lookup(ClientContext &ctx, QualifiedName name); + +private: + PostgresCopyFormat GetBinaryCopyFormatIfTypesAllow(); + +protected: + ColumnList columns; + public: //! Postgres type annotations vector postgres_types; @@ -63,6 +92,10 @@ class PostgresTableEntry : public TableCatalogEntry { //! We track these separately because of case sensitivity - Postgres allows e.g. the columns "ID" and "id" together //! We would in this case remap them to "ID" and "id:1", while postgres_names store the original names vector postgres_names; + //! The (schema qualified, quoted) names of the column types within Postgres - empty if not known + vector postgres_type_names; + //! How Postgres fills in each column when it is not specified in an INSERT + vector column_defaults; //! The approximate number of pages a table consumes in Postgres std::atomic approx_num_pages; }; diff --git a/src/include/storage/postgres_table_set.hpp b/src/include/storage/postgres_table_set.hpp index fcdff5428..2e1c32cc3 100644 --- a/src/include/storage/postgres_table_set.hpp +++ b/src/include/storage/postgres_table_set.hpp @@ -9,6 +9,8 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "storage/postgres_table_entry.hpp" namespace duckdb { @@ -17,6 +19,9 @@ class PostgresConnection; class PostgresResult; class PostgresSchemaEntry; +//! The CREATE TABLE statement for a (bound) table definition +string GetPostgresCreateTable(CreateTableInfo &info); + class PostgresTableSet : public PostgresInSchemaSet { public: explicit PostgresTableSet(PostgresSchemaEntry &schema, unique_ptr tables = nullptr); @@ -32,13 +37,18 @@ class PostgresTableSet : public PostgresInSchemaSet { void AlterTable(ClientContext &context, PostgresTransaction &transaction, AlterTableInfo &info); - static string GetInitializeQuery(const string &schema = string(), const string &table = string()); + static string GetInitializeQuery(PostgresVersion version, const string &schema, const string &table = string()); + static string GetInitializeQuery(PostgresVersion version, const vector &schemas, + const string &table = string()); + static string GetInitializeQueryInformationSchema(const string &schema = string(), const string &table = string()); + static string GetInitializeQueryInformationSchema(const vector &schemas, const string &table = string()); protected: void LoadEntries(ClientContext &context, PostgresTransaction &transaction) override; bool SupportReload() const override { return true; } + string GetStalenessQuery(ClientContext &context) const override; void AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameTableInfo &info); void AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameColumnInfo &info); @@ -46,11 +56,12 @@ class PostgresTableSet : public PostgresInSchemaSet { void AlterTable(ClientContext &context, PostgresTransaction &transaction, RemoveColumnInfo &info); static void AddColumn(optional_ptr transaction, optional_ptr schema, - PostgresResult &result, idx_t row, PostgresTableInfo &table_info); + const PostgresTypeConfig &type_config, PostgresResult &result, idx_t row, + PostgresTableInfo &table_info); static void AddConstraint(PostgresResult &result, idx_t row, PostgresTableInfo &table_info); static void AddColumnOrConstraint(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, idx_t row, - PostgresTableInfo &table_info); + optional_ptr schema, const PostgresTypeConfig &type_config, + PostgresResult &result, idx_t row, PostgresTableInfo &table_info); void CreateEntries(PostgresTransaction &transaction, PostgresResult &result, idx_t start, idx_t end); diff --git a/src/include/storage/postgres_transaction.hpp b/src/include/storage/postgres_transaction.hpp index 9cbb5eb7e..f8f0c5623 100644 --- a/src/include/storage/postgres_transaction.hpp +++ b/src/include/storage/postgres_transaction.hpp @@ -9,11 +9,14 @@ #pragma once #include "duckdb/transaction/transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "postgres_connection.hpp" #include "storage/postgres_connection_pool.hpp" namespace duckdb { class PostgresCatalog; +class PostgresCatalogSet; class PostgresSchemaEntry; class PostgresTableEntry; @@ -30,6 +33,8 @@ class PostgresTransaction : public Transaction { PostgresConnection &GetConnectionWithoutTransaction(); PostgresConnection &GetConnection(); + //! ID of the pooled connection used by this transaction + uint64_t GetPoolConnectionId(); optional_ptr GetContext(); string GetDSN(); @@ -43,13 +48,21 @@ class PostgresTransaction : public Transaction { string GetTemporarySchema(); + PostgresTransactionState GetTransactionState() const { + return transaction_state; + } + void StageStalenessSignature(PostgresCatalogSet &catalog_set, string signature); + private: PostgresPoolConnection connection; PostgresTransactionState transaction_state; AccessMode access_mode; PostgresIsolationLevel isolation_level; string temporary_schema; + mutex referenced_entries_lock; reference_map_t> referenced_entries; + mutex pending_signatures_lock; + vector, string>> pending_signatures; private: //! Retrieves the connection **without** starting a transaction if none is active diff --git a/src/include/storage/postgres_transaction_manager.hpp b/src/include/storage/postgres_transaction_manager.hpp index 24b82b435..023831f91 100644 --- a/src/include/storage/postgres_transaction_manager.hpp +++ b/src/include/storage/postgres_transaction_manager.hpp @@ -8,27 +8,13 @@ #pragma once -#include "duckdb/transaction/transaction_manager.hpp" +#include "dbconnector/storage/transaction_manager.hpp" + #include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" -#include "duckdb/common/reference_map.hpp" namespace duckdb { -class PostgresTransactionManager : public TransactionManager { -public: - PostgresTransactionManager(AttachedDatabase &db_p, PostgresCatalog &postgres_catalog); - - Transaction &StartTransaction(ClientContext &context) override; - ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; - void RollbackTransaction(Transaction &transaction) override; - - void Checkpoint(ClientContext &context, bool force = false) override; - -private: - PostgresCatalog &postgres_catalog; - mutex transaction_lock; - reference_map_t> transactions; -}; +using PostgresTransactionManager = dbconnector::storage::TransactionManager; } // namespace duckdb diff --git a/src/include/storage/postgres_type_entry.hpp b/src/include/storage/postgres_type_entry.hpp index 6ea79d3e4..861ca5538 100644 --- a/src/include/storage/postgres_type_entry.hpp +++ b/src/include/storage/postgres_type_entry.hpp @@ -9,6 +9,8 @@ #pragma once #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" #include "postgres_utils.hpp" diff --git a/src/include/storage/postgres_type_set.hpp b/src/include/storage/postgres_type_set.hpp index 9b6321b78..5df937262 100644 --- a/src/include/storage/postgres_type_set.hpp +++ b/src/include/storage/postgres_type_set.hpp @@ -9,6 +9,7 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" #include "storage/postgres_type_entry.hpp" namespace duckdb { @@ -25,8 +26,8 @@ class PostgresTypeSet : public PostgresInSchemaSet { public: optional_ptr CreateType(PostgresTransaction &transaction, CreateTypeInfo &info); - static string GetInitializeEnumsQuery(PostgresVersion version, const string &schema = string()); - static string GetInitializeCompositesQuery(const string &schema = string()); + static string GetInitializeEnumsQuery(PostgresVersion version, const vector &schemas); + static string GetInitializeCompositesQuery(const vector &schemas); protected: bool HasInternalDependencies() const override { diff --git a/src/include/storage/postgres_update.hpp b/src/include/storage/postgres_update.hpp index 1d567b978..44c184e69 100644 --- a/src/include/storage/postgres_update.hpp +++ b/src/include/storage/postgres_update.hpp @@ -9,6 +9,9 @@ #pragma once #include "duckdb/execution/physical_operator.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "duckdb/common/index_vector.hpp" namespace duckdb { diff --git a/src/postgres_attach.cpp b/src/postgres_attach.cpp deleted file mode 100644 index a9fc5a39f..000000000 --- a/src/postgres_attach.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "duckdb.hpp" - -#include "duckdb/parser/parsed_data/create_table_function_info.hpp" - -#include "postgres_filter_pushdown.hpp" -#include "postgres_scanner.hpp" -#include "postgres_result.hpp" -#include "postgres_utils.hpp" - -namespace duckdb { - -struct AttachFunctionData : public TableFunctionData { - bool finished = false; - string source_schema = "public"; - string sink_schema = "main"; - string suffix = ""; - bool overwrite = false; - bool filter_pushdown = false; - - string dsn = ""; -}; - -static unique_ptr AttachBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(); - result->dsn = input.inputs[0].GetValue(); - - for (auto &kv : input.named_parameters) { - if (kv.first == "source_schema") { - result->source_schema = StringValue::Get(kv.second); - } else if (kv.first == "sink_schema") { - result->sink_schema = StringValue::Get(kv.second); - } else if (kv.first == "overwrite") { - result->overwrite = BooleanValue::Get(kv.second); - } else if (kv.first == "filter_pushdown") { - result->filter_pushdown = BooleanValue::Get(kv.second); - } - } - - return_types.push_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - return std::move(result); -} - -static void AttachFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (AttachFunctionData &)*data_p.bind_data; - if (data.finished) { - return; - } - - auto conn = PostgresConnection::Open(data.dsn, data.dsn); - auto dconn = Connection(context.db->GetDatabase(context)); - auto fetch_table_query = StringUtil::Format( - R"( -SELECT relname -FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid -JOIN pg_attribute ON pg_class.oid = pg_attribute.attrelid -WHERE relkind = 'r' AND attnum > 0 AND nspname = %s -GROUP BY relname -ORDER BY relname; -)", - PostgresUtils::WriteLiteral(data.source_schema)); - auto res = conn.Query(context, fetch_table_query); - for (idx_t row = 0; row < PQntuples(res->res); row++) { - auto table_name = res->GetString(row, 0); - string query; - if (data.overwrite) { - query = "CREATE OR REPLACE VIEW "; - } else { - query = "CREATE VIEW IF NOT EXISTS "; - } - if (!data.sink_schema.empty()) { - query += PostgresUtils::WriteIdentifier(data.sink_schema) + "."; - } - query += PostgresUtils::WriteIdentifier(table_name); - query += " AS SELECT * FROM "; - if (data.filter_pushdown) { - query += "postgres_scan_pushdown"; - } else { - query += "postgres_scan"; - } - query += "("; - query += PostgresUtils::WriteLiteral(data.dsn); - query += ", "; - query += PostgresUtils::WriteLiteral(data.source_schema); - query += ", "; - query += PostgresUtils::WriteLiteral(table_name); - query += ");"; - dconn.Query(query); - } - res.reset(); - - data.finished = true; -} - -PostgresAttachFunction::PostgresAttachFunction() - : TableFunction("postgres_attach", {LogicalType::VARCHAR}, AttachFunction, AttachBind) { - named_parameters["overwrite"] = LogicalType::BOOLEAN; - named_parameters["filter_pushdown"] = LogicalType::BOOLEAN; - - named_parameters["source_schema"] = LogicalType::VARCHAR; - named_parameters["sink_schema"] = LogicalType::VARCHAR; - named_parameters["suffix"] = LogicalType::VARCHAR; -} - -} // namespace duckdb diff --git a/src/postgres_aws.cpp b/src/postgres_aws.cpp index 708c89889..1e1da4d0f 100644 --- a/src/postgres_aws.cpp +++ b/src/postgres_aws.cpp @@ -1,4 +1,6 @@ #include "postgres_aws.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/database.hpp" #include #include @@ -7,6 +9,7 @@ #include "dbconnector/defer.hpp" +#include "postgres_connection_string.hpp" #include "postgres_secrets.hpp" #include "postgres_utils.hpp" @@ -17,17 +20,20 @@ bool PostgresAwsRdsTokenConfig::Enabled() { } int64_t PostgresAwsRdsTokenConfig::MaxAgeSeconds() { + if (max_age_seconds_override >= 0) { + return max_age_seconds_override; + } return expiration_seconds - 60; } static std::string MakeCreateSecretQuery(const std::string &template_secret_name, const std::string &secret_name, - const KeyValueSecret &kv_secret) { + const identifier_tree_t &template_options) { std::string query("CREATE OR REPLACE TEMPORARY SECRET \""); query += secret_name; query += "\" (\n"; query += " TYPE rds,\n"; query += " PROVIDER credential_chain,\n"; - for (auto &en : kv_secret.secret_map) { + for (auto &en : template_options) { query += " " + en.first + " " + en.second.ToSQLString() + ",\n"; } query += " RDS_TEMPLATE_SECRET_NAME " + PostgresUtils::WriteLiteral(template_secret_name) + "\n"; @@ -47,9 +53,10 @@ static void RunQuery(Connection &conn, const std::string &query, const std::stri } } -static unique_ptr GetRdsSecret(SecretManager &secret_manager, Connection &conn, const std::string &name) { +static unique_ptr GetRdsSecret(SecretManager &secret_manager, Connection &conn, const std::string &name, + const std::string &storage) { auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*conn.context); - auto secret_entry = secret_manager.GetSecretByName(transaction, name); + auto secret_entry = secret_manager.GetSecretByName(transaction, name, storage); if (!secret_entry) { throw InvalidConfigurationException("Specified RDS secret with name: \"%s\" not found", name); } @@ -65,19 +72,19 @@ std::string PostgresAws::GenerateRdsAuthToken(AttachedDatabase &attached_db, auto deferred_rollback = dbconnector::Defer([&conn] { conn.Query("ROLLBACK"); }); SecretManager &secret_manager = SecretManager::Get(db); - auto template_secret = GetRdsSecret(secret_manager, conn, token_config.rds_secret_name); - const auto &kv_template_secret = dynamic_cast(*template_secret->secret); - + // The temporary secret is only looked up in (and dropped from) the temporary storage: this can be + // called when creating a new connection to the attached DB, so the lookup must not reach the + // Postgres secret storage of this DB - it would need a new connection itself and would deadlock. std::string secret_name = token_config.rds_secret_name + "_" + attached_db.GetName() + "_postgres_temp"; std::string create_secret_query = - MakeCreateSecretQuery(token_config.rds_secret_name, secret_name, kv_template_secret); + MakeCreateSecretQuery(token_config.rds_secret_name, secret_name, token_config.template_options); std::string quoted_secret_name = PostgresUtils::WriteIdentifier(secret_name); RunQuery(conn, create_secret_query, "error creating RDS secret from template: " + token_config.rds_secret_name); - auto deferred_drop_secret = - dbconnector::Defer([&conn, quoted_secret_name] { conn.Query("DROP SECRET " + quoted_secret_name); }); + auto deferred_drop_secret = dbconnector::Defer( + [&conn, quoted_secret_name] { conn.Query("DROP TEMPORARY SECRET " + quoted_secret_name + " FROM memory"); }); - auto secret = GetRdsSecret(secret_manager, conn, secret_name); + auto secret = GetRdsSecret(secret_manager, conn, secret_name, SecretManager::TEMPORARY_STORAGE_NAME); const auto &kv_secret = dynamic_cast(*secret->secret); std::string token = kv_secret.TryGetValue("session_token").ToString(); @@ -99,7 +106,8 @@ static std::string ExtractString(const KeyValueSecret &kv, const std::string nam return str; } -PostgresAwsRdsTokenConfig PostgresAws::ExtractTokenConfigFromSecret(optional_ptr secret_entry) { +PostgresAwsRdsTokenConfig PostgresAws::ExtractTokenConfigFromSecret(ClientContext &context, + optional_ptr secret_entry) { PostgresAwsRdsTokenConfig config; if (!secret_entry) { return config; @@ -107,22 +115,57 @@ PostgresAwsRdsTokenConfig PostgresAws::ExtractTokenConfigFromSecret(optional_ptr const auto &kv = dynamic_cast(*secret_entry->secret); config.rds_secret_name = ExtractString(kv, "aws_rds_secret"); + std::string max_age_str = ExtractString(kv, "aws_rds_token_max_age_seconds"); if (config.rds_secret_name.empty()) { + if (!max_age_str.empty()) { + throw BinderException("Invalid AWS RDS IAM auth secret configuration: 'AWS_RDS_TOKEN_MAX_AGE_SECONDS' " + "parameter can only be specified along with 'AWS_RDS_SECRET' parameter"); + } return config; } + if (!max_age_str.empty()) { + int64_t max_age = Value(max_age_str).DefaultCastAs(LogicalType::BIGINT).GetValue(); + int64_t max_allowed = config.MaxAgeSeconds(); + if (max_age < 0 || max_age > max_allowed) { + throw BinderException("Invalid AWS RDS IAM auth secret configuration: 'AWS_RDS_TOKEN_MAX_AGE_SECONDS' " + "must be between 0 and %d, specified: %d", + max_allowed, max_age); + } + config.max_age_seconds_override = max_age; + } + std::string password = ExtractString(kv, "password"); if (!password.empty()) { throw BinderException("Invalid AWS RDS IAM auth secret configuration: 'PASSWORD' parameters must not be " "specified - generated IAM token will be used instead of the password"); } + // Resolve the template secret while the Postgres secret storage of the attached DB is not yet registered + auto &secret_manager = SecretManager::Get(context); + auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); + auto template_secret = secret_manager.GetSecretByName(transaction, config.rds_secret_name); + if (!template_secret) { + throw InvalidConfigurationException("Specified RDS secret with name: \"%s\" not found", config.rds_secret_name); + } + const auto &kv_template_secret = dynamic_cast(*template_secret->secret); + config.template_options = kv_template_secret.secret_map; + + bool has_app_name = + !kv.TryGetValue("application_name").IsNull() || !kv.TryGetValue("fallback_application_name").IsNull(); + // Build the base connection string (without password) for (const string &opt_name : PostgresSecrets::ConnectionOptionNames()) { if (opt_name == "password" || opt_name == "passfile") { continue; } - config.base_connection_string += PostgresUtils::ExtractConnectionOption(kv, opt_name); + config.base_connection_string += PostgresConnectionString::ExtractOption(kv, opt_name); + } + + if (!has_app_name) { + PostgresConnectionString::AppendDefaultApplicationName(config.base_connection_string, + PostgresConnectionStringFormat::KEY_VALUE); + config.base_connection_string.push_back(' '); } return config; diff --git a/src/postgres_binary_copy.cpp b/src/postgres_binary_copy.cpp index 0ceafa61e..9387cb98d 100644 --- a/src/postgres_binary_copy.cpp +++ b/src/postgres_binary_copy.cpp @@ -1,4 +1,5 @@ #include "postgres_binary_copy.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_binary_writer.hpp" #include "postgres_binary_file_reader.hpp" #include "duckdb/common/serializer/buffered_file_writer.hpp" @@ -97,7 +98,7 @@ void PostgresBinaryCopyFunction::PostgresBinaryWriteFinalize(ClientContext &cont struct PostgresBinaryReadBindData : public TableFunctionData { string file_path; - vector names; + vector names; vector types; vector postgres_types; idx_t buffer_size = PostgresBinaryFileReader::DEFAULT_BUFFER_SIZE; @@ -124,7 +125,7 @@ struct PostgresBinaryReadGlobalState : public GlobalTableFunctionState { unique_ptr PostgresBinaryCopyFunction::PostgresBinaryReadBind(ClientContext &context, CopyFromFunctionBindInput &info, - vector &expected_names, + vector &expected_names, vector &expected_types) { auto result = make_uniq(); result->file_path = info.info.file_path; @@ -140,8 +141,9 @@ static unique_ptr PostgresBinaryReadInitGlobal(ClientC TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(); + PostgresTypeConfig type_config = PostgresTypeConfig::FromContext(context); result->reader = make_uniq(context, bind_data.file_path, bind_data.types, - bind_data.postgres_types, bind_data.buffer_size); + bind_data.postgres_types, type_config, bind_data.buffer_size); return std::move(result); } @@ -156,11 +158,11 @@ static void PostgresBinaryReadScan(ClientContext &context, TableFunctionInput &d } static unique_ptr ReadPostgresBinaryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); result->file_path = input.inputs[0].GetValue(); - if (!input.named_parameters.count("columns")) { + if (!input.named_parameters.contains("columns")) { throw BinderException("read_postgres_binary requires a 'columns' parameter, " "e.g. columns={col1: 'INTEGER', col2: 'VARCHAR'}"); } @@ -172,7 +174,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T auto col_type_str = column_map[i].GetValue(); auto col_type = TransformStringToLogicalType(col_type_str, context); - names.push_back(col_name.GetIdentifierName()); + names.push_back(col_name); return_types.push_back(col_type); result->postgres_types.push_back(PostgresUtils::CreateEmptyPostgresType(col_type)); } @@ -180,7 +182,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T result->names = names; result->types = return_types; - if (input.named_parameters.count("buffer_size")) { + if (input.named_parameters.contains("buffer_size")) { result->buffer_size = input.named_parameters.at("buffer_size").GetValue(); } @@ -188,7 +190,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T } static void PostgresBinaryReadSerialize(Serializer &serializer, const optional_ptr bind_data_p, - const TableFunction &function) { + const BoundTableFunction &function) { auto &bind_data = bind_data_p->Cast(); serializer.WriteProperty(100, "file_path", bind_data.file_path); serializer.WriteProperty(101, "names", bind_data.names); @@ -196,7 +198,8 @@ static void PostgresBinaryReadSerialize(Serializer &serializer, const optional_p serializer.WriteProperty(103, "buffer_size", bind_data.buffer_size); } -static unique_ptr PostgresBinaryReadDeserialize(Deserializer &deserializer, TableFunction &function) { +static unique_ptr PostgresBinaryReadDeserialize(Deserializer &deserializer, + BoundTableFunction &function) { auto result = make_uniq(); deserializer.ReadProperty(100, "file_path", result->file_path); deserializer.ReadProperty(101, "names", result->names); @@ -220,11 +223,17 @@ PostgresBinaryCopyFunction::PostgresBinaryCopyFunction() : CopyFunction("postgre copy_from_function = PostgresReadBinaryFunction(); } +static FunctionSignature ReadPostgresBinarySignature() { + FunctionSignature signature; + signature.AddParameter("file_path", LogicalType::VARCHAR).WithTypedKwargs("options", [&](TypedKwargs &options) { + options.Add("columns", LogicalType::ANY).Add("buffer_size", LogicalType::UBIGINT); + }); + return signature; +} + PostgresReadBinaryFunction::PostgresReadBinaryFunction() - : TableFunction("read_postgres_binary", {LogicalType::VARCHAR}, PostgresBinaryReadScan, ReadPostgresBinaryBind, - PostgresBinaryReadInitGlobal) { - named_parameters["columns"] = LogicalType::ANY; - named_parameters["buffer_size"] = LogicalType::UBIGINT; + : TableFunction("read_postgres_binary", ReadPostgresBinarySignature(), PostgresBinaryReadScan, + ReadPostgresBinaryBind, PostgresBinaryReadInitGlobal) { serialize = PostgresBinaryReadSerialize; deserialize = PostgresBinaryReadDeserialize; } diff --git a/src/postgres_binary_file_reader.cpp b/src/postgres_binary_file_reader.cpp index 480bcb3db..72cbe1fc6 100644 --- a/src/postgres_binary_file_reader.cpp +++ b/src/postgres_binary_file_reader.cpp @@ -1,4 +1,6 @@ #include "postgres_binary_file_reader.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/common/file_system.hpp" namespace duckdb { @@ -12,9 +14,10 @@ static vector MakeSequentialColumnIds(idx_t count) { PostgresBinaryFileReader::PostgresBinaryFileReader(ClientContext &context, const string &file_path, vector types_p, vector postgres_types_p, - idx_t buffer_size_p) - : column_ids(MakeSequentialColumnIds(types_p.size())), parser(std::move(types_p), std::move(postgres_types_p)), - buffer_size(buffer_size_p), file_offset(0), leftover(0), leftover_offset(0), finished(false) { + PostgresTypeConfig type_config, idx_t buffer_size_p) + : column_ids(MakeSequentialColumnIds(types_p.size())), + parser(std::move(types_p), std::move(postgres_types_p), type_config), buffer_size(buffer_size_p), file_offset(0), + leftover(0), leftover_offset(0), finished(false), header_scanned(false) { auto &fs = FileSystem::GetFileSystem(context); file_handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ); file_size = file_handle->GetFileSize(); @@ -29,18 +32,18 @@ PostgresBinaryFileReader::PostgresBinaryFileReader(ClientContext &context, const } bool PostgresBinaryFileReader::ReadChunk(DataChunk &output) { + bool full = true; while (output.size() < STANDARD_VECTOR_SIZE) { if (parser.ReadChunk(output, column_ids)) { - return true; - } - if (finished) { - return false; + break; } - if (!FillBuffer()) { - return false; + if (finished || !FillBuffer()) { + full = false; + break; } } - return true; + output.SetChildCardinality(output.size()); + return full; } bool PostgresBinaryFileReader::FillBuffer() { @@ -65,15 +68,25 @@ bool PostgresBinaryFileReader::FillBuffer() { return false; } - idx_t valid = FindLastCompleteRow(read_buffer.get(), total); - if (valid == 0) { - if (file_offset >= file_size) { - valid = total; - } else { - throw IOException("Postgres binary file contains a row larger than the read buffer (%llu bytes). " - "Increase buffer_size.", - buffer_size); - } + // the first buffer starts with the COPY file header, which is not a tuple - skip it while scanning for + // row boundaries, but keep it in the buffer so that CheckHeader can still validate it + idx_t scan_offset = header_scanned ? 0 : MinValue(COPY_FILE_HEADER_SIZE, total); + header_scanned = true; + + idx_t valid = FindLastCompleteRow(read_buffer.get() + scan_offset, total - scan_offset); + if (valid > 0) { + valid += scan_offset; + } else if (file_offset >= file_size) { + valid = total; + } else if (scan_offset > 0) { + // no complete row fits next to the header. The header is a boundary of its own, so hand the parser + // just the header and carry the partial row over to the next fill, where the whole buffer is + // available for it - a row only has to fit in buffer_size, not in buffer_size minus the header. + valid = scan_offset; + } else { + throw IOException("Postgres binary file contains a row larger than the read buffer (%llu bytes). " + "Increase buffer_size.", + buffer_size); } parser.SetBuffer(read_buffer.get(), valid); diff --git a/src/postgres_binary_parser.cpp b/src/postgres_binary_parser.cpp index 900a2905c..42cea0c8f 100644 --- a/src/postgres_binary_parser.cpp +++ b/src/postgres_binary_parser.cpp @@ -1,4 +1,5 @@ #include "postgres_binary_parser.hpp" +#include "duckdb/common/operator/double_cast_operator.hpp" #include "duckdb/common/types/geometry.hpp" #include "duckdb/common/vector/flat_vector.hpp" @@ -7,8 +8,9 @@ namespace duckdb { -PostgresBinaryParser::PostgresBinaryParser(vector types_p, vector postgres_types_p) - : types(std::move(types_p)), postgres_types(std::move(postgres_types_p)) { +PostgresBinaryParser::PostgresBinaryParser(vector types_p, vector postgres_types_p, + PostgresTypeConfig type_config_p) + : types(std::move(types_p)), postgres_types(std::move(postgres_types_p)), type_config(type_config_p) { } void PostgresBinaryParser::SetBuffer(data_ptr_t buf, idx_t len) { @@ -22,16 +24,19 @@ bool PostgresBinaryParser::ReadChunk(DataChunk &output, const vector & return false; } - auto tuple_count = ReadInteger(); - if (tuple_count <= 0) { - // tuple_count of -1 signifies the file trailer (i.e. footer) + auto field_count = ReadInteger(); + if (field_count <= 0) { + // a field count of -1 signifies the file trailer (i.e. footer) // clear the buffer so Ready() returns false and the caller can free it buffer_ptr = nullptr; end = nullptr; continue; } - D_ASSERT(tuple_count == column_ids.size()); + if (field_count != column_ids.size()) { + throw InvalidInputException("Postgres binary reader - expected %llu field(s) per row, got %d", + column_ids.size(), field_count); + } idx_t output_offset = output.size(); for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { @@ -45,11 +50,16 @@ bool PostgresBinaryParser::ReadChunk(DataChunk &output, const vector & ReadValue(types[col_idx], postgres_types[col_idx], out_vec, output_offset); } } - output.SetChildCardinality(output_offset + 1); + // only update the count; this runs every row, and SetChildCardinality would visit every vector each time + output.SetCardinalityUnsafe(output_offset + 1); } return true; } +void PostgresBinaryParser::ThrowOutOfBuffer(const char *function_name) { + throw IOException("Postgres binary reader - out of buffer in %s", function_name); +} + void PostgresBinaryParser::CheckHeader() { auto magic_len = PostgresConversion::COPY_HEADER_LENGTH; auto flags_len = 8; @@ -58,7 +68,9 @@ void PostgresBinaryParser::CheckHeader() { if (!buffer_ptr) { throw IOException("buffer_ptr not set in CheckHeader"); } - if (buffer_ptr + header_len >= end) { + // a buffer holding exactly the header is valid - the reader hands one over when the first row does + // not fit alongside it + if (buffer_ptr + header_len > end) { throw IOException("Unable to read binary COPY data, invalid header"); } if (memcmp(buffer_ptr, PostgresConversion::COPY_HEADER, magic_len) != 0) { @@ -79,12 +91,50 @@ PostgresDecimalConfig PostgresBinaryParser::ReadDecimalConfig() { sign == NUMERIC_NEG)) { throw NotImplementedException("Postgres numeric NA/Inf"); } + config.sign = sign; config.is_negative = sign == NUMERIC_NEG; config.scale = ReadInteger(); return config; } +PostgresDecimal PostgresBinaryParser::ReadDecimalAsDouble() { + const auto config = ReadDecimalConfig(); + if (config.sign == NUMERIC_NAN || config.sign == NUMERIC_PINF || config.sign == NUMERIC_NINF) { + return PostgresDecimal(NonFiniteDecimalKindFromSign(config.sign)); + } + if (config.ndigits == 0) { + return PostgresDecimal(0); + } + + // A scientific exponent avoids expanding implicit zero groups or applying the display scale. + const auto exponent = (int32_t(config.weight) + 1 - int32_t(config.ndigits)) * DEC_DIGITS; + const auto exponent_text = std::to_string(exponent); + // Preserve the base-10000 digits and round only once when converting to DOUBLE. + string text; + text.reserve(idx_t(config.ndigits) * DEC_DIGITS + exponent_text.size() + 1 + config.is_negative); + if (config.is_negative) { + text.push_back('-'); + } + for (idx_t i = 0; i < config.ndigits; i++) { + const auto digit = ReadInteger(); + if (digit >= NBASE) { + throw InvalidInputException("Invalid Postgres numeric digit: %d", digit); + } + text.push_back(char('0' + digit / 1000)); + text.push_back(char('0' + (digit / 100) % 10)); + text.push_back(char('0' + (digit / 10) % 10)); + text.push_back(char('0' + digit % 10)); + } + text.push_back('e'); + text += exponent_text; + double result = 0; + if (!TryDoubleCast(text.c_str(), text.size(), result, false)) { + throw InvalidInputException("Could not convert Postgres numeric to DOUBLE"); + } + return PostgresDecimal(result); +} + void PostgresBinaryParser::ReadGeometry(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset) { idx_t element_count = 0; @@ -149,47 +199,85 @@ void PostgresBinaryParser::ReadArray(const LogicalType &type, const PostgresType ListVector::SetListSize(out_vec, child_offset + child_count); } +// Defined out of line so the length checks that call it stay small enough to inline +[[noreturn]] static void ThrowValueLengthMismatch(int32_t value_len, idx_t expected_len, const LogicalType &type) { + throw InvalidInputException("Postgres binary reader - expected %llu bytes for a value of type %s, got %d", + expected_len, type.ToString(), value_len); +} + +// Check that the length of the value Postgres sent matches the length we expect. COPY format only +// tells us the length of the value, so this is the only check we have that the data matches the +// type we expect. +static void CheckValueLength(int32_t value_len, idx_t expected_len, const LogicalType &type) { + if (value_len != expected_len) { + ThrowValueLengthMismatch(value_len, expected_len, type); + } +} + void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset) { auto value_len = ReadInteger(); + ParseValue(type, postgres_type, value_len, out_vec, output_offset); +} + +void PostgresBinaryParser::ParseValue(const LogicalType &type, const PostgresType &postgres_type, int32_t value_len, + Vector &out_vec, idx_t output_offset) { if (value_len == -1) { // NULL FlatVector::SetNull(out_vec, output_offset, true); return; } switch (type.id()) { case LogicalTypeId::SMALLINT: - D_ASSERT(value_len == sizeof(int16_t)); + CheckValueLength(value_len, sizeof(int16_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::INTEGER: - D_ASSERT(value_len == sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::UINTEGER: - D_ASSERT(value_len == sizeof(uint32_t)); + CheckValueLength(value_len, sizeof(uint32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::BIGINT: if (postgres_type.info == PostgresTypeAnnotation::CTID) { - D_ASSERT(value_len == 6); + CheckValueLength(value_len, sizeof(int32_t) + sizeof(int16_t), type); int64_t page_index = ReadInteger(); int64_t row_in_page = ReadInteger(); FlatVector::GetDataMutable(out_vec)[output_offset] = (page_index << 16LL) + row_in_page; return; } - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::FLOAT: - D_ASSERT(value_len == sizeof(float)); + CheckValueLength(value_len, sizeof(float), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadFloat(); break; case LogicalTypeId::DOUBLE: { if (postgres_type.info == PostgresTypeAnnotation::NUMERIC_AS_DOUBLE) { - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + auto dec = ReadDecimalAsDouble(); + double double_value = 0; + switch (dec.kind) { + case PostgresDecimalKind::ORDINARY: + double_value = dec.value; + break; + case PostgresDecimalKind::NOT_A_NUMBER: + double_value = std::numeric_limits::quiet_NaN(); + break; + case PostgresDecimalKind::POSITIVE_INFINITY: + double_value = std::numeric_limits::infinity(); + break; + case PostgresDecimalKind::NEGATIVE_INFINITY: + double_value = -std::numeric_limits::infinity(); + break; + default: + throw InvalidInputException("Unsupported decimal kind"); + } + FlatVector::GetDataMutable(out_vec)[output_offset] = double_value; break; } - D_ASSERT(value_len == sizeof(double)); + CheckValueLength(value_len, sizeof(double), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDouble(); break; } @@ -225,7 +313,7 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType break; } case LogicalTypeId::BOOLEAN: - D_ASSERT(value_len == sizeof(bool)); + CheckValueLength(value_len, sizeof(bool), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadBoolean(); break; case LogicalTypeId::DECIMAL: { @@ -233,19 +321,34 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType throw InvalidInputException("Need at least 8 bytes to read a Postgres decimal. Got %d", value_len); } switch (type.InternalType()) { - case PhysicalType::INT16: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + case PhysicalType::INT16: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT32: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + } + case PhysicalType::INT32: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT64: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + } + case PhysicalType::INT64: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT128: - FlatVector::GetDataMutable(out_vec)[output_offset] = - ReadDecimal(); + } + case PhysicalType::INT128: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; + } default: throw InvalidInputException("Unsupported decimal storage type"); } @@ -253,24 +356,24 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType } case LogicalTypeId::DATE: { - D_ASSERT(value_len == sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int32_t), type); auto out_ptr = FlatVector::GetDataMutable(out_vec); out_ptr[output_offset] = ReadDate(); break; } case LogicalTypeId::TIME: { - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTime(); break; } case LogicalTypeId::TIME_TZ: { - D_ASSERT(value_len == sizeof(int64_t) + sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int64_t) + sizeof(int32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTimeTZ(); break; } case LogicalTypeId::TIMESTAMP_TZ: case LogicalTypeId::TIMESTAMP: { - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTimestamp(); break; } @@ -300,11 +403,26 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType break; } case LogicalTypeId::INTERVAL: { - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInterval(); + interval_t interval; + if (postgres_type.info == PostgresTypeAnnotation::INTERVAL_YEAR_TO_MONTH) { + CheckValueLength(value_len, sizeof(int32_t), type); + interval.months = ReadInteger(); + interval.days = 0; + interval.micros = 0; + } else if (postgres_type.info == PostgresTypeAnnotation::INTERVAL_DAY_TO_SECOND) { + CheckValueLength(value_len, sizeof(int64_t), type); + auto micros = ReadInteger(); + interval.months = 0; + interval.days = NumericCast(micros / Interval::MICROS_PER_DAY); + interval.micros = micros % Interval::MICROS_PER_DAY; + } else { + interval = ReadInterval(); + } + FlatVector::GetDataMutable(out_vec)[output_offset] = interval; break; } case LogicalTypeId::UUID: { - D_ASSERT(value_len == 2 * sizeof(int64_t)); + CheckValueLength(value_len, 2 * sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadUUID(); break; } @@ -329,7 +447,11 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType default: break; } - D_ASSERT(value_len >= 3 * sizeof(uint32_t)); + if (value_len < 3 * sizeof(uint32_t)) { + throw InvalidInputException( + "Postgres binary reader - expected at least %llu bytes for an array header of type %s, got %d", + 3 * sizeof(uint32_t), type.ToString(), value_len); + } auto array_dim = ReadInteger(); auto array_has_null = ReadInteger(); // whether or not the array has nulls - ignore auto value_oid = ReadInteger(); // value_oid - not necessary @@ -363,7 +485,7 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType case LogicalTypeId::STRUCT: { auto &child_entries = StructVector::GetEntries(out_vec); if (postgres_type.info == PostgresTypeAnnotation::GEOM_POINT) { - D_ASSERT(value_len == sizeof(double) * 2); + CheckValueLength(value_len, sizeof(double) * 2, type); FlatVector::GetDataMutable(child_entries[0])[output_offset] = ReadDouble(); FlatVector::GetDataMutable(child_entries[1])[output_offset] = ReadDouble(); break; @@ -385,4 +507,42 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType } } +PostgresDecimalKind PostgresBinaryParser::NonFiniteDecimalKindFromSign(uint16_t dec_sign) { + if (dec_sign == NUMERIC_NAN) { + return PostgresDecimalKind::NOT_A_NUMBER; + } + if (dec_sign == NUMERIC_PINF) { + return PostgresDecimalKind::POSITIVE_INFINITY; + } + if (dec_sign == NUMERIC_NINF) { + return PostgresDecimalKind::NEGATIVE_INFINITY; + } + throw InvalidInputException("Unsupported unbound NUMERIC sign: %u", static_cast(dec_sign)); +} + +string PostgresBinaryParser::NonFiniteDecimalKindToString(PostgresDecimalKind kind) { + switch (kind) { + case PostgresDecimalKind::NOT_A_NUMBER: + return "NaN"; + case PostgresDecimalKind::POSITIVE_INFINITY: + return "Infinity"; + case PostgresDecimalKind::NEGATIVE_INFINITY: + return "-Infinity"; + default: + throw InvalidInputException("Unsupported unbound NUMERIC kind"); + } +} + +bool PostgresBinaryParser::CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind) { + if (dec_kind == PostgresDecimalKind::ORDINARY) { + return false; + } + if (!type_config.numeric_nan_as_null) { + string kind_str = NonFiniteDecimalKindToString(dec_kind); + throw InvalidInputException("Unsupported NUMERIC value: %s", kind_str); + } + FlatVector::SetNull(out_vec, output_offset, true); + return true; +} + } // namespace duckdb diff --git a/src/postgres_binary_reader.cpp b/src/postgres_binary_reader.cpp index c41e5fe3d..58b4702b1 100644 --- a/src/postgres_binary_reader.cpp +++ b/src/postgres_binary_reader.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -8,8 +9,15 @@ namespace duckdb { PostgresBinaryReader::PostgresBinaryReader(PostgresConnection &con_p, const vector &column_ids, - const PostgresBindData &bind_data) - : PostgresResultReader(con_p, column_ids, bind_data), parser(bind_data.types, bind_data.postgres_types) { + const PostgresBindData &bind_data, + optional_ptr prepared_statement) + : PostgresResultReader(con_p, column_ids, bind_data, prepared_statement), + parser(bind_data.types, bind_data.postgres_types, bind_data.type_config) { + if (bind_data.query_mode == PostgresQueryMode::PLAIN_QUERY_BINARY) { + this->mode = PostgresBinaryReaderMode::PLAIN_QUERY; + } else { + this->mode = PostgresBinaryReaderMode::COPY; + } } PostgresBinaryReader::~PostgresBinaryReader() { @@ -17,27 +25,46 @@ PostgresBinaryReader::~PostgresBinaryReader() { } void PostgresBinaryReader::BeginCopy(ClientContext &context, const string &sql) { - con.BeginCopyFrom(context, sql, PGRES_COPY_OUT); - if (!FetchNextBuffer()) { - throw IOException("Failed to fetch header for COPY \"%s\"", sql); + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + this->result = Query(context, sql, 1); + this->row_offset = 0; + } else { + // COPY does not use the statement, it must be closed before the connection enters the COPY state + DestroyPreparedStatement(); + con.BeginCopyFrom(context, sql, PGRES_COPY_OUT); + if (!FetchNextBuffer()) { + throw IOException("Failed to fetch header for COPY \"%s\"", sql); + } + parser.CheckHeader(); } - parser.CheckHeader(); } -PostgresReadResult PostgresBinaryReader::Read(DataChunk &output) { +PostgresReadResult PostgresBinaryReader::Read(DataChunk &result) { + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + return ReadPlain(result); + } else { + return ReadCopy(result); + } +} + +PostgresReadResult PostgresBinaryReader::ReadCopy(DataChunk &output) { + auto result = PostgresReadResult::HAVE_MORE_TUPLES; while (output.size() < STANDARD_VECTOR_SIZE) { if (parser.ReadChunk(output, column_ids)) { - return PostgresReadResult::HAVE_MORE_TUPLES; + break; } FreeBuffer(); if (!FetchNextBuffer()) { - return PostgresReadResult::FINISHED; + result = PostgresReadResult::FINISHED; + break; } } - return PostgresReadResult::HAVE_MORE_TUPLES; + output.SetChildCardinality(output.size()); + return result; } bool PostgresBinaryReader::FetchNextBuffer() { + auto conn_lock = con.Lock(); char *out_buffer; int len = PQgetCopyData(con.GetConn(), &out_buffer, 0); auto new_buffer = data_ptr_cast(out_buffer); @@ -58,21 +85,71 @@ bool PostgresBinaryReader::FetchNextBuffer() { return false; } + // take ownership of the buffer before validating it, so that a short read + // does not leak it - PQgetCopyData allocates even when it returns a length + // we cannot use + buffer = new_buffer; + // len -2 is error // we expect at least 2 bytes in each message for the tuple count if (!new_buffer || len < sizeof(int16_t)) { throw IOException("Unable to read binary COPY data from Postgres: %s", string(PQerrorMessage(con.GetConn()))); } - buffer = new_buffer; parser.SetBuffer(buffer, len); return true; } void PostgresBinaryReader::FreeBuffer() { - if (buffer) { - PQfreemem(buffer); + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + result.reset(); + row_offset = 0; + } else { + if (buffer) { + PQfreemem(buffer); + } + buffer = nullptr; + } +} + +PostgresReadResult PostgresBinaryReader::ReadPlain(DataChunk &output) { + if (!result) { + return PostgresReadResult::FINISHED; } - buffer = nullptr; + + idx_t output_offset = output.size(); + for (; output_offset < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++, output_offset++) { + for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { + auto col_idx = column_ids[output_idx]; + auto &out_vec = output.data[output_idx]; + + if (result->IsNull(row_offset, output_idx)) { + FlatVector::SetNull(out_vec, output_offset, true); + continue; + } + + // Obtain the raw binary data from the result cell + auto val = result->GetStringRef(row_offset, output_idx); + parser.SetBuffer(data_ptr_cast(val.GetDataWriteable()), val.GetSize()); + + if (col_idx == COLUMN_IDENTIFIER_ROW_ID) { + PostgresType ctid_type; + ctid_type.info = PostgresTypeAnnotation::CTID; + parser.ParseValue(LogicalType::BIGINT, ctid_type, NumericCast(val.GetSize()), out_vec, + output_offset); + } else { + parser.ParseValue(bind_data.types[col_idx], bind_data.postgres_types[col_idx], + NumericCast(val.GetSize()), out_vec, output_offset); + } + } + } + output.SetChildCardinality(output_offset); + + bool finished = row_offset >= result->Count(); + if (finished) { + FreeBuffer(); + return PostgresReadResult::FINISHED; + } + return PostgresReadResult::HAVE_MORE_TUPLES; } } // namespace duckdb diff --git a/src/postgres_connection.cpp b/src/postgres_connection.cpp index b5a10689f..1d5ca2aaa 100644 --- a/src/postgres_connection.cpp +++ b/src/postgres_connection.cpp @@ -1,29 +1,22 @@ +#include "postgres_connection.hpp" + #include "duckdb/storage/table_storage_info.hpp" +#include "duckdb/logging/logger.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/column_list.hpp" #include "duckdb/parser/parser.hpp" -#include "postgres_connection.hpp" -#include "postgres_logging.hpp" #include "duckdb/common/types/uuid.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/helper.hpp" +#include "postgres_logging.hpp" +#include "postgres_prepared_statement.hpp" + namespace duckdb { static bool debug_postgres_print_queries = false; -OwnedPostgresConnection::OwnedPostgresConnection(PGconn *conn) : connection(conn) { -} - -OwnedPostgresConnection::~OwnedPostgresConnection() { - if (!connection) { - return; - } - PQfinish(connection); - connection = nullptr; -} - -PostgresConnection::PostgresConnection(shared_ptr connection_p) - : connection(std::move(connection_p)) { +PostgresConnection::PostgresConnection(PGconn *connection_p) : connection(connection_p) { } PostgresConnection::~PostgresConnection() { @@ -31,11 +24,18 @@ PostgresConnection::~PostgresConnection() { } PostgresConnection::PostgresConnection(PostgresConnection &&other) noexcept { + lock_guard guard(other.connection_lock); std::swap(connection, other.connection); std::swap(dsn, other.dsn); } PostgresConnection &PostgresConnection::operator=(PostgresConnection &&other) noexcept { + if (this == &other) { + return *this; + } + std::lock(connection_lock, other.connection_lock); + lock_guard guard(connection_lock, std::adopt_lock); + lock_guard other_guard(other.connection_lock, std::adopt_lock); std::swap(connection, other.connection); std::swap(dsn, other.dsn); return *this; @@ -43,7 +43,7 @@ PostgresConnection &PostgresConnection::operator=(PostgresConnection &&other) no PostgresConnection PostgresConnection::Open(const string &dsn, const string &attach_path) { PostgresConnection result; - result.connection = make_shared_ptr(PostgresUtils::PGConnect(dsn, attach_path)); + result.connection = PostgresUtils::PGConnect(dsn, attach_path); result.dsn = dsn; return result; } @@ -62,7 +62,7 @@ static bool ResultHasError(PGresult *result) { } PGresult *PostgresConnection::PQExecute(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { + const PostgresParameters ¶ms, int format) { if (PostgresConnection::DebugPrintQueries()) { Printer::Print(query + "\n"); } @@ -73,11 +73,10 @@ PGresult *PostgresConnection::PQExecute(optional_ptr context, con PGconn *conn = GetConn(); PGresult *res = nullptr; - if (params.Empty()) { + if (params.Empty() && format == 0) { res = PQexec(GetConn(), query.c_str()); } else { // Unlike PQexec, PQexecParams allows at most one SQL command in the given string. - int format = 0; // text format res = PQexecParams(conn, query.c_str(), params.Count(), params.Types(), params.Values(), params.Lengths(), params.Formats(), format); } @@ -93,14 +92,14 @@ PGresult *PostgresConnection::PQExecute(optional_ptr context, con unique_ptr PostgresConnection::TryQuery(optional_ptr context, const string &query, optional_ptr error_message, - const PostgresParameters ¶ms) { - lock_guard guard(connection->connection_lock); - auto result = PQExecute(context, query.c_str(), params); + const PostgresParameters ¶ms, int format) { + lock_guard guard(connection_lock); + auto result = PQExecute(context, query.c_str(), params, format); if (ResultHasError(result)) { if (error_message) { - *error_message = StringUtil::Format("Failed to execute query \"" + query + - "\": " + string(PQresultErrorMessage(result))); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + *error_message = StringUtil::Format("Failed to execute query \"" + query + "\": " + string(error)); } PQclear(result); return nullptr; @@ -109,18 +108,51 @@ unique_ptr PostgresConnection::TryQuery(optional_ptr PostgresConnection::Query(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { + const PostgresParameters ¶ms, int format) { string error_msg; - auto result = TryQuery(context, query, &error_msg, params); + auto result = TryQuery(context, query, &error_msg, params, format); if (!result) { throw std::runtime_error(error_msg); } return result; } +unique_ptr PostgresConnection::QueryStmt(optional_ptr context, + const PostgresPreparedStatement &stmt, + const PostgresParameters ¶ms, int format) { + auto &query = stmt.GetQuery(); + if (PostgresConnection::DebugPrintQueries()) { + Printer::Print(query + "\n"); + } + int64_t start_time = std::chrono::time_point_cast(std::chrono::steady_clock::now()) + .time_since_epoch() + .count(); + + lock_guard guard(connection_lock); + PGconn *conn = GetConn(); + PGresult *result = PQexecPrepared(conn, stmt.GetName().c_str(), params.Count(), params.Values(), params.Lengths(), + params.Formats(), format); + + int64_t end_time = std::chrono::time_point_cast(std::chrono::steady_clock::now()) + .time_since_epoch() + .count(); + if (context) { + DUCKDB_LOG(*context, PostgresQueryLogType, query, end_time - start_time); + } + + if (ResultHasError(result)) { + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(conn); + auto error_message = StringUtil::Format("Failed to execute prepared statement \"%s\" for query \"%s\": %s", + stmt.GetName(), query, string(error)); + PQclear(result); + throw std::runtime_error(error_message); + } + return make_uniq(result); +} + void PostgresConnection::Execute(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { - Query(context, query, params); + const PostgresParameters ¶ms, int format) { + Query(context, query, params, format); } vector> PostgresConnection::ExecuteQueries(ClientContext &context, const string &queries) { @@ -130,11 +162,13 @@ vector> PostgresConnection::ExecuteQueries(ClientCont int64_t start_time = std::chrono::time_point_cast(std::chrono::steady_clock::now()) .time_since_epoch() .count(); + lock_guard guard(connection_lock); auto res = PQsendQuery(GetConn(), queries.c_str()); if (res == 0) { throw std::runtime_error("Failed to execute query \"" + queries + "\": " + string(PQerrorMessage(GetConn()))); } vector> results; + string error_message; while (true) { auto res = PQgetResult(GetConn()); if (!res) { @@ -142,14 +176,19 @@ vector> PostgresConnection::ExecuteQueries(ClientCont } auto result = make_uniq(res); if (ResultHasError(res)) { - throw std::runtime_error("Failed to execute query \"" + queries + - "\": " + string(PQresultErrorMessage(res))); + if (error_message.empty()) { + error_message = "Failed to execute query \"" + queries + "\": " + string(PQresultErrorMessage(res)); + } + continue; } if (PQresultStatus(res) != PGRES_TUPLES_OK) { continue; } results.push_back(std::move(result)); } + if (!error_message.empty()) { + throw std::runtime_error(error_message); + } int64_t end_time = std::chrono::time_point_cast(std::chrono::steady_clock::now()) .time_since_epoch() .count(); @@ -176,17 +215,24 @@ PostgresVersion PostgresConnection::GetPostgresVersion(ClientContext &context) { } bool PostgresConnection::IsOpen() { - return connection.get(); + return connection != nullptr; } void PostgresConnection::Close() { - if (!IsOpen()) { + lock_guard guard(connection_lock); + if (!connection) { return; } + PQfinish(connection); connection = nullptr; } bool PostgresConnection::PingServer(const std::string &health_check_query) { + lock_guard guard(connection_lock); + return PingServerInternal(health_check_query); +} + +bool PostgresConnection::PingServerInternal(const std::string &health_check_query) { if (!IsOpen()) { return false; } @@ -203,6 +249,7 @@ bool PostgresConnection::PingServer(const std::string &health_check_query) { } void PostgresConnection::Reset(const std::string &health_check_query) { + lock_guard guard(connection_lock); if (!IsOpen()) { throw InternalException("Cannot reset a connection that is not open"); } @@ -220,7 +267,7 @@ void PostgresConnection::Reset(const std::string &health_check_query) { } } PQreset(conn); - if (!PingServer(health_check_query)) { + if (!PingServerInternal(health_check_query)) { throw InternalException("Connection reset failure"); } } diff --git a/src/postgres_connection_string.cpp b/src/postgres_connection_string.cpp new file mode 100644 index 000000000..0ac225637 --- /dev/null +++ b/src/postgres_connection_string.cpp @@ -0,0 +1,105 @@ +#include "postgres_connection_string.hpp" + +#include +#include +#include +#include + +#include "duckdb.hpp" + +#include "dbconnector/query/query_writer.hpp" + +namespace duckdb { + +string PostgresConnectionString::Escape(const string &input) { + using namespace dbconnector::query; + auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::BACKSLASH); + return QueryWriter::WriteQuotedAndEscaped(config, input); +} + +string PostgresConnectionString::ExtractOption(const KeyValueSecret &kv_secret, const string &name) { + Value input_val = kv_secret.TryGetValue(Identifier(name)); + if (input_val.IsNull()) { + // not provided + return string(); + } + string result; + result += name; + result += "="; + result += Escape(input_val.ToString()); + result += " "; + return result; +} + +bool PostgresConnectionString::ContainsApplicationName(const std::string &conninfo_str) { + char *errmsg = nullptr; + + // Parse the connection string into an array of PQconninfoOption structs + PQconninfoOption *options = PQconninfoParse(conninfo_str.c_str(), &errmsg); + + if (!options) { + std::string err_str; + if (errmsg) { + err_str = std::string(errmsg); + PQfreemem(errmsg); + } + throw InvalidInputException("Failed to parse connection string: \"%s\"", err_str); + } + + bool found = false; + + for (PQconninfoOption *opt = options; opt->keyword != nullptr; ++opt) { + if (std::strcmp(opt->keyword, "application_name") == 0 || + std::strcmp(opt->keyword, "fallback_application_name") == 0) { + // opt->val will be non-null if it was explicitly defined in the string + if (opt->val != nullptr && std::strlen(opt->val) > 0) { + found = true; + break; + } + } + } + + // cleanup + PQconninfoFree(options); + if (errmsg) { + PQfreemem(errmsg); + } + + return found; +} + +// libpq allows '?' in the user info, so the query starts at the first '?' after it +static std::string::size_type FindURIQueryStart(const string &uri) { + auto scheme_end = uri.find("://"); + auto start = scheme_end == std::string::npos ? 0 : scheme_end + 3; + auto user_info_end = uri.find_first_of("@/", start); + if (user_info_end != std::string::npos && uri[user_info_end] == '@') { + start = user_info_end + 1; + } + return uri.find('?', start); +} + +bool PostgresConnectionString::IsURI(const string &conn_string) { + return StringUtil::StartsWith(conn_string, "postgresql://") || StringUtil::StartsWith(conn_string, "postgres://"); +} + +void PostgresConnectionString::AppendDefaultApplicationName(string &conn_string, + PostgresConnectionStringFormat format) { + // a fallback name lets PGAPPNAME and user-specified names take precedence + string app_name = "fallback_application_name=DuckDB"; + if (format == PostgresConnectionStringFormat::KEY_VALUE) { + if (conn_string.length() == 0 || conn_string[conn_string.length() - 1] != ' ') { + conn_string.push_back(' '); + } + } else { + auto query_start = FindURIQueryStart(conn_string); + if (query_start == std::string::npos) { + conn_string.push_back('?'); + } else if (query_start + 1 < conn_string.length() && conn_string.back() != '&') { + conn_string.push_back('&'); + } + } + conn_string += app_name; +} + +} // namespace duckdb diff --git a/src/postgres_copy_from.cpp b/src/postgres_copy_from.cpp index a39fe419d..bca80954c 100644 --- a/src/postgres_copy_from.cpp +++ b/src/postgres_copy_from.cpp @@ -1,13 +1,16 @@ #include "postgres_connection.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_binary_reader.hpp" namespace duckdb { void PostgresConnection::BeginCopyFrom(ClientContext &context, const string &query, ExecStatusType expected_result) { + lock_guard guard(connection_lock); PostgresResult pg_res(PQExecute(context, query.c_str())); auto result = pg_res.res; if (!result || PQresultStatus(result) != expected_result) { - throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(PQresultErrorMessage(result))); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(error)); } } diff --git a/src/postgres_copy_to.cpp b/src/postgres_copy_to.cpp index 7796a126a..88d64a892 100644 --- a/src/postgres_copy_to.cpp +++ b/src/postgres_copy_to.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -60,10 +61,14 @@ void PostgresConnection::BeginCopyTo(ClientContext &context, PostgresCopyState & } query += ")"; - PostgresResult pg_res(PQExecute(context, query.c_str())); - auto result = pg_res.res; - if (!result || PQresultStatus(result) != PGRES_COPY_IN) { - throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(PQresultErrorMessage(result))); + { + lock_guard guard(connection_lock); + PostgresResult pg_res(PQExecute(context, query.c_str())); + auto result = pg_res.res; + if (!result || PQresultStatus(result) != PGRES_COPY_IN) { + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(error)); + } } if (state.format == PostgresCopyFormat::BINARY) { // binary copy requires a header @@ -74,6 +79,7 @@ void PostgresConnection::BeginCopyTo(ClientContext &context, PostgresCopyState & } void PostgresConnection::CopyData(data_ptr_t buffer, idx_t size) { + lock_guard guard(connection_lock); int result; do { result = PQputCopyData(GetConn(), (const char *)buffer, int(size)); @@ -104,6 +110,7 @@ void PostgresConnection::FinishCopyTo(PostgresCopyState &state) { CopyData(writer); } + lock_guard guard(connection_lock); auto result_code = PQputCopyEnd(GetConn(), nullptr); if (result_code != 1) { throw InternalException("Error during PQputCopyEnd: %s", PQerrorMessage(GetConn())); @@ -112,8 +119,8 @@ void PostgresConnection::FinishCopyTo(PostgresCopyState &state) { PostgresResult pg_res(PQgetResult(GetConn())); auto result = pg_res.res; if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) { - string error_msg(PQresultErrorMessage(result)); - throw std::runtime_error("Failed to copy data: " + error_msg); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to copy data: " + string(error)); } } diff --git a/src/postgres_execute.cpp b/src/postgres_execute.cpp deleted file mode 100644 index 46caebb1f..000000000 --- a/src/postgres_execute.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "duckdb.hpp" - -#include "duckdb/parser/parsed_data/create_table_function_info.hpp" -#include "postgres_scanner.hpp" -#include "duckdb/main/database_manager.hpp" -#include "duckdb/main/attached_database.hpp" -#include "storage/postgres_catalog.hpp" -#include "storage/postgres_transaction.hpp" - -namespace duckdb { - -struct PGExecuteBindData : public TableFunctionData { - explicit PGExecuteBindData(PostgresCatalog &pg_catalog, string query_p, bool use_transaction) - : pg_catalog(pg_catalog), query(std::move(query_p)), use_transaction(use_transaction) { - } - - bool finished = false; - PostgresCatalog &pg_catalog; - string query; - bool use_transaction = true; -}; - -static duckdb::unique_ptr PGExecuteBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - return_types.emplace_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - - // look up the database to query - auto db_name = input.inputs[0].GetValue(); - auto &db_manager = DatabaseManager::Get(context); - auto db = db_manager.GetDatabase(context, Identifier(db_name)); - if (!db) { - throw BinderException("Failed to find attached database \"%s\" referenced in postgres_query", db_name); - } - auto &catalog = db->GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - throw BinderException("Attached database \"%s\" does not refer to a Postgres database", db_name); - } - auto &pg_catalog = catalog.Cast(); - - bool use_transaction = true; - for (auto &kv : input.named_parameters) { - if (kv.first == "use_transaction") { - use_transaction = BooleanValue::Get(kv.second); - } - } - - return make_uniq(pg_catalog, input.inputs[1].GetValue(), use_transaction); -} - -static void PGExecuteFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = data_p.bind_data->CastNoConst(); - if (data.finished) { - return; - } - auto &transaction = Transaction::Get(context, data.pg_catalog).Cast(); - if (data.use_transaction) { - transaction.Query(data.query); - } else { - transaction.QueryWithoutTransaction(data.query); - } - - data.finished = true; -} - -PostgresExecuteFunction::PostgresExecuteFunction() - : TableFunction("postgres_execute", {LogicalType::VARCHAR, LogicalType::VARCHAR}, PGExecuteFunction, - PGExecuteBind) { - named_parameters["use_transaction"] = LogicalType::BOOLEAN; -} - -} // namespace duckdb diff --git a/src/postgres_extension.cpp b/src/postgres_extension.cpp index debc9e01c..65920aa58 100644 --- a/src/postgres_extension.cpp +++ b/src/postgres_extension.cpp @@ -1,7 +1,7 @@ #define DUCKDB_BUILD_LOADABLE_EXTENSION #include "duckdb.hpp" -#include -#include +#include "duckdb/logging/log_manager.hpp" +#include "duckdb/main/config.hpp" #include "postgres_scanner.hpp" #include "postgres_storage.hpp" @@ -83,6 +83,10 @@ static void SetPostgresConnectionLimit(ClientContext &context, SetScope scope, V } } +static void ValidateQueryMode(ClientContext &, SetScope, Value ¶meter) { + PostgresUtils::ExtractQueryMode(parameter); +} + static void DisablePool(ClientContext &context, SetScope scope, Value ¶meter) { if (scope == SetScope::LOCAL) { throw InvalidInputException("pg_connection_cache can only be set globally"); @@ -118,24 +122,9 @@ static std::string CreatePoolNote(const std::string &option) { } static void LoadInternal(ExtensionLoader &loader) { - // Prevent a race with OpenSSL init that manifests itself with - // the following message on the first connection attempt: - // "port 5432 failed: could not create SSL context: unknown option" - OPENSSL_init_crypto(0, nullptr); - OPENSSL_init_ssl(0, nullptr); - // Register the OAuth bearer token hook before any connections are made PostgresInitOAuthHook(); - PostgresScanFunction postgres_fun; - loader.RegisterFunction(postgres_fun); - - PostgresScanFunctionFilterPushdown postgres_fun_filter_pushdown; - loader.RegisterFunction(postgres_fun_filter_pushdown); - - PostgresAttachFunction attach_func; - loader.RegisterFunction(attach_func); - PostgresClearCacheFunction clear_cache_func; loader.RegisterFunction(clear_cache_func); @@ -154,6 +143,12 @@ static void LoadInternal(ExtensionLoader &loader) { PostgresConfigurePoolFunction configure_pool_function; loader.RegisterFunction(configure_pool_function); + PostgresPinConnectionFunction pin_connection_function; + loader.RegisterFunction(pin_connection_function); + + PostgresClosePinnedConnectionFunction postgres_close_pinned_connection_function; + loader.RegisterFunction(postgres_close_pinned_connection_function); + RegisterHstoreFunctions(loader); // Register the new type @@ -168,6 +163,9 @@ static void LoadInternal(ExtensionLoader &loader) { auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); StorageExtension::Register(config, "postgres_scanner", make_shared_ptr()); + config.AddExtensionOption("pg_query_mode", + "Query mode, one of: 'AUTO', 'COPY_BINARY', 'PLAIN_QUERY_BINARY', 'PLAIN_QUERY_TEXT'", + LogicalType::VARCHAR, Value("AUTO"), ValidateQueryMode); config.AddExtensionOption("pg_use_binary_copy", "Whether or not to use BINARY copy to read data", LogicalType::BOOLEAN, Value::BOOLEAN(true)); config.AddExtensionOption("pg_use_ctid_scan", "Whether or not to parallelize scanning using table ctids", @@ -185,6 +183,13 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption( "pg_array_as_varchar", "Read Postgres arrays as varchar - enables reading mixed dimensional arrays", LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_numeric_as_varchar", + "Read Postgres numerics without precision and scale or precision > 38 as varchar instead of double", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_numeric_nan_as_null", "Read Postgres numeric NaN value as NULL instead of throwing an error", + LogicalType::BOOLEAN, Value::BOOLEAN(true), PostgresClearCacheFunction::ClearCacheOnSetting); config.AddExtensionOption( "pg_connection_cache", "Whether or not to use the connection pooling." @@ -193,9 +198,10 @@ static void LoadInternal(ExtensionLoader &loader) { "postgres_configure_pool(catalog_name='my_attached_postgres_db', " "max_connections=0)\" for already attached Postgres databases.", LogicalType::BOOLEAN, Value::BOOLEAN(true), DisablePool); - config.AddExtensionOption("pg_experimental_filter_pushdown", "Whether or not to use filter pushdown", + config.AddExtensionOption("pg_enable_remote_pushdown", + "Whether or not to push down whole queries into Postgres for newly attached databases", LogicalType::BOOLEAN, Value::BOOLEAN(true)); - config.AddExtensionOption("pg_order_pushdown", "Push ORDER BY and LIMIT clauses to Postgres (default: true)", + config.AddExtensionOption("pg_enable_filter_pushdown", "Whether or not to use filter pushdown", LogicalType::BOOLEAN, Value::BOOLEAN(true)); config.AddExtensionOption("pg_null_byte_replacement", "When writing NULL bytes to Postgres, replace them with the given character", @@ -210,6 +216,23 @@ static void LoadInternal(ExtensionLoader &loader) { "Whether or not to use TEXT protocol to read data. This is slower, but provides better " "compatibility with non-Postgres systems", LogicalType::BOOLEAN, Value::BOOLEAN(false)); + config.AddExtensionOption( + "pg_use_information_schema_introspection", + "Use SQL-standard information_schema views for ATTACH-time schema discovery instead of pg_catalog." + "For compatibility with pg protocol compatible databases like spanner, cockroach, redshift, ...", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_staleness_query_enabled", + "Whether or not the table catalog cache checks Postgres for external DDL changes before serving a " + "cache hit. Defaults to the opposite of pg_use_information_schema_introspection when not explicitly set " + "(off for pg protocol compatible databases that may not support the underlying query, on for Postgres).", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting, SetScope::GLOBAL); + config.AddExtensionOption( + "pg_staleness_query", + "Custom query used in place of the default table staleness query when pg_staleness_query_enabled " + "resolves to true. Must contain a ${SCHEMA} placeholder and return at least 3 columns " + "(identity, name, revision marker). Empty (default) uses the built-in pg_class/xmin query.", + LogicalType::VARCHAR, Value(), PostgresClearCacheFunction::ClearCacheOnSetting, SetScope::GLOBAL); config.AddExtensionOption("pg_statement_timeout_millis", "Postgres statement timeout in milliseconds to set on scan connections", LogicalType::UINTEGER, Value()); @@ -270,8 +293,7 @@ static void LoadInternal(ExtensionLoader &loader) { "The query that is used to check that the connection is healthy. Setting this option to " "an empty string disables the health check. " + CreatePoolNote("health_check_query=SELECT 42"), - LogicalType::VARCHAR, PostgresConnectionPool::DefaultHealthCheckQuery(), nullptr, - SetScope::GLOBAL); + LogicalType::VARCHAR, default_pool_config.health_check_query, nullptr, SetScope::GLOBAL); OptimizerExtension postgres_optimizer; postgres_optimizer.optimize_function = PostgresOptimizer::Optimize; diff --git a/src/postgres_filter_pushdown.cpp b/src/postgres_filter_pushdown.cpp index 907b4ad82..b9a173339 100644 --- a/src/postgres_filter_pushdown.cpp +++ b/src/postgres_filter_pushdown.cpp @@ -1,24 +1,49 @@ #include "postgres_filter_pushdown.hpp" -#include "duckdb/parser/keyword_helper.hpp" -#include "duckdb/function/scalar/struct_utils.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_conjunction_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/planner/expression/bound_operator_expression.hpp" -#include "duckdb/planner/filter/table_filter_functions.hpp" #include "duckdb/common/enum_util.hpp" +#include "duckdb/common/string_util.hpp" #include "dbconnector/table_scan/filter_pushdown.hpp" #include "dbconnector/table_scan/filter_util.hpp" -#include "postgres_utils.hpp" - namespace duckdb { +static string WriteDistinctFrom(ExpressionType distinct_type, const string &column_name, + const string &constant_string) { + switch (distinct_type) { + case ExpressionType::COMPARE_DISTINCT_FROM: + return StringUtil::Format("%s %s %s", column_name, "IS DISTINCT FROM", constant_string); + case ExpressionType::COMPARE_NOT_DISTINCT_FROM: + return StringUtil::Format("%s %s %s", column_name, "IS NOT DISTINCT FROM", constant_string); + default: + throw InvalidInputException("Unsupported DISTINCT FROM comparion type: %s", EnumUtil::ToString(distinct_type)); + } +} + +static dbconnector::table_scan::FilterPushdown::Config +CreatePostgresConfig(PostgresInstanceType instance_type = PostgresInstanceType::POSTGRES) { + using namespace dbconnector; + + if (instance_type == PostgresInstanceType::REDSHIFT) { + // Redshift does not understand the '\x...' bytea hex format, '\x41'::BYTEA is read as the bytes "x41" + return table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "FROM_HEX('", + "')", "C", WriteDistinctFrom); + } + return table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "'\\x", + "'::BYTEA", "C", WriteDistinctFrom); +} + +bool PostgresFilterPushdown::CanPushExpressionDown(ClientContext &, const LogicalGet &, Expression &expr) { + using dbconnector::table_scan::FilterPushdown; + + auto config = CreatePostgresConfig(); + string filter = FilterPushdown::TransformFilterExpression(config, "dummy", expr); + return !filter.empty(); +} + string PostgresFilterPushdown::TransformFilters(const vector &column_ids, - optional_ptr filters, const vector &names) { + optional_ptr filters, const vector &names, + PostgresInstanceType instance_type) { using namespace dbconnector; if (!filters || !filters->HasFilters()) { // no filters @@ -34,8 +59,7 @@ string PostgresFilterPushdown::TransformFilters(const vector &column_i column_name = names[column_id]; } auto &filter = entry.Filter(); - auto config = table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "'\\x", - "::BYTEA"); + auto config = CreatePostgresConfig(instance_type); auto filter_text = table_scan::FilterPushdown::TransformFilter(config, column_name, filter, column_id); if (filter_text.empty()) { @@ -43,7 +67,7 @@ string PostgresFilterPushdown::TransformFilters(const vector &column_i continue; } throw NotImplementedException( - "Unsupported filter pushdown, use 'pg_experimental_filter_pushdown=FALSE' to disable pushdowns." + "Unsupported filter pushdown, use 'pg_enable_filter_pushdown=FALSE' to disable pushdowns." " Problematic filter: \"%s\"", table_scan::FilterUtil::ToString(filter)); } diff --git a/src/postgres_hstore.cpp b/src/postgres_hstore.cpp index c1fe02ada..860e74e18 100644 --- a/src/postgres_hstore.cpp +++ b/src/postgres_hstore.cpp @@ -216,10 +216,12 @@ void PostgresHstoreToJsonFun(DataChunk &args, ExpressionState &state, Vector &re void RegisterHstoreFunctions(ExtensionLoader &loader) { auto hstore_get = ScalarFunction("postgres_hstore_get", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, PostgresHstoreGetFun); + hstore_get.SetFallible(); loader.RegisterFunction(hstore_get); auto hstore_to_json = ScalarFunction("postgres_hstore_to_json", {LogicalType::VARCHAR}, LogicalType::JSON(), PostgresHstoreToJsonFun); + hstore_to_json.SetFallible(); loader.RegisterFunction(hstore_to_json); } diff --git a/src/postgres_oauth.cpp b/src/postgres_oauth.cpp index 619532aec..a9cc073aa 100644 --- a/src/postgres_oauth.cpp +++ b/src/postgres_oauth.cpp @@ -1,4 +1,5 @@ #include "postgres_oauth.hpp" +#include "duckdb/main/client_context.hpp" #include #include diff --git a/src/postgres_pin_connection.cpp b/src/postgres_pin_connection.cpp new file mode 100644 index 000000000..527f3bf62 --- /dev/null +++ b/src/postgres_pin_connection.cpp @@ -0,0 +1,64 @@ +#include "postgres_scanner.hpp" + +#include "storage/postgres_catalog.hpp" + +namespace duckdb { + +static void PostgresPinConnection(DataChunk &args, ExpressionState &state, Vector &result) { + UnifiedVectorFormat catalog_name_data; + args.data[0].ToUnifiedFormat(catalog_name_data); + if (!catalog_name_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified attached database name must be not null"); + } + + auto catalog_name_strt = UnifiedVectorFormat::GetData(catalog_name_data)[0]; + string catalog_name(catalog_name_strt.GetData(), catalog_name_strt.GetSize()); + + auto &catalog = PostgresUtils::GetCatalogByName(state.GetContext(), catalog_name); + auto &pool = catalog.GetConnectionPool(); + + auto conn = pool.ForceAcquire(); + uint64_t conn_id = pool.PinConnection(std::move(conn)); + + auto result_data = FlatVector::GetDataMutable(result); + result_data[0] = conn_id; +} + +PostgresPinConnectionFunction::PostgresPinConnectionFunction() + : ScalarFunction("postgres_pin_connection", {LogicalType::VARCHAR}, LogicalType::UBIGINT, PostgresPinConnection) { + SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + SetFallible(); + SetVolatile(); +} + +static void PostgresClosePinnedConnection(DataChunk &args, ExpressionState &state, Vector &result) { + UnifiedVectorFormat catalog_name_data; + args.data[0].ToUnifiedFormat(catalog_name_data); + if (!catalog_name_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified catalog name must be not null"); + } + auto catalog_name_strt = UnifiedVectorFormat::GetData(catalog_name_data)[0]; + string catalog_name(catalog_name_strt.GetData(), catalog_name_strt.GetSize()); + + UnifiedVectorFormat conn_id_data; + args.data[1].ToUnifiedFormat(conn_id_data); + if (!conn_id_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified connection ID must be not null"); + } + auto conn_id = UnifiedVectorFormat::GetData(conn_id_data)[0]; + + auto &catalog = PostgresUtils::GetCatalogByName(state.GetContext(), catalog_name); + { auto conn = catalog.GetConnectionPool().UnpinConnection(conn_id); } + + auto &result_validity = FlatVector::ValidityMutable(result); + result_validity.SetInvalid(0); +} + +PostgresClosePinnedConnectionFunction::PostgresClosePinnedConnectionFunction() + : ScalarFunction("postgres_close_pinned_connection", {LogicalType::VARCHAR, LogicalType::UBIGINT}, + LogicalType::BOOLEAN, PostgresClosePinnedConnection) { + SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + SetFallible(); + SetVolatile(); +} +} // namespace duckdb diff --git a/src/postgres_prepared_statement.cpp b/src/postgres_prepared_statement.cpp new file mode 100644 index 000000000..11cb92b2f --- /dev/null +++ b/src/postgres_prepared_statement.cpp @@ -0,0 +1,63 @@ +#include "postgres_prepared_statement.hpp" +#include "postgres_connection.hpp" + +#include + +namespace duckdb { + +PostgresPreparedStatement::PostgresPreparedStatement(PGconn *conn, uint64_t pool_connection_id_p, string name_p, + string query_p, PGresult *prepare_result_p) + : pool_connection_id(pool_connection_id_p), name(std::move(name_p)), query(std::move(query_p)), + prepare_result(prepare_result_p) { + if (!prepare_result.res) { + const char *error = conn ? PQerrorMessage(conn) : ""; + throw BinderException("Failed to prepare query \"%s\" (no result returned): %s", query, error); + } + if (PQresultStatus(prepare_result.res) != PGRES_COMMAND_OK) { + throw BinderException("Failed to prepare query \"%s\": %s", query, PQresultErrorMessage(prepare_result.res)); + } +} + +string PostgresPreparedStatement::GenerateName() { + // statement names only need to be unique within a Postgres session, a process-wide counter guarantees this + static std::atomic counter {0}; + return "duckdb_stmt_" + std::to_string(counter.fetch_add(1, std::memory_order_relaxed)); +} + +uint64_t PostgresPreparedStatement::GetPoolConnectionId() const { + return pool_connection_id; +} + +const string &PostgresPreparedStatement::GetName() const { + return name; +} + +const string &PostgresPreparedStatement::GetQuery() const { + return query; +} + +PGresult *PostgresPreparedStatement::GetPrepareResult() const { + return prepare_result.res; +} + +void PostgresPreparedStatement::Destroy(PostgresConnection &con) { + if (destroyed) { + return; + } + // mark the statement as destroyed even if closing fails, so it is never executed again + destroyed = true; + auto conn_lock = con.Lock(); + auto conn = con.GetConn(); + PostgresResult close_result(PQclosePrepared(conn, name.c_str())); + if (!close_result.res || PQresultStatus(close_result.res) != PGRES_COMMAND_OK) { + auto error = close_result.res ? PQresultErrorMessage(close_result.res) : PQerrorMessage(conn); + throw std::runtime_error( + StringUtil::Format("Failed to close prepared statement \"%s\": %s", name, string(error))); + } +} + +bool PostgresPreparedStatement::IsDestroyed() const { + return destroyed; +} + +} // namespace duckdb diff --git a/src/postgres_query.cpp b/src/postgres_query.cpp index 8096f14fd..69b85d35c 100644 --- a/src/postgres_query.cpp +++ b/src/postgres_query.cpp @@ -1,7 +1,11 @@ #include "duckdb.hpp" +#include "dbconnector/defer.hpp" + +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "postgres_parameters.hpp" +#include "postgres_prepared_statement.hpp" #include "postgres_scanner.hpp" #include "duckdb/main/database_manager.hpp" #include "duckdb/main/attached_database.hpp" @@ -10,132 +14,321 @@ namespace duckdb { -static unique_ptr PGQueryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(context); +static bool ExtractFlag(TableFunctionBindInput &input, const string &name, bool default_val) { + auto it = input.named_parameters.find(Identifier(name)); + if (it != input.named_parameters.end()) { + Value &bool_val = it->second; + if (!bool_val.IsNull()) { + return BooleanValue::Get(bool_val); + } + } + return default_val; +} - if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) { - throw BinderException("Parameters to postgres_query cannot be NULL"); +static unique_ptr BindDML(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names, + PostgresCatalog &catalog, PostgresConnection &con, + vector preliminary_queries, std::string sql, bool use_transaction, + uint64_t pinned_connection_id, PostgresParameters params, + unique_ptr prepared_statement) { + // The statement returns no result columns: it's a command (DDL, or DML without RETURNING). + // Instead of failing, run it as a command and return a single-row Success result. We reuse + // the prepare/describe just done — no extra round-trip — and defer execution to + // InitGlobalState (execution time, not bind, so EXPLAIN does not run it). + auto result = make_uniq(context, catalog.GetName()); + result->command_only = true; + result->clear_catalog_cache = ExtractFlag(input, "clear_catalog_cache", false); + result->pinned_connection_id = pinned_connection_id; + if (ExtractFlag(input, "suppress_dml_output", false)) { + // This invocation wraps a command with no result set (DDL, or DML without RETURNING). Tell the + // binder via the return-type modifier so that when this is routed through CONNECT the outer + // statement is reported as NOTHING and displays like a native command (no spurious result table). + input.table_function.call_return_type = StatementReturnType::NOTHING; + } else if (ExtractFlag(input, "return_changed_rows", false)) { + // The command is a DML statement that is executed remotely in place of a DuckDB statement - report the + // number of changed rows, like DuckDB reports them for its own DML statements + input.table_function.call_return_type = StatementReturnType::CHANGED_ROWS; } + return_types.emplace_back(LogicalType::BIGINT); + names.emplace_back(Identifier("rowcount")); + result->dsn = con.GetDSN(); + result->types = return_types; + result->names.emplace_back(names[0].GetIdentifierName()); + result->read_only = false; + result->preliminary_queries = std::move(preliminary_queries); + result->sql = std::move(sql); + result->params = std::move(params); + result->use_transaction = use_transaction; + result->prepared_statement = std::move(prepared_statement); + PostgresScanFunction::PrepareBind(catalog.GetPostgresVersion(), context, *result, 0, catalog); + return std::move(result); +} - // look up the database to query - auto db_name = input.inputs[0].GetValue(); - auto &db_manager = DatabaseManager::Get(context); - auto db = db_manager.GetDatabase(context, Identifier(db_name)); - if (!db) { - throw BinderException("Failed to find attached database \"%s\" referenced in postgres_query", db_name); - } - auto &catalog = db->GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - throw BinderException("Attached database \"%s\" does not refer to a Postgres database", db_name); - } - auto &pg_catalog = catalog.Cast(); - auto &transaction = Transaction::Get(context, catalog).Cast(); - auto sql = input.inputs[1].GetValue(); - // strip any trailing semicolons +static PostgresQueryMode ResolveQueryMode(ClientContext &context, TableFunctionBindInput &input, + const vector ¶ms) { + auto it = input.named_parameters.find(Identifier("query_mode")); + if (it != input.named_parameters.end()) { + Value &str_val = it->second; + if (!str_val.IsNull()) { + if (str_val.type().id() != LogicalTypeId::VARCHAR) { + throw BinderException("Invalid non-VARCHAR 'query_mode' parameter specified"); + } + PostgresQueryMode mode = PostgresUtils::ExtractQueryMode(str_val); + if (mode != PostgresQueryMode::AUTO) { + return mode; + } + } + } + PostgresQueryMode mode = PostgresUtils::ExtractQueryMode(context); + if (mode != PostgresQueryMode::AUTO) { + return mode; + } + Value text_proto_val; + if (context.TryGetCurrentSetting("pg_use_text_protocol", text_proto_val) && !text_proto_val.IsNull() && + BooleanValue::Get(text_proto_val)) { + return PostgresQueryMode::PLAIN_QUERY_TEXT; + } + if (params.size() > 0) { + return PostgresQueryMode::PLAIN_QUERY_BINARY; + } + return PostgresQueryMode::AUTO; +} + +static string StripTrailingSemicolon(string sql) { StringUtil::RTrim(sql); while (!sql.empty() && sql.back() == ';') { sql = sql.substr(0, sql.size() - 1); StringUtil::RTrim(sql); } + return sql; +} - bool use_transaction = true; - for (auto &kv : input.named_parameters) { - if (kv.first == "use_transaction") { - use_transaction = BooleanValue::Get(kv.second); - } - } - - vector param_values; +static vector ExtractParams(TableFunctionBindInput &input) { auto params_it = input.named_parameters.find("params"); if (params_it != input.named_parameters.end()) { Value &struct_val = params_it->second; if (struct_val.IsNull()) { throw BinderException("Parameters to postgres_query cannot be NULL"); } - if (struct_val.type().id() != LogicalTypeId::STRUCT) { + if (struct_val.type().id() != LogicalTypeId::STRUCT && struct_val.type().id() != LogicalTypeId::TUPLE) { throw BinderException("Query parameters must be specified in a STRUCT"); } - param_values = StructValue::GetChildren(struct_val); + return StructValue::GetChildren(struct_val); } + return vector(); +} - auto &con = use_transaction ? transaction.GetConnection() : transaction.GetConnectionWithoutTransaction(); +static vector ExtractPreliminaryQueries(TableFunctionBindInput &input) { + vector queries; + auto it = input.named_parameters.find("preliminary_queries"); + if (it != input.named_parameters.end()) { + Value &list_val = it->second; + if (list_val.IsNull()) { + throw BinderException("Specified preliminary queries LIST must be not NULL"); + } + if (list_val.type().id() != LogicalTypeId::LIST) { + throw BinderException("Preliminary queries must be specified in a LIST"); + } + for (const Value &val : ListValue::GetChildren(list_val)) { + if (val.IsNull()) { + throw BinderException("All preliminary queries must be not NULL"); + } + if (val.type().id() != LogicalTypeId::VARCHAR) { + throw BinderException("All preliminary queries must be of type VARCHAR"); + } + queries.push_back(StringValue::Get(val)); + } + } + return queries; +} +static uint64_t ExtractPinnedConnId(TableFunctionBindInput &input) { + uint64_t pinned_connection_id = 0; + auto conn_it = input.named_parameters.find("connection"); + if (conn_it != input.named_parameters.end()) { + Value &conn_val = conn_it->second; + if (conn_val.IsNull()) { + throw BinderException("Specified connection must be not null"); + } + pinned_connection_id = UBigIntValue::Get(conn_val); + } + return pinned_connection_id; +} + +static unique_ptr PGQueryBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) { + throw BinderException("Parameters to postgres_query cannot be NULL"); + } + + auto db_name = input.inputs[0].GetValue(); + auto &catalog = PostgresUtils::GetCatalogByName(context, db_name); + auto input_sql = input.inputs[1].GetValue(); + auto sql = StripTrailingSemicolon(std::move(input_sql)); + bool use_transaction = ExtractFlag(input, "use_transaction", true); + vector param_values = ExtractParams(input); + vector preliminary_queries = ExtractPreliminaryQueries(input); + uint64_t pinned_connection_id = ExtractPinnedConnId(input); + + optional_ptr con_ptr = nullptr; + PostgresPoolConnection pinned_connection; + uint64_t pool_connection_id = 0; + + if (pinned_connection_id > 0) { + pinned_connection = catalog.GetConnectionPool().UnpinConnection(pinned_connection_id); + PostgresConnection &conn = pinned_connection.GetConnection(); + con_ptr = &conn; + pool_connection_id = pinned_connection.Id(); + } else { + auto &transaction = Transaction::Get(context, catalog).Cast(); + if (use_transaction) { + con_ptr = &transaction.GetConnection(); + } else { + con_ptr = &transaction.GetConnectionWithoutTransaction(); + } + pool_connection_id = transaction.GetPoolConnectionId(); + } + + auto deferred_pin = dbconnector::Defer([&pinned_connection, &catalog] { + if (!pinned_connection) { + return; + } + try { + catalog.GetConnectionPool().PinConnection(std::move(pinned_connection)); + } catch (...) { + // suppress + } + }); + + PostgresConnection &con = *con_ptr; + + if (!ExtractFlag(input, "prepare", true)) { + if (param_values.size() > 0) { + throw BinderException("query parameters cannot be used with 'prepare=FALSE'"); + } + return BindDML(context, input, return_types, names, catalog, con, std::move(preliminary_queries), + std::move(sql), use_transaction, pinned_connection_id, PostgresParameters(), nullptr); + } + + auto conn_lock = con.Lock(); auto conn = con.GetConn(); // prepare execution of the query to figure out the result types and names - auto prepared = PQprepare(conn, "", sql.c_str(), 0, nullptr); - PostgresResult prepared_wrapper(prepared); - if (!prepared) { - throw BinderException("Failed to prepare query \"%s\" (no result returned): %s", sql, PQerrorMessage(conn)); - } - if (PQresultStatus(prepared) != PGRES_COMMAND_OK) { - throw BinderException("Failed to prepare query \"%s\": %s", sql, PQresultErrorMessage(prepared)); + string stmt_name = PostgresPreparedStatement::GenerateName(); + if (PostgresConnection::DebugPrintQueries()) { + Printer::Print("PREPARE " + stmt_name + ": " + sql + "\n"); } + PGresult *prepared_res_ptr = PQprepare(conn, stmt_name.c_str(), sql.c_str(), 0, nullptr); + auto prepared = make_uniq(conn, pool_connection_id, stmt_name, sql, prepared_res_ptr); // use describe_prepared - auto describe_prepared = PQdescribePrepared(conn, ""); + auto describe_prepared = PQdescribePrepared(conn, prepared->GetName().c_str()); PostgresResult describe_wrapper(describe_prepared); if (!describe_prepared || PQresultStatus(describe_prepared) != PGRES_COMMAND_OK) { auto extended_err = describe_prepared ? PQresultErrorMessage(describe_prepared) : PQerrorMessage(conn); throw BinderException("Failed to describe prepared statement: %s", extended_err); } + conn_lock.unlock(); + int nparams = PQnparams(describe_prepared); + if (nparams != param_values.size()) { + throw BinderException("Incorrect number of parameters specified, expected: %d, actual: %zu, query: \"%s\"", + nparams, param_values.size(), sql); + } + vector param_types; + for (idx_t p = 0; p < nparams; p++) { + Oid ptype = PQparamtype(describe_prepared, p); + param_types.emplace_back(ptype); + } int nfields = PQnfields(describe_prepared); if (nfields <= 0) { - // The statement returns no result columns: it's a command (DDL, or DML without RETURNING). - // Instead of failing, run it as a command and return a single-row Success result. We reuse - // the prepare/describe just done — no extra round-trip — and defer execution to - // InitGlobalState (execution time, not bind, so EXPLAIN does not run it). - result->command_only = true; - return_types.emplace_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - result->SetCatalog(pg_catalog); - result->dsn = con.GetDSN(); - result->types = return_types; - result->names = names; - result->read_only = false; - result->SetTablePages(0); - result->sql = std::move(sql); - result->use_transaction = use_transaction; - return std::move(result); + return BindDML(context, input, return_types, names, catalog, con, std::move(preliminary_queries), + std::move(sql), use_transaction, pinned_connection_id, + PostgresParameters(std::move(param_types), std::move(param_values)), std::move(prepared)); } + auto result = make_uniq(context, catalog.GetName()); + result->pinned_connection_id = pinned_connection_id; + auto type_config = PostgresTypeConfig::FromContext(context); for (idx_t c = 0; c < nfields; c++) { PostgresType postgres_type; postgres_type.oid = PQftype(describe_prepared, c); PostgresTypeData type_data; - type_data.type_name = PostgresUtils::PostgresOidToName(postgres_type.oid); + type_data.instance_type = catalog.GetPostgresVersion().type_v; + if (type_data.instance_type == PostgresInstanceType::REDSHIFT) { + type_data.type_name = PostgresUtils::RedshiftOidToName(postgres_type.oid); + } + if (type_data.type_name.empty()) { + type_data.type_name = PostgresUtils::PostgresOidToName(postgres_type.oid); + } type_data.type_modifier = PQfmod(describe_prepared, c); - auto converted_type = PostgresUtils::TypeToLogicalType(nullptr, nullptr, type_data, postgres_type); + auto converted_type = PostgresUtils::TypeToLogicalType(nullptr, nullptr, type_config, type_data, postgres_type); result->postgres_types.push_back(postgres_type); return_types.emplace_back(converted_type); - names.emplace_back(PQfname(describe_prepared, c)); - } - int nparams = PQnparams(describe_prepared); - if (nparams != param_values.size()) { - throw BinderException("Incorrect number of parameters specified, expected: %d, actual: %zu, query: \"%s\"", - nparams, param_values.size(), sql); - } - vector param_types; - for (idx_t p = 0; p < nparams; p++) { - Oid ptype = PQparamtype(describe_prepared, p); - param_types.emplace_back(ptype); + char *name_ptr = PQfname(describe_prepared, c); + string name = name_ptr ? name_ptr : "unnamed"; + names.emplace_back(name); } + PostgresUtils::DeduplicateColumns(names); // set up the bind data - result->SetCatalog(pg_catalog); + result->type_config = type_config; result->dsn = con.GetDSN(); result->types = return_types; - result->names = names; + for (auto &nm : names) { + result->names.emplace_back(nm.GetIdentifierName()); + } result->read_only = false; - result->SetTablePages(0); + result->preliminary_queries = std::move(preliminary_queries); result->sql = std::move(sql); + result->query_mode = ResolveQueryMode(context, input, param_values); result->params = PostgresParameters(std::move(param_types), std::move(param_values)); result->use_transaction = use_transaction; + result->prepared_statement = std::move(prepared); + result->clear_catalog_cache = ExtractFlag(input, "clear_catalog_cache", false); + PostgresScanFunction::PrepareBind(catalog.GetPostgresVersion(), context, *result, 0, catalog); return std::move(result); } +static InsertionOrderPreservingMap PostgresQueryToString(TableFunctionToStringInput &input) { + D_ASSERT(input.bind_data); + InsertionOrderPreservingMap result; + auto &bind_data = input.bind_data->Cast(); + if (!bind_data.preliminary_queries.empty()) { + result["Preliminary Queries"] = StringUtil::Join(bind_data.preliminary_queries, "\n"); + } + result["SQL"] = bind_data.sql; + return result; +} + +static FunctionSignature PostgresQuerySignature(bool suppress_dml_output) { + FunctionSignature signature; + signature.AddParameter("database", LogicalType::VARCHAR) + .AddParameter("sql", LogicalType::VARCHAR) + .WithTypedKwargs("options", [&](TypedKwargs &options) { + options.Add("use_transaction", LogicalType::BOOLEAN).Add("params", LogicalType::ANY); + if (suppress_dml_output) { + options.Add("suppress_dml_output", LogicalType::BOOLEAN); + } + options.Add("prepare", LogicalType::BOOLEAN); + options.Add("query_mode", LogicalType::VARCHAR); + options.Add("connection", LogicalType::UBIGINT); + options.Add("preliminary_queries", LogicalType::LIST(LogicalType::VARCHAR)); + options.Add("clear_catalog_cache", LogicalType::BOOLEAN); + options.Add("return_changed_rows", LogicalType::BOOLEAN); + }); + return signature; +} + PostgresQueryFunction::PostgresQueryFunction() - : TableFunction("postgres_query", {LogicalType::VARCHAR, LogicalType::VARCHAR}, nullptr, PGQueryBind) { - named_parameters["use_transaction"] = LogicalType::BOOLEAN; - named_parameters["params"] = LogicalType::ANY; + : TableFunction("postgres_query", PostgresQuerySignature(true), nullptr, PGQueryBind) { + PostgresScanFunction scan_function; + init_global = scan_function.init_global; + init_local = scan_function.init_local; + function = scan_function.function; + to_string = PostgresQueryToString; + projection_pushdown = true; + global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; +} + +PostgresExecuteFunction::PostgresExecuteFunction() + : TableFunction("postgres_execute", PostgresQuerySignature(false), nullptr, PGQueryBind) { PostgresScanFunction scan_function; init_global = scan_function.init_global; init_local = scan_function.init_local; diff --git a/src/postgres_result_reader.cpp b/src/postgres_result_reader.cpp new file mode 100644 index 000000000..1756a4228 --- /dev/null +++ b/src/postgres_result_reader.cpp @@ -0,0 +1,26 @@ +#include "postgres_result_reader.hpp" + +#include "postgres_connection.hpp" +#include "postgres_prepared_statement.hpp" +#include "postgres_scanner.hpp" + +namespace duckdb { + +unique_ptr PostgresResultReader::Query(ClientContext &context, const string &sql, int format) { + if (prepared_statement && !prepared_statement->IsDestroyed() && prepared_statement->GetQuery() == sql) { + auto result = con.QueryStmt(context, *prepared_statement, bind_data.params, format); + prepared_statement->Destroy(con); + return result; + } + // the query was modified, the statement will not be used + DestroyPreparedStatement(); + return con.Query(context, sql, bind_data.params, format); +} + +void PostgresResultReader::DestroyPreparedStatement() { + if (prepared_statement && !prepared_statement->IsDestroyed()) { + prepared_statement->Destroy(con); + } +} + +} // namespace duckdb diff --git a/src/postgres_scanner.cpp b/src/postgres_scanner.cpp index 1673c4b0b..ddb6a8478 100644 --- a/src/postgres_scanner.cpp +++ b/src/postgres_scanner.cpp @@ -1,7 +1,10 @@ #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include +#include "dbconnector/attached.hpp" + #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/helper.hpp" @@ -31,11 +34,18 @@ struct PostgresLocalState : public LocalTableFunctionState { vector column_ids; TableFilterSet *filters; string col_names; - PostgresConnection connection; + //! Whether this scan reads using the text protocol (see PostgresBindData::text_protocol_fallback) + bool use_text_protocol = false; + //! Connection owned by the global state (pinned or transaction connection), if this thread uses it + optional_ptr global_connection; + //! Pool connection ID of global_connection + uint64_t global_connection_id = 0; idx_t batch_idx = 0; + //! Connection of this thread, used when global_connection is not set PostgresPoolConnection pool_connection; unique_ptr reader; + PostgresConnection &GetConnection(); void ScanChunk(ClientContext &context, const PostgresBindData &bind_data, PostgresGlobalState &gstate, DataChunk &output); }; @@ -44,6 +54,8 @@ struct PostgresGlobalState : public GlobalTableFunctionState { explicit PostgresGlobalState(idx_t max_threads) : page_idx(0), batch_idx(0), max_threads(max_threads) { } + ~PostgresGlobalState(); + mutable mutex lock; idx_t page_idx; idx_t batch_idx; @@ -53,9 +65,12 @@ struct PostgresGlobalState : public GlobalTableFunctionState { bool used_main_thread = false; string snapshot; - PostgresConnection &GetConnection(); - void SetConnection(PostgresConnection connection); - void SetConnection(shared_ptr connection); + //! Returns the pinned connection if there is one, otherwise the connection of the current transaction + PostgresConnection &GetConnection(ClientContext &context); + //! Returns the pool connection ID of the connection returned by GetConnection + uint64_t GetConnectionId(ClientContext &context); + void SetConnectionSource(dbconnector::attached::AttachedCatalog attached_catalog, bool use_transaction, + PostgresPoolConnection pinned_connection); bool TryOpenNewConnection(ClientContext &context, PostgresLocalState &lstate, const PostgresBindData &bind_data); idx_t MaxThreads() const override { @@ -63,53 +78,53 @@ struct PostgresGlobalState : public GlobalTableFunctionState { } private: - PostgresConnection connection; + dbconnector::attached::AttachedCatalog attached_catalog; + bool use_transaction = true; + PostgresPoolConnection pinned_connection; }; -static void PostgresGetSnapshot(ClientContext &context, PostgresVersion version, const PostgresBindData &bind_data, +static void PostgresGetSnapshot(ClientContext &context, const PostgresBindData &bind_data, PostgresGlobalState &gstate) { - unique_ptr result; // by default disable snapshotting gstate.snapshot = string(); if (gstate.max_threads <= 1) { return; } - if (version.type_v == PostgresInstanceType::AURORA) { - return; - } // SET TRANSACTION SNAPSHOT requires REPEATABLE READ or SERIALIZABLE - auto pg_catalog = bind_data.GetCatalog(); - if (pg_catalog && pg_catalog->isolation_level == PostgresIsolationLevel::READ_COMMITTED) { + D_ASSERT(!bind_data.catalog_name.empty()); + auto attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + auto &pg_catalog = attached_catalog.Get(); + if (pg_catalog.isolation_level == PostgresIsolationLevel::READ_COMMITTED) { return; } // reader threads can use the same snapshot - auto &con = gstate.GetConnection(); - // pg_stat_wal_receiver was introduced in PostgreSQL 9.6 - if (version < PostgresVersion(9, 6, 0)) { - result = con.TryQuery(context, "SELECT pg_is_in_recovery(), pg_export_snapshot()"); - if (result) { - auto in_recovery = result->GetBool(0, 0); - if (!in_recovery) { - gstate.snapshot = result->GetString(0, 1); - } - } - return; - } - - result = con.TryQuery( - context, "SELECT pg_is_in_recovery(), pg_export_snapshot(), (select count(*) from pg_stat_wal_receiver)"); + auto &con = gstate.GetConnection(context); + // snapshot export works on replicas since PostgreSQL 10 and on RDS/Aurora, so no recovery check is needed + auto result = con.TryQuery(context, "SELECT pg_export_snapshot()"); if (result) { - auto in_recovery = result->GetBool(0, 0) || result->GetInt64(0, 2) > 0; - gstate.snapshot = ""; - if (!in_recovery) { - gstate.snapshot = result->GetString(0, 1); - } - return; + gstate.snapshot = result->GetString(0, 0); } } void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &context, PostgresBindData &bind_data, - int64_t approx_num_pages) { + int64_t approx_num_pages, optional_ptr pg_catalog) { + // resolve the protocol before any decision that depends on it (ctid scan, max threads) + // note that bind_data.use_text_protocol holds the value of the pg_use_text_protocol setting here + bind_data.instance_type = version.type_v; + if (pg_catalog && + pg_catalog->UseBinaryPlainQueryWithTextFallback(bind_data.query_mode, bind_data.use_text_protocol)) { + // Redshift does not support the binary COPY protocol, but plain query results in the binary format are + // faster to transfer than the text ones + bind_data.query_mode = PostgresQueryMode::PLAIN_QUERY_BINARY; + bind_data.use_text_protocol = false; + bind_data.text_protocol_fallback = true; + } else if (pg_catalog) { + // for an attached database the mode set at ATTACH time decides + bind_data.use_text_protocol = pg_catalog->UseTextProtocol(bind_data.query_mode, bind_data.use_text_protocol); + } else if (version.type_v == PostgresInstanceType::REDSHIFT) { + // no attached database (postgres_scan over a dsn) - Redshift does not support the binary COPY protocol + bind_data.use_text_protocol = true; + } Value pages_per_task; if (context.TryGetCurrentSetting("pg_pages_per_task", pages_per_task)) { bind_data.pages_per_task = UBigIntValue::Get(pages_per_task); @@ -122,6 +137,9 @@ void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &c if (context.TryGetCurrentSetting("pg_use_ctid_scan", pg_use_ctid_scan)) { use_ctid_scan = BooleanValue::Get(pg_use_ctid_scan); } + if (pg_catalog && pg_catalog->use_ctid_scan != PostgresUseCtidScan::DEFAULT) { + use_ctid_scan = pg_catalog->use_ctid_scan == PostgresUseCtidScan::ENABLED; + } if (bind_data.use_text_protocol) { // ctid scan is only supported for binary copy use_ctid_scan = false; @@ -140,13 +158,10 @@ void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &c approx_num_pages = 0; } bind_data.SetTablePages(static_cast(approx_num_pages)); - bind_data.version = version; - if (version.type_v == PostgresInstanceType::REDSHIFT) { - bind_data.use_text_protocol = true; - } } -PostgresBindData::PostgresBindData(ClientContext &context) { +PostgresBindData::PostgresBindData(ClientContext &context, const Identifier &catalog_name_p) + : catalog_name(catalog_name_p), query_mode(PostgresUtils::ExtractQueryMode(context)) { Value text_protocol; if (context.TryGetCurrentSetting("pg_use_text_protocol", text_protocol)) { if (BooleanValue::Get(text_protocol)) { @@ -164,52 +179,36 @@ void PostgresBindData::SetTablePages(idx_t approx_num_pages) { } } -PostgresConnection &PostgresGlobalState::GetConnection() { - return connection; -} - -void PostgresGlobalState::SetConnection(PostgresConnection connection) { - this->connection = std::move(connection); -} - -void PostgresGlobalState::SetConnection(shared_ptr connection) { - this->connection = PostgresConnection(std::move(connection)); +uint64_t PostgresGlobalState::GetConnectionId(ClientContext &context) { + if (pinned_connection) { + return pinned_connection.Id(); + } + PostgresCatalog &catalog = attached_catalog.Get(); + return PostgresTransaction::Get(context, catalog).GetPoolConnectionId(); } -void PostgresBindData::SetCatalog(PostgresCatalog &catalog) { - this->pg_catalog = &catalog; +PostgresConnection &PostgresGlobalState::GetConnection(ClientContext &context) { + if (pinned_connection) { + return pinned_connection.GetConnection(); + } + PostgresCatalog &catalog = attached_catalog.Get(); + auto &transaction = PostgresTransaction::Get(context, catalog); + if (use_transaction) { + return transaction.GetConnection(); + } + return transaction.GetConnectionWithoutTransaction(); } -void PostgresBindData::SetTable(PostgresTableEntry &table) { - this->pg_table = &table; +void PostgresGlobalState::SetConnectionSource(dbconnector::attached::AttachedCatalog attached_catalog, + bool use_transaction, PostgresPoolConnection pinned_connection) { + this->attached_catalog = std::move(attached_catalog); + this->use_transaction = use_transaction; + this->pinned_connection = std::move(pinned_connection); } -static unique_ptr PostgresBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(context); - - bind_data->dsn = input.inputs[0].GetValue(); - bind_data->schema_name = input.inputs[1].GetValue(); - bind_data->table_name = input.inputs[2].GetValue(); - bind_data->attach_path = bind_data->dsn; - - auto con = PostgresConnection::Open(bind_data->dsn, bind_data->attach_path); - auto version = con.GetPostgresVersion(context); - // query the table schema so we can interpret the bits in the pages - auto info = PostgresTableSet::GetTableInfo(context, con, bind_data->schema_name, bind_data->table_name); - - bind_data->postgres_types = info->postgres_types; - for (auto &col : info->create_info->columns.Logical()) { - names.push_back(col.GetName().GetIdentifierName()); - return_types.push_back(col.GetType()); - } - bind_data->names = info->postgres_names; - bind_data->types = return_types; - bind_data->can_use_main_thread = true; - bind_data->requires_materialization = false; - - PostgresScanFunction::PrepareBind(version, context, *bind_data, info->approx_num_pages); - return std::move(bind_data); +static unique_ptr PostgresBind(ClientContext &, TableFunctionBindInput &, vector &, + vector &) { + throw InternalException("PostgresBind"); } static bool ContainsCastToVarchar(const PostgresType &type) { @@ -224,55 +223,88 @@ static bool ContainsCastToVarchar(const PostgresType &type) { return false; } -static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p, - PostgresLocalState &lstate, idx_t task_min, idx_t task_max) { - D_ASSERT(bind_data_p); - D_ASSERT(task_min <= task_max); - - auto bind_data = (const PostgresBindData *)bind_data_p; +static bool RequiresTextProtocol(const PostgresBindData &bind_data, const vector &column_ids) { + for (auto column_id : column_ids) { + if (column_id < bind_data.postgres_types.size() && + bind_data.postgres_types[column_id].info == PostgresTypeAnnotation::TEXT_PROTOCOL_ONLY) { + return true; + } + } + return false; +} +static string WriteColumnNames(const PostgresBindData &bind_data, const PostgresLocalState &lstate, + bool &col_names_required) { string col_names; - for (auto &column_id : lstate.column_ids) { + col_names_required = bind_data.names.size() != lstate.column_ids.size(); + for (idx_t i = 0; i < lstate.column_ids.size(); i++) { + auto column_id = lstate.column_ids[i]; + if (column_id != i) { + // the projection reorders the columns, the readers decode them in the projected order + col_names_required = true; + } if (!col_names.empty()) { col_names += ", "; } if (column_id == COLUMN_IDENTIFIER_ROW_ID) { - if (bind_data->table_name.empty() || !bind_data->emit_ctid) { + if (bind_data.table_name.empty() || !bind_data.emit_ctid) { // count(*) over postgres_query col_names += "NULL"; } else { col_names += "ctid"; } + col_names_required = true; } else { - col_names += PostgresUtils::WriteIdentifier(bind_data->names[column_id]); - if (bind_data->postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { + col_names += PostgresUtils::WriteIdentifier(bind_data.names[column_id]); + if (bind_data.postgres_types[column_id].info == PostgresTypeAnnotation::TEXT_PROTOCOL_ONLY && + !lstate.use_text_protocol) { + throw NotImplementedException("Error reading table \"%s\" - column \"%s\" can only be read using the " + "text protocol, use pg_query_mode = 'PLAIN_QUERY_TEXT'", + bind_data.table_name, bind_data.names[column_id]); + } + if (bind_data.postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR"; - } else if (bind_data->types[column_id].id() == LogicalTypeId::LIST) { - if (bind_data->postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) { + col_names_required = true; + } else if (bind_data.types[column_id].id() == LogicalTypeId::LIST) { + if (bind_data.postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) { continue; } - if (bind_data->postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { + if (bind_data.postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR[]"; + col_names_required = true; } } else { - if (ContainsCastToVarchar(bind_data->postgres_types[column_id])) { + if (ContainsCastToVarchar(bind_data.postgres_types[column_id])) { throw NotImplementedException("Error reading table \"%s\" - cast to varchar not implemented for " "composite column \"%s\" (type %s)", - bind_data->table_name, bind_data->names[column_id], - bind_data->types[column_id].ToString()); + bind_data.table_name, bind_data.names[column_id], + bind_data.types[column_id].ToString()); } } } } + return col_names; +} - string filter_string = - PostgresFilterPushdown::TransformFilters(lstate.column_ids, lstate.filters, bind_data->names); +static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p, + PostgresLocalState &lstate, idx_t task_min, idx_t task_max) { + D_ASSERT(bind_data_p); + D_ASSERT(task_min <= task_max); + + auto &bind_data = bind_data_p->Cast(); + + lstate.use_text_protocol = bind_data.use_text_protocol || + (bind_data.text_protocol_fallback && RequiresTextProtocol(bind_data, lstate.column_ids)); + bool col_names_required = false; + string col_names = WriteColumnNames(bind_data, lstate, col_names_required); + string filter_string = PostgresFilterPushdown::TransformFilters(lstate.column_ids, lstate.filters, bind_data.names, + bind_data.instance_type); string filter; lstate.exec = false; lstate.done = false; - if (bind_data->pages_approx > 0) { + if (bind_data.pages_approx > 0) { filter = StringUtil::Format("WHERE ctid BETWEEN '(%d,0)'::tid AND '(%d,0)'::tid", task_min, task_max); } if (!filter_string.empty()) { @@ -283,28 +315,28 @@ static void PostgresInitInternal(ClientContext &context, const PostgresBindData } filter += filter_string; } - string query; - if (bind_data->table_name.empty()) { - D_ASSERT(!bind_data->sql.empty()); - query = - StringUtil::Format(R"(SELECT %s FROM (%s) AS __unnamed_subquery %s)", col_names, bind_data->sql, filter); + string query; + if (bind_data.table_name.empty()) { + D_ASSERT(!bind_data.sql.empty()); + if (col_names_required) { + query = + StringUtil::Format("SELECT %s FROM (%s) AS __unnamed_subquery %s", col_names, bind_data.sql, filter); + } else if (!filter.empty()) { + query = StringUtil::Format("%s %s", bind_data.sql, filter); + } else { + query = bind_data.sql; + } } else { - query = StringUtil::Format(R"(SELECT %s FROM %s.%s %s)", col_names, - PostgresUtils::WriteIdentifier(bind_data->schema_name), - PostgresUtils::WriteIdentifier(bind_data->table_name), filter); - } - if (!bind_data->order_by_and_limit_bind_data.order_by_clause.empty()) { - query += bind_data->order_by_and_limit_bind_data.order_by_clause; - query += " NULLS LAST"; + query = StringUtil::Format("SELECT %s FROM %s.%s %s", col_names, + PostgresUtils::WriteIdentifier(bind_data.schema_name), + PostgresUtils::WriteIdentifier(bind_data.table_name), filter); } - if (!bind_data->order_by_and_limit_bind_data.limit_clause.empty()) { - query += bind_data->order_by_and_limit_bind_data.limit_clause; + if (!bind_data.limit_clause.empty()) { + query += bind_data.limit_clause; } - if (!bind_data->use_text_protocol) { - query = StringUtil::Format(R"(COPY (%s) TO STDOUT (FORMAT "binary");)", query); - } else { - query += ";"; + if (!(lstate.use_text_protocol || bind_data.query_mode == PostgresQueryMode::PLAIN_QUERY_BINARY)) { + query = StringUtil::Format("COPY (%s) TO STDOUT (FORMAT \"binary\");", query); } lstate.sql = std::move(query); } @@ -339,49 +371,67 @@ static void PostgresScanConnect(ClientContext &context, PostgresConnection &conn } } +static void HandleCatalogCache(ClientContext &context, const PostgresBindData &bind_data) { + if (!bind_data.clear_catalog_cache || bind_data.pinned_connection_id > 0) { + return; + } + auto attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + PostgresCatalog &catalog = attached_catalog.Get(); + catalog.ClearCache(); +} + static unique_ptr PostgresInitGlobalState(ClientContext &context, TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(PostgresMaxThreads(context, input.bind_data.get())); + optional_ptr pg_catalog = nullptr; + dbconnector::attached::AttachedCatalog attached_catalog; + D_ASSERT(!bind_data.catalog_name.empty()); + attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + pg_catalog = &attached_catalog.Get(); + + PostgresPoolConnection pinned_connection; + if (bind_data.pinned_connection_id > 0) { + pinned_connection = pg_catalog->GetConnectionPool().UnpinConnection(bind_data.pinned_connection_id); + } + result->SetConnectionSource(std::move(attached_catalog), bind_data.use_transaction, std::move(pinned_connection)); + PostgresConnection &con = result->GetConnection(context); + + // the statement prepared in bind is only offered to the reader that uses the global connection, close it if + // there is no such reader. If it was prepared on another connection, that session is already gone. + auto &prepared_statement = bind_data.prepared_statement; + if (prepared_statement && !prepared_statement->IsDestroyed() && + prepared_statement->GetPoolConnectionId() == result->GetConnectionId(context) && + (bind_data.command_only || !bind_data.can_use_main_thread)) { + prepared_statement->Destroy(con); + } + + for (const string &query : bind_data.preliminary_queries) { + con.Execute(context, query); + } + if (bind_data.command_only) { // A command (no result columns). Execute it here — at execution time, so EXPLAIN does not - // fire it — and hand back a single-row Success result through the materialized-collection + // fire it — and hand back a rowcount result through the materialized-collection // fast path (PostgresScan returns gstate.collection directly, and GetLocalState short-circuits // when a collection is present, so no postgres scan connection is opened). - auto command_catalog = bind_data.GetCatalog(); - D_ASSERT(command_catalog); - auto &transaction = Transaction::Get(context, *command_catalog).Cast(); - if (bind_data.use_transaction) { - transaction.Query(bind_data.sql); - } else { - transaction.QueryWithoutTransaction(bind_data.sql); - } - vector success_types {LogicalType::BOOLEAN}; - auto materialized = make_uniq(Allocator::Get(context), success_types); - DataChunk success_chunk; - success_chunk.Initialize(Allocator::Get(context), success_types); - success_chunk.SetChildCardinality(1); - success_chunk.data[0].SetValue(0, Value::BOOLEAN(true)); + unique_ptr pg_res = con.Query(context, bind_data.sql, bind_data.params); + HandleCatalogCache(context, bind_data); + vector rowcount_types {LogicalType::BIGINT}; + auto materialized = make_uniq(Allocator::Get(context), rowcount_types); + DataChunk rowcount_chunk; + rowcount_chunk.Initialize(Allocator::Get(context), rowcount_types); + rowcount_chunk.SetChildCardinality(1); + int64_t *vec_data = FlatVector::GetDataMutable(rowcount_chunk.data[0]); + vec_data[0] = pg_res->AffectedRows(); ColumnDataAppendState append_state; materialized->InitializeAppend(append_state); - materialized->Append(append_state, success_chunk); + materialized->Append(append_state, rowcount_chunk); result->collection = std::move(materialized); result->collection->InitializeScan(result->scan_state); return std::move(result); } - auto pg_catalog = bind_data.GetCatalog(); - if (pg_catalog) { - auto &transaction = Transaction::Get(context, *pg_catalog).Cast(); - auto &con = - bind_data.use_transaction ? transaction.GetConnection() : transaction.GetConnectionWithoutTransaction(); - result->SetConnection(con.GetConnection()); - } else { - auto con = PostgresConnection::Open(bind_data.dsn, bind_data.attach_path); - if (bind_data.use_transaction) { - PostgresScanConnect(context, con, string(), AccessMode::READ_ONLY, PostgresIsolationLevel::REPEATABLE_READ); - } - result->SetConnection(std::move(con)); - } + if (bind_data.requires_materialization) { // if requires_materialization is enabled we scan and materialize the table in its entirety up-front vector types; @@ -408,7 +458,7 @@ static unique_ptr PostgresInitGlobalState(ClientContex result->collection->InitializeScan(result->scan_state); } else { // we create a transaction here, and get the snapshot id to enable transaction-safe parallelism - PostgresGetSnapshot(context, bind_data.version, bind_data, *result); + PostgresGetSnapshot(context, bind_data, *result); } return std::move(result); } @@ -437,40 +487,42 @@ static bool PostgresParallelStateNext(ClientContext &context, const FunctionData bool PostgresGlobalState::TryOpenNewConnection(ClientContext &context, PostgresLocalState &lstate, const PostgresBindData &bind_data) { - auto pg_catalog = bind_data.GetCatalog(); + optional_ptr pg_catalog = nullptr; + dbconnector::attached::AttachedCatalog attached_catalog; + D_ASSERT(!bind_data.catalog_name.empty()); + attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + pg_catalog = &attached_catalog.Get(); { lock_guard parallel_lock(lock); if (!used_main_thread) { if (bind_data.can_use_main_thread) { - lstate.connection = PostgresConnection(GetConnection().GetConnection()); + lstate.global_connection = &GetConnection(context); + lstate.global_connection_id = GetConnectionId(context); } else { // we cannot use the main thread but we haven't initiated ANY scan yet // we HAVE to open a new connection { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - lstate.pool_connection = pg_catalog->GetConnectionPool().ForceGetConnection(); + lstate.pool_connection = pg_catalog->GetConnectionPool().ForceAcquire(); } - lstate.connection = PostgresConnection(lstate.pool_connection.GetConnection().GetConnection()); + // unlike the main-thread connection this one is not in a transaction yet, so we pin it to + // the snapshot to prevent reads at its own instant + PostgresScanConnect(context, lstate.GetConnection(), snapshot, pg_catalog->access_mode, + pg_catalog->isolation_level); } used_main_thread = true; return true; } } - if (pg_catalog) { - { - auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - if (!pg_catalog->GetConnectionPool().TryGetConnection(lstate.pool_connection)) { - return false; - } + { + auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); + if (!pg_catalog->GetConnectionPool().TryGetConnection(lstate.pool_connection)) { + return false; } - lstate.connection = PostgresConnection(lstate.pool_connection.GetConnection().GetConnection()); - PostgresScanConnect(context, lstate.connection, snapshot, pg_catalog->access_mode, pg_catalog->isolation_level); - } else { - lstate.connection = PostgresConnection::Open(bind_data.dsn, bind_data.attach_path); - PostgresScanConnect(context, lstate.connection, snapshot, AccessMode::READ_ONLY, - PostgresIsolationLevel::REPEATABLE_READ); } + PostgresScanConnect(context, lstate.GetConnection(), snapshot, pg_catalog->access_mode, + pg_catalog->isolation_level); return true; } @@ -500,6 +552,17 @@ static unique_ptr GetLocalState(ClientContext &context, return std::move(local_state); } +PostgresGlobalState::~PostgresGlobalState() { + if (pinned_connection) { + try { + PostgresCatalog &catalog = attached_catalog.Get(); + catalog.GetConnectionPool().PinConnection(std::move(pinned_connection)); + } catch (...) { + // suppress + } + } +} + static unique_ptr PostgresInitLocalState(ExecutionContext &context, TableFunctionInitInput &input, GlobalTableFunctionState *global_state) { @@ -507,14 +570,30 @@ static unique_ptr PostgresInitLocalState(ExecutionConte return GetLocalState(context.client, input, gstate); } +PostgresConnection &PostgresLocalState::GetConnection() { + if (global_connection) { + return *global_connection; + } + if (pool_connection) { + return pool_connection.GetConnection(); + } + throw InternalException("PostgresLocalState::GetConnection - no connection available"); +} + void PostgresLocalState::ScanChunk(ClientContext &context, const PostgresBindData &bind_data, PostgresGlobalState &gstate, DataChunk &output) { idx_t output_offset = 0; if (!reader) { - if (bind_data.use_text_protocol) { - reader = make_uniq(context, connection, column_ids, bind_data); + // the statement prepared in bind can only be reused on the connection it was prepared on + optional_ptr prepared_statement; + if (global_connection && bind_data.prepared_statement && + bind_data.prepared_statement->GetPoolConnectionId() == global_connection_id) { + prepared_statement = bind_data.prepared_statement.get(); + } + if (use_text_protocol) { + reader = make_uniq(context, GetConnection(), column_ids, bind_data, prepared_statement); } else { - reader = make_uniq(connection, column_ids, bind_data); + reader = make_uniq(GetConnection(), column_ids, bind_data, prepared_statement); } } while (true) { @@ -525,6 +604,7 @@ void PostgresLocalState::ScanChunk(ClientContext &context, const PostgresBindDat if (!exec) { reader->BeginCopy(context, sql); exec = true; + HandleCatalogCache(context, bind_data); } auto read_result = reader->Read(output); if (read_result == PostgresReadResult::FINISHED) { @@ -542,7 +622,10 @@ static void PostgresScan(ClientContext &context, TableFunctionInput &data, DataC auto &gstate = data.global_state->Cast(); if (gstate.collection) { - gstate.collection->Scan(gstate.scan_state, output); + bool scan_happened = gstate.collection->Scan(gstate.scan_state, output); + if (!scan_happened) { + output.SetChildCardinality(0); + } return; } auto &local_state = data.local_state->Cast(); @@ -550,7 +633,7 @@ static void PostgresScan(ClientContext &context, TableFunctionInput &data, DataC // Release the PostgreSQL connection early so it is not held // idle during post-scan processing (joins, aggregations, etc.) local_state.reader.reset(); - local_state.connection = PostgresConnection(); + local_state.global_connection = nullptr; local_state.pool_connection = PostgresPoolConnection(); return; } @@ -601,38 +684,30 @@ double PostgresScanProgress(ClientContext &context, const FunctionData *bind_dat } static void PostgresScanSerialize(Serializer &serializer, const optional_ptr bind_data_p, - const TableFunction &function) { + const BoundTableFunction &function) { throw NotImplementedException("PostgresScanSerialize"); } -static unique_ptr PostgresScanDeserialize(Deserializer &deserializer, TableFunction &function) { +static unique_ptr PostgresScanDeserialize(Deserializer &deserializer, BoundTableFunction &function) { throw NotImplementedException("PostgresScanDeserialize"); } -static BindInfo PostgresGetBindInfo(const optional_ptr bind_data_p) { - auto &bind_data = bind_data_p->Cast(); - auto table = bind_data.GetTable(); - BindInfo info(ScanType::EXTERNAL); - info.table = table.get(); - return info; +static optional_ptr PostgresGetTableEntry(optional_ptr bind_data_p) { + auto &bdata = bind_data_p->Cast(); + shared_ptr ctx = bdata.context_ptr.lock(); + if (!ctx) { // cannot fail in known scenarios + return nullptr; + } + auto attached_table = PostgresTableEntry::Lookup(*ctx, bdata.qualified_table_name); + return attached_table.Get(); } PostgresScanFunction::PostgresScanFunction() - : TableFunction("postgres_scan", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, PostgresScan, - PostgresBind, PostgresInitGlobalState, PostgresInitLocalState) { - to_string = PostgresScanToString; - serialize = PostgresScanSerialize; - deserialize = PostgresScanDeserialize; - get_partition_data = PostgresGetPartitionData; - cardinality = PostgresScanCardinality; - table_scan_progress = PostgresScanProgress; - get_bind_info = PostgresGetBindInfo; - projection_pushdown = true; - global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; -} - -PostgresScanFunctionFilterPushdown::PostgresScanFunctionFilterPushdown() - : TableFunction("postgres_scan_pushdown", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + : TableFunction("postgres_scan", + FunctionSignature() + .AddPositionalOnly("dsn", LogicalType::VARCHAR) + .AddPositionalOnly("schema_name", LogicalType::VARCHAR) + .AddPositionalOnly("table_name", LogicalType::VARCHAR), PostgresScan, PostgresBind, PostgresInitGlobalState, PostgresInitLocalState) { to_string = PostgresScanToString; serialize = PostgresScanSerialize; @@ -640,9 +715,8 @@ PostgresScanFunctionFilterPushdown::PostgresScanFunctionFilterPushdown() get_partition_data = PostgresGetPartitionData; cardinality = PostgresScanCardinality; table_scan_progress = PostgresScanProgress; - get_bind_info = PostgresGetBindInfo; + get_table_entry = PostgresGetTableEntry; projection_pushdown = true; - filter_pushdown = true; global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; } diff --git a/src/postgres_secrets.cpp b/src/postgres_secrets.cpp index 8d3dddc55..2729db126 100644 --- a/src/postgres_secrets.cpp +++ b/src/postgres_secrets.cpp @@ -1,4 +1,5 @@ #include "postgres_secrets.hpp" +#include "duckdb/main/client_context.hpp" #include #include @@ -69,7 +70,8 @@ static const std::unordered_map connection_option_alia static const std::vector other_option_names = { "uri", - "aws_rds_secret" + "aws_rds_secret", + "aws_rds_token_max_age_seconds" }; // clang-format on @@ -128,6 +130,7 @@ void PostgresSecrets::SetSecretParameters(CreateSecretFunction &function) { // other options function.named_parameters["uri"] = LogicalType::VARCHAR; function.named_parameters["aws_rds_secret"] = LogicalType::VARCHAR; + function.named_parameters["aws_rds_token_max_age_seconds"] = LogicalType::UBIGINT; } } // namespace duckdb diff --git a/src/postgres_sql_writer.cpp b/src/postgres_sql_writer.cpp new file mode 100644 index 000000000..ad2dd608f --- /dev/null +++ b/src/postgres_sql_writer.cpp @@ -0,0 +1,4007 @@ +#include "postgres_sql_writer.hpp" + +#include "duckdb.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/parser/expression/between_expression.hpp" +#include "duckdb/parser/expression/case_expression.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/comparison_expression.hpp" +#include "duckdb/parser/expression/conjunction_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/operator_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/constraints/unique_constraint.hpp" +#include "duckdb/parser/expression/default_expression.hpp" +#include "duckdb/parser/query_node/delete_query_node.hpp" +#include "duckdb/parser/query_node/insert_query_node.hpp" +#include "duckdb/parser/query_node/merge_query_node.hpp" +#include "duckdb/parser/statement/merge_into_statement.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/update_query_node.hpp" +#include "duckdb/parser/statement/alter_statement.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/parsed_data/alter_table_info.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/planner/operator/logical_create_table.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "storage/postgres_table_set.hpp" +#include "duckdb/parser/statement/update_statement.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/emptytableref.hpp" +#include "duckdb/parser/tableref/expressionlistref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" + +#include "duckdb/common/type_visitor.hpp" +#include "postgres_utils.hpp" +#include "storage/postgres_table_entry.hpp" +#include "storage/postgres_capabilities.hpp" +#include "storage/postgres_catalog.hpp" + +namespace duckdb { + +PostgresSQLWriter::PostgresSQLWriter(ClientContext &context, PostgresCatalog &catalog) + : context(context), catalog(catalog) { +} + +PostgresWriterResult PostgresSQLWriter::WriteQuery(ClientContext &context, PostgresCatalog &catalog, QueryNode &node, + optional_ptr> result_types) { + PostgresSQLWriter writer(context, catalog); + writer.CollectNames(node); + writer.CollectResultNodes(node); + writer.result_types = result_types; + PostgresWriterResult result; + result.sql = writer.WriteQueryNode(node); + result.requires_time_zone = writer.requires_time_zone; + if (writer.requires_iso_date_style) { + // reverted at the end of the transaction + result.preliminary_queries.push_back("SET LOCAL DateStyle = 'ISO'"); + } + return result; +} + +//===--------------------------------------------------------------------===// +// Local binding +//===--------------------------------------------------------------------===// +static void PostgresCollectCTENamesExpr(ParsedExpression &expr, identifier_set_t &cte_names) { + if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { + PostgresSQLWriter::CollectCTENames(*expr.Cast().SubqueryMutable()->node, cte_names); + } + ParsedExpressionIterator::EnumerateChildren( + expr, [&](ParsedExpression &child) { PostgresCollectCTENamesExpr(child, cte_names); }); +} + +void PostgresSQLWriter::CollectCTENames(QueryNode &node, identifier_set_t &cte_names) { + for (auto &cte : node.cte_map.map) { + cte_names.insert(cte.first); + } + ParsedExpressionIterator::EnumerateQueryNodeChildren( + node, [&](unique_ptr &child) { PostgresCollectCTENamesExpr(*child, cte_names); }); +} + +static void PostgresQualifyTablesExpr(ParsedExpression &expr, const Identifier &catalog_name, + const identifier_set_t &cte_names) { + if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { + PostgresSQLWriter::QualifyTables(*expr.Cast().SubqueryMutable()->node, catalog_name, + cte_names); + } + ParsedExpressionIterator::EnumerateChildren( + expr, [&](ParsedExpression &child) { PostgresQualifyTablesExpr(child, catalog_name, cte_names); }); +} + +void PostgresSQLWriter::QualifyTables(QueryNode &node, const Identifier &catalog_name, + const identifier_set_t &cte_names) { + if (node.type == QueryNodeType::INSERT_QUERY_NODE) { + auto &insert = node.Cast(); + vector schema_path; + if (!insert.qualified_name.Schema().empty()) { + schema_path.push_back(insert.qualified_name.Schema()); + } + insert.qualified_name = + QualifiedName::FromCatalogSchema(catalog_name, std::move(schema_path), insert.qualified_name.Name()); + if (insert.table_ref && insert.table_ref->type == TableReferenceType::BASE_TABLE) { + // the catalog name is only stripped from the qualified name of the INSERT target + insert.table_ref->Cast().SetQualifiedName(insert.qualified_name); + } + } + ParsedExpressionIterator::EnumerateQueryNodeChildren( + node, [&](unique_ptr &child) { PostgresQualifyTablesExpr(*child, catalog_name, cte_names); }, + [&](TableRef &ref) { + if (ref.type != TableReferenceType::BASE_TABLE) { + return; + } + auto &base = ref.Cast(); + auto &name = base.GetQualifiedName(); + if (name.Path().size() == 1 && cte_names.count(name.Name()) > 0) { + // (possibly) a CTE reference + return; + } + if (name.Path().size() > 2) { + return; + } + vector schema_path; + if (!name.Schema().empty()) { + schema_path.push_back(name.Schema()); + } + base.SetQualifiedName(QualifiedName::FromCatalogSchema(catalog_name, std::move(schema_path), name.Name())); + }); +} + +void PostgresSQLWriter::CollectScopeTables(const TableRef &ref, vector> &tables, + bool &only_base_tables) { + switch (ref.type) { + case TableReferenceType::BASE_TABLE: + if (IsCTEReference(ref.Cast())) { + only_base_tables = false; + } else { + tables.push_back(ref.Cast()); + } + break; + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + if (join.type == JoinType::SEMI || join.type == JoinType::ANTI || !join.using_columns.empty() || + join.ref_type == JoinRefType::NATURAL) { + // the bindings of the join differ from the bindings of the joined tables + only_base_tables = false; + } + CollectScopeTables(*join.left, tables, only_base_tables); + CollectScopeTables(*join.right, tables, only_base_tables); + break; + } + default: + only_base_tables = false; + break; + } +} + +optional_ptr PostgresSQLWriter::FindBindingTable(const Identifier &binding) { + for (idx_t i = table_scopes.size(); i > 0; i--) { + vector> tables; + bool only_base_tables = true; + for (auto &table_ref : table_scopes[i - 1]) { + if (!table_ref.get()) { + continue; + } + if (table_ref.get()->alias == binding && table_ref.get()->type != TableReferenceType::BASE_TABLE) { + // e.g. a subquery + return nullptr; + } + CollectScopeTables(*table_ref.get(), tables, only_base_tables); + } + for (auto &base : tables) { + auto base_binding = base.get().alias.empty() ? base.get().Table() : base.get().alias; + if (base_binding == binding) { + return LookupTable(base.get()); + } + } + } + // e.g. a reference to a table in an outer query of a subquery in the FROM clause + auto entry = binding_tables.find(binding.GetIdentifierName()); + if (entry != binding_tables.end()) { + return &entry->second.get(); + } + return nullptr; +} + +LogicalType PostgresSQLWriter::InferType(const ParsedExpression &expr) { + auto result = InferTypeInternal(expr); + if (result.id() == LogicalTypeId::TIMESTAMP_TZ || result.id() == LogicalTypeId::TIME_TZ) { + requires_time_zone = true; + } + return result; +} + +LogicalType PostgresSQLWriter::InferTypeInternal(const ParsedExpression &expr) { + if (expr.GetExpressionClass() == ExpressionClass::CONSTANT) { + return expr.Cast().GetLiteral().ToValue().type(); + } + if (expr.GetExpressionClass() == ExpressionClass::COLUMN_REF && !table_scopes.empty()) { + // fast path: a column of one of the base tables in scope + auto &names = expr.Cast().ColumnNames(); + if (names.size() == 1 && on_conflict_target && query_depth == on_conflict_depth && + on_conflict_target->ColumnExists(names[0])) { + // unqualified columns in ON CONFLICT DO UPDATE refer to the existing row + return on_conflict_target->GetColumn(names[0]).Type(); + } + vector> tables; + bool only_base_tables = true; + for (auto &table_ref : table_scopes.back()) { + if (table_ref.get()) { + CollectScopeTables(*table_ref.get(), tables, only_base_tables); + } + } + optional_ptr found; + idx_t matches = 0; + for (auto &base : tables) { + auto table = LookupTable(base.get()); + if (!table) { + only_base_tables = false; + continue; + } + auto binding = base.get().alias.empty() ? base.get().Table() : base.get().alias; + if (names.size() == 1 && table->ColumnExists(names[0])) { + found = table; + matches++; + } else if (names.size() == 2 && names[0] == binding && table->ColumnExists(names[1])) { + return table->GetColumn(names[1]).Type(); + } + } + if (names.size() == 1 && only_base_tables && matches == 1) { + return found->GetColumn(names[0]).Type(); + } + } + interval_t interval; + if (PostgresCapabilities::TryGetIntervalLiteral(expr, interval)) { + return LogicalType::INTERVAL; + } + auto entry = type_cache.find(&expr); + if (entry != type_cache.end()) { + return entry->second; + } + // bind a probe query that evaluates the expression over the FROM clause of the enclosing SELECT + auto probe = make_uniq(); + probe->select_list.push_back(expr.Copy()); + if (!table_scopes.empty()) { + for (auto &table : table_scopes.back()) { + auto table_copy = table.get()->Copy(); + if (!probe->from_table) { + probe->from_table = std::move(table_copy); + continue; + } + auto join = make_uniq(JoinRefType::CROSS); + join->left = std::move(probe->from_table); + join->right = std::move(table_copy); + probe->from_table = std::move(join); + } + } + if (!probe->from_table) { + probe->from_table = make_uniq(); + } + for (idx_t i = cte_scopes.size(); i > 0; i--) { + // inner CTEs shadow outer ones - insert those first + for (auto &cte : cte_scopes[i - 1].get().map) { + probe->cte_map.map.insert(cte.first, cte.second->Copy()); + } + } + identifier_set_t cte_names; + CollectCTENames(*probe, cte_names); + QualifyTables(*probe, catalog.GetName(), cte_names); + LogicalType result = LogicalType::INVALID; + try { + auto binder = Binder::CreateBinder(context); + auto bound = binder->Bind(*probe); + if (bound.types.size() == 1) { + result = bound.types[0]; + } + } catch (std::exception &) { + // e.g. a correlated column reference or a reference to a select-list alias + } + if (result.id() == LogicalTypeId::INVALID && expr.GetExpressionClass() == ExpressionClass::COLUMN_REF) { + result = InferOuterColumnType(expr.Cast()); + } + type_cache[&expr] = result; + return result; +} + +vector PostgresSQLWriter::InferQueryTypes(const QueryNode &node) { + auto probe = node.Copy(); + probe->modifiers.clear(); + for (idx_t i = cte_scopes.size(); i > 0; i--) { + for (auto &cte : cte_scopes[i - 1].get().map) { + if (probe->cte_map.map.find(cte.first) == probe->cte_map.map.end()) { + probe->cte_map.map.insert(cte.first, cte.second->Copy()); + } + } + } + identifier_set_t cte_names; + CollectCTENames(*probe, cte_names); + QualifyTables(*probe, catalog.GetName(), cte_names); + try { + auto binder = Binder::CreateBinder(context); + return binder->Bind(*probe).types; + } catch (std::exception &) { + // e.g. a correlated column reference + return vector(); + } +} + +LogicalType PostgresSQLWriter::InferOuterColumnType(const ColumnRefExpression &column_ref) { + auto &names = column_ref.ColumnNames(); + if (names.size() == 2) { + auto table = FindBindingTable(names[0]); + if (table && table->ColumnExists(names[1])) { + return table->GetColumn(names[1]).Type(); + } + return LogicalType::INVALID; + } + if (names.size() != 1 || table_scopes.size() < 2) { + return LogicalType::INVALID; + } + // a correlated reference - the innermost enclosing query that has the column decides + for (idx_t i = table_scopes.size() - 1; i > 0; i--) { + vector> tables; + bool only_base_tables = true; + for (auto &table_ref : table_scopes[i - 1]) { + if (table_ref.get()) { + CollectScopeTables(*table_ref.get(), tables, only_base_tables); + } + } + optional_ptr found; + idx_t matches = 0; + for (auto &base : tables) { + auto table = LookupTable(base.get()); + if (!table) { + only_base_tables = false; + continue; + } + if (table->ColumnExists(names[0])) { + found = table; + matches++; + } + } + if (matches == 1) { + // the query was bound by DuckDB - the reference is not ambiguous + return found->GetColumn(names[0]).Type(); + } + if (matches > 1 || !only_base_tables) { + // the column might be defined by a subquery of this scope + return LogicalType::INVALID; + } + } + return LogicalType::INVALID; +} + +//===--------------------------------------------------------------------===// +// Identifier spelling +//===--------------------------------------------------------------------===// +void PostgresSQLWriter::AddCatalogColumnName(const string &name) { + catalog_columns.insert(name); + AddCatalogName(name); +} + +void PostgresSQLWriter::AddCatalogName(const string &name) { + if (ambiguous_identifiers.count(name) > 0) { + return; + } + auto entry = identifier_spelling.find(name); + if (entry == identifier_spelling.end()) { + identifier_spelling[name] = name; + } else if (entry->second != name) { + // e.g. a table with both a column "a" and a column "A" + identifier_spelling.erase(entry); + ambiguous_identifiers.insert(name); + } +} + +void PostgresSQLWriter::AddQueryName(const Identifier &name) { + auto &str = name.GetIdentifierName(); + if (str.empty() || ambiguous_identifiers.count(str) > 0) { + return; + } + // names defined in the query (aliases, CTE names, ...) never override catalog names - the first + // spelling wins + if (identifier_spelling.find(str) == identifier_spelling.end()) { + identifier_spelling[str] = str; + } +} + +optional_ptr PostgresSQLWriter::LookupTable(const BaseTableRef &ref) { + return LookupTable(ref.GetQualifiedName()); +} + +optional_ptr PostgresSQLWriter::LookupTable(const QualifiedName &qualified_name) { + if (qualified_name.Path().size() > 2) { + // the catalog name has been stripped by the optimizer, nested schemas are not supported + return nullptr; + } + Identifier schema_name = qualified_name.Schema(); + if (schema_name.empty()) { + auto default_schema = catalog.GetDefaultSchema(); + schema_name = default_schema ? *default_schema : Identifier("public"); + } + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, + QualifiedName(catalog.GetName(), schema_name, qualified_name.Name())); + auto entry = Catalog::GetEntry(context, lookup, OnEntryNotFound::RETURN_NULL); + if (!entry || entry->type != CatalogType::TABLE_ENTRY) { + return nullptr; + } + return &entry->Cast(); +} + +bool PostgresSQLWriter::IsCTEReference(const BaseTableRef &ref) { + if (ref.GetQualifiedName().Path().size() > 1) { + return false; + } + for (auto &scope : cte_scopes) { + if (scope.get().map.find(ref.Table()) != scope.get().map.end()) { + return true; + } + } + return false; +} + +void PostgresSQLWriter::CollectNames(ParsedExpression &expr) { + AddQueryName(expr.GetAlias()); + if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { + auto &subquery = expr.Cast(); + CollectNames(*subquery.SubqueryMutable()->node); + } + ParsedExpressionIterator::EnumerateChildren(expr, [&](ParsedExpression &child) { CollectNames(child); }); +} + +void PostgresSQLWriter::CollectNames(TableRef &ref) { + AddQueryName(ref.alias); + if (!ref.alias.empty()) { + table_bindings.insert(ref.alias.GetIdentifierName()); + } else if (ref.type == TableReferenceType::BASE_TABLE) { + table_bindings.insert(ref.Cast().Table().GetIdentifierName()); + } else if (ref.type == TableReferenceType::TABLE_FUNCTION && + ref.Cast().function->GetExpressionClass() == ExpressionClass::FUNCTION) { + // the single column of the table function is named after the function + table_bindings.insert( + ref.Cast().function->Cast().FunctionName().GetIdentifierName()); + } else if (ref.type == TableReferenceType::EXPRESSION_LIST) { + table_bindings.insert("valueslist"); + } + for (auto &alias : ref.column_name_alias) { + AddQueryName(alias); + } + if (ref.type == TableReferenceType::TABLE_FUNCTION && !ref.alias.empty() && ref.column_name_alias.empty() && + ref.Cast().function->GetExpressionClass() == ExpressionClass::FUNCTION) { + // DuckDB names the column of an aliased table function after the function and the alias, Postgres only + // after the alias + auto &function_name = ref.Cast().function->Cast().FunctionName(); + table_function_aliases[ref.alias.GetIdentifierName()] = function_name.GetIdentifierName(); + } + if (ref.type == TableReferenceType::BASE_TABLE) { + auto table = LookupTable(ref.Cast()); + if (table) { + AddTableNames(*table); + auto binding = ref.alias.empty() ? ref.Cast().Table() : ref.alias; + binding_tables.emplace(binding.GetIdentifierName(), *table); + } + } +} + +static string PostgresNativeColumnType(TableCatalogEntry &table, const ColumnDefinition &column, bool &known); + +//! DuckDB reads some Postgres types as different types (e.g. JSONB or INET as VARCHAR, unconstrained NUMERIC as DOUBLE) +//! - references to these columns are cast to the type DuckDB reads them as, so they behave the same +static string PostgresColumnReadType(TableCatalogEntry &table, const ColumnDefinition &column) { + auto &pg_table = table.Cast(); + auto index = column.Logical().index; + if (index < pg_table.postgres_types.size() && + pg_table.postgres_types[index].info == PostgresTypeAnnotation::NUMERIC_AS_DOUBLE) { + return "DOUBLE PRECISION"; + } + if (column.Type().id() == LogicalTypeId::VARCHAR) { + bool known; + if (!PostgresNativeColumnType(table, column, known).empty()) { + return "TEXT"; + } + } + return string(); +} + +void PostgresSQLWriter::AddTableNames(TableCatalogEntry &table) { + AddCatalogName(table.ParentSchema().name.GetIdentifierName()); + AddCatalogName(table.name.GetIdentifierName()); + auto &pg_table = table.Cast(); + for (auto &column : table.GetColumns().Logical()) { + auto &name = column.Name().GetIdentifierName(); + auto index = column.Logical().index; + if (TypeVisitor::Contains(column.Type(), [](const LogicalType &type) { + return type.id() == LogicalTypeId::TIMESTAMP_TZ || type.id() == LogicalTypeId::TIME_TZ; + })) { + // operations on the column depend on the time zone + requires_time_zone = true; + } + auto read_type = PostgresColumnReadType(table, column); + auto entry = column_read_types.find(name); + if (entry == column_read_types.end()) { + column_read_types[name] = read_type; + } else if (entry->second != read_type) { + // columns with this name are read differently in different tables + entry->second = "?"; + } + if (index < pg_table.postgres_names.size() && pg_table.postgres_names[index] != name) { + // DuckDB renames columns whose names only differ in case (e.g. "ID" and "id" -> "ID" and "id:1") + identifier_spelling[name] = pg_table.postgres_names[index]; + catalog_columns.insert(name); + continue; + } + AddCatalogColumnName(name); + } +} + +string PostgresSQLWriter::WriteColumnName(TableCatalogEntry &table, const ColumnDefinition &column) { + auto &pg_table = table.Cast(); + auto index = column.Logical().index; + auto &name = + index < pg_table.postgres_names.size() ? pg_table.postgres_names[index] : column.Name().GetIdentifierName(); + return PostgresUtils::WriteIdentifier(name); +} + +void PostgresSQLWriter::CollectNames(QueryNode &node) { + if (node.type == QueryNodeType::INSERT_QUERY_NODE) { + auto table = LookupTable(node.Cast().qualified_name); + if (table) { + AddTableNames(*table); + table_bindings.insert(table->name.GetIdentifierName()); + } + table_bindings.insert("excluded"); + } + for (auto &cte : node.cte_map.map) { + AddQueryName(cte.first); + table_bindings.insert(cte.first.GetIdentifierName()); + for (auto &alias : cte.second->aliases) { + AddQueryName(alias); + } + } + ParsedExpressionIterator::EnumerateQueryNodeChildren( + node, [&](unique_ptr &child) { CollectNames(*child); }, + [&](TableRef &ref) { CollectNames(ref); }); +} + +string PostgresSQLWriter::WriteIdentifier(const string &identifier) { + auto entry = identifier_spelling.find(identifier); + if (entry != identifier_spelling.end()) { + return PostgresUtils::WriteIdentifier(entry->second); + } + return PostgresUtils::WriteIdentifier(identifier); +} + +string PostgresSQLWriter::WriteIdentifier(const Identifier &identifier) { + return WriteIdentifier(identifier.GetIdentifierName()); +} + +void PostgresSQLWriter::CollectResultNodes(const QueryNode &node) { + switch (node.type) { + case QueryNodeType::SELECT_NODE: + result_nodes.insert(&node); + break; + case QueryNodeType::SET_OPERATION_NODE: + for (auto &child : node.Cast().children) { + CollectResultNodes(*child); + } + break; + default: + break; + } +} + +//! Postgres produces unconstrained NUMERIC values for e.g. sum(BIGINT) or arithmetic on decimals, which are read back +//! as DOUBLE - cast these to the exact NUMERIC type that corresponds to the type DuckDB binds the expression to +static string PostgresExactNumericType(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::HUGEINT: + return "NUMERIC(38,0)"; + case LogicalTypeId::UHUGEINT: + return "NUMERIC(39,0)"; + case LogicalTypeId::DECIMAL: + return StringUtil::Format("NUMERIC(%d,%d)", DecimalType::GetWidth(type), DecimalType::GetScale(type)); + case LogicalTypeId::LIST: { + auto child = PostgresExactNumericType(ListType::GetChildType(type)); + return child.empty() ? child : child + "[]"; + } + default: + return string(); + } +} + +string PostgresSQLWriter::WriteResultCast(const string &expr, const LogicalType &type) { + auto exact_type = PostgresExactNumericType(type); + return exact_type.empty() ? expr : "CAST(" + expr + " AS " + exact_type + ")"; +} + +string PostgresSQLWriter::WriteResultCast(const string &expr, idx_t column_index) { + if (!result_types || column_index >= result_types->size()) { + return expr; + } + return WriteResultCast(expr, (*result_types)[column_index]); +} + +//===--------------------------------------------------------------------===// +// Constants / Types +//===--------------------------------------------------------------------===// +string PostgresSQLWriter::WriteConstant(const Value &value) { + if (value.IsNull()) { + if (value.type().id() == LogicalTypeId::SQLNULL) { + return "NULL"; + } + // Postgres resolves untyped NULLs to TEXT in e.g. subqueries - keep the type + try { + return "CAST(NULL AS " + WriteType(value.type()) + ")"; + } catch (PostgresPushdownNotSupported &) { + return "NULL"; + } + } + if (value.type().id() == LogicalTypeId::TIMESTAMP_TZ) { + requires_time_zone = true; + } + switch (value.type().id()) { + case LogicalTypeId::BOOLEAN: + return BooleanValue::Get(value) ? "TRUE" : "FALSE"; + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::UHUGEINT: { + // literals beyond the BIGINT range are NUMERIC in Postgres + auto str = value.ToString(); + return StringUtil::StartsWith(str, "-") ? "(" + str + ")" : str; + } + case LogicalTypeId::DECIMAL: + // keep the width and scale - an unconstrained NUMERIC is read back as DOUBLE + return "CAST(" + PostgresUtils::WriteLiteral(value.ToString()) + " AS " + WriteType(value.type()) + ")"; + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + // Postgres reads literals with a decimal point as NUMERIC - write them as typed literals + // (this also covers inf and nan) + return "CAST(" + PostgresUtils::WriteLiteral(value.ToString()) + " AS " + WriteType(value.type()) + ")"; + case LogicalTypeId::VARCHAR: { + // untyped string literals are coerced to the type of the other operand, like DuckDB does + auto &str = StringValue::Get(value); + if (str.find('\\') != string::npos) { + // escape string syntax - independent of the standard_conforming_strings setting + return "E" + PostgresUtils::WriteLiteral(StringUtil::Replace(str, "\\", "\\\\")); + } + return PostgresUtils::WriteLiteral(str); + } + case LogicalTypeId::BLOB: { + static const char *HEX_DIGITS = "0123456789abcdef"; + auto &data = StringValue::Get(value); + string hex; + for (auto c : data) { + auto byte = static_cast(c); + hex += HEX_DIGITS[byte >> 4]; + hex += HEX_DIGITS[byte & 0x0F]; + } + return "decode('" + hex + "', 'hex')"; + } + case LogicalTypeId::DATE: + return "DATE " + PostgresUtils::WriteLiteral(value.ToString()); + case LogicalTypeId::TIME: + return "TIME " + PostgresUtils::WriteLiteral(value.ToString()); + case LogicalTypeId::TIMESTAMP: + return "TIMESTAMP " + PostgresUtils::WriteLiteral(value.ToString()); + case LogicalTypeId::TIMESTAMP_TZ: + // without a client context the value is formatted with an explicit UTC offset + return "TIMESTAMPTZ " + PostgresUtils::WriteLiteral(value.ToString()); + case LogicalTypeId::INTERVAL: + return "INTERVAL " + PostgresUtils::WriteLiteral(value.ToString()); + case LogicalTypeId::UUID: + return "UUID " + PostgresUtils::WriteLiteral(value.ToString()); + default: + throw InternalException("PostgresSQLWriter: unsupported constant type %s - should have been blocked by " + "SupportsPushdown", + value.type().ToString()); + } +} + +string PostgresSQLWriter::WriteType(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + return "BOOLEAN"; + case LogicalTypeId::SMALLINT: + return "SMALLINT"; + case LogicalTypeId::INTEGER: + return "INTEGER"; + case LogicalTypeId::BIGINT: + return "BIGINT"; + case LogicalTypeId::FLOAT: + return "REAL"; + case LogicalTypeId::DOUBLE: + return "DOUBLE PRECISION"; + case LogicalTypeId::DECIMAL: + return StringUtil::Format("NUMERIC(%d,%d)", DecimalType::GetWidth(type), DecimalType::GetScale(type)); + case LogicalTypeId::VARCHAR: + return "TEXT"; + case LogicalTypeId::DATE: + return "DATE"; + case LogicalTypeId::TIME: + return "TIME"; + case LogicalTypeId::TIMESTAMP: + return "TIMESTAMP"; + case LogicalTypeId::TIMESTAMP_TZ: + return "TIMESTAMPTZ"; + case LogicalTypeId::INTERVAL: + return "INTERVAL"; + case LogicalTypeId::UUID: + return "UUID"; + case LogicalTypeId::BLOB: + return "BYTEA"; + default: + // e.g. user-defined types, which are verified when writing assignments + throw PostgresPushdownNotSupported("unsupported type " + type.ToString()); + } +} + +//===--------------------------------------------------------------------===// +// Type dependent translation +//===--------------------------------------------------------------------===// +static bool PostgresIsIntegerType(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::UHUGEINT: + return true; + default: + return false; + } +} + +static bool PostgresIsNumericType(const LogicalType &type) { + return PostgresIsIntegerType(type) || type.id() == LogicalTypeId::DECIMAL || type.id() == LogicalTypeId::FLOAT || + type.id() == LogicalTypeId::DOUBLE; +} + +static bool PostgresIsDateTimeType(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + return true; + default: + return false; + } +} + +//! Whether Postgres formats values of this type as text exactly like DuckDB casts them to VARCHAR - when used as +//! a concat / || argument (the output function is used) or explicitly cast to TEXT (booleans are cast to +//! 'true' / 'false', but output as 't' / 'f') +static bool PostgresHasIdenticalTextFormat(const LogicalType &type, bool explicit_cast = false) { + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + return explicit_cast; + case LogicalTypeId::SQLNULL: + case LogicalTypeId::VARCHAR: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::UUID: + case LogicalTypeId::ENUM: + return true; + default: + // doubles (DuckDB writes 3.0, Postgres 3), intervals (2 months vs 2 mons), blobs (escapes vs hex) differ + // date / time types require the ISO DateStyle, see WriteCast + return PostgresIsIntegerType(type); + } +} + +string PostgresSQLWriter::WriteCast(const ParsedExpression &child, const LogicalType &target_type) { + auto source_type = InferType(child); + if (source_type.id() == LogicalTypeId::INVALID) { + throw PostgresPushdownNotSupported("cannot determine the source type of a cast"); + } + return WriteCast(WriteExpression(child), source_type, target_type); +} + +string PostgresSQLWriter::WriteCast(const string &child_str, const LogicalType &source_type, + const LogicalType &target_type) { + if (source_type == target_type) { + return child_str; + } + auto cast = [&](const string &expr) { + return "CAST(" + expr + " AS " + WriteType(target_type) + ")"; + }; + auto source_id = source_type.id(); + if (source_id == LogicalTypeId::SQLNULL) { + return cast(child_str); + } + switch (target_type.id()) { + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + if (source_id == LogicalTypeId::BOOLEAN) { + // Postgres only has a BOOLEAN -> INTEGER cast + return cast("CAST(" + child_str + " AS INTEGER)"); + } + // note: both DuckDB and Postgres round doubles to the nearest even integer + if (PostgresIsNumericType(source_type) || source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::DECIMAL: + if (PostgresIsNumericType(source_type) || source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + case LogicalTypeId::BOOLEAN: + if (PostgresIsIntegerType(source_type)) { + return "(" + child_str + " <> 0)"; + } + if (source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + case LogicalTypeId::VARCHAR: + if (PostgresHasIdenticalTextFormat(source_type, true)) { + return cast(child_str); + } + if (PostgresIsDateTimeType(source_type)) { + // DuckDB formats dates and times like Postgres' ISO DateStyle + requires_iso_date_style = true; + return cast(child_str); + } + break; + case LogicalTypeId::DATE: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + if (source_id == LogicalTypeId::DATE || source_id == LogicalTypeId::TIMESTAMP || + source_id == LogicalTypeId::TIMESTAMP_TZ || source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + case LogicalTypeId::TIME: + if (source_id == LogicalTypeId::TIMESTAMP || source_id == LogicalTypeId::TIMESTAMP_TZ || + source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + case LogicalTypeId::UUID: + if (source_id == LogicalTypeId::VARCHAR) { + return cast(child_str); + } + break; + default: + break; + } + throw PostgresPushdownNotSupported("unsupported cast from " + source_type.ToString() + " to " + + target_type.ToString()); +} + +string PostgresSQLWriter::WriteLength(const string &name, const ParsedExpression &child) { + auto type = InferType(child); + auto child_str = WriteExpression(child); + switch (type.id()) { + case LogicalTypeId::VARCHAR: + return name + "(" + child_str + ")"; + case LogicalTypeId::LIST: + if (name == "length" && ListType::GetChildType(type).id() != LogicalTypeId::LIST) { + // the number of elements of a one-dimensional array + return "cardinality(" + child_str + ")"; + } + break; + default: + break; + } + throw PostgresPushdownNotSupported("unsupported length function argument"); +} + +//===--------------------------------------------------------------------===// +// Scalar function translation +//===--------------------------------------------------------------------===// +static string PostgresFloat8(const string &expr) { + return "CAST(" + expr + " AS DOUBLE PRECISION)"; +} + +//! Normalize a DuckDB date part name, returns an empty string for unsupported parts +static string PostgresNormalizeDatePart(const string &part) { + static const case_insensitive_map_t DATE_PARTS = { + {"year", "year"}, + {"years", "year"}, + {"month", "month"}, + {"months", "month"}, + {"day", "day"}, + {"days", "day"}, + {"hour", "hour"}, + {"hours", "hour"}, + {"minute", "minute"}, + {"minutes", "minute"}, + {"second", "second"}, + {"seconds", "second"}, + {"millisecond", "millisecond"}, + {"milliseconds", "millisecond"}, + {"microsecond", "microsecond"}, + {"microseconds", "microsecond"}, + {"quarter", "quarter"}, + {"quarters", "quarter"}, + {"week", "week"}, + {"weeks", "week"}, + {"weekofyear", "week"}, + {"dow", "dow"}, + {"dayofweek", "dow"}, + {"isodow", "isodow"}, + {"doy", "doy"}, + {"dayofyear", "doy"}, + {"isoyear", "isoyear"}, + {"decade", "decade"}, + {"decades", "decade"}, + {"century", "century"}, + {"centuries", "century"}, + {"millennium", "millennium"}, + {"millennia", "millennium"}, + {"millenniums", "millennium"}, + {"epoch", "epoch"}, + {"julian", "julian"}, + }; + auto entry = DATE_PARTS.find(part); + return entry == DATE_PARTS.end() ? string() : entry->second; +} + +static bool PostgresIsTimestampType(const LogicalType &type) { + return type.id() == LogicalTypeId::DATE || type.id() == LogicalTypeId::TIMESTAMP || + type.id() == LogicalTypeId::TIMESTAMP_TZ; +} + +string PostgresSQLWriter::WriteDatePart(const string &part, const ParsedExpression &arg) { + auto type = InferType(arg); + auto arg_str = WriteExpression(arg); + bool supported; + switch (type.id()) { + case LogicalTypeId::DATE: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + supported = true; + break; + case LogicalTypeId::TIME: + supported = part == "hour" || part == "minute" || part == "second" || part == "millisecond" || + part == "microsecond" || part == "epoch"; + break; + case LogicalTypeId::INTERVAL: + supported = part == "year" || part == "month" || part == "day" || part == "hour" || part == "minute" || + part == "second" || part == "millisecond" || part == "microsecond" || part == "epoch"; + break; + default: + supported = false; + break; + } + if (!supported) { + throw PostgresPushdownNotSupported("unsupported date part"); + } + if (part == "epoch" || part == "julian") { + // DuckDB returns a DOUBLE + return PostgresFloat8("EXTRACT(" + StringUtil::Upper(part) + " FROM " + arg_str + ")"); + } + string field = part; + if (part == "millisecond" || part == "microsecond") { + field += "s"; + } + string extract = "EXTRACT(" + StringUtil::Upper(field) + " FROM " + arg_str + ")"; + if (part == "second" || part == "millisecond") { + // Postgres includes the fractional part + extract = "trunc(" + extract + ")"; + } + if (part == "year" && type.id() != LogicalTypeId::INTERVAL) { + // DuckDB uses astronomical year numbering (0 is 1 BC), Postgres uses negative years for BC years + extract = "(CASE WHEN " + extract + " < 0 THEN " + extract + " + 1 ELSE " + extract + " END)"; + } + // DuckDB's date parts are BIGINTs, EXTRACT returns NUMERIC + return "CAST(" + extract + " AS BIGINT)"; +} + +string PostgresSQLWriter::WriteDateDiff(const string &part, const ParsedExpression &start, + const ParsedExpression &end) { + auto start_type = InferType(start); + auto end_type = InferType(end); + auto is_supported = [](const LogicalType &type) { + return type.id() == LogicalTypeId::DATE || type.id() == LogicalTypeId::TIMESTAMP; + }; + if (!is_supported(start_type) || !is_supported(end_type)) { + // note: DuckDB computes differences of TIMESTAMP WITH TIME ZONE values with calendar arithmetic + throw PostgresPushdownNotSupported("unsupported date_diff argument"); + } + auto start_str = WriteExpression(start); + auto end_str = WriteExpression(end); + auto extract = [](const string &field, const string &expr) { + return "CAST(EXTRACT(" + field + " FROM " + expr + ") AS BIGINT)"; + }; + auto diff = [&](const string &field) { + return "(" + extract(field, end_str) + " - " + extract(field, start_str) + ")"; + }; + auto months = [&](const string &expr) { + return "(" + extract("YEAR", expr) + " * 12 + " + extract("MONTH", expr) + " - 1)"; + }; + // DuckDB counts the boundaries crossed - for units of time with a floor division of the epoch + auto epoch_units = [&](const string &units) { + return "CAST(floor(EXTRACT(EPOCH FROM " + end_str + ") * " + units + ") - floor(EXTRACT(EPOCH FROM " + + start_str + ") * " + units + ") AS BIGINT)"; + }; + auto days = "(CAST(" + end_str + " AS DATE) - CAST(" + start_str + " AS DATE))"; + if (part == "year" || part == "isoyear") { + return diff(StringUtil::Upper(part)); + } + if (part == "decade" || part == "century" || part == "millennium") { + auto divisor = part == "decade" ? "10" : part == "century" ? "100" : "1000"; + return "(" + extract("YEAR", end_str) + " / " + divisor + " - " + extract("YEAR", start_str) + " / " + divisor + + ")"; + } + if (part == "month") { + return "(" + months(end_str) + " - " + months(start_str) + ")"; + } + if (part == "quarter") { + return "(" + months(end_str) + " / 3 - " + months(start_str) + " / 3)"; + } + if (part == "day") { + return "CAST(" + days + " AS BIGINT)"; + } + if (part == "week") { + // weeks count the distance in days, not the Monday crossings + return "CAST(" + days + " / 7 AS BIGINT)"; + } + if (part == "hour") { + return epoch_units("1 / 3600.0"); + } + if (part == "minute") { + return epoch_units("1 / 60.0"); + } + if (part == "second") { + return epoch_units("1"); + } + if (part == "millisecond") { + return epoch_units("1000"); + } + if (part == "microsecond") { + return epoch_units("1000000"); + } + throw PostgresPushdownNotSupported("unsupported date_diff part"); +} + +static const char *PostgresDateTruncParts[] = {"microseconds", "milliseconds", "second", "minute", "hour", + "day", "week", "month", "quarter", "year", + "decade", "century", "millennium", nullptr}; + +string PostgresSQLWriter::WriteRound(const vector &children, bool half_even) { + auto &arg = children[0].GetExpression(); + auto type = InferType(arg); + int64_t precision = 0; + if (children.size() == 2) { + Value value; + if (!PostgresCapabilities::TryGetConstantValue(children[1].GetExpression(), value)) { + throw InternalException("PostgresSQLWriter: round precision must be a constant"); + } + precision = BigIntValue::Get(value.DefaultCastAs(LogicalType::BIGINT)); + } + auto arg_str = WriteExpression(arg); + if (PostgresIsIntegerType(type)) { + if (precision >= 0) { + return arg_str; + } + if (half_even) { + throw PostgresPushdownNotSupported("unsupported round_even argument"); + } + // NUMERIC rounding rounds half away from zero, like DuckDB - the result keeps the integer type + return "CAST(round(" + arg_str + ", " + to_string(precision) + ") AS " + WriteType(type) + ")"; + } + if (type.id() == LogicalTypeId::DECIMAL && !half_even) { + return "round(" + arg_str + ", " + to_string(precision) + ")"; + } + if (type.id() != LogicalTypeId::DOUBLE || precision > 22 || precision < -22) { + throw PostgresPushdownNotSupported("unsupported round argument"); + } + // DuckDB rounds doubles as std::round(x * 10^p) / 10^p - Postgres' round(DOUBLE PRECISION) rounds half to + // even (like DuckDB's round_even), half away from zero is emulated on the (exact) fractional part + string modifier = "CAST('1e" + to_string(precision < 0 ? -precision : precision) + "' AS DOUBLE PRECISION)"; + string scaled = precision == 0 ? arg_str : "(" + arg_str + (precision > 0 ? " * " : " / ") + modifier + ")"; + string rounded; + if (half_even) { + rounded = "round(" + scaled + ")"; + } else { + rounded = "(CASE WHEN abs(" + scaled + " - trunc(" + scaled + ")) >= 0.5 THEN trunc(" + scaled + ") + sign(" + + scaled + ") ELSE trunc(" + scaled + ") END)"; + } + if (precision == 0) { + return rounded; + } + return "(" + rounded + (precision > 0 ? " / " : " * ") + modifier + ")"; +} + +//! DuckDB's division of numbers is a DOUBLE division (Postgres performs integer division on integers) - and +//! dividing by zero yields inf / NaN where Postgres raises an error +bool PostgresSQLWriter::HasDefaultDivisionSemantics() { + Value ieee_floating_point_ops; + if (context.TryGetCurrentSetting("ieee_floating_point_ops", ieee_floating_point_ops) && + !ieee_floating_point_ops.IsNull() && !BooleanValue::Get(ieee_floating_point_ops)) { + return false; + } + Value error_on_division_by_zero; + if (context.TryGetCurrentSetting("error_on_division_by_zero", error_on_division_by_zero) && + !error_on_division_by_zero.IsNull() && !BooleanValue::Get(error_on_division_by_zero)) { + return false; + } + return true; +} + +string PostgresSQLWriter::WriteDivision(const ParsedExpression &left, const ParsedExpression &right) { + if (!HasDefaultDivisionSemantics()) { + throw PostgresPushdownNotSupported("non-default division semantics"); + } + auto x = PostgresFloat8(WriteExpression(left)); + auto y = WriteExpression(right); + Value value; + if (PostgresCapabilities::TryGetConstantValue(right, value)) { + auto double_value = value.DefaultTryCastAs(LogicalType::DOUBLE); + if (double_value && !double_value->IsNull() && DoubleValue::Get(*double_value) != 0) { + // division by a non-zero constant + return "(" + x + " / " + y + ")"; + } + } + y = PostgresFloat8(y); + // note: Postgres considers NaN equal to itself and larger than any other value + return "(CASE WHEN " + y + " = 0 THEN (CASE WHEN " + x + " = 'NaN' OR " + x + + " = 0 THEN CAST('NaN' AS DOUBLE PRECISION) WHEN " + x + + " > 0 THEN CAST('Infinity' AS DOUBLE PRECISION) WHEN " + x + + " < 0 THEN CAST('-Infinity' AS DOUBLE PRECISION) END) ELSE " + x + " / " + y + " END)"; +} + +bool PostgresSQLWriter::TryWriteSpecialFunction(const string &name, const vector &children, + string &result) { + auto arg = [&](idx_t i) -> const ParsedExpression & { + return children[i].GetExpression(); + }; + auto require_types = [&](std::function check) { + for (auto &child : children) { + if (!check(InferType(child.GetExpression()))) { + throw PostgresPushdownNotSupported("unsupported argument type for " + name); + } + } + }; + auto is_string = [](const LogicalType &type) { + return type.id() == LogicalTypeId::VARCHAR || type.id() == LogicalTypeId::SQLNULL; + }; + auto is_number = [](const LogicalType &type) { + return PostgresIsNumericType(type) || type.id() == LogicalTypeId::SQLNULL; + }; + auto is_integer = [](const LogicalType &type) { + return PostgresIsIntegerType(type) || type.id() == LogicalTypeId::SQLNULL; + }; + auto literal_string = [&](idx_t i) { + Value value; + if (!PostgresCapabilities::TryGetConstantValue(arg(i), value) || value.IsNull()) { + throw InternalException("PostgresSQLWriter: expected a string literal"); + } + return StringValue::Get(value.DefaultCastAs(LogicalType::VARCHAR)); + }; + if (name == "lower" || name == "lcase") { + require_types(is_string); + result = "lower(" + WriteExpression(arg(0)) + ")"; + return true; + } + if (name == "~~*" || name == "!~~*") { + require_types(is_string); + result = "(" + WriteExpression(arg(0)) + (name == "~~*" ? " ILIKE " : " NOT ILIKE ") + WriteExpression(arg(1)) + + " ESCAPE '')"; + return true; + } + if (name == "contains") { + // DuckDB's contains also checks list / map membership + require_types(is_string); + result = "(strpos(" + WriteExpression(arg(0)) + ", " + WriteExpression(arg(1)) + ") > 0)"; + return true; + } + if (name == "ends_with" || name == "suffix") { + require_types(is_string); + auto str = WriteExpression(arg(0)); + auto suffix = WriteExpression(arg(1)); + result = "(right(" + str + ", length(" + suffix + ")) = " + suffix + ")"; + return true; + } + if (name == "^@") { + require_types(is_string); + result = WriteFunctionCall("starts_with", children, false, string()); + return true; + } + if (name == "translate") { + require_types(is_string); + result = name + "(" + WriteArgumentList(children) + ")"; + return true; + } + if (name == "unicode") { + require_types(is_string); + // DuckDB returns -1 for an empty string, Postgres' ascii returns 0 + auto x = WriteExpression(arg(0)); + result = "(CASE WHEN " + x + " = '' THEN -1 ELSE ascii(" + x + ") END)"; + return true; + } + if (name == "trunc" && children.size() == 1) { + auto type = InferType(arg(0)); + if (PostgresIsIntegerType(type)) { + // DuckDB returns the integer itself - Postgres would compute in DOUBLE PRECISION + result = WriteExpression(arg(0)); + return true; + } + return false; + } + if (name == "chr") { + require_types(is_integer); + result = "chr(CAST(" + WriteExpression(arg(0)) + " AS INTEGER))"; + return true; + } + if (name == "sha256") { + auto type = InferType(arg(0)); + auto arg_str = WriteExpression(arg(0)); + if (type.id() == LogicalTypeId::VARCHAR) { + arg_str = "convert_to(" + arg_str + ", 'UTF8')"; + } else if (type.id() != LogicalTypeId::BLOB) { + throw PostgresPushdownNotSupported("unsupported sha256 argument"); + } + result = "encode(sha256(" + arg_str + "), 'hex')"; + return true; + } + if (name == "sqrt") { + require_types(is_number); + // Postgres raises an error where DuckDB yields NaN + auto x = PostgresFloat8(WriteExpression(arg(0))); + result = "(CASE WHEN " + x + " < 0 THEN CAST('NaN' AS DOUBLE PRECISION) ELSE sqrt(" + x + ") END)"; + return true; + } + if (name == "ln" || ((name == "log" || name == "log10") && children.size() == 1)) { + require_types(is_number); + // Postgres raises an error where DuckDB yields -inf / NaN + auto x = PostgresFloat8(WriteExpression(arg(0))); + result = "(CASE WHEN " + x + " = 0 THEN CAST('-Infinity' AS DOUBLE PRECISION) WHEN " + x + + " < 0 THEN CAST('NaN' AS DOUBLE PRECISION) ELSE " + (name == "ln" ? "ln(" : "log(") + x + ") END)"; + return true; + } + if (name == "log") { + // the two argument variant is only available for NUMERIC in Postgres + throw PostgresPushdownNotSupported("unsupported log"); + } + if (name == "exp") { + require_types(is_number); + // Postgres raises an error on underflow / overflow where DuckDB yields 0 / inf + auto x = PostgresFloat8(WriteExpression(arg(0))); + result = "(CASE WHEN " + x + " < -745.2 THEN CAST(0 AS DOUBLE PRECISION) WHEN " + x + " > 709.79 AND " + x + + " <> 'NaN' THEN CAST('Infinity' AS DOUBLE PRECISION) ELSE exp(" + x + ") END)"; + return true; + } + if (name == "pow" || name == "power" || name == "^" || name == "**") { + require_types(is_number); + auto x = PostgresFloat8(WriteExpression(arg(0))); + auto y = PostgresFloat8(WriteExpression(arg(1))); + // Postgres raises an error where DuckDB yields inf / NaN + // (C's pow(-inf, y) is inf for a positive and 0 for a negative non-integer y) + auto fractional = y + " <> trunc(" + y + ")"; + result = "(CASE WHEN " + x + " = 0 AND " + y + " < 0 THEN CAST('Infinity' AS DOUBLE PRECISION) WHEN " + x + + " = '-Infinity' AND " + fractional + " THEN (CASE WHEN " + y + + " > 0 THEN CAST('Infinity' AS DOUBLE PRECISION) ELSE CAST(0 AS DOUBLE PRECISION) END) WHEN " + x + + " < 0 AND " + fractional + " THEN CAST('NaN' AS DOUBLE PRECISION) ELSE power(" + x + ", " + y + + ") END)"; + return true; + } + if (name == "isnan" || name == "isinf" || name == "isfinite") { + auto type = InferType(arg(0)); + auto x = WriteExpression(arg(0)); + if (type.id() == LogicalTypeId::FLOAT || type.id() == LogicalTypeId::DOUBLE) { + // Postgres considers NaN equal to itself + auto is_nan = "(" + x + " = 'NaN')"; + auto is_inf = "(abs(" + x + ") = 'Infinity')"; + if (name == "isnan") { + result = is_nan; + } else if (name == "isinf") { + result = is_inf; + } else { + result = "(NOT (" + is_nan + " OR " + is_inf + "))"; + } + return true; + } + if (PostgresIsTimestampType(type) && name != "isnan") { + result = name == "isfinite" ? "isfinite(" + x + ")" : "(NOT isfinite(" + x + "))"; + return true; + } + throw PostgresPushdownNotSupported("unsupported argument for " + name); + } + if (name == "gcd" || name == "lcm" || name == "greatest_common_divisor" || name == "least_common_multiple") { + require_types(is_integer); + result = string(name == "gcd" || name == "greatest_common_divisor" ? "gcd(" : "lcm(") + + WriteArgumentList(children) + ")"; + return true; + } + if (name == "xor" || name == "&" || name == "|" || name == "~") { + require_types(is_integer); + if (children.size() == 1) { + result = "(~" + WriteExpression(arg(0)) + ")"; + } else { + auto op = name == "xor" ? string("#") : name; + result = "(" + WriteExpression(arg(0)) + " " + op + " " + WriteExpression(arg(1)) + ")"; + } + return true; + } + if (name == "round" || name == "round_even" || name == "roundbankers") { + result = WriteRound(children, name != "round"); + return true; + } + if (name == "date_part" || name == "datepart") { + auto part = PostgresNormalizeDatePart(literal_string(0)); + if (part.empty()) { + throw PostgresPushdownNotSupported("unsupported date part"); + } + result = WriteDatePart(part, arg(1)); + return true; + } + static const case_insensitive_map_t PART_FUNCTIONS = {{"year", "year"}, + {"month", "month"}, + {"day", "day"}, + {"dayofmonth", "day"}, + {"hour", "hour"}, + {"minute", "minute"}, + {"second", "second"}, + {"millisecond", "millisecond"}, + {"microsecond", "microsecond"}, + {"quarter", "quarter"}, + {"dayofweek", "dow"}, + {"isodow", "isodow"}, + {"dayofyear", "doy"}, + {"week", "week"}, + {"weekofyear", "week"}, + {"isoyear", "isoyear"}, + {"decade", "decade"}, + {"century", "century"}, + {"millennium", "millennium"}, + {"epoch", "epoch"}}; + auto part_entry = PART_FUNCTIONS.find(name); + if (part_entry != PART_FUNCTIONS.end()) { + result = WriteDatePart(part_entry->second, arg(0)); + return true; + } + if (name == "epoch_ms") { + auto type = InferType(arg(0)); + if (!PostgresIsTimestampType(type)) { + // epoch_ms(BIGINT) converts milliseconds to a timestamp + throw PostgresPushdownNotSupported("unsupported epoch_ms argument"); + } + result = "CAST(floor(EXTRACT(EPOCH FROM " + WriteExpression(arg(0)) + ") * 1000) AS BIGINT)"; + return true; + } + if (name == "date_diff" || name == "datediff") { + auto part = PostgresNormalizeDatePart(literal_string(0)); + result = WriteDateDiff(part, arg(1), arg(2)); + return true; + } + if (name == "date_trunc" || name == "datetrunc") { + auto part = PostgresNormalizeDatePart(literal_string(0)); + if (part == "millisecond" || part == "microsecond") { + part += "s"; + } + bool supported_part = false; + for (idx_t i = 0; PostgresDateTruncParts[i]; i++) { + if (part == PostgresDateTruncParts[i]) { + supported_part = true; + } + } + auto type = InferType(arg(1)); + auto x = WriteExpression(arg(1)); + if (!supported_part || !PostgresIsTimestampType(type)) { + throw PostgresPushdownNotSupported("unsupported date_trunc"); + } + if (type.id() == LogicalTypeId::DATE) { + // DuckDB truncates dates to a TIMESTAMP - Postgres would pick the TIMESTAMP WITH TIME ZONE overload + x = "CAST(" + x + " AS TIMESTAMP)"; + } + result = "date_trunc('" + part + "', " + x + ")"; + return true; + } + if (name == "last_day") { + auto type = InferType(arg(0)); + if (type.id() != LogicalTypeId::DATE && type.id() != LogicalTypeId::TIMESTAMP) { + throw PostgresPushdownNotSupported("unsupported last_day argument"); + } + result = "CAST(date_trunc('month', CAST(" + WriteExpression(arg(0)) + + " AS TIMESTAMP)) + INTERVAL '1 month' - INTERVAL '1 day' AS DATE)"; + return true; + } + if (name == "make_date") { + require_types(is_integer); + if (children.size() != 3) { + throw PostgresPushdownNotSupported("unsupported make_date"); + } + // DuckDB uses astronomical year numbering (0 is 1 BC), Postgres uses negative years for BC years (-1 is 1 BC) + auto year = "CAST(" + WriteExpression(arg(0)) + " AS INTEGER)"; + result = "make_date((CASE WHEN " + year + " <= 0 THEN " + year + " - 1 ELSE " + year + " END), CAST(" + + WriteExpression(arg(1)) + " AS INTEGER), CAST(" + WriteExpression(arg(2)) + " AS INTEGER))"; + return true; + } + if (name == "to_timestamp") { + require_types(is_number); + result = "to_timestamp(" + PostgresFloat8(WriteExpression(arg(0))) + ")"; + return true; + } + return false; +} + +//===--------------------------------------------------------------------===// +// Expressions +//===--------------------------------------------------------------------===// +string PostgresSQLWriter::WriteExpressionList(const vector> &expressions, + const string &separator) { + string result; + for (auto &expr : expressions) { + if (!result.empty()) { + result += separator; + } + result += WriteExpression(*expr); + } + return result; +} + +string PostgresSQLWriter::WriteColumnRef(const ColumnRefExpression &column_ref) { + auto &names = column_ref.ColumnNames(); + if (names.size() == 1 && catalog_columns.count(names[0].GetIdentifierName()) == 0) { + // SQL value functions without parentheses are parsed as column references - they are evaluated by Postgres + // (at the start of the Postgres transaction, in the DuckDB time zone) + static const case_insensitive_map_t VALUE_FUNCTIONS = {{"current_date", "CURRENT_DATE"}, + {"current_timestamp", "CURRENT_TIMESTAMP"}, + {"localtimestamp", "LOCALTIMESTAMP"}}; + auto entry = VALUE_FUNCTIONS.find(names[0].GetIdentifierName()); + if (entry != VALUE_FUNCTIONS.end()) { + requires_time_zone = true; + return entry->second; + } + } + if (names.size() == 2) { + auto entry = table_function_aliases.find(names[0].GetIdentifierName()); + if (entry != table_function_aliases.end() && + StringUtil::CIEquals(entry->second, names[1].GetIdentifierName())) { + return WriteIdentifier(names[0]) + "." + WriteIdentifier(names[0]); + } + } + if (names.size() == 1 && catalog_columns.count(names[0].GetIdentifierName()) == 0) { + for (auto &entry : table_function_aliases) { + if (StringUtil::CIEquals(entry.second, names[0].GetIdentifierName())) { + return WriteIdentifier(entry.first); + } + } + } + if (names.size() == 1 && on_conflict_target && query_depth == on_conflict_depth && + on_conflict_target->ColumnExists(names[0])) { + // unqualified columns in ON CONFLICT DO UPDATE refer to the existing row in DuckDB - in Postgres they could + // also refer to the excluded row + auto &column = on_conflict_target->GetColumn(names[0]); + auto result = on_conflict_binding + "." + WriteColumnName(*on_conflict_target, column); + auto read_type = PostgresColumnReadType(*on_conflict_target, column); + return read_type.empty() ? result : "CAST(" + result + " AS " + read_type + ")"; + } + // figure out how many leading names qualify the column (schema.table.column / table.column) - any + // remaining names are struct (composite type) field accesses + idx_t column_index = 0; + if (names.size() >= 2 && table_bindings.count(names[0].GetIdentifierName()) > 0) { + column_index = 1; + } else if (names.size() >= 3 && table_bindings.count(names[1].GetIdentifierName()) > 0) { + column_index = 2; + } + optional_ptr binding_table; + if (column_index > 0) { + binding_table = FindBindingTable(names[column_index - 1]); + if (binding_table && !binding_table->ColumnExists(names[column_index])) { + binding_table = nullptr; + } + } else if (ambiguous_identifiers.count(names[0].GetIdentifierName()) > 0) { + // the column has different spellings in different tables - which one is referenced is not known + throw PostgresPushdownNotSupported("ambiguous column name " + names[0].GetIdentifierName()); + } + string result; + for (idx_t i = 0; i <= column_index; i++) { + if (i > 0) { + result += "."; + } + if (i == column_index && binding_table) { + result += WriteColumnName(*binding_table, binding_table->GetColumn(names[i])); + } else { + result += WriteIdentifier(names[i]); + } + } + for (idx_t i = column_index + 1; i < names.size(); i++) { + // Postgres requires parentheses for field accesses + result = "(" + result + ")." + WriteIdentifier(names[i]); + } + if (column_index + 1 == names.size()) { + string read_type; + auto &column_name = names[column_index]; + if (column_index > 0) { + if (binding_table) { + read_type = PostgresColumnReadType(*binding_table, binding_table->GetColumn(column_name)); + } + } else { + auto entry = column_read_types.find(column_name.GetIdentifierName()); + if (entry != column_read_types.end()) { + read_type = entry->second; + } + } + if (read_type == "?") { + throw PostgresPushdownNotSupported("ambiguous column reference " + column_name.GetIdentifierName()); + } + if (!read_type.empty()) { + result = "CAST(" + result + " AS " + read_type + ")"; + } + } + return result; +} + +string PostgresSQLWriter::WriteArgumentList(const vector &arguments) { + string result; + for (auto &argument : arguments) { + if (!result.empty()) { + result += ", "; + } + result += WriteExpression(argument.GetExpression()); + } + return result; +} + +static bool PostgresIsAggregateName(const string &name) { + static const case_insensitive_set_t REWRITTEN_AGGREGATES = {"count_if", "countif", "median", "quantile", + "quantile_cont", "quantile_disc", "arg_min", "arg_max", + "min_by", "max_by"}; + return REWRITTEN_AGGREGATES.count(name) > 0; +} + +//! Whether Postgres arrays of this type are read back as the corresponding DuckDB list +static bool PostgresSupportsArrayElement(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::VARCHAR: + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::UUID: + return true; + default: + return false; + } +} + +//! How values of a type are ordered with respect to strings +enum class PostgresStringOrdering : uint8_t { + //! no strings are compared + NONE, + //! strings (or lists of strings) - the collation can be pinned + COLLATABLE, + //! strings nested in other types (or an unknown type) - the collation cannot be pinned + NOT_COLLATABLE +}; + +static PostgresStringOrdering PostgresGetStringOrdering(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::VARCHAR: + return PostgresStringOrdering::COLLATABLE; + case LogicalTypeId::LIST: + // arrays of text are collatable + return PostgresGetStringOrdering(ListType::GetChildType(type)); + case LogicalTypeId::ARRAY: + return PostgresGetStringOrdering(ArrayType::GetChildType(type)); + case LogicalTypeId::INVALID: + case LogicalTypeId::UNKNOWN: + case LogicalTypeId::ANY: + return PostgresStringOrdering::NOT_COLLATABLE; + default: + if (TypeVisitor::Contains(type, LogicalTypeId::VARCHAR)) { + return PostgresStringOrdering::NOT_COLLATABLE; + } + return PostgresStringOrdering::NONE; + } +} + +string PostgresSQLWriter::WriteFunctionCall(const string &function_name, const vector &children, + bool distinct, const string &order_by) { + auto name = StringUtil::Lower(function_name); + static const case_insensitive_map_t> INTEGER_ARGUMENTS = { + {"left", {1}}, {"right", {1}}, {"lpad", {1}}, {"rpad", {1}}, + {"repeat", {1}}, {"substring", {1, 2}}, {"substr", {1, 2}}}; + auto integer_arguments = INTEGER_ARGUMENTS.find(name); + if (integer_arguments != INTEGER_ARGUMENTS.end()) { + // these functions only accept INTEGER arguments in Postgres + if ((name == "lpad" || name == "rpad") && children.size() == 3) { + Value fill; + if (!PostgresCapabilities::TryGetConstantValue(children[2].GetExpression(), fill) || fill.IsNull() || + StringValue::Get(fill.DefaultCastAs(LogicalType::VARCHAR)).empty()) { + // DuckDB raises an error for an empty fill string, Postgres returns the string + throw PostgresPushdownNotSupported("unsupported padding"); + } + } + vector args; + for (idx_t i = 0; i < children.size(); i++) { + auto &child = children[i].GetExpression(); + auto arg = WriteExpression(child); + bool is_integer_argument = false; + for (auto index : integer_arguments->second) { + is_integer_argument = is_integer_argument || index == i; + } + if (is_integer_argument) { + Value value; + if (PostgresCapabilities::TryGetConstantValue(child, value)) { + auto integer_value = value.DefaultTryCastAs(LogicalType::INTEGER); + if (!integer_value) { + throw PostgresPushdownNotSupported("argument out of range"); + } + arg = WriteConstant(*integer_value); + } else { + auto type = InferType(child); + if (!PostgresIsIntegerType(type)) { + throw PostgresPushdownNotSupported("unsupported argument type for " + name); + } + if (type.id() != LogicalTypeId::INTEGER) { + arg = "CAST(" + arg + " AS INTEGER)"; + } + } + } + args.push_back(std::move(arg)); + } + if (name == "substring") { + name = "substr"; + } + return name + "(" + StringUtil::Join(args, ", ") + ")"; + } + if ((name == "count_star" || name == "count") && children.empty()) { + return "count(*)"; + } + if (name == "string_agg" || name == "group_concat" || name == "listagg") { + if (InferType(children[0].GetExpression()).id() != LogicalTypeId::VARCHAR) { + throw PostgresPushdownNotSupported("unsupported string_agg argument"); + } + string result = + "string_agg(" + string(distinct ? "DISTINCT " : "") + WriteExpression(children[0].GetExpression()) + ", "; + // DuckDB's default separator is a comma + result += children.size() > 1 ? WriteExpression(children[1].GetExpression()) : "','"; + return result + order_by + ")"; + } + if (name == "array_agg" || name == "list") { + if (!PostgresSupportsArrayElement(InferType(children[0].GetExpression()))) { + throw PostgresPushdownNotSupported("unsupported array_agg argument"); + } + name = "array_agg"; + } + static const case_insensitive_set_t TYPE_CHECKED_AGGREGATES = { + "min", "max", "sum", "avg", "stddev", "variance", "stddev_pop", + "stddev_samp", "var_pop", "var_samp", "bool_and", "bool_or", "corr", "covar_pop", + "covar_samp", "regr_avgx", "regr_avgy", "regr_count", "regr_intercept", "regr_r2", "regr_slope", + "regr_sxx", "regr_sxy", "regr_syy"}; + if (TYPE_CHECKED_AGGREGATES.count(name) > 0 && !children.empty()) { + // Postgres does not provide all aggregates for all types (e.g. min(BOOLEAN), avg(TIMESTAMP)) + for (auto &child : children) { + auto type = InferType(child.GetExpression()); + bool supported; + if (name == "min" || name == "max") { + switch (type.id()) { + case LogicalTypeId::VARCHAR: + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::INTERVAL: + case LogicalTypeId::ENUM: + supported = true; + break; + case LogicalTypeId::BOOLEAN: + // DuckDB orders false before true + name = name == "min" ? "bool_and" : "bool_or"; + supported = true; + break; + default: + supported = PostgresIsNumericType(type); + break; + } + } else if (name == "sum" || name == "avg") { + supported = PostgresIsNumericType(type) || type.id() == LogicalTypeId::INTERVAL; + } else if (name == "bool_and" || name == "bool_or") { + supported = type.id() == LogicalTypeId::BOOLEAN; + } else { + supported = PostgresIsNumericType(type); + } + if (!supported) { + throw PostgresPushdownNotSupported("unsupported argument type for " + name); + } + } + if (name == "sum" && children.size() == 1 && + InferType(children[0].GetExpression()).id() == LogicalTypeId::FLOAT) { + // Postgres sums REAL values in single precision, DuckDB in double precision + return "sum(" + string(distinct ? "DISTINCT " : "") + "CAST(" + + WriteExpression(children[0].GetExpression()) + " AS DOUBLE PRECISION)" + order_by + ")"; + } + } + if (name == "bit_and" || name == "bit_or" || name == "bit_xor") { + auto type = InferType(children[0].GetExpression()); + if (type.id() != LogicalTypeId::SMALLINT && type.id() != LogicalTypeId::INTEGER && + type.id() != LogicalTypeId::BIGINT) { + throw PostgresPushdownNotSupported("unsupported bitwise aggregate argument"); + } + } + if ((name == "length" || name == "len" || name == "char_length" || name == "character_length") && + children.size() == 1) { + return WriteLength("length", children[0].GetExpression()); + } + if ((name == "strlen" || name == "octet_length") && children.size() == 1) { + auto type = InferType(children[0].GetExpression()); + if (type.id() != LogicalTypeId::VARCHAR && type.id() != LogicalTypeId::BLOB) { + throw PostgresPushdownNotSupported("unsupported octet_length argument"); + } + return "octet_length(" + WriteExpression(children[0].GetExpression()) + ")"; + } + if ((name == "concat" || name == "concat_ws") && !children.empty()) { + for (auto &child : children) { + if (!PostgresHasIdenticalTextFormat(InferType(child.GetExpression()))) { + // non-string arguments are formatted as text, which differs for e.g. doubles + throw PostgresPushdownNotSupported("unsupported concat argument"); + } + } + } + if ((name == "starts_with" || name == "prefix") && children.size() == 2) { + auto version = catalog.GetPostgresVersion(); + if (version < PostgresVersion(11, 0)) { + return "(strpos(" + WriteArgumentList(children) + ") = 1)"; + } + name = "starts_with"; + } else if (name == "trim" && children.size() == 2) { + // DuckDB's trim(string, characters) is btrim in Postgres + name = "btrim"; + } else if (name == "grouping_id") { + name = "grouping"; + } else if (name == "rank_dense") { + name = "dense_rank"; + } else if (name == "instr" || name == "position") { + name = "strpos"; + } else if (name == "ifnull") { + name = "coalesce"; + } else if (name == "substring") { + name = "substr"; + } + string result = name + "("; + if (distinct) { + result += "DISTINCT "; + } + if ((name == "min" || name == "max" || name == "greatest" || name == "least") && !catalog.HasBinaryCollation()) { + // the result depends on the string ordering + for (idx_t i = 0; i < children.size(); i++) { + result += (i > 0 ? ", " : "") + WriteOrderedExpression(children[i].GetExpression()); + } + } else { + result += WriteArgumentList(children); + } + result += order_by; + result += ")"; + return result; +} + +string PostgresSQLWriter::WriteAggregate(const FunctionExpression &func, const string &name) { + auto &children = func.GetArguments(); + vector filters; + if (func.Filter()) { + filters.push_back(WriteExpression(*func.Filter())); + } + string order_by; + if (func.OrderBy() && !func.OrderBy()->orders.empty()) { + if (func.Distinct() && !catalog.HasBinaryCollation()) { + // the ORDER BY expressions of a DISTINCT aggregate must appear in its arguments - the collation of + // string keys cannot be pinned + for (auto &order : func.OrderBy()->orders) { + if (PostgresGetStringOrdering(InferType(*order.expression)) != PostgresStringOrdering::NONE) { + throw PostgresPushdownNotSupported("the string ordering of the database differs from DuckDB's"); + } + } + } + order_by = " ORDER BY " + WriteOrderList(func.OrderBy()->orders); + } + string result; + if (name == "count_if" || name == "countif") { + result = "count(CASE WHEN " + WriteExpression(children[0].GetExpression()) + " THEN 1 END)"; + } else if (name == "median" || name == "quantile" || name == "quantile_cont" || name == "quantile_disc") { + auto &arg = children[0].GetExpression(); + auto type = InferType(arg); + // DuckDB interpolates decimals (with truncation) and temporal types, while Postgres only interpolates + // doubles - and DuckDB's median is only discrete for types that cannot be interpolated + bool interpolated_as_double = PostgresIsNumericType(type) && type.id() != LogicalTypeId::DECIMAL; + bool continuous = name == "quantile_cont" || name == "median"; + if (continuous && !interpolated_as_double && !(name == "median" && type.id() == LogicalTypeId::VARCHAR)) { + throw PostgresPushdownNotSupported("unsupported quantile argument"); + } + if (type.id() == LogicalTypeId::INVALID) { + throw PostgresPushdownNotSupported("unsupported quantile argument"); + } + continuous = continuous && interpolated_as_double; + auto quantile = name == "median" ? string("0.5") : WriteExpression(children[1].GetExpression()); + result = string(continuous ? "percentile_cont(" : "percentile_disc(") + quantile + ") WITHIN GROUP (ORDER BY " + + WriteOrderedOperand(WriteExpression(arg), type) + ")"; + } else if (name == "arg_min" || name == "arg_max" || name == "min_by" || name == "max_by") { + // the argument of the row with the minimum / maximum value - rows where either is NULL are ignored + auto &arg = children[0].GetExpression(); + auto &val = children[1].GetExpression(); + if (!PostgresSupportsArrayElement(InferType(arg))) { + throw PostgresPushdownNotSupported("unsupported arg_min / arg_max argument"); + } + auto arg_str = WriteExpression(arg); + auto val_str = WriteExpression(val); + bool is_max = name == "arg_max" || name == "max_by"; + filters.push_back("(" + arg_str + " IS NOT NULL)"); + filters.push_back("(" + val_str + " IS NOT NULL)"); + result = "array_agg(" + arg_str + " ORDER BY " + WriteOrderedOperand(val_str, InferType(val)) + + (is_max ? " DESC" : " ASC") + ")"; + result += " FILTER (WHERE " + StringUtil::Join(filters, " AND ") + ")"; + return "(" + result + ")[1]"; + } else { + result = WriteFunctionCall(name, children, func.Distinct(), order_by); + } + if (!filters.empty()) { + result += " FILTER (WHERE " + StringUtil::Join(filters, " AND ") + ")"; + } + return result; +} + +string PostgresSQLWriter::WriteFunction(const FunctionExpression &func) { + auto &children = func.GetArguments(); + auto name = StringUtil::Lower(func.FunctionName().GetIdentifierName()); + if (func.Filter() || (func.OrderBy() && !func.OrderBy()->orders.empty()) || PostgresIsAggregateName(name)) { + return WriteAggregate(func, name); + } + string special_result; + if (TryWriteSpecialFunction(name, children, special_result)) { + return special_result; + } + if (name == "~~" || name == "!~~") { + // [NOT] LIKE - DuckDB's LIKE has no default escape character, while Postgres' default + // escape character is backslash; disable it with an empty ESCAPE clause + D_ASSERT(children.size() == 2); + return "(" + WriteStringOperand(children[0].GetExpression()) + (name == "~~" ? " LIKE " : " NOT LIKE ") + + WriteStringOperand(children[1].GetExpression()) + " ESCAPE '')"; + } + static const case_insensitive_set_t OPERATOR_NAMES = {"+", "-", "*", "/", "%", "||"}; + if (func.IsOperator() || OPERATOR_NAMES.count(name) > 0) { + if (children.size() == 1) { + return "(" + name + WriteExpression(children[0].GetExpression()) + ")"; + } + if (children.size() == 2) { + auto &left_expr = children[0].GetExpression(); + auto &right_expr = children[1].GetExpression(); + if (name == "/" || name == "%" || name == "||") { + auto left_type = InferType(left_expr); + auto right_type = InferType(right_expr); + if (name == "/") { + if (!PostgresIsNumericType(left_type) || !PostgresIsNumericType(right_type)) { + throw PostgresPushdownNotSupported("unsupported division"); + } + return WriteDivision(left_expr, right_expr); + } + if (name == "%") { + if (!HasDefaultDivisionSemantics()) { + throw PostgresPushdownNotSupported("non-default division semantics"); + } + // Postgres has no modulo for floating point numbers + auto is_exact = [](const LogicalType &type) { + return PostgresIsIntegerType(type) || type.id() == LogicalTypeId::DECIMAL; + }; + if (!is_exact(left_type) || !is_exact(right_type)) { + throw PostgresPushdownNotSupported("unsupported modulo"); + } + } + if (name == "||") { + // non-string operands are formatted as text, which differs for e.g. doubles - and DuckDB's + // || on lists concatenates the lists + if (!PostgresHasIdenticalTextFormat(left_type) || !PostgresHasIdenticalTextFormat(right_type) || + (left_type.id() != LogicalTypeId::VARCHAR && right_type.id() != LogicalTypeId::VARCHAR)) { + throw PostgresPushdownNotSupported("unsupported concatenation"); + } + } + } + return "(" + WriteExpression(left_expr) + " " + name + " " + WriteExpression(right_expr) + ")"; + } + throw InternalException("PostgresSQLWriter: unsupported operator %s - should have been blocked by " + "SupportsPushdown", + name); + } + return WriteFunctionCall(func.FunctionName().GetIdentifierName(), children, func.Distinct(), string()); +} + +static string PostgresWriteWindowBoundary(WindowBoundary boundary, const string &boundary_expr) { + switch (boundary) { + case WindowBoundary::UNBOUNDED_PRECEDING: + return "UNBOUNDED PRECEDING"; + case WindowBoundary::UNBOUNDED_FOLLOWING: + return "UNBOUNDED FOLLOWING"; + case WindowBoundary::CURRENT_ROW_ROWS: + case WindowBoundary::CURRENT_ROW_RANGE: + case WindowBoundary::CURRENT_ROW_GROUPS: + return "CURRENT ROW"; + case WindowBoundary::EXPR_PRECEDING_ROWS: + case WindowBoundary::EXPR_PRECEDING_RANGE: + case WindowBoundary::EXPR_PRECEDING_GROUPS: + return boundary_expr + " PRECEDING"; + case WindowBoundary::EXPR_FOLLOWING_ROWS: + case WindowBoundary::EXPR_FOLLOWING_RANGE: + case WindowBoundary::EXPR_FOLLOWING_GROUPS: + return boundary_expr + " FOLLOWING"; + default: + throw InternalException("PostgresSQLWriter: unsupported window boundary - should have been blocked by " + "SupportsPushdown"); + } +} + +static bool PostgresIsRowsBoundary(WindowBoundary boundary) { + return boundary == WindowBoundary::CURRENT_ROW_ROWS || boundary == WindowBoundary::EXPR_PRECEDING_ROWS || + boundary == WindowBoundary::EXPR_FOLLOWING_ROWS; +} + +static bool PostgresIsGroupsBoundary(WindowBoundary boundary) { + return boundary == WindowBoundary::CURRENT_ROW_GROUPS || boundary == WindowBoundary::EXPR_PRECEDING_GROUPS || + boundary == WindowBoundary::EXPR_FOLLOWING_GROUPS; +} + +//! Postgres requires RANGE offsets to have a type that matches the ORDER BY key (e.g. an integer offset for an integer +//! key, an interval for a date) - other offsets are cast to the type of the key like DuckDB does +string PostgresSQLWriter::WriteFrameOffset(const WindowExpression &window, const ParsedExpression &offset, + const string &mode) { + if (mode != "RANGE") { + return WriteExpression(offset); + } + if (window.OrderBy().size() != 1) { + throw PostgresPushdownNotSupported("RANGE frame with an offset requires a single ORDER BY key"); + } + auto key_type = InferType(*window.OrderBy()[0].expression); + Value value; + interval_t interval; + if (PostgresCapabilities::TryGetIntervalLiteral(offset, interval)) { + value = Value::INTERVAL(interval); + } else if (!PostgresCapabilities::TryGetConstantValue(offset, value)) { + throw PostgresPushdownNotSupported("non-constant RANGE frame offset"); + } + if (value.type().id() == LogicalTypeId::INTERVAL) { + if (!PostgresIsDateTimeType(key_type) && key_type.id() != LogicalTypeId::INTERVAL) { + throw PostgresPushdownNotSupported("unsupported RANGE frame offset"); + } + return WriteConstant(value); + } + if (!PostgresIsNumericType(key_type)) { + throw PostgresPushdownNotSupported("unsupported RANGE frame offset"); + } + auto cast_value = value.TryCastAs(context, key_type); + if (!cast_value || !PostgresCapabilities::SupportsValue(*cast_value)) { + throw PostgresPushdownNotSupported("unsupported RANGE frame offset"); + } + return WriteConstant(*cast_value); +} + +string PostgresSQLWriter::WriteWindow(const WindowExpression &window) { + string result = WriteFunctionCall(window.FunctionName().GetIdentifierName(), window.GetArguments(), + window.Distinct(), string()); + if (window.Filter()) { + result += " FILTER (WHERE " + WriteExpression(*window.Filter()) + ")"; + } + result += " OVER ("; + string sep; + if (!window.Partitions().empty()) { + result += "PARTITION BY " + WriteExpressionList(window.Partitions()); + sep = " "; + } + if (!window.OrderBy().empty()) { + result += sep + "ORDER BY " + WriteOrderList(window.OrderBy()); + sep = " "; + } + auto window_start = window.WindowStart(); + auto window_end = window.WindowEnd(); + if (!(window_start == WindowBoundary::UNBOUNDED_PRECEDING && window_end == WindowBoundary::CURRENT_ROW_RANGE && + window.WindowExclude() == WindowExcludeMode::NO_OTHER)) { + // non-default frame + string mode = "RANGE"; + if (PostgresIsRowsBoundary(window_start) || PostgresIsRowsBoundary(window_end)) { + mode = "ROWS"; + } else if (PostgresIsGroupsBoundary(window_start) || PostgresIsGroupsBoundary(window_end)) { + mode = "GROUPS"; + } + string start_expr = window.StartExpr() ? WriteFrameOffset(window, *window.StartExpr(), mode) : string(); + string end_expr = window.EndExpr() ? WriteFrameOffset(window, *window.EndExpr(), mode) : string(); + result += sep + mode; + result += " BETWEEN " + PostgresWriteWindowBoundary(window_start, start_expr); + result += " AND " + PostgresWriteWindowBoundary(window_end, end_expr); + switch (window.WindowExclude()) { + case WindowExcludeMode::NO_OTHER: + break; + case WindowExcludeMode::CURRENT_ROW: + result += " EXCLUDE CURRENT ROW"; + break; + case WindowExcludeMode::GROUP: + result += " EXCLUDE GROUP"; + break; + case WindowExcludeMode::TIES: + result += " EXCLUDE TIES"; + break; + } + } + result += ")"; + return result; +} + +string PostgresSQLWriter::WriteOperator(const OperatorExpression &op) { + auto &children = op.GetChildren(); + switch (op.GetExpressionType()) { + case ExpressionType::OPERATOR_IS_NULL: + return "(" + WriteExpression(*children[0]) + " IS NULL)"; + case ExpressionType::OPERATOR_IS_NOT_NULL: + return "(" + WriteExpression(*children[0]) + " IS NOT NULL)"; + case ExpressionType::OPERATOR_NOT: + return "(NOT " + WriteExpressionList(children) + ")"; + case ExpressionType::OPERATOR_COALESCE: + return "COALESCE(" + WriteExpressionList(children) + ")"; + case ExpressionType::OPERATOR_NULLIF: + return "NULLIF(" + WriteExpressionList(children) + ")"; + case ExpressionType::GROUPING_FUNCTION: + return "GROUPING(" + WriteExpressionList(children) + ")"; + case ExpressionType::COMPARE_IN: + case ExpressionType::COMPARE_NOT_IN: { + vector> operands; + for (auto &child : children) { + operands.push_back(*child); + } + auto written = WriteComparisonOperands(operands); + string result = "(" + written[0]; + result += op.GetExpressionType() == ExpressionType::COMPARE_IN ? " IN (" : " NOT IN ("; + for (idx_t i = 1; i < written.size(); i++) { + if (i > 1) { + result += ", "; + } + result += written[i]; + } + result += "))"; + return result; + } + default: + throw InternalException("PostgresSQLWriter: unsupported operator type - should have been blocked by " + "SupportsPushdown"); + } +} + +string PostgresSQLWriter::WriteCase(const CaseExpression &case_expr) { + string result = "CASE"; + for (auto &check : case_expr.CaseChecks()) { + result += " WHEN " + WriteExpression(*check.when_expr); + result += " THEN " + WriteExpression(*check.then_expr); + } + result += " ELSE " + WriteExpression(case_expr.Else()); + result += " END"; + return result; +} + +static string PostgresComparisonOperator(ExpressionType type) { + switch (type) { + case ExpressionType::COMPARE_EQUAL: + return "="; + case ExpressionType::COMPARE_NOTEQUAL: + return "<>"; + case ExpressionType::COMPARE_LESSTHAN: + return "<"; + case ExpressionType::COMPARE_GREATERTHAN: + return ">"; + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + return "<="; + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + return ">="; + case ExpressionType::COMPARE_DISTINCT_FROM: + return "IS DISTINCT FROM"; + case ExpressionType::COMPARE_NOT_DISTINCT_FROM: + return "IS NOT DISTINCT FROM"; + default: + throw InternalException("PostgresSQLWriter: unsupported comparison type - should have been blocked by " + "SupportsPushdown"); + } +} + +//! Write the operands of a comparison (or of IN / BETWEEN) - constants that are compared with an expression of a +//! different type are cast to the type DuckDB compares them as (with DuckDB's cast semantics) +vector PostgresSQLWriter::WriteComparisonOperands(const vector> &operands, + bool ordering) { + // ordered strings require the type the operands are compared as + bool pin_collation = ordering && !catalog.HasBinaryCollation(); + vector values(operands.size()); + vector is_constant(operands.size()); + bool any_constant = false; + bool any_expression = false; + for (idx_t i = 0; i < operands.size(); i++) { + interval_t interval; + if (PostgresCapabilities::TryGetIntervalLiteral(operands[i].get(), interval)) { + values[i] = Value::INTERVAL(interval); + is_constant[i] = true; + } else { + is_constant[i] = PostgresCapabilities::TryGetConstantValue(operands[i].get(), values[i]); + } + any_constant = any_constant || is_constant[i]; + any_expression = any_expression || !is_constant[i]; + } + vector result; + if ((!any_constant || !any_expression) && !pin_collation) { + for (auto &operand : operands) { + result.push_back(WriteExpression(operand.get())); + } + return result; + } + vector types; + for (idx_t i = 0; i < operands.size(); i++) { + types.push_back(is_constant[i] ? values[i].type() : InferType(operands[i].get())); + if (types.back().id() == LogicalTypeId::INVALID) { + throw PostgresPushdownNotSupported("cannot determine the type of a comparison operand"); + } + } + // the type DuckDB compares the operands as + LogicalType target = types[0]; + for (idx_t i = 1; i < types.size(); i++) { + if (!BoundComparisonExpression::TryBindComparison(context, target, types[i], target, + ExpressionType::COMPARE_EQUAL)) { + throw PostgresPushdownNotSupported("operands cannot be compared"); + } + } + for (idx_t i = 0; i < operands.size(); i++) { + if (is_constant[i]) { + if (values[i].type() == target || values[i].IsNull()) { + result.push_back(WriteExpression(operands[i].get())); + continue; + } + auto cast_value = values[i].TryCastAs(context, target); + if (!cast_value || !PostgresCapabilities::SupportsValue(*cast_value)) { + // DuckDB raises an error + throw PostgresPushdownNotSupported("constant cannot be converted to the compared type"); + } + result.push_back(WriteConstant(*cast_value)); + } else { + auto expr_str = WriteExpression(operands[i].get()); + result.push_back(types[i] == target ? expr_str : WriteCast(expr_str, types[i], target)); + } + } + if (pin_collation) { + // the collation of one operand applies to the comparison + result[0] = WriteOrderedOperand(result[0], target); + } + return result; +} + +//! Write an operand of a string operation (e.g. LIKE) - DuckDB converts other types to VARCHAR +string PostgresSQLWriter::WriteStringOperand(const ParsedExpression &expr) { + auto type = InferType(expr); + auto expr_str = WriteExpression(expr); + if (type.id() == LogicalTypeId::VARCHAR || type.id() == LogicalTypeId::SQLNULL) { + return expr_str; + } + if (type.id() == LogicalTypeId::INVALID) { + throw PostgresPushdownNotSupported("cannot determine the type of a string operand"); + } + return WriteCast(expr_str, type, LogicalType::VARCHAR); +} + +string PostgresSQLWriter::WriteOrderedOperand(const string &expr_str, const LogicalType &type) { + if (catalog.HasBinaryCollation()) { + // the database orders strings like DuckDB + return expr_str; + } + switch (PostgresGetStringOrdering(type)) { + case PostgresStringOrdering::NONE: + return expr_str; + case PostgresStringOrdering::COLLATABLE: { + // the cast makes the expression collatable - values that DuckDB reads as VARCHAR are not necessarily + // strings in Postgres (e.g. JSON or enum columns) + bool is_list = type.id() == LogicalTypeId::LIST || type.id() == LogicalTypeId::ARRAY; + return "CAST(" + expr_str + (is_list ? " AS TEXT[])" : " AS TEXT)") + " COLLATE \"C\""; + } + default: + throw PostgresPushdownNotSupported("the string ordering of the database differs from DuckDB's"); + } +} + +string PostgresSQLWriter::WriteOrderedExpression(const ParsedExpression &expr) { + auto expr_str = WriteExpression(expr); + if (catalog.HasBinaryCollation()) { + return expr_str; + } + return WriteOrderedOperand(expr_str, InferType(expr)); +} + +string PostgresSQLWriter::WriteSubquery(const SubqueryExpression &subquery) { + auto &node = *subquery.Subquery()->node; + switch (subquery.GetSubqueryType()) { + case SubqueryType::EXISTS: + return "EXISTS(" + WriteQueryNode(node) + ")"; + case SubqueryType::NOT_EXISTS: + return "NOT EXISTS(" + WriteQueryNode(node) + ")"; + case SubqueryType::SCALAR: + return "(" + WriteQueryNode(node) + ")"; + case SubqueryType::ANY: { + auto comparison_type = subquery.GetComparisonType(); + bool ordering = + comparison_type != ExpressionType::COMPARE_EQUAL && comparison_type != ExpressionType::COMPARE_NOTEQUAL; + auto child = ordering ? WriteOrderedExpression(*subquery.GetChild()) : WriteExpression(*subquery.GetChild()); + return "(" + child + " " + PostgresComparisonOperator(comparison_type) + " ANY(" + WriteQueryNode(node) + "))"; + } + default: + throw InternalException("PostgresSQLWriter: unsupported subquery type - should have been blocked by " + "SupportsPushdown"); + } +} + +string PostgresSQLWriter::WriteExpression(const ParsedExpression &expr) { + switch (expr.GetExpressionClass()) { + case ExpressionClass::CONSTANT: + return WriteConstant(expr.Cast().GetLiteral().ToValue()); + case ExpressionClass::COLUMN_REF: + return WriteColumnRef(expr.Cast()); + case ExpressionClass::STAR: { + auto &star = expr.Cast(); + if (!star.RelationName().empty()) { + // qualified star (e.g. t.*) + return WriteIdentifier(star.RelationName()) + ".*"; + } + return "*"; + } + case ExpressionClass::FUNCTION: { + interval_t interval; + if (PostgresCapabilities::TryGetIntervalLiteral(expr, interval)) { + return WriteConstant(Value::INTERVAL(interval)); + } + return WriteFunction(expr.Cast()); + } + case ExpressionClass::WINDOW: + return WriteWindow(expr.Cast()); + case ExpressionClass::COMPARISON: { + auto &comparison = expr.Cast(); + vector> operands {comparison.Left(), comparison.Right()}; + bool ordering = expr.GetExpressionType() != ExpressionType::COMPARE_EQUAL && + expr.GetExpressionType() != ExpressionType::COMPARE_NOTEQUAL && + expr.GetExpressionType() != ExpressionType::COMPARE_DISTINCT_FROM && + expr.GetExpressionType() != ExpressionType::COMPARE_NOT_DISTINCT_FROM; + auto written = WriteComparisonOperands(operands, ordering); + return "(" + written[0] + " " + PostgresComparisonOperator(expr.GetExpressionType()) + " " + written[1] + ")"; + } + case ExpressionClass::CONJUNCTION: { + auto &conjunction = expr.Cast(); + auto op = expr.GetExpressionType() == ExpressionType::CONJUNCTION_AND ? " AND " : " OR "; + return "(" + WriteExpressionList(conjunction.GetChildren(), op) + ")"; + } + case ExpressionClass::OPERATOR: + return WriteOperator(expr.Cast()); + case ExpressionClass::CASE: + return WriteCase(expr.Cast()); + case ExpressionClass::CAST: { + auto &cast_expr = expr.Cast(); + auto target_type = PostgresCapabilities::ResolveType(cast_expr.TargetType()); + if (cast_expr.Child().GetExpressionClass() == ExpressionClass::CONSTANT) { + // casts of literals are evaluated locally and serialized as the resulting value + auto value = cast_expr.Child().Cast().GetLiteral().ToValue(); + auto cast_result = value.DefaultTryCastAs(target_type); + if (!cast_result) { + throw InternalException("PostgresSQLWriter: cast of literal failed - should have been blocked by " + "SupportsPushdown"); + } + return WriteConstant(*cast_result); + } + return WriteCast(cast_expr.Child(), target_type); + } + case ExpressionClass::BETWEEN: { + auto &between = expr.Cast(); + vector> operands {between.Input(), between.LowerBound(), + between.UpperBound()}; + auto written = WriteComparisonOperands(operands, true); + return "(" + written[0] + " BETWEEN " + written[1] + " AND " + written[2] + ")"; + } + case ExpressionClass::SUBQUERY: + return WriteSubquery(expr.Cast()); + default: + throw InternalException("PostgresSQLWriter: unsupported expression class - should have been blocked by " + "SupportsPushdown"); + } +} + +//===--------------------------------------------------------------------===// +// Table references +//===--------------------------------------------------------------------===// +string PostgresSQLWriter::WriteTableAlias(const TableRef &ref) { + string result; + if (!ref.alias.empty()) { + result += " AS " + WriteIdentifier(ref.alias); + } + if (!ref.column_name_alias.empty()) { + result += "("; + for (idx_t i = 0; i < ref.column_name_alias.size(); i++) { + if (i > 0) { + result += ", "; + } + result += WriteIdentifier(ref.column_name_alias[i]); + } + result += ")"; + } + return result; +} + +string PostgresSQLWriter::WriteTableRef(const TableRef &ref, bool lateral) { + switch (ref.type) { + case TableReferenceType::BASE_TABLE: { + auto &base = ref.Cast(); + string result; + if (IsCTEReference(base)) { + result = WriteIdentifier(base.Table()); + } else { + auto table = LookupTable(base); + if (table) { + result = PostgresUtils::WriteIdentifier(table->ParentSchema().name.GetIdentifierName()) + "." + + PostgresUtils::WriteIdentifier(table->name.GetIdentifierName()); + } else { + if (!base.GetQualifiedName().Schema().empty()) { + result += WriteIdentifier(base.GetQualifiedName().Schema()) + "."; + } + result += WriteIdentifier(base.Table()); + } + } + auto alias = WriteTableAlias(ref); + if (alias.empty() && !ref.column_name_alias.empty()) { + // a column alias list requires a table alias + alias = " AS " + WriteIdentifier(base.Table()) + alias; + } + return result + alias; + } + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + if (join.type == JoinType::SEMI || join.type == JoinType::ANTI) { + // a SEMI / ANTI join filters the left side - which keeps the name it is bound to + auto &left = *join.left; + Identifier binding_name = left.alias.empty() && left.type == TableReferenceType::BASE_TABLE + ? left.Cast().Table() + : left.alias; + string result = "(SELECT * FROM " + WriteTableRef(left) + " WHERE "; + result += join.type == JoinType::SEMI ? "EXISTS" : "NOT EXISTS"; + result += " (SELECT 1 FROM " + WriteTableRef(*join.right, true) + " WHERE " + + WriteExpression(*join.condition) + ")) AS " + WriteIdentifier(binding_name); + if (!left.column_name_alias.empty()) { + throw PostgresPushdownNotSupported("SEMI / ANTI join with column aliases"); + } + return result + WriteTableAlias(ref); + } + string result = "(" + WriteTableRef(*join.left) + " "; + bool lateral_right = false; + if (join.ref_type == JoinRefType::CROSS) { + result += "CROSS JOIN "; + lateral_right = true; + } else if (join.ref_type == JoinRefType::REGULAR || join.ref_type == JoinRefType::NATURAL) { + if (join.ref_type == JoinRefType::NATURAL) { + result += "NATURAL "; + } + switch (join.type) { + case JoinType::INNER: + result += "INNER JOIN "; + lateral_right = true; + break; + case JoinType::LEFT: + result += "LEFT JOIN "; + lateral_right = true; + break; + case JoinType::RIGHT: + result += "RIGHT JOIN "; + break; + case JoinType::OUTER: + result += "FULL OUTER JOIN "; + break; + default: + throw InternalException("PostgresSQLWriter: unsupported join type - should have been blocked by " + "SupportsPushdown"); + } + } else { + throw InternalException("PostgresSQLWriter: unsupported join ref type - should have been blocked by " + "SupportsPushdown"); + } + // DuckDB detects lateral references automatically - Postgres requires the LATERAL keyword + result += WriteTableRef(*join.right, lateral_right); + if (join.condition) { + result += " ON " + WriteExpression(*join.condition); + } else if (!join.using_columns.empty()) { + result += " USING ("; + for (idx_t i = 0; i < join.using_columns.size(); i++) { + if (i > 0) { + result += ", "; + } + result += WriteIdentifier(join.using_columns[i]); + } + result += ")"; + } else if (join.ref_type == JoinRefType::REGULAR) { + // an INNER JOIN without a condition + result += " ON TRUE"; + } + result += ")"; + return result + WriteTableAlias(ref); + } + case TableReferenceType::SUBQUERY: { + auto &subquery = ref.Cast(); + string result = lateral ? "LATERAL (" : "("; + result += WriteQueryNode(*subquery.subquery->node) + ")"; + if (subquery.alias.empty()) { + // older Postgres versions require an alias for every subquery + string alias = "unnamed_subquery"; + if (unnamed_subquery_index > 0) { + alias += to_string(unnamed_subquery_index); + } + unnamed_subquery_index++; + result += " AS " + WriteIdentifier(alias); + } + return result + WriteTableAlias(ref); + } + case TableReferenceType::EXPRESSION_LIST: { + auto &expression_list = ref.Cast(); + string result = "(VALUES "; + for (idx_t row = 0; row < expression_list.values.size(); row++) { + if (row > 0) { + result += ", "; + } + result += "(" + WriteExpressionList(expression_list.values[row]) + ")"; + } + result += ")"; + // name the relation and its columns the way DuckDB does (Postgres names the columns column1, ...) + result += " AS " + WriteIdentifier(ref.alias.empty() ? Identifier("valueslist") : ref.alias) + "("; + auto column_count = expression_list.values[0].size(); + for (idx_t col = 0; col < column_count; col++) { + if (col > 0) { + result += ", "; + } + if (col < ref.column_name_alias.size()) { + result += WriteIdentifier(ref.column_name_alias[col]); + } else if (col < expression_list.expected_names.size()) { + result += WriteIdentifier(expression_list.expected_names[col]); + } else { + result += WriteIdentifier("col" + to_string(col)); + } + } + return result + ")"; + } + case TableReferenceType::TABLE_FUNCTION: { + auto &table_function = ref.Cast(); + auto &func = table_function.function->Cast(); + auto name = StringUtil::Lower(func.FunctionName().GetIdentifierName()); + auto &children = func.GetArguments(); + vector types; + for (auto &child : children) { + types.push_back(InferType(child.GetExpression())); + } + bool integers = true; + for (auto &type : types) { + integers = integers && PostgresIsIntegerType(type); + } + auto bigint = [&](idx_t i) { + return "CAST(" + WriteExpression(children[i].GetExpression()) + " AS BIGINT)"; + }; + string result = lateral ? "LATERAL " : ""; + if (name == "range" && integers) { + // DuckDB's range excludes the upper bound + string start = children.size() == 1 ? "CAST(0 AS BIGINT)" : bigint(0); + string end = children.size() == 1 ? bigint(0) : bigint(1); + int64_t step = 1; + if (children.size() == 3) { + Value step_value; + PostgresCapabilities::TryGetConstantValue(children[2].GetExpression(), step_value); + step = BigIntValue::Get(step_value.DefaultCastAs(LogicalType::BIGINT)); + } + result += "generate_series(" + start + ", " + end + (step > 0 ? " - 1" : " + 1"); + result += children.size() == 3 ? ", " + bigint(2) + ")" : ")"; + } else if (name == "generate_series" && integers) { + result += "generate_series(" + bigint(0) + ", " + bigint(1) + + (children.size() == 3 ? ", " + bigint(2) : "") + ")"; + } else if (name == "generate_series" && children.size() == 3 && types[0].id() == LogicalTypeId::TIMESTAMP && + types[1].id() == LogicalTypeId::TIMESTAMP && types[2].id() == LogicalTypeId::INTERVAL) { + result += "generate_series(" + WriteArgumentList(children) + ")"; + } else { + throw PostgresPushdownNotSupported("unsupported table function arguments"); + } + if (table_function.with_ordinality == OrdinalityType::WITH_ORDINALITY) { + result += " WITH ORDINALITY"; + } + // a single-column table function is named after the function by default + result += " AS " + WriteIdentifier(ref.alias.empty() ? Identifier(name) : ref.alias); + if (!ref.column_name_alias.empty()) { + result += "("; + for (idx_t i = 0; i < ref.column_name_alias.size(); i++) { + if (i > 0) { + result += ", "; + } + result += WriteIdentifier(ref.column_name_alias[i]); + } + result += ")"; + } + return result; + } + default: + throw InternalException("PostgresSQLWriter: unsupported table ref type - should have been blocked by " + "SupportsPushdown"); + } +} + +//===--------------------------------------------------------------------===// +// Result modifiers +//===--------------------------------------------------------------------===// +//! Integer literals in ORDER BY / GROUP BY / DISTINCT ON are positional references - other constants (e.g. a cast of +//! an integer literal) are written as typed literals, so that Postgres does not interpret them as positions either +string PostgresSQLWriter::WriteOrderingKey(const ParsedExpression &expr) { + Value value; + if (expr.GetExpressionClass() != ExpressionClass::CONSTANT && + PostgresCapabilities::TryGetConstantValue(expr, value)) { + return "CAST(" + WriteConstant(value) + " AS " + WriteType(value.type()) + ")"; + } + return WriteExpression(expr); +} + +//! The select-list entry an ORDER BY key refers to (a position or an alias), nullptr if it does not refer to one or +//! if the entry cannot be determined ("is_reference" is set then) +static optional_ptr +PostgresOrderedSelectEntry(const ParsedExpression &key, const vector> &select_list, + bool &is_reference) { + is_reference = false; + if (key.GetExpressionClass() == ExpressionClass::CONSTANT) { + auto value = key.Cast().GetLiteral().ToValue(); + if (!value.type().IsIntegral()) { + return nullptr; + } + is_reference = true; + auto position = value.GetValue(); + for (auto &entry : select_list) { + if (entry->GetExpressionClass() == ExpressionClass::STAR) { + // the position refers to an expanded column + return nullptr; + } + } + if (position < 1 || position > static_cast(select_list.size())) { + return nullptr; + } + return select_list[static_cast(position - 1)].get(); + } + if (key.GetExpressionClass() == ExpressionClass::COLUMN_REF && !key.Cast().IsQualified()) { + auto &name = key.Cast().GetColumnName(); + for (auto &entry : select_list) { + if (!entry->GetAlias().empty() && entry->GetAlias() == name) { + is_reference = true; + return entry.get(); + } + } + for (auto &entry : select_list) { + // a result column named after the column it selects + if (entry->GetAlias().empty() && entry->GetExpressionClass() == ExpressionClass::COLUMN_REF && + entry->Cast().GetColumnName() == name) { + is_reference = true; + return entry.get(); + } + } + } + return nullptr; +} + +//! The leftmost SELECT of a set operation, which determines the names and types of its result columns +static optional_ptr PostgresLeftmostSelect(const QueryNode &node) { + switch (node.type) { + case QueryNodeType::SELECT_NODE: + return &node.Cast(); + case QueryNodeType::SET_OPERATION_NODE: { + auto &setop = node.Cast(); + return setop.children.empty() ? nullptr : PostgresLeftmostSelect(*setop.children[0]); + } + case QueryNodeType::RECURSIVE_CTE_NODE: + return PostgresLeftmostSelect(*node.Cast().left); + default: + return nullptr; + } +} + +string PostgresSQLWriter::WriteOrderList(const vector &orders, optional_ptr node) { + auto &config = DBConfig::GetConfig(context); + optional_ptr>> select_list; + bool is_select = node && node->type == QueryNodeType::SELECT_NODE; + bool has_distinct = false; + // the result types of the leftmost SELECT of a set operation - its scope is not active anymore + vector leftmost_types; + optional_ptr leftmost; + if (node) { + leftmost = PostgresLeftmostSelect(*node); + if (leftmost) { + select_list = &leftmost->select_list; + } + for (auto &modifier : node->modifiers) { + has_distinct = has_distinct || modifier->type == ResultModifierType::DISTINCT_MODIFIER; + } + } + // the type of a select-list entry - binding the whole SELECT also determines the types of aggregates and + // GROUPING() entries + bool types_inferred = false; + auto entry_type = [&](const ParsedExpression &entry) { + if (!types_inferred) { + types_inferred = true; + bool has_star = false; + for (auto &select_entry : *select_list) { + has_star = has_star || select_entry->GetExpressionClass() == ExpressionClass::STAR; + } + if (!has_star) { + leftmost_types = InferQueryTypes(*leftmost); + } + } + for (idx_t i = 0; i < select_list->size() && i < leftmost_types.size(); i++) { + if ((*select_list)[i].get() == &entry) { + return leftmost_types[i]; + } + } + return is_select ? InferType(entry) : LogicalType(LogicalType::INVALID); + }; + string result; + for (auto &order : orders) { + auto order_type = config.ResolveOrder(context, order.type); + auto null_order = config.ResolveNullOrder(context, order_type, order.null_order); + vector keys; + if (order.expression->GetExpressionClass() == ExpressionClass::STAR) { + // ORDER BY ALL - order by all select-list positions + if (!select_list || !is_select) { + throw PostgresPushdownNotSupported("ORDER BY ALL without a select list"); + } + for (idx_t i = 0; i < select_list->size(); i++) { + auto &entry = *(*select_list)[i]; + if (catalog.HasBinaryCollation()) { + keys.push_back(to_string(i + 1)); + continue; + } + auto type = entry.GetExpressionClass() == ExpressionClass::STAR ? LogicalType(LogicalType::INVALID) + : entry_type(entry); + if (PostgresGetStringOrdering(type) == PostgresStringOrdering::NONE) { + keys.push_back(to_string(i + 1)); + } else if (has_distinct || entry.GetExpressionClass() == ExpressionClass::STAR) { + throw PostgresPushdownNotSupported("the string ordering of the database differs from DuckDB's"); + } else { + // a position cannot be combined with a collation - order by the expression + keys.push_back(WriteOrderedOperand(WriteExpression(entry), type)); + } + } + } else if (catalog.HasBinaryCollation()) { + keys.push_back(WriteOrderingKey(*order.expression)); + } else if (!node) { + // window / aggregate ORDER BY + keys.push_back(WriteOrderedOperand(WriteOrderingKey(*order.expression), InferType(*order.expression))); + } else { + bool is_reference = false; + optional_ptr entry; + if (select_list) { + entry = PostgresOrderedSelectEntry(*order.expression, *select_list, is_reference); + } + if (!is_select && !is_reference) { + throw PostgresPushdownNotSupported("unsupported ORDER BY of a set operation"); + } + optional_ptr ordered = is_reference ? entry : order.expression.get(); + LogicalType ordered_type = LogicalType::INVALID; + if (ordered) { + ordered_type = is_reference ? entry_type(*ordered) : InferType(*ordered); + } + auto ordering = PostgresGetStringOrdering(ordered_type); + if (ordering == PostgresStringOrdering::NONE) { + keys.push_back(WriteOrderingKey(*order.expression)); + } else if (!is_select || has_distinct) { + // a set operation can only be ordered by its result columns, with DISTINCT the ORDER BY expressions + // must appear in the select list - neither allows pinning the collation + throw PostgresPushdownNotSupported("the string ordering of the database differs from DuckDB's"); + } else { + // a position / alias cannot be combined with a collation - order by the referenced expression + auto &ordered_expr = *ordered; + keys.push_back(WriteOrderedOperand( + is_reference ? WriteExpression(ordered_expr) : WriteOrderingKey(ordered_expr), ordered_type)); + } + } + for (auto &key : keys) { + if (!result.empty()) { + result += ", "; + } + result += key; + result += order_type == OrderType::DESCENDING ? " DESC" : " ASC"; + // Postgres sorts NULLs as larger than any value, DuckDB sorts them last by default + result += null_order == OrderByNullType::NULLS_FIRST ? " NULLS FIRST" : " NULLS LAST"; + } + } + return result; +} + +string PostgresSQLWriter::WriteResultModifiers(const QueryNode &node, + optional_ptr>> select_list) { + string result; + for (auto &modifier : node.modifiers) { + switch (modifier->type) { + case ResultModifierType::ORDER_MODIFIER: { + auto &order_modifier = modifier->Cast(); + result += " ORDER BY " + WriteOrderList(order_modifier.orders, &node); + break; + } + case ResultModifierType::LIMIT_MODIFIER: { + auto &limit_modifier = modifier->Cast(); + if (limit_modifier.limit) { + result += " LIMIT " + WriteExpression(*limit_modifier.limit); + } + if (limit_modifier.offset) { + result += " OFFSET " + WriteExpression(*limit_modifier.offset); + } + break; + } + case ResultModifierType::DISTINCT_MODIFIER: + // handled in WriteSelectNode + break; + default: + throw InternalException("PostgresSQLWriter: unsupported result modifier - should have been blocked by " + "SupportsPushdown"); + } + } + return result; +} + +//===--------------------------------------------------------------------===// +// DML +//===--------------------------------------------------------------------===// +//! Postgres truncates identifiers to 63 bytes - longer names are made unique instead +static string PostgresColumnAlias(const string &name, idx_t index) { + static constexpr idx_t MAX_IDENTIFIER_LENGTH = 63; + if (name.size() <= MAX_IDENTIFIER_LENGTH) { + return name; + } + auto suffix = "_" + to_string(index); + idx_t length = MAX_IDENTIFIER_LENGTH - suffix.size(); + // do not cut a UTF-8 sequence in half + while (length > 0 && (static_cast(name[length]) & 0xC0) == 0x80) { + length--; + } + return name.substr(0, length) + suffix; +} + +//! The Postgres type of a column, if DuckDB reads it with a type that does not correspond to its Postgres type +//! (e.g. JSON columns read as VARCHAR, enums, composite types) - values assigned to these columns must be +//! converted to the Postgres type. Returns an empty string for regular columns. +static string PostgresNativeColumnType(TableCatalogEntry &table, const ColumnDefinition &column, bool &known) { + auto &pg_table = table.Cast(); + auto index = column.Logical().index; + known = index < pg_table.postgres_type_names.size(); + string type_name = known ? pg_table.postgres_type_names[index] : string(); + bool requires_native = false; + if (index < pg_table.postgres_types.size()) { + auto info = pg_table.postgres_types[index].info; + requires_native = info != PostgresTypeAnnotation::STANDARD && info != PostgresTypeAnnotation::FIXED_LENGTH_CHAR; + } + if (column.Type().id() == LogicalTypeId::VARCHAR && known) { + static const case_insensitive_set_t TEXT_TYPES = {"\"pg_catalog\".\"text\"", + "\"pg_catalog\".\"varchar\"", + "\"pg_catalog\".\"bpchar\"", + "\"pg_catalog\".\"name\"", + "\"text\"", + "\"varchar\"", + "\"bpchar\""}; + requires_native = requires_native || TEXT_TYPES.count(type_name) == 0; + } + requires_native = + requires_native || TypeVisitor::Contains(column.Type(), [](const LogicalType &type) { + return type.HasAlias() || type.id() == LogicalTypeId::ENUM || type.id() == LogicalTypeId::STRUCT; + }); + if (!requires_native) { + return string(); + } + if (!known || type_name.empty()) { + // the Postgres type is not known + known = false; + return "?"; + } + return type_name; +} + +string PostgresSQLWriter::WriteAssignment(const ParsedExpression &expr, TableCatalogEntry &table, + const ColumnDefinition &column, + optional_ptr source_type_override) { + if (expr.GetExpressionClass() == ExpressionClass::DEFAULT) { + return "DEFAULT"; + } + auto &target_type = column.Type(); + Value value; + if (PostgresCapabilities::TryGetConstantValue(expr, value) && value.IsNull()) { + // NULL is converted to the column type + return "NULL"; + } + auto expr_str = WriteExpression(expr); + if (PostgresCapabilities::TryGetConstantValue(expr, value)) { + bool known; + if (value.IsNull() || !PostgresNativeColumnType(table, column, known).empty()) { + // untyped literals are converted to the column type by Postgres - verify that DuckDB can convert + // the value as well (e.g. that it is a valid enum value) + if (!value.IsNull() && !value.TryCastAs(context, target_type)) { + throw PostgresPushdownNotSupported("invalid value for a column of type " + target_type.ToString()); + } + if (value.IsNull() || value.type().id() == LogicalTypeId::VARCHAR) { + return expr_str; + } + } else { + // cast constants locally - with DuckDB's cast semantics + auto cast_result = value.TryCastAs(context, target_type); + if (cast_result && PostgresCapabilities::SupportsValue(*cast_result)) { + return WriteConstant(*cast_result); + } + } + } + auto source_type = source_type_override ? *source_type_override : InferType(expr); + if (source_type.id() == LogicalTypeId::INVALID) { + throw PostgresPushdownNotSupported("cannot determine the type of an assigned value"); + } + return WriteAssignmentCast(expr_str, source_type, table, column); +} + +string PostgresSQLWriter::WriteAssignmentCast(const string &expr_str, const LogicalType &source_type, + TableCatalogEntry &table, const ColumnDefinition &column) { + auto &target_type = column.Type(); + bool known; + auto native_type = PostgresNativeColumnType(table, column, known); + if (!native_type.empty()) { + if (source_type == target_type && (target_type.id() != LogicalTypeId::VARCHAR || source_type.HasAlias())) { + // e.g. an enum value of the same type, or excluded. + return expr_str; + } + if (!known || (target_type.id() != LogicalTypeId::VARCHAR && target_type.id() != LogicalTypeId::ENUM)) { + throw PostgresPushdownNotSupported("unsupported assignment to a column of type " + target_type.ToString()); + } + // DuckDB converts the value to a string (or enum value) - Postgres parses the text representation + auto text = source_type.id() == LogicalTypeId::VARCHAR ? expr_str + : WriteCast(expr_str, source_type, LogicalType::VARCHAR); + return "CAST(" + text + " AS " + native_type + ")"; + } + if (source_type == target_type) { + return expr_str; + } + // DuckDB casts the value to the column type - Postgres would apply an assignment cast + return WriteCast(expr_str, source_type, target_type); +} + +string PostgresSQLWriter::WriteReturningList(const vector> &returning_list) { + if (returning_list.empty()) { + return string(); + } + bool cast_results = true; + for (auto &expr : returning_list) { + if (expr->GetExpressionClass() == ExpressionClass::STAR) { + cast_results = false; + } + } + string result = " RETURNING "; + for (idx_t i = 0; i < returning_list.size(); i++) { + if (i > 0) { + result += ", "; + } + auto &expr = *returning_list[i]; + auto expr_str = WriteExpression(expr); + result += cast_results ? WriteResultCast(expr_str, i) : expr_str; + if (!expr.GetAlias().empty()) { + result += " AS " + WriteIdentifier(expr.GetAlias()); + } else if (expr.GetExpressionClass() != ExpressionClass::STAR && + expr.GetExpressionClass() != ExpressionClass::COLUMN_REF) { + result += " AS " + PostgresUtils::WriteIdentifier(PostgresColumnAlias(expr.ToString(), i)); + } + } + return result; +} + +TableCatalogEntry &PostgresSQLWriter::GetTargetTable(const TableRef &ref) { + if (ref.type != TableReferenceType::BASE_TABLE || IsCTEReference(ref.Cast())) { + throw PostgresPushdownNotSupported("unsupported DML target"); + } + auto table = LookupTable(ref.Cast()); + if (!table) { + throw PostgresPushdownNotSupported("DML target table not found"); + } + return *table; +} + +string PostgresSQLWriter::WriteSetClause(const UpdateSetInfo &set_info, TableCatalogEntry &table, bool on_conflict) { + string result; + for (idx_t i = 0; i < set_info.columns.size(); i++) { + if (i > 0) { + result += ", "; + } + auto &column_name = set_info.columns[i]; + if (!table.ColumnExists(column_name)) { + throw PostgresPushdownNotSupported("column not found"); + } + auto &column = table.GetColumn(column_name); + auto &expr = *set_info.expressions[i]; + optional_ptr source_type; + if (on_conflict && expr.GetExpressionClass() == ExpressionClass::COLUMN_REF) { + // excluded. has the type of the column + auto &colref = expr.Cast(); + auto &names = colref.ColumnNames(); + if (names.size() == 2 && names[0] == "excluded" && table.ColumnExists(names[1])) { + source_type = &table.GetColumn(names[1]).Type(); + } + } + result += WriteColumnName(table, column) + " = " + WriteAssignment(expr, table, column, source_type); + } + return result; +} + +string PostgresSQLWriter::WriteUpdateNode(const UpdateQueryNode &node) { + auto &table = GetTargetTable(*node.table); + string result = WriteCTEMap(node.cte_map); + table_scopes.emplace_back(); + table_scopes.back().push_back(node.table); + if (node.from_table) { + table_scopes.back().push_back(node.from_table); + } + result += "UPDATE " + WriteTableRef(*node.table); + result += " SET " + WriteSetClause(*node.set_info, table, false); + if (node.from_table) { + result += " FROM " + WriteTableRef(*node.from_table); + } + if (node.set_info->condition) { + result += " WHERE " + WriteExpression(*node.set_info->condition); + } + result += WriteReturningList(node.returning_list); + table_scopes.pop_back(); + return result; +} + +string PostgresSQLWriter::WriteDeleteNode(const DeleteQueryNode &node) { + GetTargetTable(*node.table); + string result = WriteCTEMap(node.cte_map); + table_scopes.emplace_back(); + table_scopes.back().push_back(node.table); + for (auto &using_clause : node.using_clauses) { + table_scopes.back().push_back(using_clause); + } + result += "DELETE FROM " + WriteTableRef(*node.table); + for (idx_t i = 0; i < node.using_clauses.size(); i++) { + result += i == 0 ? " USING " : ", "; + result += WriteTableRef(*node.using_clauses[i]); + } + if (node.condition) { + result += " WHERE " + WriteExpression(*node.condition); + } + result += WriteReturningList(node.returning_list); + table_scopes.pop_back(); + return result; +} + +//! The types of the result columns of a query, as bound by DuckDB +static vector PostgresBindQueryTypes(ClientContext &context, const Identifier &catalog_name, + const QueryNode &node, const CommonTableExpressionMap &cte_map) { + auto bind_node = node.Copy(); + for (auto &cte : cte_map.map) { + bind_node->cte_map.map.insert(cte.first, cte.second->Copy()); + } + identifier_set_t cte_names; + PostgresSQLWriter::CollectCTENames(*bind_node, cte_names); + PostgresSQLWriter::QualifyTables(*bind_node, catalog_name, cte_names); + try { + auto binder = Binder::CreateBinder(context); + return binder->Bind(*bind_node).types; + } catch (std::exception &) { + throw PostgresPushdownNotSupported("cannot determine the types of the inserted values"); + } +} + +static PostgresColumnDefault PostgresGetColumnDefault(TableCatalogEntry &table, const ColumnDefinition &column) { + auto &pg_table = table.Cast(); + auto index = column.Logical().index; + return index < pg_table.column_defaults.size() ? pg_table.column_defaults[index] : PostgresColumnDefault::NONE; +} + +static bool PostgresIsNullOrDefault(const ParsedExpression &expr) { + if (expr.GetExpressionClass() == ExpressionClass::DEFAULT) { + return true; + } + Value value; + return PostgresCapabilities::TryGetConstantValue(expr, value) && value.IsNull(); +} + +string PostgresSQLWriter::WriteInsertNode(const InsertQueryNode &node) { + auto table_ptr = LookupTable(node.qualified_name); + if (!table_ptr) { + throw PostgresPushdownNotSupported("INSERT target table not found"); + } + auto &table = *table_ptr; + // the columns that are inserted into - like the regular INSERT into a Postgres table, values for generated + // columns must be NULL (and are not sent), and NULL values for columns with a Postgres-side default are + // replaced with the default + vector> columns; + if (node.columns.empty()) { + for (auto &column : table.GetColumns().Physical()) { + columns.push_back(column); + } + } else { + for (auto &column_name : node.columns) { + if (!table.ColumnExists(column_name)) { + throw PostgresPushdownNotSupported("INSERT column not found"); + } + columns.push_back(table.GetColumn(column_name)); + } + } + vector defaults; + vector> target_columns; + for (auto &column : columns) { + defaults.push_back(PostgresGetColumnDefault(table, column)); + if (defaults.back() != PostgresColumnDefault::GENERATED) { + target_columns.push_back(column); + } + } + string result = WriteCTEMap(node.cte_map); + result += "INSERT INTO " + PostgresUtils::WriteIdentifier(table.ParentSchema().name.GetIdentifierName()) + "." + + PostgresUtils::WriteIdentifier(table.name.GetIdentifierName()); + if (node.table_ref && !node.table_ref->alias.empty()) { + result += " AS " + WriteIdentifier(node.table_ref->alias); + } + auto values_list = node.GetValuesList(); + if (!node.default_values && !target_columns.empty()) { + result += " ("; + for (idx_t i = 0; i < target_columns.size(); i++) { + result += (i > 0 ? ", " : "") + WriteColumnName(table, target_columns[i]); + } + result += ")"; + } + if (node.default_values) { + result += " DEFAULT VALUES"; + } else if (values_list) { + // a VALUES list - every value is converted to the type of its column (this also supports DEFAULT) + vector rows; + table_scopes.emplace_back(); + for (auto &values : values_list->values) { + if (values.size() != columns.size()) { + throw PostgresPushdownNotSupported("INSERT column count mismatch"); + } + string row; + for (idx_t col = 0; col < values.size(); col++) { + auto &value = *values[col]; + string value_str; + switch (defaults[col]) { + case PostgresColumnDefault::GENERATED: + if (!PostgresIsNullOrDefault(value)) { + // an error is raised by the regular INSERT + throw PostgresPushdownNotSupported("INSERT into a generated column"); + } + continue; + case PostgresColumnDefault::DEFAULT_VALUE: + if (PostgresIsNullOrDefault(value)) { + value_str = "DEFAULT"; + break; + } + if (value.GetExpressionClass() != ExpressionClass::CONSTANT && + value.GetExpressionClass() != ExpressionClass::CAST) { + // the value could be NULL - which is replaced with the default + throw PostgresPushdownNotSupported( + "INSERT of a non-constant value into a column with a default"); + } + value_str = WriteAssignment(value, table, columns[col]); + break; + default: + value_str = WriteAssignment(value, table, columns[col]); + break; + } + row += (row.empty() ? "" : ", ") + value_str; + } + rows.push_back("(" + row + ")"); + } + table_scopes.pop_back(); + if (target_columns.empty()) { + result += " DEFAULT VALUES"; + if (rows.size() != 1) { + throw PostgresPushdownNotSupported("INSERT of multiple rows of default values"); + } + } else { + result += " VALUES " + StringUtil::Join(rows, ", "); + } + } else { + for (auto column_default : defaults) { + if (column_default != PostgresColumnDefault::NONE) { + // NULL values would have to be replaced with the Postgres-side default + throw PostgresPushdownNotSupported("INSERT ... SELECT into a column with a default"); + } + } + auto &select = *node.select_statement->node; + auto source_types = PostgresBindQueryTypes(context, catalog.GetName(), select, node.cte_map); + if (source_types.size() != columns.size()) { + throw PostgresPushdownNotSupported("INSERT column count mismatch"); + } + bool requires_casts = false; + for (idx_t i = 0; i < columns.size(); i++) { + bool known; + if (source_types[i] != columns[i].get().Type() || + !PostgresNativeColumnType(table, columns[i], known).empty()) { + requires_casts = true; + } + } + auto select_str = WriteQueryNode(select); + if (!requires_casts) { + result += " " + select_str; + } else { + // DuckDB casts the inserted values to the column types - Postgres would apply assignment casts + string select_list; + string aliases; + for (idx_t i = 0; i < columns.size(); i++) { + auto name = PostgresUtils::WriteIdentifier("c" + to_string(i)); + select_list += (i > 0 ? ", " : "") + WriteAssignmentCast(name, source_types[i], table, columns[i]); + aliases += (i > 0 ? ", " : "") + name; + } + result += " SELECT " + select_list + " FROM (" + select_str + ") AS " + + PostgresUtils::WriteIdentifier("insert_values") + "(" + aliases + ")"; + } + } + if (node.on_conflict_info) { + auto target_binding = node.table_ref ? node.table_ref->alias : Identifier(); + result += + WriteOnConflict(*node.on_conflict_info, table, target_binding, target_columns, values_list.get(), columns); + } + table_scopes.emplace_back(); + table_scopes.back().push_back(node.table_ref); + result += WriteReturningList(node.returning_list); + table_scopes.pop_back(); + return result; +} + +string PostgresSQLWriter::WriteOnConflict(const OnConflictInfo &info, TableCatalogEntry &table, + const Identifier &target_binding, + const vector> &columns, + optional_ptr values_list, + const vector> &value_columns) { + if (info.action_type == OnConflictAction::THROW) { + return string(); + } + // the conflict target - DuckDB defaults to the primary key, Postgres requires it for DO UPDATE + vector conflict_columns; + for (auto &column_name : info.indexed_columns) { + if (!table.ColumnExists(column_name)) { + throw PostgresPushdownNotSupported("ON CONFLICT column not found"); + } + conflict_columns.push_back(WriteColumnName(table, table.GetColumn(column_name))); + } + if (conflict_columns.empty() && info.action_type != OnConflictAction::NOTHING) { + for (auto &constraint : table.GetConstraints()) { + if (constraint->type != ConstraintType::UNIQUE) { + continue; + } + auto &unique = constraint->Cast(); + if (!unique.is_primary_key) { + continue; + } + if (unique.index.index != DConstants::INVALID_INDEX) { + conflict_columns.push_back(WriteColumnName(table, table.GetColumn(unique.index))); + } else { + for (auto &column_name : unique.columns) { + conflict_columns.push_back(WriteColumnName(table, table.GetColumn(column_name))); + } + } + } + if (conflict_columns.empty()) { + throw PostgresPushdownNotSupported("ON CONFLICT DO UPDATE without a primary key"); + } + } + string result = " ON CONFLICT"; + if (!conflict_columns.empty()) { + result += " (" + StringUtil::Join(conflict_columns, ", ") + ")"; + } + if (info.action_type == OnConflictAction::NOTHING) { + return result + " DO NOTHING"; + } + // Postgres raises an error when a row is updated twice by the same statement - only push down VALUES lists + // where the values of the conflict target are constants that are distinct + if (!values_list) { + throw PostgresPushdownNotSupported("ON CONFLICT DO UPDATE with a query"); + } + vector key_indexes; + for (auto &conflict_column : conflict_columns) { + bool found = false; + for (idx_t i = 0; i < value_columns.size(); i++) { + if (WriteColumnName(table, value_columns[i]) == conflict_column) { + key_indexes.push_back(i); + found = true; + } + } + if (!found) { + throw PostgresPushdownNotSupported("ON CONFLICT column is not inserted"); + } + } + vector> keys; + for (auto &row : values_list->values) { + vector key; + for (auto index : key_indexes) { + Value value; + if (index >= row.size() || !PostgresCapabilities::TryGetConstantValue(*row[index], value)) { + throw PostgresPushdownNotSupported("ON CONFLICT DO UPDATE with non-constant keys"); + } + auto cast_value = value.TryCastAs(context, value_columns[index].get().Type()); + if (!cast_value) { + throw PostgresPushdownNotSupported("ON CONFLICT DO UPDATE with invalid keys"); + } + key.push_back(std::move(*cast_value)); + } + for (auto &other : keys) { + bool equal = true; + for (idx_t k = 0; k < key.size(); k++) { + equal = equal && !key[k].IsNull() && Value::NotDistinctFrom(key[k], other[k]); + } + if (equal) { + throw PostgresPushdownNotSupported("ON CONFLICT DO UPDATE with duplicate keys"); + } + } + keys.push_back(std::move(key)); + } + if (info.action_type == OnConflictAction::REPLACE) { + // INSERT OR REPLACE updates all inserted columns that are not part of the conflict target + string set_clause; + for (auto &column : columns) { + auto name = WriteColumnName(table, column); + bool is_conflict_column = false; + for (auto &conflict_column : conflict_columns) { + is_conflict_column = is_conflict_column || conflict_column == name; + } + if (is_conflict_column) { + continue; + } + set_clause += set_clause.empty() ? "" : ", "; + set_clause += name + " = " + PostgresUtils::WriteIdentifier("excluded") + "." + name; + } + if (set_clause.empty()) { + return result + " DO NOTHING"; + } + return result + " DO UPDATE SET " + set_clause; + } + // DO UPDATE - the SET expressions can reference the target table and the excluded values + auto target_ref = make_uniq(); + target_ref->SetQualifiedName(QualifiedName(Identifier(), table.ParentSchema().name, table.name)); + auto excluded_ref = target_ref->Copy(); + excluded_ref->alias = Identifier("excluded"); + unique_ptr excluded_table = std::move(excluded_ref); + unique_ptr target_table = std::move(target_ref); + table_scopes.emplace_back(); + table_scopes.back().push_back(target_table); + table_scopes.back().push_back(excluded_table); + on_conflict_target = &table; + on_conflict_depth = query_depth; + on_conflict_binding = target_binding.empty() ? PostgresUtils::WriteIdentifier(table.name.GetIdentifierName()) + : WriteIdentifier(target_binding); + result += " DO UPDATE SET " + WriteSetClause(*info.set_info, table, true); + if (info.set_info->condition) { + result += " WHERE " + WriteExpression(*info.set_info->condition); + } + on_conflict_target = nullptr; + table_scopes.pop_back(); + return result; +} + +string PostgresSQLWriter::WriteMergeNode(const MergeQueryNode &node) { + auto &table = GetTargetTable(*node.target); + string result = WriteCTEMap(node.cte_map); + table_scopes.emplace_back(); + table_scopes.back().push_back(node.target); + table_scopes.back().push_back(node.source); + result += "MERGE INTO " + WriteTableRef(*node.target) + " USING " + WriteTableRef(*node.source) + " ON "; + if (node.join_condition) { + result += WriteExpression(*node.join_condition); + } else if (!node.using_columns.empty()) { + // Postgres has no USING clause for MERGE + auto target_binding = + node.target->alias.empty() ? node.target->Cast().Table() : node.target->alias; + if (node.source->alias.empty()) { + throw PostgresPushdownNotSupported("MERGE USING with an unnamed source"); + } + for (idx_t i = 0; i < node.using_columns.size(); i++) { + auto &column = node.using_columns[i]; + result += (i > 0 ? " AND " : "") + WriteIdentifier(target_binding) + "." + WriteIdentifier(column) + " = " + + WriteIdentifier(node.source->alias) + "." + WriteIdentifier(column); + } + } else { + result += "TRUE"; + } + for (auto &entry : node.actions) { + for (auto &action : entry.second) { + switch (entry.first) { + case MergeActionCondition::WHEN_MATCHED: + result += " WHEN MATCHED"; + break; + case MergeActionCondition::WHEN_NOT_MATCHED_BY_TARGET: + result += " WHEN NOT MATCHED"; + break; + case MergeActionCondition::WHEN_NOT_MATCHED_BY_SOURCE: + result += " WHEN NOT MATCHED BY SOURCE"; + break; + } + if (action->condition) { + result += " AND " + WriteExpression(*action->condition); + } + result += " THEN "; + switch (action->action_type) { + case MergeActionType::MERGE_UPDATE: + result += "UPDATE SET " + WriteSetClause(*action->update_info, table, false); + break; + case MergeActionType::MERGE_DELETE: + result += "DELETE"; + break; + case MergeActionType::MERGE_DO_NOTHING: + result += "DO NOTHING"; + break; + case MergeActionType::MERGE_INSERT: { + if (action->default_values) { + result += "INSERT DEFAULT VALUES"; + break; + } + vector> columns; + if (action->insert_columns.empty()) { + for (auto &column : table.GetColumns().Physical()) { + columns.push_back(column); + } + } else { + for (auto &column_name : action->insert_columns) { + if (!table.ColumnExists(column_name)) { + throw PostgresPushdownNotSupported("MERGE INSERT column not found"); + } + columns.push_back(table.GetColumn(column_name)); + } + } + if (columns.size() != action->expressions.size()) { + throw PostgresPushdownNotSupported("MERGE INSERT column count mismatch"); + } + string column_list; + string value_list; + for (idx_t i = 0; i < columns.size(); i++) { + auto &value = *action->expressions[i]; + string value_str; + switch (PostgresGetColumnDefault(table, columns[i])) { + case PostgresColumnDefault::GENERATED: + if (!PostgresIsNullOrDefault(value)) { + throw PostgresPushdownNotSupported("MERGE INSERT into a generated column"); + } + continue; + case PostgresColumnDefault::DEFAULT_VALUE: + if (PostgresIsNullOrDefault(value)) { + value_str = "DEFAULT"; + break; + } + if (value.GetExpressionClass() != ExpressionClass::CONSTANT && + value.GetExpressionClass() != ExpressionClass::CAST) { + // the value could be NULL - which is replaced with the default by the regular INSERT + throw PostgresPushdownNotSupported( + "MERGE INSERT of a non-constant value into a column with " + "a default"); + } + value_str = WriteAssignment(value, table, columns[i]); + break; + default: + value_str = WriteAssignment(value, table, columns[i]); + break; + } + column_list += (column_list.empty() ? "" : ", ") + WriteColumnName(table, columns[i]); + value_list += (value_list.empty() ? "" : ", ") + value_str; + } + if (column_list.empty()) { + result += "INSERT DEFAULT VALUES"; + } else { + result += "INSERT (" + column_list + ") VALUES (" + value_list + ")"; + } + break; + } + default: + throw InternalException("PostgresSQLWriter: unsupported MERGE action - should have been blocked by " + "SupportsPushdown"); + } + } + } + result += WriteReturningList(node.returning_list); + table_scopes.pop_back(); + return result; +} + +//===--------------------------------------------------------------------===// +// Statements +//===--------------------------------------------------------------------===// +PostgresWriterResult PostgresSQLWriter::WriteStatement(ClientContext &context, PostgresCatalog &catalog, + SQLStatement &statement) { + PostgresSQLWriter writer(context, catalog); + PostgresWriterResult result; + switch (statement.type) { + case StatementType::CREATE_STATEMENT: { + auto &create = statement.Cast(); + switch (create.info->type) { + case CatalogType::TABLE_ENTRY: + writer.WriteCreateTable(create, result); + break; + case CatalogType::VIEW_ENTRY: + writer.WriteCreateView(create.info->Cast(), result); + break; + default: + throw InternalException("PostgresSQLWriter: unsupported CREATE statement - should have been blocked by " + "SupportsPushdown"); + } + break; + } + case StatementType::DROP_STATEMENT: + result.sql = writer.WriteDrop(*statement.Cast().info, result); + break; + case StatementType::ALTER_STATEMENT: + result.sql = writer.WriteAlter(*statement.Cast().info, result); + break; + default: + throw InternalException("PostgresSQLWriter: unsupported statement type - should have been blocked by " + "SupportsPushdown"); + } + result.requires_time_zone = writer.requires_time_zone; + if (writer.requires_iso_date_style) { + result.preliminary_queries.push_back("SET LOCAL DateStyle = 'ISO'"); + } + return result; +} + +static optional_ptr PostgresFindCreateTable(LogicalOperator &op) { + if (op.type == LogicalOperatorType::LOGICAL_CREATE_TABLE) { + return op.Cast(); + } + for (auto &child : op.children) { + auto result = PostgresFindCreateTable(*child); + if (result) { + return result; + } + } + return nullptr; +} + +void PostgresSQLWriter::WriteCreateTable(CreateStatement &statement, PostgresWriterResult &result) { + auto &info = statement.info->Cast(); + // bind the statement to resolve the column types (and the types of the query of a CREATE TABLE AS) + auto bind_statement = statement.Copy(); + auto &bind_info = bind_statement->Cast().info->Cast(); + vector schema_path; + if (!info.GetQualifiedName().Schema().empty()) { + schema_path.push_back(info.GetQualifiedName().Schema()); + } + bind_info.SetQualifiedName( + QualifiedName::FromCatalogSchema(catalog.GetName(), std::move(schema_path), info.GetQualifiedName().Name())); + if (bind_info.query) { + identifier_set_t cte_names; + CollectCTENames(*bind_info.query->node, cte_names); + QualifyTables(*bind_info.query->node, catalog.GetName(), cte_names); + } + auto binder = Binder::CreateBinder(context); + auto bound = binder->Bind(*bind_statement); + auto create_table = PostgresFindCreateTable(*bound.plan); + if (!create_table) { + throw InternalException("PostgresSQLWriter: CREATE TABLE statement was not bound to a LogicalCreateTable"); + } + auto &bound_info = create_table->info->Base(); + vector source_types; + for (auto &column : bound_info.columns.Logical()) { + source_types.push_back(column.Type()); + } + auto table_name = PostgresUtils::WriteIdentifier(create_table->schema.name.GetIdentifierName()) + "." + + PostgresUtils::WriteIdentifier(bound_info.GetTableName().GetIdentifierName()); + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + result.preliminary_queries.push_back("DROP TABLE IF EXISTS " + table_name); + } + // the tables of the schema are reloaded - so that the table is also visible when scanning the schema + result.invalidations.push_back({CatalogType::TABLE_ENTRY, create_table->schema.name.GetIdentifierName(), + bound_info.GetTableName().GetIdentifierName(), true}); + // generate the same CREATE TABLE statement as a regular CREATE TABLE + bound_info.on_conflict = info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT + ? OnCreateConflict::IGNORE_ON_CONFLICT + : OnCreateConflict::ERROR_ON_CONFLICT; + auto create_sql = GetPostgresCreateTable(bound_info); + if (StringUtil::EndsWith(create_sql, ";")) { + create_sql = create_sql.substr(0, create_sql.size() - 1); + } + if (!info.query) { + throw InternalException("PostgresSQLWriter: CREATE TABLE without a query should have been blocked by " + "SupportsPushdown"); + } + // CREATE TABLE AS - create the table, and fill it with the result of the query + result.preliminary_queries.push_back(std::move(create_sql)); + auto &query = *info.query->node; + CollectNames(query); + auto query_str = WriteQueryNode(query); + string columns; + string select_list; + bool requires_casts = false; + idx_t index = 0; + for (auto &column : bound_info.columns.Logical()) { + auto name = PostgresUtils::WriteIdentifier(column.Name().GetIdentifierName()); + auto alias = PostgresUtils::WriteIdentifier("c" + to_string(index)); + columns += (index > 0 ? ", " : "") + name; + // the column types are converted to types that are supported by Postgres (e.g. HUGEINT -> DOUBLE) + auto &source_type = source_types[index]; + auto &target_type = column.Type(); + string value = alias; + if (source_type != target_type) { + requires_casts = true; + value = WriteCast(alias, source_type, target_type); + } + select_list += (index > 0 ? ", " : "") + value; + index++; + } + result.sql = "INSERT INTO " + table_name + " (" + columns + ") "; + if (!requires_casts) { + result.sql += query_str; + } else { + string aliases; + for (idx_t i = 0; i < index; i++) { + aliases += (i > 0 ? ", " : "") + PostgresUtils::WriteIdentifier("c" + to_string(i)); + } + result.sql += "SELECT " + select_list + " FROM (" + query_str + ") AS " + + PostgresUtils::WriteIdentifier("insert_values") + "(" + aliases + ")"; + } + result.changed_rows = true; +} + +void PostgresSQLWriter::WriteCreateView(CreateViewInfo &info, PostgresWriterResult &result) { + auto &query = *info.query->node; + // bind the query to determine the types DuckDB binds the result to - the view returns these types + auto result_types = PostgresBindQueryTypes(context, catalog.GetName(), query, CommonTableExpressionMap()); + CollectNames(query); + // all tables must be tables of this database + identifier_set_t cte_names; + auto query_copy = query.Copy(); + CollectCTENames(*query_copy, cte_names); + ParsedExpressionIterator::EnumerateQueryNodeChildren( + *query_copy, [&](unique_ptr &) {}, + [&](TableRef &ref) { + if (ref.type != TableReferenceType::BASE_TABLE) { + return; + } + auto &base = ref.Cast(); + if (base.GetQualifiedName().Path().size() == 1 && cte_names.count(base.Table()) > 0) { + return; + } + if (!LookupTable(base)) { + throw BinderException("Views in Postgres can only reference tables of the same database - table \"%s\" " + "is not a table of database \"%s\"", + base.Table(), catalog.GetName()); + } + }); + CollectResultNodes(query); + this->result_types = &result_types; + auto entry_name = ResolveEntryName(info.GetQualifiedName()); + auto view_name = + PostgresUtils::WriteIdentifier(entry_name.first) + "." + PostgresUtils::WriteIdentifier(entry_name.second); + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + // DuckDB replaces the view regardless of its columns + result.preliminary_queries.push_back("DROP VIEW IF EXISTS " + view_name); + } + // the tables of the schema are reloaded - so that the view is also visible when scanning the schema + result.invalidations.push_back({CatalogType::VIEW_ENTRY, entry_name.first, entry_name.second, true}); + result.sql = "CREATE VIEW " + view_name; + if (!info.aliases.empty()) { + result.sql += " ("; + for (idx_t i = 0; i < info.aliases.size(); i++) { + result.sql += (i > 0 ? ", " : "") + PostgresUtils::WriteIdentifier(info.aliases[i].GetIdentifierName()); + } + result.sql += ")"; + } + result.sql += " AS " + WriteQueryNode(query); + this->result_types = nullptr; +} + +pair PostgresSQLWriter::ResolveEntryName(const QualifiedName &name) { + auto table = LookupTable(name); + if (table) { + return make_pair(table->ParentSchema().name.GetIdentifierName(), table->name.GetIdentifierName()); + } + Identifier schema_name = name.Schema(); + if (schema_name.empty()) { + auto default_schema = catalog.GetDefaultSchema(); + schema_name = default_schema ? *default_schema : Identifier("public"); + } + return make_pair(schema_name.GetIdentifierName(), name.Name().GetIdentifierName()); +} + +string PostgresSQLWriter::WriteQualifiedTableName(const QualifiedName &name) { + auto entry_name = ResolveEntryName(name); + return PostgresUtils::WriteIdentifier(entry_name.first) + "." + PostgresUtils::WriteIdentifier(entry_name.second); +} + +void PostgresSQLWriter::VerifyEntryExists(CatalogType type, const QualifiedName &name) { + // raise the same error as the regular statement when the entry does not exist + if (type == CatalogType::SCHEMA_ENTRY) { + Catalog::GetSchema(context, catalog.GetName(), name.Name(), OnEntryNotFound::THROW_EXCEPTION); + return; + } + Identifier schema_name = name.Schema(); + if (schema_name.empty()) { + auto default_schema = catalog.GetDefaultSchema(); + schema_name = default_schema ? *default_schema : Identifier("public"); + } + EntryLookupInfo lookup(type, QualifiedName(catalog.GetName(), schema_name, name.Name())); + Catalog::GetEntry(context, lookup, OnEntryNotFound::THROW_EXCEPTION); +} + +string PostgresSQLWriter::WriteDrop(const DropInfo &info, PostgresWriterResult &write_result) { + if (info.if_not_found != OnEntryNotFound::RETURN_NULL) { + VerifyEntryExists(info.type, info.GetQualifiedName()); + } + PostgresCacheInvalidation invalidation; + invalidation.type = info.type; + if (info.type == CatalogType::SCHEMA_ENTRY) { + invalidation.name = info.GetQualifiedName().Name().GetIdentifierName(); + } else { + auto entry_name = ResolveEntryName(info.GetQualifiedName()); + invalidation.schema = entry_name.first; + invalidation.name = entry_name.second; + } + if (info.type != CatalogType::SEQUENCE_ENTRY) { + // sequences are not cached + write_result.invalidations.push_back(std::move(invalidation)); + } + string result = "DROP "; + switch (info.type) { + case CatalogType::TABLE_ENTRY: + result += "TABLE "; + break; + case CatalogType::VIEW_ENTRY: + result += "VIEW "; + break; + case CatalogType::SCHEMA_ENTRY: + result += "SCHEMA "; + break; + case CatalogType::INDEX_ENTRY: + result += "INDEX "; + break; + case CatalogType::SEQUENCE_ENTRY: + result += "SEQUENCE "; + break; + case CatalogType::TYPE_ENTRY: + result += "TYPE "; + break; + default: + throw InternalException("PostgresSQLWriter: unsupported DROP statement - should have been blocked by " + "SupportsPushdown"); + } + if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { + result += "IF EXISTS "; + } + if (info.type == CatalogType::SCHEMA_ENTRY) { + result += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Name().GetIdentifierName()); + } else { + result += WriteQualifiedTableName(info.GetQualifiedName()); + } + if (info.cascade) { + result += " CASCADE"; + } + return result; +} + +string PostgresSQLWriter::WriteAlter(const AlterInfo &info, PostgresWriterResult &write_result) { + auto &alter_info = info.Cast(); + if (alter_info.if_not_found != OnEntryNotFound::RETURN_NULL) { + VerifyEntryExists(CatalogType::TABLE_ENTRY, alter_info.GetQualifiedName()); + } + // like a regular ALTER TABLE, all tables of the schema are reloaded (e.g. because of renamed tables) + PostgresCacheInvalidation invalidation; + invalidation.type = CatalogType::TABLE_ENTRY; + invalidation.schema = ResolveEntryName(alter_info.GetQualifiedName()).first; + invalidation.all_tables = true; + write_result.invalidations.push_back(std::move(invalidation)); + auto table = LookupTable(alter_info.GetQualifiedName()); + auto column_name = [&](const Identifier &name) { + if (table && table->ColumnExists(name)) { + return WriteColumnName(*table, table->GetColumn(name)); + } + return PostgresUtils::WriteIdentifier(name.GetIdentifierName()); + }; + string result = "ALTER TABLE "; + if (alter_info.if_not_found == OnEntryNotFound::RETURN_NULL) { + result += "IF EXISTS "; + } + result += WriteQualifiedTableName(alter_info.GetQualifiedName()); + switch (alter_info.alter_table_type) { + case AlterTableType::RENAME_TABLE: + return result + " RENAME TO " + + PostgresUtils::WriteIdentifier(alter_info.Cast().new_table_name.GetIdentifierName()); + case AlterTableType::RENAME_COLUMN: { + auto &rename_info = alter_info.Cast(); + return result + " RENAME COLUMN " + column_name(rename_info.old_name) + " TO " + + PostgresUtils::WriteIdentifier(rename_info.new_name.GetIdentifierName()); + } + case AlterTableType::ADD_COLUMN: { + auto &add_info = alter_info.Cast(); + auto &column = add_info.new_column; + LogicalType type = column.Type(); + if (type.id() == LogicalTypeId::UNBOUND) { + type = PostgresCapabilities::ResolveType(UnboundType::GetTypeExpression(type)->Cast()); + } + result += " ADD COLUMN "; + if (add_info.if_column_not_exists) { + result += "IF NOT EXISTS "; + } + return result + PostgresUtils::WriteIdentifier(column.Name().GetIdentifierName()) + " " + + PostgresUtils::TypeToString(PostgresUtils::ToPostgresType(type)); + } + case AlterTableType::REMOVE_COLUMN: { + auto &remove_info = alter_info.Cast(); + result += " DROP COLUMN "; + if (remove_info.if_column_exists) { + result += "IF EXISTS "; + } + result += column_name(remove_info.removed_column); + if (remove_info.cascade) { + result += " CASCADE"; + } + return result; + } + case AlterTableType::SET_NOT_NULL: + return result + " ALTER COLUMN " + column_name(alter_info.Cast().column_path[0]) + + " SET NOT NULL"; + case AlterTableType::DROP_NOT_NULL: + return result + " ALTER COLUMN " + column_name(alter_info.Cast().column_path[0]) + + " DROP NOT NULL"; + default: + throw InternalException("PostgresSQLWriter: unsupported ALTER statement - should have been blocked by " + "SupportsPushdown"); + } +} + +//===--------------------------------------------------------------------===// +// Query nodes +//===--------------------------------------------------------------------===// +string PostgresSQLWriter::WriteCTEMap(const CommonTableExpressionMap &cte_map) { + if (cte_map.map.empty()) { + return string(); + } + bool has_recursive = false; + for (auto &kv : cte_map.map) { + if (kv.second->query_node && kv.second->query_node->type == QueryNodeType::RECURSIVE_CTE_NODE) { + has_recursive = true; + break; + } + } + string result = "WITH "; + if (has_recursive) { + result += "RECURSIVE "; + } + bool first_cte = true; + for (auto &kv : cte_map.map) { + if (!first_cte) { + result += ", "; + } + auto &cte = *kv.second; + result += WriteIdentifier(kv.first); + if (!cte.aliases.empty()) { + result += " ("; + for (idx_t k = 0; k < cte.aliases.size(); k++) { + if (k > 0) { + result += ", "; + } + result += WriteIdentifier(cte.aliases[k]); + } + result += ")"; + } + result += " AS "; + switch (cte.materialized) { + case CTEMaterialize::CTE_MATERIALIZE_ALWAYS: + result += "MATERIALIZED "; + break; + case CTEMaterialize::CTE_MATERIALIZE_NEVER: + result += "NOT MATERIALIZED "; + break; + default: + break; + } + result += "(" + WriteQueryNode(*cte.query_node) + ")"; + first_cte = false; + } + result += " "; + return result; +} + +string PostgresSQLWriter::WriteSelectNode(const SelectNode &node) { + string result = WriteCTEMap(node.cte_map); + table_scopes.emplace_back(); + if (node.from_table) { + table_scopes.back().push_back(node.from_table); + } + result += "SELECT "; + bool has_distinct = false; + for (auto &modifier : node.modifiers) { + if (modifier->type == ResultModifierType::DISTINCT_MODIFIER) { + has_distinct = true; + auto &distinct_modifier = modifier->Cast(); + result += "DISTINCT "; + if (!distinct_modifier.distinct_on_targets.empty()) { + result += "ON ("; + for (idx_t i = 0; i < distinct_modifier.distinct_on_targets.size(); i++) { + // the targets must match the leading ORDER BY keys - including their collation + auto &target = *distinct_modifier.distinct_on_targets[i]; + auto target_str = WriteOrderingKey(target); + if (!catalog.HasBinaryCollation()) { + target_str = WriteOrderedOperand(target_str, InferType(target)); + } + result += (i > 0 ? ", " : "") + target_str; + } + result += ") "; + } + break; + } + } + // casts are not applied with DISTINCT if the ORDER BY contains expressions (Postgres requires the ORDER BY + // expressions to appear in the select list then) + bool allow_casts = !has_distinct; + if (has_distinct) { + allow_casts = true; + for (auto &modifier : node.modifiers) { + if (modifier->type != ResultModifierType::ORDER_MODIFIER) { + continue; + } + for (auto &order : modifier->Cast().orders) { + auto expr_class = order.expression->GetExpressionClass(); + if (expr_class != ExpressionClass::CONSTANT && expr_class != ExpressionClass::COLUMN_REF) { + allow_casts = false; + } + } + } + } + // the types DuckDB binds the result to are known for the result nodes - if the select-list entries map + // one-to-one to the result columns + bool use_result_types = result_nodes.count(&node) > 0; + for (auto &select_expr : node.select_list) { + if (select_expr->GetExpressionClass() == ExpressionClass::STAR) { + use_result_types = false; + } + } + for (idx_t i = 0; i < node.select_list.size(); i++) { + if (i > 0) { + result += ", "; + } + auto &select_expr = *node.select_list[i]; + auto expr_str = WriteExpression(select_expr); + auto expr_class = select_expr.GetExpressionClass(); + if (!allow_casts || expr_class == ExpressionClass::STAR || expr_class == ExpressionClass::COLUMN_REF) { + result += expr_str; + } else if (use_result_types) { + result += WriteResultCast(expr_str, i); + } else { + auto type = InferType(select_expr); + result += type.id() == LogicalTypeId::INVALID ? expr_str : WriteResultCast(expr_str, type); + } + if (!select_expr.GetAlias().empty()) { + result += " AS " + WriteIdentifier(select_expr.GetAlias()); + } else if (select_expr.GetExpressionClass() != ExpressionClass::STAR && + select_expr.GetExpressionClass() != ExpressionClass::COLUMN_REF) { + // name the column the way DuckDB does - Postgres names e.g. CAST(x AS INTEGER) after "x", which can + // make references to "x" (e.g. in the ORDER BY) ambiguous, or break references to the column by name + result += " AS " + PostgresUtils::WriteIdentifier(PostgresColumnAlias(select_expr.ToString(), i)); + } + } + if (node.from_table && node.from_table->type != TableReferenceType::EMPTY_FROM) { + result += " FROM " + WriteTableRef(*node.from_table); + } + if (node.where_clause) { + result += " WHERE " + WriteExpression(*node.where_clause); + } + if (!node.groups.grouping_sets.empty()) { + result += " GROUP BY "; + if (node.groups.grouping_sets.size() == 1) { + bool first = true; + for (auto &group_index : node.groups.grouping_sets[0]) { + if (!first) { + result += ", "; + } + result += WriteOrderingKey(*node.groups.group_expressions[group_index]); + first = false; + } + if (node.groups.grouping_sets[0].empty()) { + result += "()"; + } + } else { + result += "GROUPING SETS ("; + bool first_set = true; + for (auto &grouping_set : node.groups.grouping_sets) { + if (!first_set) { + result += ", "; + } + result += "("; + bool first = true; + for (auto &group_index : grouping_set) { + if (!first) { + result += ", "; + } + result += WriteOrderingKey(*node.groups.group_expressions[group_index]); + first = false; + } + result += ")"; + first_set = false; + } + result += ")"; + } + } + if (node.having) { + result += " HAVING " + WriteExpression(*node.having); + } + if (node.qualify || node.sample) { + throw InternalException("PostgresSQLWriter: QUALIFY / SAMPLE should have been blocked by SupportsPushdown"); + } + result += WriteResultModifiers(node, &node.select_list); + table_scopes.pop_back(); + return result; +} + +string PostgresSQLWriter::WriteSetOperationNode(const SetOperationNode &node) { + string result = WriteCTEMap(node.cte_map); + for (idx_t i = 0; i < node.children.size(); i++) { + if (i > 0) { + switch (node.setop_type) { + case SetOperationType::UNION: + result += node.setop_all ? " UNION ALL " : " UNION "; + break; + case SetOperationType::EXCEPT: + result += node.setop_all ? " EXCEPT ALL " : " EXCEPT "; + break; + case SetOperationType::INTERSECT: + result += node.setop_all ? " INTERSECT ALL " : " INTERSECT "; + break; + default: + throw InternalException("PostgresSQLWriter: unsupported set operation - should have been blocked by " + "SupportsPushdown"); + } + } + result += "(" + WriteQueryNode(*node.children[i]) + ")"; + } + result += WriteResultModifiers(node); + return result; +} + +string PostgresSQLWriter::WriteRecursiveCTENode(const RecursiveCTENode &node) { + string result = WriteCTEMap(node.cte_map); + result += "(" + WriteQueryNode(*node.left) + ")"; + result += node.union_all ? " UNION ALL " : " UNION "; + result += "(" + WriteQueryNode(*node.right) + ")"; + result += WriteResultModifiers(node); + return result; +} + +string PostgresSQLWriter::WriteQueryNode(const QueryNode &node) { + query_depth++; + bool has_ctes = !node.cte_map.map.empty(); + if (has_ctes) { + cte_scopes.push_back(node.cte_map); + } + string result; + switch (node.type) { + case QueryNodeType::SELECT_NODE: + result = WriteSelectNode(node.Cast()); + break; + case QueryNodeType::SET_OPERATION_NODE: + result = WriteSetOperationNode(node.Cast()); + break; + case QueryNodeType::RECURSIVE_CTE_NODE: + result = WriteRecursiveCTENode(node.Cast()); + break; + case QueryNodeType::UPDATE_QUERY_NODE: + result = WriteUpdateNode(node.Cast()); + break; + case QueryNodeType::DELETE_QUERY_NODE: + result = WriteDeleteNode(node.Cast()); + break; + case QueryNodeType::INSERT_QUERY_NODE: + result = WriteInsertNode(node.Cast()); + break; + case QueryNodeType::MERGE_QUERY_NODE: + result = WriteMergeNode(node.Cast()); + break; + default: + throw InternalException("PostgresSQLWriter: unsupported query node type - should have been blocked by " + "SupportsPushdown"); + } + if (has_ctes) { + cte_scopes.pop_back(); + } + query_depth--; + return result; +} + +} // namespace duckdb diff --git a/src/postgres_storage.cpp b/src/postgres_storage.cpp index 23a0cf1c6..a7d8779cf 100644 --- a/src/postgres_storage.cpp +++ b/src/postgres_storage.cpp @@ -1,13 +1,115 @@ #include "duckdb.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/main/config.hpp" #include "duckdb/main/settings.hpp" #include "postgres_storage.hpp" #include "storage/postgres_catalog.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" #include "storage/postgres_transaction_manager.hpp" +#include "utf8proc_wrapper.hpp" namespace duckdb { +static vector ExtractSchemas(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"SCHEMA\" option must not be null"); + } + switch (value.type().id()) { + case LogicalTypeId::VARCHAR: { + vector res; + const string &name = StringValue::Get(value); + if (name.empty()) { + throw BinderException("Value \"SCHEMA\" option must be not empty"); + } + res.push_back(name); + return res; + } + case LogicalTypeId::LIST: { + if (ListType::GetChildType(value.type()).id() != LogicalTypeId::VARCHAR) { + throw BinderException( + "Value for \"SCHEMA\" option must be either \"VARCHAR\" or \"VARCHAR[]\", was: \"%s\"", + value.type().ToString()); + } + vector res; + for (const Value &en : ListValue::GetChildren(value)) { + if (en.IsNull()) { + throw BinderException("Values for \"SCHEMA\" option must not be null"); + } + const string &name = StringValue::Get(en); + if (name.empty()) { + throw BinderException("Values \"SCHEMA\" option must be not empty"); + } + res.push_back(name); + } + return res; + } + default: + throw BinderException("Value for `SCHEMA` option must be either \"VARCHAR\" or \"VARCHAR[]\", was: \"%s\"", + value.type().ToString()); + } +} + +//! The connect display is rendered verbatim into the shell prompt, so reject anything that could take over the +//! terminal or trip up the code that renders it +static string ExtractConnectDisplay(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must not be null"); + } + // generous upper bound - this is only here to keep pathological input from making the prompt unusable, + // every realistic endpoint is far below it + const idx_t max_width = 1024; + auto display = value.ToString(); + // this has to happen before anything below inspects the string - RenderWidth walks UTF-8 sequences and + // relies on them being well formed + if (!Utf8Proc::IsValid(display.c_str(), display.size())) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must be valid UTF-8"); + } + for (idx_t i = 0; i < display.size(); i++) { + auto c = static_cast(display[i]); + // C0 control characters and DEL + bool is_control = c < 0x20 || c == 0x7f; + // C1 control characters - these are encoded as 0xC2 0x80 through 0xC2 0x9F in UTF-8 + if (c == 0xc2 && i + 1 < display.size()) { + auto next = static_cast(display[i + 1]); + is_control = is_control || (next >= 0x80 && next <= 0x9f); + } + if (is_control) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must not contain control characters - the " + "value is rendered directly into the shell prompt"); + } + } + auto width = Utf8Proc::RenderWidth(display); + if (width > max_width) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must be at most %d columns wide, was %d", max_width, + idx_t(width)); + } + return display; +} + +static PostgresTextProtocolMode ExtractTextProtocolMode(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"USE_TEXT_PROTOCOL\" option must not be null"); + } + if (value.type().id() == LogicalTypeId::BOOLEAN) { + return BooleanValue::Get(value) ? PostgresTextProtocolMode::TEXT : PostgresTextProtocolMode::BINARY; + } + auto param = value.ToString(); + auto lparam = StringUtil::Lower(param); + if (lparam == "auto") { + return PostgresTextProtocolMode::AUTO; + } + if (lparam == "true") { + return PostgresTextProtocolMode::TEXT; + } + if (lparam == "false") { + return PostgresTextProtocolMode::BINARY; + } + throw BinderException("Invalid value \"%s\" for use_text_protocol, expected TRUE, FALSE or AUTO", param); +} + static unique_ptr PostgresAttach(optional_ptr storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AttachOptions &attach_options) { @@ -18,16 +120,19 @@ static unique_ptr PostgresAttach(optional_ptr sto string attach_path = info.path; string secret_name; - string schema_to_load; + vector schemas_to_load; PostgresIsolationLevel isolation_level = PostgresIsolationLevel::REPEATABLE_READ; + PostgresTextProtocolMode text_protocol_mode = PostgresTextProtocolMode::AUTO; string secret_storage_table_name; bool secret_storage_table_specified_explicitly = false; + string connect_display; + PostgresUseCtidScan use_ctid_scan = PostgresUseCtidScan::DEFAULT; for (auto &entry : attach_options.options) { auto lower_name = StringUtil::Lower(entry.first); if (lower_name == "secret") { secret_name = entry.second.ToString(); } else if (lower_name == "schema") { - schema_to_load = entry.second.ToString(); + schemas_to_load = ExtractSchemas(entry.second); } else if (lower_name == "isolation_level") { auto param = entry.second.ToString(); auto lparam = StringUtil::Lower(param); @@ -38,22 +143,32 @@ static unique_ptr PostgresAttach(optional_ptr sto } else if (lparam == "serializable") { isolation_level = PostgresIsolationLevel::SERIALIZABLE; } else { - throw InvalidInputException("Invalid value \"%s\" for isolation_level, expected READ COMMITTED, " - "REPEATABLE READ or SERIALIZABLE", - param); + throw BinderException("Invalid value \"%s\" for isolation_level, expected READ COMMITTED, " + "REPEATABLE READ or SERIALIZABLE", + param); } + } else if (lower_name == "use_text_protocol") { + text_protocol_mode = ExtractTextProtocolMode(entry.second); } else if (lower_name == "secret_storage_table") { secret_storage_table_name = entry.second.ToString(); secret_storage_table_specified_explicitly = true; + } else if (lower_name == "connect_display") { + connect_display = ExtractConnectDisplay(entry.second); + } else if (lower_name == "use_ctid_scan") { + use_ctid_scan = BooleanValue::Get(entry.second.DefaultCastAs(LogicalType::BOOLEAN)) + ? PostgresUseCtidScan::ENABLED + : PostgresUseCtidScan::DISABLED; } else { throw BinderException("Unrecognized option for Postgres attach: %s", entry.first); } } SecretStorageTable secret_storage_table(std::move(secret_storage_table_name), secret_storage_table_specified_explicitly); - return make_uniq(context, db, std::move(attach_path), attach_options.access_mode, - std::move(schema_to_load), isolation_level, secret_name, - std::move(secret_storage_table)); + auto catalog = make_uniq( + context, db, std::move(attach_path), attach_options.access_mode, std::move(schemas_to_load), isolation_level, + secret_name, std::move(secret_storage_table), text_protocol_mode, std::move(connect_display)); + catalog->use_ctid_scan = use_ctid_scan; + return catalog; } static unique_ptr PostgresCreateTransactionManager(optional_ptr storage_info, diff --git a/src/postgres_text_reader.cpp b/src/postgres_text_reader.cpp index f1c70640f..39e0aec76 100644 --- a/src/postgres_text_reader.cpp +++ b/src/postgres_text_reader.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -9,8 +10,9 @@ namespace duckdb { PostgresTextReader::PostgresTextReader(ClientContext &context, PostgresConnection &con_p, - const vector &column_ids, const PostgresBindData &bind_data) - : PostgresResultReader(con_p, column_ids, bind_data), context(context) { + const vector &column_ids, const PostgresBindData &bind_data, + optional_ptr prepared_statement) + : PostgresResultReader(con_p, column_ids, bind_data, prepared_statement), context(context) { } PostgresTextReader::~PostgresTextReader() { @@ -18,7 +20,7 @@ PostgresTextReader::~PostgresTextReader() { } void PostgresTextReader::BeginCopy(ClientContext &context, const string &sql) { - result = con.Query(context, sql, bind_data.params); + result = Query(context, sql, 0); row_offset = 0; } @@ -275,13 +277,13 @@ void PostgresTextReader::ConvertCTID(Vector &source, Vector &target, idx_t count } } -void PostgresTextReader::ConvertBlob(Vector &source, Vector &target, idx_t count) { - // ctids have the format (page_index, row_in_page) +static void ConvertHexBlob(Vector &source, Vector &target, idx_t count, bool has_hex_prefix) { UnifiedVectorFormat vdata; source.ToUnifiedFormat(vdata); auto strings = UnifiedVectorFormat::GetData(vdata); auto result = FlatVector::GetDataMutable(target); + idx_t prefix_size = has_hex_prefix ? 2 : 0; for (idx_t i = 0; i < count; i++) { if (!vdata.validity.RowIsValid(i)) { // NULL value - skip @@ -291,14 +293,14 @@ void PostgresTextReader::ConvertBlob(Vector &source, Vector &target, idx_t count auto blob_str = strings[i]; auto str = blob_str.GetData(); auto size = blob_str.GetSize(); - if (size < 2 || str[0] != '\\' || str[1] != 'x') { + if (has_hex_prefix && (size < 2 || str[0] != '\\' || str[1] != 'x')) { throw InvalidInputException("Incorrect blob format - expected \\x... for blob"); } if (size % 2 != 0) { throw InvalidInputException("Blob size must be modulo 2 (\\xAA)"); } string result_blob; - for (idx_t i = 2; i < size; i += 2) { + for (idx_t i = prefix_size; i < size; i += 2) { int byte_a = Blob::HEX_MAP[static_cast(str[i])]; int byte_b = Blob::HEX_MAP[static_cast(str[i + 1])]; result_blob += UnsafeNumericCast((byte_a << 4) + byte_b); @@ -307,6 +309,11 @@ void PostgresTextReader::ConvertBlob(Vector &source, Vector &target, idx_t count } } +void PostgresTextReader::ConvertBlob(Vector &source, Vector &target, idx_t count) { + // bytea in the hex format: \x0123... + ConvertHexBlob(source, target, count, true); +} + static void ConvertGeometry(Vector &source, Vector &target, idx_t count) { // Geometry is encoded in HEXWKB format @@ -361,6 +368,10 @@ void PostgresTextReader::ConvertVector(Vector &source, Vector &target, const Pos ConvertCTID(source, target, count); return; } + if (postgres_type.info == PostgresTypeAnnotation::HEX_BLOB) { + ConvertHexBlob(source, target, count, false); + return; + } switch (target.GetType().id()) { case LogicalTypeId::LIST: ConvertList(source, target, postgres_type, count); @@ -392,8 +403,8 @@ PostgresReadResult PostgresTextReader::Read(DataChunk &output) { scan_chunk.Initialize(context, types); } scan_chunk.Reset(); - for (; scan_chunk.size() < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++) { - idx_t output_offset = scan_chunk.size(); + idx_t output_offset = 0; + for (; output_offset < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++, output_offset++) { for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { auto col_idx = column_ids[output_idx]; auto &out_vec = scan_chunk.data[output_idx]; @@ -405,8 +416,8 @@ PostgresReadResult PostgresTextReader::Read(DataChunk &output) { col_data[output_offset] = StringVector::AddStringOrBlob(out_vec, result->GetStringRef(row_offset, output_idx)); } - scan_chunk.SetChildCardinality(scan_chunk.size() + 1); } + scan_chunk.SetChildCardinality(output_offset); for (idx_t c = 0; c < output.ColumnCount(); c++) { auto col_idx = column_ids[c]; if (col_idx == COLUMN_IDENTIFIER_ROW_ID) { @@ -414,7 +425,7 @@ PostgresReadResult PostgresTextReader::Read(DataChunk &output) { ctid_type.info = PostgresTypeAnnotation::CTID; ConvertVector(scan_chunk.data[c], output.data[c], ctid_type, scan_chunk.size()); } else { - ConvertVector(scan_chunk.data[c], output.data[c], bind_data.postgres_types[c], scan_chunk.size()); + ConvertVector(scan_chunk.data[c], output.data[c], bind_data.postgres_types[col_idx], scan_chunk.size()); } } output.SetChildCardinality(scan_chunk.size()); diff --git a/src/postgres_utils.cpp b/src/postgres_utils.cpp index e60bccd4a..0fcecce69 100644 --- a/src/postgres_utils.cpp +++ b/src/postgres_utils.cpp @@ -1,31 +1,92 @@ #include "postgres_utils.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "dbconnector/query/query_writer.hpp" +#include "storage/postgres_catalog.hpp" #include "storage/postgres_schema_entry.hpp" #include "storage/postgres_transaction.hpp" +#include "postgres_scanner.hpp" #include "postgres_type_oids.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/geometry_crs.hpp" namespace duckdb { +mutex PostgresUtils::libpq_init_lock; +bool PostgresUtils::libpq_connect_succeeded_at_least_once = false; + static void PGNoticeProcessor(void *arg, const char *message) { } PGconn *PostgresUtils::PGConnect(const string &dsn, const string &attach_path) { + bool lock_needed = true; + { + lock_guard guard(libpq_init_lock); + if (libpq_connect_succeeded_at_least_once) { + lock_needed = false; + } + } + + if (lock_needed) { + lock_guard guard(libpq_init_lock); + idx_t max_attempts = 2; + for (idx_t i = 0; i < max_attempts; i++) { // observed sporadic OpenSSL failures on the first connection + PostgresConnectCheck check = i < max_attempts - 1 ? PostgresConnectCheck::RETURN_ON_CONNECTION_BAD + : PostgresConnectCheck::THROW_ON_CONNECTION_BAD; + PGconn *conn = PGConnectInternal(dsn, attach_path, check); + if (PQstatus(conn) == CONNECTION_BAD) { + PQfinish(conn); + continue; + } + libpq_connect_succeeded_at_least_once = true; + return conn; + } + } + + return PGConnectInternal(dsn, attach_path, PostgresConnectCheck::THROW_ON_CONNECTION_BAD); +} + +PGconn *PostgresUtils::PGConnectInternal(const string &dsn, const string &attach_path, + PostgresConnectCheck check_behaviour) { PGconn *conn = PQconnectdb(dsn.c_str()); // both PQStatus and PQerrorMessage check for nullptr if (PQstatus(conn) == CONNECTION_BAD) { - char *msg_cstr = PQerrorMessage(conn); - string msg = msg_cstr != nullptr ? string(msg_cstr) : string(); - PQfinish(conn); - throw IOException("Unable TODO:REMOVEME to connect to Postgres at \"%s\": %s", attach_path, msg); + if (check_behaviour == PostgresConnectCheck::THROW_ON_CONNECTION_BAD) { + char *msg_cstr = PQerrorMessage(conn); + string msg = msg_cstr != nullptr ? string(msg_cstr) : string(); + PQfinish(conn); + throw IOException("Unable to connect to Postgres at \"%s\": %s", attach_path, msg); + } else { + return conn; + } } PQsetNoticeProcessor(conn, PGNoticeProcessor, nullptr); return conn; } string PostgresUtils::TypeToString(const LogicalType &input) { + // Handle GEOMETRY('EPSG:N') first: the CRS-bearing GEOMETRY type can have + // an alias of "GEOMETRY", which would otherwise short-circuit the alias + // branch below and emit an untyped `geometry` column (typmod -1). When + // CRS is present, emit a typmod-bearing PostGIS column type so the SRID + // survives a CREATE TABLE round trip; otherwise fall through. + if (input.id() == LogicalTypeId::GEOMETRY && GeoType::HasCRS(input)) { + auto &crs = GeoType::GetCRS(input); + auto &id = crs.GetIdentifier(); + auto colon = id.find(':'); + if (colon != string::npos) { + try { + int32_t srid = std::stoi(id.substr(colon + 1)); + if (srid > 0) { + return "geometry(Geometry, " + std::to_string(srid) + ")"; + } + } catch (...) { + // fall through to plain GEOMETRY + } + } + } if (input.HasAlias()) { return input.GetAlias(); } @@ -78,6 +139,77 @@ LogicalType PostgresUtils::RemoveAlias(const LogicalType &type) { } } +bool PostgresUtils::UseInformationSchemaIntrospection(optional_ptr context) { + if (!context) { + return false; + } + Value value; + if (context->TryGetCurrentSetting("pg_use_information_schema_introspection", value)) { + return BooleanValue::Get(value); + } + return false; +} + +bool PostgresUtils::StalenessQueryEnabled(ClientContext &context) { + Value value; + if (context.TryGetCurrentSetting("pg_staleness_query_enabled", value) && !value.IsNull()) { + return BooleanValue::Get(value); + } + return !UseInformationSchemaIntrospection(&context); +} + +string PostgresUtils::GetCustomStalenessQuery(ClientContext &context) { + Value value; + if (context.TryGetCurrentSetting("pg_staleness_query", value) && !value.IsNull()) { + return StringValue::Get(value); + } + return string(); +} + +string PostgresUtils::DataTypeToTypeName(const string &data_type) { + // Map SQL-standard information_schema.columns.data_type values to the + // pg_type.typname + auto type_name = StringUtil::Lower(data_type); + if (type_name == "boolean") { + return "bool"; + } else if (type_name == "smallint") { + return "int2"; + } else if (type_name == "integer") { + return "int4"; + } else if (type_name == "bigint") { + return "int8"; + } else if (type_name == "real") { + return "float4"; + } else if (type_name == "double precision") { + return "float8"; + } else if (type_name == "character varying") { + return "varchar"; + } else if (type_name == "character") { + return "bpchar"; + } else if (type_name == "\"char\"") { + return "char"; + } else if (type_name == "time without time zone") { + return "time"; + } else if (type_name == "time with time zone") { + return "timetz"; + } else if (type_name == "timestamp without time zone") { + return "timestamp"; + } else if (type_name == "timestamp with time zone") { + return "timestamptz"; + } else if (type_name == "bit varying") { + return "varbit"; + } else if (type_name == "binary varying") { + // Redshift VARBYTE as reported by pg_get_late_binding_view_cols() + return "varbyte"; + } else if (type_name == "interval year to month") { + // Redshift interval types as reported by pg_get_late_binding_view_cols() + return "intervaly2m"; + } else if (type_name == "interval day to second") { + return "intervald2s"; + } + return data_type; +} + uint32_t PostgresUtils::TypeNameToPostgresOid(const string &type_name) { if (type_name == "bool") { return BOOLOID; @@ -117,25 +249,75 @@ uint32_t PostgresUtils::TypeNameToPostgresOid(const string &type_name) { return 0; } +PostgresTypeConfig PostgresTypeConfig::FromContext(optional_ptr context) { + PostgresTypeConfig result; + if (!context) { + return result; + } + Value setting; + if (context->TryGetCurrentSetting("pg_array_as_varchar", setting)) { + result.array_as_varchar = BooleanValue::Get(setting); + } + if (context->TryGetCurrentSetting("pg_numeric_as_varchar", setting)) { + result.numeric_as_varchar = BooleanValue::Get(setting); + } + if (context->TryGetCurrentSetting("pg_numeric_nan_as_null", setting)) { + result.numeric_nan_as_null = BooleanValue::Get(setting); + } + return result; +} + +//! Redshift-specific types, all of them are read using the text protocol (Redshift does not support binary COPY) +static bool TryRedshiftTypeToLogicalType(const PostgresTypeData &type_info, PostgresType &postgres_type, + LogicalType &result) { + auto &pgtypename = type_info.type_name; + if (pgtypename == "geometry") { + // the Redshift type modifier does not follow the PostGIS layout and does not encode the SRID, + // the text output is hex EWKB + result = LogicalType::GEOMETRY(); + } else if (pgtypename == "geography") { + // the text output is hex EWKB, geography values are always in WGS 84 + result = LogicalType::GEOMETRY("EPSG:4326"); + } else if (pgtypename == "intervaly2m") { + postgres_type.info = PostgresTypeAnnotation::INTERVAL_YEAR_TO_MONTH; + result = LogicalType::INTERVAL; + } else if (pgtypename == "intervald2s") { + postgres_type.info = PostgresTypeAnnotation::INTERVAL_DAY_TO_SECOND; + result = LogicalType::INTERVAL; + } else if (pgtypename == "varbyte") { + postgres_type.info = PostgresTypeAnnotation::HEX_BLOB; + result = LogicalType::BLOB; + } else if (pgtypename == "super") { + // the output is JSON - SUPER must not be cast to VARCHAR, SUPER::VARCHAR returns NULL for non-string values + result = LogicalType::JSON(); + } else if (pgtypename == "hllsketch") { + // the text output is JSON, HLLSKETCH cannot be cast to VARCHAR and has no binary output + postgres_type.info = PostgresTypeAnnotation::TEXT_PROTOCOL_ONLY; + result = LogicalType::JSON(); + } else { + return false; + } + return true; +} + LogicalType PostgresUtils::TypeToLogicalType(optional_ptr transaction, optional_ptr schema, - const PostgresTypeData &type_info, PostgresType &postgres_type) { + const PostgresTypeConfig &type_config, const PostgresTypeData &type_info, + PostgresType &postgres_type) { auto &pgtypename = type_info.type_name; + if (type_info.instance_type == PostgresInstanceType::REDSHIFT) { + LogicalType result; + if (TryRedshiftTypeToLogicalType(type_info, postgres_type, result)) { + return result; + } + } + // postgres array types start with an _ if (StringUtil::StartsWith(pgtypename, "_")) { - if (transaction) { - auto context = transaction->GetContext(); - if (!context) { - throw InternalException("Context is destroyed!?"); - } - Value array_as_varchar; - if (context->TryGetCurrentSetting("pg_array_as_varchar", array_as_varchar)) { - if (BooleanValue::Get(array_as_varchar)) { - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; - } - } + if (type_config.array_as_varchar) { + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; } // get the array dimension information idx_t dimensions = type_info.array_dimensions; @@ -148,7 +330,8 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t child_type_info.type_modifier = type_info.type_modifier; child_type_info.type_schema = type_info.type_schema; PostgresType child_pg_type; - auto child_type = PostgresUtils::TypeToLogicalType(transaction, schema, child_type_info, child_pg_type); + auto child_type = + PostgresUtils::TypeToLogicalType(transaction, schema, type_config, child_type_info, child_pg_type); // populate the child OID from the actual Postgres type name if (child_pg_type.oid == 0) { child_pg_type.oid = TypeNameToPostgresOid(child_type_info.type_name); @@ -183,7 +366,10 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t auto width = ((type_info.type_modifier - sizeof(int32_t)) >> 16) & 0xffff; auto scale = (((type_info.type_modifier - sizeof(int32_t)) & 0x7ff) ^ 1024) - 1024; if (type_info.type_modifier == -1 || width < 0 || scale < 0 || width > 38) { - // fallback to double + if (type_config.numeric_as_varchar) { + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; + } postgres_type.info = PostgresTypeAnnotation::NUMERIC_AS_DOUBLE; return LogicalType::DOUBLE; } @@ -197,6 +383,20 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t postgres_type.info = PostgresTypeAnnotation::JSONB; return LogicalType::VARCHAR; } else if (pgtypename == "geometry") { + // PostGIS encodes the column-level SRID in the type modifier of + // `geometry(TYPE, SRID)` columns. The bit layout is: + // bits 0 : has-M + // bits 1 : has-Z + // bits 2-7 : geometry type code (POINT=1, LINESTRING=2, ...) + // bits 8-29 : SRID + // Untyped `geometry` columns carry typmod -1 and we fall back to + // plain GEOMETRY (no CRS), matching the previous behavior. + if (type_info.type_modifier > 0) { + int32_t srid = static_cast((static_cast(type_info.type_modifier) & 0x0FFFFF00u) >> 8); + if (srid > 0) { + return LogicalType::GEOMETRY("EPSG:" + std::to_string(srid)); + } + } return LogicalType::GEOMETRY(); } else if (pgtypename == "date") { return LogicalType::DATE; @@ -238,34 +438,44 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t } else if (pgtypename == "circle") { postgres_type.info = PostgresTypeAnnotation::GEOM_CIRCLE; return LogicalType::LIST(LogicalType::DOUBLE); + } + + if (!transaction || !schema || type_info.type_schema.empty()) { + // unsupported, or no type schema to resolve custom types in + // (information_schema introspection) - fallback to varchar + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; + } + + auto context = transaction->GetContext(); + if (!context) { + throw InternalException("Context is destroyed!?"); + } + + Identifier type_schema = Identifier(type_info.type_schema); + optional_ptr lookup_schema; + if (type_schema == schema->name) { + lookup_schema = schema.get(); } else { - if (!transaction) { - // unsupported so fallback to varchar - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; - } - auto context = transaction->GetContext(); - if (!context) { - throw InternalException("Context is destroyed!?"); - } - optional_ptr lookup_schema = - Identifier(type_info.type_schema) != schema->name - ? schema->ParentCatalog().GetSchema(*context, Identifier(type_info.type_schema), - OnEntryNotFound::THROW_EXCEPTION) - : schema.get(); + Catalog &parent_catalog = schema->ParentCatalog(); + lookup_schema = parent_catalog.GetSchema(*context, type_schema, OnEntryNotFound::RETURN_NULL); + } + + if (lookup_schema) { auto entry = lookup_schema->GetEntry(CatalogTransaction(lookup_schema->ParentCatalog(), *context), CatalogType::TYPE_ENTRY, Identifier(pgtypename)); - if (!entry) { - // unsupported so fallback to varchar - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; + if (entry) { + // custom type (e.g. composite or enum) + auto &type_entry = entry->Cast(); + auto result_type = RemoveAlias(type_entry.user_type); + postgres_type = type_entry.postgres_type; + return result_type; } - // custom type (e.g. composite or enum) - auto &type_entry = entry->Cast(); - auto result_type = RemoveAlias(type_entry.user_type); - postgres_type = type_entry.postgres_type; - return result_type; } + + // either schema or type is not available so fallback to varchar + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; } LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { @@ -297,9 +507,7 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { auto &type = StructType::GetChildType(input, c); new_types.push_back(make_pair(name, ToPostgresType(type))); } - auto result = LogicalType::STRUCT(std::move(new_types)); - result.SetAlias(input.GetAlias()); - return result; + return LogicalType::STRUCT(std::move(new_types)).WithAlias(input.GetAlias()); } case LogicalTypeId::TIMESTAMP_SEC: case LogicalTypeId::TIMESTAMP_MS: @@ -316,7 +524,11 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { case LogicalTypeId::HUGEINT: return LogicalType::DOUBLE; case LogicalTypeId::GEOMETRY: - return LogicalType::GEOMETRY(); + // Preserve CRS metadata so downstream TypeToString can emit a typmod- + // bearing PostGIS column type and the SRID survives a CREATE TABLE + // AS round trip. Returning a CRS-less GEOMETRY here strips the CRS + // before it ever reaches the schema-emission path. + return input; default: return LogicalType::VARCHAR; } @@ -369,6 +581,29 @@ bool PostgresUtils::SupportedPostgresOid(const LogicalType &input) { } } +string PostgresUtils::RedshiftOidToName(uint32_t oid) { + switch (oid) { + case 1188: + return "intervaly2m"; + case 1190: + return "intervald2s"; + case 2935: + return "hllsketch"; + case 3000: + case 3999: + // query results report GEOMETRY columns with OID 3999 + return "geometry"; + case 3001: + return "geography"; + case 4000: + return "super"; + case 6551: + return "varbyte"; + default: + return string(); + } +} + string PostgresUtils::PostgresOidToName(uint32_t oid) { switch (oid) { case BOOLOID: @@ -549,36 +784,92 @@ string PostgresUtils::QuotePostgresIdentifier(const string &text) { return PostgresUtils::WriteIdentifier(text); } -string PostgresUtils::EscapeConnectionString(const string &input) { - using namespace dbconnector::query; - auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::BACKSLASH); - return QueryWriter::WriteQuotedAndEscaped(config, input); -} - -string PostgresUtils::ExtractConnectionOption(const KeyValueSecret &kv_secret, const string &name) { - Value input_val = kv_secret.TryGetValue(Identifier(name)); - if (input_val.IsNull()) { - // not provided - return string(); - } - string result; - result += name; - result += "="; - result += EscapeConnectionString(input_val.ToString()); - result += " "; - return result; -} - string PostgresUtils::WriteLiteral(const string &literal) { using namespace dbconnector::query; auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::DOUBLE_QUOTE); return QueryWriter::WriteQuotedAndEscaped(config, literal); } +string PostgresUtils::WriteLiteralsCommaSeparated(const vector &literals) { + string res; + for (idx_t i = 0; i < literals.size(); i++) { + const string &lit = literals[i]; + if (i > 0) { + res.append(", "); + } + res.append(WriteLiteral(lit)); + } + return res; +} + string PostgresUtils::WriteIdentifier(const string &identifier) { using namespace dbconnector::query; auto config = QueryWriter::CreateConfig('"', QuoteEscapeStyle::DOUBLE_QUOTE); return QueryWriter::WriteQuotedAndEscaped(config, identifier); } +PostgresQueryMode PostgresUtils::ExtractQueryMode(ClientContext &ctx) { + Value val; + if (ctx.TryGetCurrentSetting("pg_query_mode", val)) { + return ExtractQueryMode(val); + } + return PostgresQueryMode::AUTO; +} + +PostgresQueryMode PostgresUtils::ExtractQueryMode(Value &value) { + if (value.IsNull()) { + throw InvalidInputException("Invalid NULL query mode specified"); + } + string str_raw = value.ToString(); + string str = StringUtil::Upper(str_raw); + if (str == "AUTO") { + return PostgresQueryMode::AUTO; + } else if (str == "COPY_BINARY") { + return PostgresQueryMode::COPY_BINARY; + + } else if (str == "PLAIN_QUERY_BINARY") { + return PostgresQueryMode::PLAIN_QUERY_BINARY; + + } else if (str == "PLAIN_QUERY_TEXT") { + return PostgresQueryMode::PLAIN_QUERY_TEXT; + + } else { + throw InvalidInputException("Invalid query mode specified, must be on of: 'AUTO', 'COPY_BINARY', " + "'PLAIN_QUERY_BINARY', 'PLAIN_QUERY_TEXT'"); + } +} + +PostgresCatalog &PostgresUtils::GetCatalogByName(ClientContext &context, const std::string &name) { + auto &db_manager = DatabaseManager::Get(context); + auto db = db_manager.GetDatabase(context, Identifier(name)); + if (!db) { + throw BinderException("Failed to find attached database \"%s\"", name); + } + auto &catalog = db->GetCatalog(); + if (catalog.GetCatalogType() != "postgres") { + throw BinderException("Attached database \"%s\" does not refer to a Postgres database", name); + } + return catalog.Cast(); +} + +void PostgresUtils::DeduplicateColumns(vector &names) { + identifier_map_t name_map; + for (auto &column_name : names) { + if (name_map.find(column_name) == name_map.end()) { + // Name does not exist yet + name_map[column_name]++; + } else { + // Name already exists, we add _x where x is the repetition number + Identifier new_column_name(column_name + "_" + std::to_string(name_map[column_name])); + while (name_map.find(new_column_name) != name_map.end()) { + // This name is already here due to a previous definition + name_map[column_name]++; + new_column_name = Identifier(column_name + "_" + std::to_string(name_map[column_name])); + } + column_name = new_column_name; + name_map[new_column_name]++; + } + } +} + } // namespace duckdb diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 24c903a85..fb31bac63 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -1,10 +1,10 @@ add_library( postgres_ext_storage OBJECT + postgres_capabilities.cpp postgres_catalog.cpp postgres_catalog_set.cpp postgres_connection_pool.cpp postgres_clear_cache.cpp - postgres_configure_pool.cpp postgres_delete.cpp postgres_index.cpp postgres_index_entry.cpp @@ -18,7 +18,6 @@ add_library( postgres_table_entry.cpp postgres_table_set.cpp postgres_transaction.cpp - postgres_transaction_manager.cpp postgres_type_entry.cpp postgres_type_set.cpp postgres_update.cpp) diff --git a/src/storage/postgres_capabilities.cpp b/src/storage/postgres_capabilities.cpp new file mode 100644 index 000000000..d62ec4810 --- /dev/null +++ b/src/storage/postgres_capabilities.cpp @@ -0,0 +1,1882 @@ +#include "storage/postgres_capabilities.hpp" + +#include "duckdb.hpp" +#include "duckdb/catalog/default/default_types.hpp" +#include "duckdb/common/logical_type_info.hpp" +#include "duckdb/common/types/decimal.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/query_node/delete_query_node.hpp" +#include "duckdb/parser/query_node/insert_query_node.hpp" +#include "duckdb/parser/query_node/merge_query_node.hpp" +#include "duckdb/parser/statement/merge_into_statement.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/query_node/update_query_node.hpp" +#include "duckdb/parser/statement/alter_statement.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/parsed_data/alter_table_info.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/statement/update_statement.hpp" +#include "duckdb/parser/result_modifier.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/expressionlistref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" + +namespace duckdb { + +//===--------------------------------------------------------------------===// +// Helpers +//===--------------------------------------------------------------------===// +//! Get the value of a constant - folded constants can also be represented as a cast of a literal +bool PostgresCapabilities::TryGetConstantValue(const ParsedExpression &expr, Value &result) { + if (expr.GetExpressionClass() == ExpressionClass::CONSTANT) { + result = expr.Cast().GetLiteral().ToValue(); + return true; + } + if (expr.GetExpressionClass() == ExpressionClass::CAST) { + auto &cast_expr = expr.Cast(); + if (cast_expr.IsTryCast() || cast_expr.Child().GetExpressionClass() != ExpressionClass::CONSTANT) { + return false; + } + auto target_type = PostgresCapabilities::ResolveType(cast_expr.TargetType()); + if (target_type.id() == LogicalTypeId::INVALID) { + return false; + } + auto value = cast_expr.Child().Cast().GetLiteral().ToValue(); + auto cast_result = value.DefaultTryCastAs(target_type); + if (!cast_result) { + return false; + } + result = std::move(*cast_result); + return true; + } + return false; +} + +static bool PostgresTryGetConstantValue(const ParsedExpression &expr, Value &result) { + return PostgresCapabilities::TryGetConstantValue(expr, result); +} + +static bool PostgresTryGetIntegerLiteral(const ParsedExpression &expr, int64_t &result) { + Value val; + if (!PostgresTryGetConstantValue(expr, val)) { + return false; + } + if (!val.type().IsIntegral() || val.IsNull()) { + return false; + } + auto bigint_value = val.DefaultTryCastAs(LogicalType::BIGINT); + if (!bigint_value || bigint_value->IsNull()) { + return false; + } + result = BigIntValue::Get(*bigint_value); + return true; +} + +//! Whether an expression is an integer literal - in ORDER BY / GROUP BY these are positional references +static bool PostgresIsIntegerLiteral(const ParsedExpression &expr) { + int64_t value; + return expr.GetExpressionClass() == ExpressionClass::CONSTANT && PostgresTryGetIntegerLiteral(expr, value); +} + +static bool PostgresIsIntegerLiteralAtLeast(const ParsedExpression &expr, int64_t minimum_value) { + int64_t value; + return PostgresTryGetIntegerLiteral(expr, value) && value >= minimum_value; +} + +static bool PostgresIsStringLiteral(const ParsedExpression &expr) { + Value val; + if (!PostgresTryGetConstantValue(expr, val)) { + return false; + } + return val.type().id() == LogicalTypeId::VARCHAR && !val.IsNull(); +} + +static bool PostgresSupportsValue(const Value &value) { + return PostgresCapabilities::SupportsValue(value); +} + +//! Years outside of [1, 9999] are formatted differently by DuckDB (negative years) and Postgres (BC) +static bool PostgresSupportsYear(int32_t year) { + return year >= 1 && year <= 9999; +} + +bool PostgresCapabilities::SupportsValue(const Value &value) { + if (value.IsNull()) { + return true; + } + switch (value.type().id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::UHUGEINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::TIME: + case LogicalTypeId::INTERVAL: + case LogicalTypeId::UUID: + return true; + case LogicalTypeId::VARCHAR: + case LogicalTypeId::BLOB: { + // Postgres text values cannot hold NUL bytes + auto &str = StringValue::Get(value); + return value.type().id() == LogicalTypeId::BLOB || str.find('\0') == string::npos; + } + case LogicalTypeId::DATE: { + auto date = DateValue::Get(value); + if (!Value::IsFinite(date)) { + return true; + } + return PostgresSupportsYear(Date::ExtractYear(date)); + } + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: { + auto ts = value.GetValue(); + if (!Value::IsFinite(ts)) { + return true; + } + return PostgresSupportsYear(Date::ExtractYear(Timestamp::GetDate(ts))); + } + default: + // nested types, enums, ... are not supported + return false; + } +} + +LogicalType PostgresCapabilities::ResolveType(const TypeExpression &type_expr) { + if (!type_expr.GetCatalog().empty() || !type_expr.GetSchema().empty()) { + // a qualified (user-defined) type + return LogicalType::INVALID; + } + auto type_id = DefaultTypeGenerator::GetDefaultType(type_expr.GetTypeName()); + auto &children = type_expr.GetChildren(); + switch (type_id) { + case LogicalTypeId::INVALID: + return LogicalType::INVALID; + case LogicalTypeId::DECIMAL: { + int64_t width = 18; + int64_t scale = 3; + if (children.size() > 2) { + return LogicalType::INVALID; + } + if (!children.empty() && !PostgresTryGetIntegerLiteral(*children[0], width)) { + return LogicalType::INVALID; + } + if (children.size() > 1 && !PostgresTryGetIntegerLiteral(*children[1], scale)) { + return LogicalType::INVALID; + } + if (children.size() == 1) { + scale = 0; + } + if (width < 1 || width > Decimal::MAX_WIDTH_DECIMAL || scale < 0 || scale > width) { + return LogicalType::INVALID; + } + return LogicalType::DECIMAL(NumericCast(width), NumericCast(scale)); + } + case LogicalTypeId::VARCHAR: + // VARCHAR(n) does not truncate in DuckDB - the length modifier is ignored + for (auto &child : children) { + if (!PostgresIsIntegerLiteral(*child)) { + return LogicalType::INVALID; + } + } + return LogicalType::VARCHAR; + default: + if (!children.empty()) { + return LogicalType::INVALID; + } + return LogicalType(type_id); + } +} + +//! Whether a function expression is a call to the built-in function with the given name, qualified with the +//! system catalog (as produced by ConstantExpression::FromValue) +static bool PostgresIsSystemFunction(const ParsedExpression &expr, const char *name, idx_t argument_count) { + if (expr.GetExpressionClass() != ExpressionClass::FUNCTION) { + return false; + } + auto &func = expr.Cast(); + auto &qualified_name = func.GetQualifiedName(); + if (qualified_name.Catalog() != Identifier::SystemCatalog() || qualified_name.Schema() != DEFAULT_SCHEMA || + qualified_name.Name() != name) { + return false; + } + return func.GetArguments().size() == argument_count && !func.Distinct() && !func.Filter() && + !func.GetArguments()[0].HasName(); +} + +static bool PostgresTryGetIntervalPart(const ParsedExpression &expr, const char *name, int64_t &result) { + if (!PostgresIsSystemFunction(expr, name, 1)) { + return false; + } + return PostgresTryGetIntegerLiteral(expr.Cast().GetArguments()[0].GetExpression(), result); +} + +bool PostgresCapabilities::TryGetIntervalLiteral(const ParsedExpression &expr, interval_t &result) { + if (!PostgresIsSystemFunction(expr, "add", 2)) { + return false; + } + auto &arguments = expr.Cast().GetArguments(); + auto &months_and_days = arguments[0].GetExpression(); + if (!PostgresIsSystemFunction(months_and_days, "add", 2)) { + return false; + } + auto &parts = months_and_days.Cast().GetArguments(); + int64_t months, days, micros; + if (!PostgresTryGetIntervalPart(parts[0].GetExpression(), "to_months", months) || + !PostgresTryGetIntervalPart(parts[1].GetExpression(), "to_days", days) || + !PostgresTryGetIntervalPart(arguments[1].GetExpression(), "to_microseconds", micros)) { + return false; + } + result.months = NumericCast(months); + result.days = NumericCast(days); + result.micros = micros; + return true; +} + +bool PostgresCapabilities::SupportsCastType(const LogicalType &type) { + // whether a cast is supported also depends on the source type - which is checked when serializing + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::VARCHAR: + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::UUID: + return true; + default: + // Postgres has no unsigned / TINYINT types, intervals are formatted differently, ... + return false; + } +} + +//===--------------------------------------------------------------------===// +// Functions +//===--------------------------------------------------------------------===// +//! Functions whose result depends on the string ordering +static bool PostgresIsOrderSensitiveFunction(const string &name) { + static const case_insensitive_set_t ORDER_SENSITIVE = {"min", "max", "greatest", "least", + "median", "quantile", "quantile_disc", "quantile_cont", + "arg_min", "arg_max", "min_by", "max_by"}; + return ORDER_SENSITIVE.count(name) > 0; +} + +//! Aggregate functions that can be pushed into Postgres +//! Aggregate functions that can be pushed into Postgres - also as window functions +static const case_insensitive_set_t &PostgresAggregateFunctions() { + // "count_star" - serialized as count(*) + // "stddev"/"variance" - sample variants, identical in Postgres + // "string_agg"/"group_concat"/"listagg" - serialized as string_agg with an explicit separator + // "list" - serialized as array_agg + // "corr"/"covar_*"/"regr_*" - note: Postgres uses different algorithms, results can differ in the + // last digits of floating point precision + // Excluded: + // "mode" - ties are broken differently + // "first"/"last" - depend on the (undefined) order of rows + // "product"/"entropy"/"kurtosis"/"skewness"/"approx_*"/"histogram" - not available in Postgres + static const case_insensitive_set_t AGGREGATES = { + "count", "count_star", "sum", "avg", "min", "max", "stddev", + "variance", "stddev_pop", "stddev_samp", "var_pop", "var_samp", "bool_and", "bool_or", + "string_agg", "group_concat", "listagg", "array_agg", "list", "bit_and", "bit_or", + "bit_xor", "corr", "covar_pop", "covar_samp", "regr_avgx", "regr_avgy", "regr_count", + "regr_intercept", "regr_r2", "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", + }; + return AGGREGATES; +} + +//! Aggregate functions that are rewritten into an expression, and can therefore not be used as window functions +static const case_insensitive_set_t &PostgresRewrittenAggregateFunctions() { + // "count_if" - serialized as count(CASE WHEN x THEN 1 END) + // "median"/"quantile_*" - serialized as the percentile_cont / percentile_disc ordered-set aggregates + // "arg_min"/"arg_max" - serialized as the first element of an ordered array_agg + // "any_value" - available from Postgres 16 + static const case_insensitive_set_t AGGREGATES = { + "count_if", "countif", "median", "quantile", "quantile_cont", "quantile_disc", + "arg_min", "arg_max", "min_by", "max_by", "any_value", + }; + return AGGREGATES; +} + +static bool PostgresIsAggregateFunction(const string &name) { + return PostgresAggregateFunctions().count(name) > 0 || PostgresRewrittenAggregateFunctions().count(name) > 0; +} + +bool PostgresCapabilities::SupportsFunction(const FunctionExpression &func) { + auto &function_name = func.FunctionName().GetIdentifierName(); + // Scalar functions that can be pushed into Postgres - either because name and semantics are + // identical, or because PostgresSQLWriter rewrites them during serialization: + // "/" - DuckDB's division always yields a DOUBLE, serialized with a float8 cast + // "~~"/"!~~" - [NOT] LIKE, serialized with an empty ESCAPE clause (Postgres' default escape + // character is backslash, DuckDB's LIKE has no default escape character) + // "len" - serialized as length + // "strlen" - serialized as octet_length + // "instr"/"position" - serialized as strpos + // "prefix" - serialized as starts_with + // "ifnull" - serialized as coalesce + // "year"/"month"/... - serialized as CAST(EXTRACT( FROM x) AS BIGINT) + // Excluded (verified mismatches that cannot be fixed by rewriting): + // "//" - DuckDB's integer division on doubles does not floor + // "sqrt"/"ln"/"log"/"log2"/"log10"/"asin"/"acos"/"exp"/"pow"/"power"/"cot" - Postgres raises + // errors on domain / range errors where DuckDB yields nan / inf + // "round" - Postgres rounds doubles half-to-even, DuckDB half away from zero + // "reverse" - DuckDB reverses grapheme clusters, Postgres code points + // "regexp_*" - different regex dialects (RE2 vs Postgres ARE) + static const case_insensitive_set_t SCALAR_FUNCTIONS = { + // operators + "+", + "-", + "*", + "/", + "%", + "||", + "~~", + "!~~", + // string + "concat", + "concat_ws", + "length", + "len", + "char_length", + "character_length", + "strlen", + "octet_length", + "bit_length", + "trim", + "ltrim", + "rtrim", + "replace", + "left", + "right", + "lpad", + "rpad", + "substring", + "substr", + "strpos", + "instr", + "position", + "starts_with", + "prefix", + "md5", + "ascii", + "unicode", + "chr", + "repeat", + "translate", + "contains", + "ends_with", + "suffix", + "^@", + "sha256", + "lower", + "lcase", + "~~*", + "!~~*", + // math + "abs", + "ceil", + "ceiling", + "floor", + "trunc", + "sign", + "cbrt", + "sin", + "cos", + "tan", + "atan", + "atan2", + "degrees", + "radians", + "pi", + "sqrt", + "ln", + "log", + "log10", + "exp", + "pow", + "power", + "^", + "**", + "isnan", + "isinf", + "isfinite", + "gcd", + "lcm", + "greatest_common_divisor", + "least_common_multiple", + "xor", + "&", + "|", + "~", + "round", + "round_even", + "roundbankers", + // date / time + "year", + "month", + "day", + "dayofmonth", + "hour", + "minute", + "quarter", + "dayofweek", + "isodow", + "dayofyear", + "week", + "weekofyear", + "second", + "millisecond", + "microsecond", + "isoyear", + "decade", + "century", + "millennium", + "epoch", + "epoch_ms", + "date_part", + "datepart", + "date_trunc", + "datetrunc", + "date_diff", + "datediff", + "last_day", + "make_date", + "to_timestamp", + // grouping sets + "grouping", + "grouping_id", + // conditional + "coalesce", + "nullif", + "ifnull", + "greatest", + "least", + }; + if (PostgresIsOrderSensitiveFunction(function_name) && StringOrderingDiffers()) { + // the result depends on the string ordering, which differs from DuckDB's for this database + return false; + } + static const case_insensitive_set_t CASE_MAPPING_FUNCTIONS = {"lower", "lcase", "~~*", "!~~*"}; + if (CASE_MAPPING_FUNCTIONS.count(function_name) > 0 && !info.unicode_case_mapping) { + // the server only maps ASCII characters to lower case + return false; + } + if (function_name == "sha256" && info.version < PostgresVersion(11, 0)) { + return false; + } + static const case_insensitive_set_t PG13_FUNCTIONS = {"gcd", "lcm", "greatest_common_divisor", + "least_common_multiple"}; + if (PG13_FUNCTIONS.count(function_name) > 0 && info.version < PostgresVersion(13, 0)) { + return false; + } + if ((function_name == "date_part" || function_name == "datepart" || function_name == "date_diff" || + function_name == "datediff" || function_name == "epoch" || function_name == "epoch_ms") && + info.version < PostgresVersion(14, 0)) { + // EXTRACT returns an exact NUMERIC from Postgres 14 + return false; + } + return SCALAR_FUNCTIONS.count(function_name) > 0 || PostgresIsAggregateFunction(function_name); +} + +static bool PostgresIsLikeFunction(const string &name) { + return name == "~~" || name == "!~~"; +} + +//===--------------------------------------------------------------------===// +// Select alias usage +//===--------------------------------------------------------------------===// +static void PostgresCollectColumnRefs(const QueryNode &node, case_insensitive_set_t &names); + +//! Collect the names of all unqualified column references in an expression tree, descending into subqueries +static void PostgresCollectColumnRefs(const ParsedExpression &expr, case_insensitive_set_t &names) { + if (expr.GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &colref = expr.Cast(); + if (!colref.IsQualified()) { + names.insert(colref.GetColumnName().GetIdentifierName()); + } + } + if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { + auto &subquery = expr.Cast(); + if (subquery.Subquery() && subquery.Subquery()->node) { + PostgresCollectColumnRefs(*subquery.Subquery()->node, names); + } + } + ParsedExpressionIterator::EnumerateChildren( + expr, [&](const ParsedExpression &child) { PostgresCollectColumnRefs(child, names); }); +} + +static void PostgresCollectColumnRefs(const TableRef &ref, case_insensitive_set_t &names) { + switch (ref.type) { + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + PostgresCollectColumnRefs(*join.left, names); + PostgresCollectColumnRefs(*join.right, names); + if (join.condition) { + PostgresCollectColumnRefs(*join.condition, names); + } + break; + } + case TableReferenceType::SUBQUERY: { + auto &subquery = ref.Cast(); + if (subquery.subquery && subquery.subquery->node) { + PostgresCollectColumnRefs(*subquery.subquery->node, names); + } + break; + } + default: + break; + } +} + +static void PostgresCollectColumnRefs(const QueryNode &node, case_insensitive_set_t &names) { + for (auto &cte_pair : node.cte_map.map) { + if (cte_pair.second->query_node) { + PostgresCollectColumnRefs(*cte_pair.second->query_node, names); + } + } + for (auto &modifier : node.modifiers) { + if (modifier->type == ResultModifierType::ORDER_MODIFIER) { + for (auto &order : modifier->Cast().orders) { + PostgresCollectColumnRefs(*order.expression, names); + } + } + } + switch (node.type) { + case QueryNodeType::SELECT_NODE: { + auto &select = node.Cast(); + for (auto &expr : select.select_list) { + PostgresCollectColumnRefs(*expr, names); + } + if (select.from_table) { + PostgresCollectColumnRefs(*select.from_table, names); + } + if (select.where_clause) { + PostgresCollectColumnRefs(*select.where_clause, names); + } + for (auto &group : select.groups.group_expressions) { + PostgresCollectColumnRefs(*group, names); + } + if (select.having) { + PostgresCollectColumnRefs(*select.having, names); + } + if (select.qualify) { + PostgresCollectColumnRefs(*select.qualify, names); + } + break; + } + case QueryNodeType::SET_OPERATION_NODE: { + auto &setop = node.Cast(); + for (auto &child : setop.children) { + PostgresCollectColumnRefs(*child, names); + } + break; + } + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &cte = node.Cast(); + PostgresCollectColumnRefs(*cte.left, names); + PostgresCollectColumnRefs(*cte.right, names); + break; + } + default: + break; + } +} + +//! Postgres can only resolve select-list aliases as bare names in GROUP BY and ORDER BY - not in the +//! WHERE / HAVING clauses, not within other select-list expressions and not within GROUP BY / ORDER BY +//! expressions, while DuckDB can. Both systems prefer a real table column over an alias, so an alias +//! reference is only a problem when no column with that name exists - which cannot be known pre-bind. +//! We approximate: a name is assumed to be a real column if it is also referenced at a position where +//! alias resolution is impossible (in a select-list item at or before the item that defines the alias). +//! Otherwise the query is conservatively not pushed down. +bool PostgresCapabilities::SupportsSelectAliasUsage(const SelectNode &select) { + // find the first select-list item that defines each alias + case_insensitive_map_t alias_definitions; + for (idx_t i = 0; i < select.select_list.size(); i++) { + auto &alias = select.select_list[i]->GetAlias().GetIdentifierName(); + if (!alias.empty() && alias_definitions.find(alias) == alias_definitions.end()) { + alias_definitions[alias] = i; + } + } + if (alias_definitions.empty()) { + return true; + } + // collect the column references per select-list item + vector item_refs(select.select_list.size()); + for (idx_t i = 0; i < select.select_list.size(); i++) { + PostgresCollectColumnRefs(*select.select_list[i], item_refs[i]); + } + // a reference at or before the defining item cannot be an alias reference - it must be a column + case_insensitive_set_t column_evidence; + for (idx_t i = 0; i < select.select_list.size(); i++) { + for (auto &name : item_refs[i]) { + auto entry = alias_definitions.find(name); + if (entry == alias_definitions.end() || entry->second >= i) { + column_evidence.insert(name); + } + } + } + // collect the names that DuckDB might resolve as an alias where Postgres cannot + case_insensitive_set_t suspects; + if (select.where_clause) { + PostgresCollectColumnRefs(*select.where_clause, suspects); + } + if (select.having) { + PostgresCollectColumnRefs(*select.having, suspects); + } + for (auto &group : select.groups.group_expressions) { + if (group->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + PostgresCollectColumnRefs(*group, suspects); + } + } + for (auto &modifier : select.modifiers) { + if (modifier->type != ResultModifierType::ORDER_MODIFIER) { + continue; + } + for (auto &order : modifier->Cast().orders) { + if (order.expression->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + PostgresCollectColumnRefs(*order.expression, suspects); + } + } + } + for (idx_t i = 0; i < select.select_list.size(); i++) { + for (auto &name : item_refs[i]) { + auto entry = alias_definitions.find(name); + if (entry != alias_definitions.end() && entry->second < i) { + suspects.insert(name); + } + } + } + for (auto &name : suspects) { + if (alias_definitions.find(name) != alias_definitions.end() && column_evidence.count(name) == 0) { + return false; + } + } + return true; +} + +//! Check whether a join tree contains a NATURAL or USING join - these coalesce the join columns when +//! expanding an unqualified *, and Postgres places the coalesced columns first while DuckDB keeps them +//! in the position of the left table's column order +static bool PostgresJoinTreeHasCoalescingJoin(const TableRef &ref) { + if (ref.type != TableReferenceType::JOIN) { + return false; + } + auto &join = ref.Cast(); + if (join.ref_type == JoinRefType::NATURAL || !join.using_columns.empty()) { + return true; + } + return PostgresJoinTreeHasCoalescingJoin(*join.left) || PostgresJoinTreeHasCoalescingJoin(*join.right); +} + +//===--------------------------------------------------------------------===// +// Result modifiers +//===--------------------------------------------------------------------===// +static bool PostgresIsPlainStar(const ParsedExpression &expr) { + auto &star = expr.Cast(); + return star.RelationName().empty() && star.ExcludeList().empty() && star.ReplaceList().empty() && + star.RenameList().empty() && !star.Expression(); +} + +//! Whether an expression tree contains a COLUMNS(*) expression +static bool PostgresContainsColumnsStar(const ParsedExpression &expr) { + if (expr.GetExpressionClass() == ExpressionClass::STAR && expr.Cast().IsColumns()) { + return true; + } + bool result = false; + ParsedExpressionIterator::EnumerateChildren(expr, [&](const ParsedExpression &child) { + if (PostgresContainsColumnsStar(child)) { + result = true; + } + }); + return result; +} + +bool PostgresCapabilities::SupportsOrderEntries(const vector &orders, + optional_ptr>> select_list) { + if (!orders.empty() && StringOrderingDiffers()) { + // string ordering differs from DuckDB's for this database + return false; + } + for (auto &order : orders) { + auto &expr = *order.expression; + if (expr.GetExpressionClass() == ExpressionClass::STAR) { + // ORDER BY ALL is serialized as an ORDER BY of all select-list positions - which requires the + // select list to not contain any stars itself + if (!select_list || orders.size() != 1 || !PostgresIsPlainStar(expr)) { + return false; + } + for (auto &select_expr : *select_list) { + if (select_expr->GetExpressionClass() == ExpressionClass::STAR) { + return false; + } + } + } + if (expr.GetExpressionClass() == ExpressionClass::CONSTANT && !PostgresIsIntegerLiteral(expr)) { + // ORDER BY with a non-integer literal is a binder error in DuckDB (unless + // order_by_non_integer_literal is set) + return false; + } + } + return true; +} + +bool PostgresCapabilities::SupportsResultModifiers( + const QueryNode &node, optional_ptr>> select_list) { + bool has_order = false; + bool has_distinct = false; + bool has_distinct_on = false; + for (auto &modifier : node.modifiers) { + switch (modifier->type) { + case ResultModifierType::ORDER_MODIFIER: { + auto &order_mod = modifier->Cast(); + if (!SupportsOrderEntries(order_mod.orders, select_list)) { + return false; + } + has_order = true; + break; + } + case ResultModifierType::LIMIT_MODIFIER: { + auto &limit_mod = modifier->Cast(); + // negative values are a binder error in DuckDB - keep them local so the user gets DuckDB's error + if (limit_mod.limit_type != LimitValueType::ROW_COUNT) { + // Postgres has no LIMIT n PERCENT + return false; + } + if (limit_mod.limit && !PostgresIsIntegerLiteralAtLeast(*limit_mod.limit, 0)) { + return false; + } + if (limit_mod.offset && !PostgresIsIntegerLiteralAtLeast(*limit_mod.offset, 0)) { + return false; + } + break; + } + case ResultModifierType::DISTINCT_MODIFIER: { + auto &distinct_mod = modifier->Cast(); + if (!select_list) { + // DISTINCT is only serialized as part of a SELECT node + return false; + } + if (!distinct_mod.distinct_on_targets.empty()) { + has_distinct_on = true; + } else { + has_distinct = true; + } + break; + } + default: + // unknown result modifier + return false; + } + } + if (has_order && has_distinct_on) { + // Postgres requires the DISTINCT ON expressions to match the leading ORDER BY expressions + return false; + } + if (has_order && has_distinct && select_list) { + // Postgres requires the ORDER BY expressions of a SELECT DISTINCT to appear in the select list + for (auto &modifier : node.modifiers) { + if (modifier->type != ResultModifierType::ORDER_MODIFIER) { + continue; + } + for (auto &order : modifier->Cast().orders) { + auto &expr = *order.expression; + if (PostgresIsIntegerLiteral(expr) || expr.GetExpressionClass() == ExpressionClass::STAR) { + continue; + } + bool in_select_list = false; + for (auto &select_expr : *select_list) { + if (select_expr->Equals(expr) || + (expr.GetExpressionClass() == ExpressionClass::COLUMN_REF && + !expr.Cast().IsQualified() && + select_expr->GetAlias() == expr.Cast().GetColumnName())) { + in_select_list = true; + } + } + if (!in_select_list) { + return false; + } + } + } + } + return true; +} + +bool PostgresCapabilities::SupportsCTEMap(const CommonTableExpressionMap &cte_map) { + for (auto &cte_pair : cte_map.map) { + auto &cte_info = *cte_pair.second; + if (!cte_info.key_targets.empty()) { + // USING KEY is DuckDB-specific + return false; + } + if (cte_info.materialized != CTEMaterialize::CTE_MATERIALIZE_DEFAULT && info.version < PostgresVersion(12, 0)) { + // AS [NOT] MATERIALIZED requires Postgres 12 + return false; + } + } + return true; +} + +bool PostgresCapabilities::SupportsWindow(const WindowExpression &window) { + if (window.IgnoreNulls()) { + // IGNORE NULLS is not supported by all Postgres versions + return false; + } + if (window.Distinct()) { + // Postgres window functions do not support DISTINCT + return false; + } + if (!window.ArgOrders().empty()) { + // ORDER BY within the argument list is DuckDB-specific + return false; + } + if (!window.GetQualifiedName().Path().empty() && window.GetQualifiedName().Path().size() > 1) { + return false; + } + bool has_frame_extensions = window.WindowExclude() != WindowExcludeMode::NO_OTHER; + for (auto boundary : {window.WindowStart(), window.WindowEnd()}) { + switch (boundary) { + case WindowBoundary::UNBOUNDED_PRECEDING: + case WindowBoundary::UNBOUNDED_FOLLOWING: + case WindowBoundary::CURRENT_ROW_RANGE: + case WindowBoundary::CURRENT_ROW_ROWS: + case WindowBoundary::EXPR_PRECEDING_ROWS: + case WindowBoundary::EXPR_FOLLOWING_ROWS: + break; + case WindowBoundary::CURRENT_ROW_GROUPS: + case WindowBoundary::EXPR_PRECEDING_GROUPS: + case WindowBoundary::EXPR_FOLLOWING_GROUPS: + case WindowBoundary::EXPR_PRECEDING_RANGE: + case WindowBoundary::EXPR_FOLLOWING_RANGE: + has_frame_extensions = true; + break; + default: + return false; + } + } + if (has_frame_extensions && info.version < PostgresVersion(11, 0)) { + // GROUPS frames, RANGE offsets and EXCLUDE require Postgres 11 + return false; + } + // frame offsets must be constants in Postgres + for (auto offset : {window.StartExpr().get(), window.EndExpr().get()}) { + Value value; + interval_t interval; + if (offset && !TryGetConstantValue(*offset, value) && !TryGetIntervalLiteral(*offset, interval)) { + return false; + } + } + if (!SupportsOrderEntries(window.OrderBy())) { + return false; + } + auto &children = window.GetArguments(); + for (auto &child : children) { + if (child.HasName()) { + // named arguments are not supported + return false; + } + } + auto &function_name = window.FunctionName().GetIdentifierName(); + if (window.Filter() && PostgresAggregateFunctions().count(function_name) == 0) { + // Postgres only supports FILTER for aggregate window functions + return false; + } + if (function_name == "lead" || function_name == "lag") { + // only push down literal non-negative offsets + if (children.size() >= 2 && !PostgresIsIntegerLiteralAtLeast(children[1].GetExpression(), 0)) { + return false; + } + if (children.empty() || children.size() > 3) { + return false; + } + } else if (function_name == "ntile") { + if (children.size() != 1 || !PostgresIsIntegerLiteralAtLeast(children[0].GetExpression(), 1)) { + return false; + } + } else if (function_name == "nth_value") { + if (children.size() != 2 || !PostgresIsIntegerLiteralAtLeast(children[1].GetExpression(), 1)) { + return false; + } + } + static const case_insensitive_set_t WINDOW_FUNCTIONS = { + "row_number", "rank", "dense_rank", "rank_dense", "percent_rank", "cume_dist", + "ntile", "lead", "lag", "first_value", "last_value", "nth_value", + }; + if (WINDOW_FUNCTIONS.count(function_name) > 0) { + return true; + } + if (PostgresIsOrderSensitiveFunction(function_name) && StringOrderingDiffers()) { + return false; + } + return PostgresAggregateFunctions().count(function_name) > 0; +} + +//===--------------------------------------------------------------------===// +// Capabilities +//===--------------------------------------------------------------------===// +bool PostgresCapabilities::Supports(RemoteCapability capability) const { + switch (capability) { + case RemoteCapability::IS_REMOTE: + case RemoteCapability::CONNECT: + return true; + case RemoteCapability::EXECUTE_QUERY_NODE: + case RemoteCapability::EXECUTE_STATEMENT: + // Redshift speaks a different SQL dialect + return info.version.type_v != PostgresInstanceType::REDSHIFT; + default: + return false; + } +} + +bool PostgresCapabilities::SupportsPushdown(const ParsedExpression &expr) { + switch (expr.GetExpressionClass()) { + case ExpressionClass::CONSTANT: { + auto value = expr.Cast().GetLiteral().ToValue(); + return PostgresSupportsValue(value); + } + case ExpressionClass::CAST: { + auto &cast_expr = expr.Cast(); + if (cast_expr.IsTryCast()) { + // Postgres has no TRY_CAST + return false; + } + auto target_type = ResolveType(cast_expr.TargetType()); + if (cast_expr.Child().GetExpressionClass() == ExpressionClass::CONSTANT) { + // casts of literals are evaluated locally and serialized as the resulting value - note that casting + // a string without an explicit UTC offset to TIMESTAMP WITH TIME ZONE fails here (it requires the + // session time zone), so those are not pushed down + Value cast_result; + return TryGetConstantValue(expr, cast_result) && PostgresSupportsValue(cast_result); + } + return SupportsCastType(target_type); + } + case ExpressionClass::FUNCTION: { + auto &func = expr.Cast(); + auto &function_name = func.FunctionName().GetIdentifierName(); + interval_t interval; + if (TryGetIntervalLiteral(expr, interval)) { + return true; + } + if (func.GetQualifiedName().Path().size() > 1) { + // qualified function names refer to (user-defined) functions in the remote catalog + return false; + } + if (func.ExportState()) { + return false; + } + bool has_order_by = func.OrderBy() && !func.OrderBy()->orders.empty(); + if ((func.Filter() || has_order_by) && !PostgresIsAggregateFunction(function_name)) { + return false; + } + if (has_order_by && (PostgresRewrittenAggregateFunctions().count(function_name) > 0 || + !SupportsOrderEntries(func.OrderBy()->orders))) { + return false; + } + if (func.Distinct()) { + static const case_insensitive_set_t DISTINCT_AGGREGATES = { + "count", "sum", "avg", "min", "max", "string_agg", "group_concat", "listagg", "array_agg", "list"}; + if (DISTINCT_AGGREGATES.count(function_name) == 0) { + return false; + } + } + auto &children = func.GetArguments(); + if (func.Distinct() && has_order_by && (function_name == "array_agg" || function_name == "list")) { + for (auto &order : func.OrderBy()->orders) { + if (children.empty() || !order.expression->Equals(children[0].GetExpression())) { + return false; + } + } + } + static const case_insensitive_set_t STRING_AGG_FUNCTIONS = {"string_agg", "group_concat", "listagg"}; + if (STRING_AGG_FUNCTIONS.count(function_name) > 0) { + // a NULL separator yields NULL in DuckDB - only push down literal separators + if (children.empty() || children.size() > 2 || + (children.size() == 2 && !PostgresIsStringLiteral(children[1].GetExpression()))) { + return false; + } + if (func.Distinct() && has_order_by) { + // Postgres requires the ORDER BY expressions to match the arguments for DISTINCT aggregates + for (auto &order : func.OrderBy()->orders) { + if (!order.expression->Equals(children[0].GetExpression())) { + return false; + } + } + } + } + if (function_name == "median" && children.size() != 1) { + return false; + } + static const case_insensitive_set_t QUANTILE_FUNCTIONS = {"quantile", "quantile_cont", "quantile_disc"}; + if (QUANTILE_FUNCTIONS.count(function_name) > 0) { + // only a single literal quantile is supported (DuckDB returns a list for a list of quantiles) + Value quantile; + if (children.size() != 2 || !PostgresTryGetConstantValue(children[1].GetExpression(), quantile)) { + return false; + } + auto quantile_value = quantile.DefaultTryCastAs(LogicalType::DOUBLE); + if (!quantile_value || quantile_value->IsNull() || DoubleValue::Get(*quantile_value) < 0 || + DoubleValue::Get(*quantile_value) > 1) { + return false; + } + } + static const case_insensitive_set_t ARG_FUNCTIONS = {"arg_min", "arg_max", "min_by", "max_by"}; + if (ARG_FUNCTIONS.count(function_name) > 0 && (children.size() != 2 || func.Distinct())) { + return false; + } + if ((function_name == "count_if" || function_name == "countif") && (children.size() != 1 || func.Distinct())) { + return false; + } + if (function_name == "any_value" && info.version < PostgresVersion(16, 0)) { + return false; + } + if (function_name == "bit_xor" && info.version < PostgresVersion(14, 0)) { + return false; + } + for (auto &child : children) { + if (child.HasName()) { + // named arguments are not supported + return false; + } + } + if (PostgresIsLikeFunction(function_name) && children.size() != 2) { + return false; + } + static const case_insensitive_set_t PART_FUNCTIONS = {"date_part", "datepart", "date_trunc", + "datetrunc", "date_diff", "datediff"}; + if (PART_FUNCTIONS.count(function_name) > 0) { + // the date part must be a literal + auto expected_children = function_name == "date_diff" || function_name == "datediff" ? 3 : 2; + if (children.size() != expected_children || !PostgresIsStringLiteral(children[0].GetExpression())) { + return false; + } + } + if (function_name == "round" || function_name == "round_even" || function_name == "roundbankers") { + // only literal precisions are supported + if (children.empty() || children.size() > 2 || + (children.size() == 2 && !PostgresIsIntegerLiteral(children[1].GetExpression()))) { + return false; + } + } + if ((function_name == "count_star" && !children.empty()) || (function_name == "count" && children.size() > 1)) { + return false; + } + if ((StringUtil::CIEquals(function_name, "substring") || StringUtil::CIEquals(function_name, "substr"))) { + // DuckDB counts negative offsets from the end of the string - only push down literal non-negative + // offsets (and a literal non-negative length) + if (children.size() < 2 || children.size() > 3) { + return false; + } + if (!PostgresIsIntegerLiteralAtLeast(children[1].GetExpression(), 0)) { + return false; + } + if (children.size() == 3 && !PostgresIsIntegerLiteralAtLeast(children[2].GetExpression(), 0)) { + return false; + } + } + if (StringUtil::CIEquals(function_name, "trunc") && children.size() != 1) { + // the two-argument trunc only exists for NUMERIC in Postgres + return false; + } + static const case_insensitive_set_t DATE_PART_FUNCTIONS = {"year", "month", "day", "dayofmonth", + "hour", "minute", "quarter", "dayofweek", + "isodow", "dayofyear", "week", "weekofyear"}; + if (DATE_PART_FUNCTIONS.count(function_name) > 0 && children.size() != 1) { + return false; + } + return SupportsFunction(func); + } + case ExpressionClass::WINDOW: + return SupportsWindow(expr.Cast()); + case ExpressionClass::COMPARISON: + switch (expr.GetExpressionType()) { + case ExpressionType::COMPARE_EQUAL: + case ExpressionType::COMPARE_NOTEQUAL: + case ExpressionType::COMPARE_DISTINCT_FROM: + case ExpressionType::COMPARE_NOT_DISTINCT_FROM: + return true; + case ExpressionType::COMPARE_LESSTHAN: + case ExpressionType::COMPARE_GREATERTHAN: + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + // string ordering must match DuckDB's binary ordering + return !StringOrderingDiffers(); + default: + return false; + } + case ExpressionClass::BETWEEN: + return !StringOrderingDiffers(); + case ExpressionClass::OPERATOR: + switch (expr.GetExpressionType()) { + case ExpressionType::OPERATOR_IS_NULL: + case ExpressionType::OPERATOR_IS_NOT_NULL: + case ExpressionType::OPERATOR_NOT: + case ExpressionType::OPERATOR_COALESCE: + case ExpressionType::OPERATOR_NULLIF: + case ExpressionType::COMPARE_IN: + case ExpressionType::COMPARE_NOT_IN: + case ExpressionType::GROUPING_FUNCTION: + return true; + default: + // TRY(...), list / struct operators, ... are DuckDB-specific + return false; + } + case ExpressionClass::SUBQUERY: { + auto &subquery = expr.Cast(); + switch (subquery.GetSubqueryType()) { + case SubqueryType::EXISTS: + case SubqueryType::NOT_EXISTS: + case SubqueryType::SCALAR: + return true; + case SubqueryType::ANY: + switch (subquery.GetComparisonType()) { + case ExpressionType::COMPARE_EQUAL: + case ExpressionType::COMPARE_NOTEQUAL: + return true; + case ExpressionType::COMPARE_LESSTHAN: + case ExpressionType::COMPARE_GREATERTHAN: + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + return !StringOrderingDiffers(); + default: + return false; + } + default: + return false; + } + } + case ExpressionClass::STAR: { + auto &star = expr.Cast(); + if (!star.ExcludeList().empty() || !star.ReplaceList().empty() || !star.RenameList().empty()) { + // EXCLUDE / REPLACE / RENAME are DuckDB-specific + return false; + } + if (star.Expression()) { + // COLUMNS(...) expressions are DuckDB-specific + return false; + } + // COLUMNS(*) is DuckDB-specific, but ORDER BY ALL is represented the same way - those are verified + // by SupportsPushdown(QueryNode) + return true; + } + case ExpressionClass::COLUMN_REF: { + auto &colref = expr.Cast(); + if (!colref.IsQualified()) { + // SQL value functions without parentheses (CURRENT_DATE, ...) are parsed as column references + static const case_insensitive_set_t UNSUPPORTED_VALUE_FUNCTIONS = { + "current_time", "localtime", "current_user", "user", + "session_user", "current_role", "current_catalog", "current_schema"}; + if (UNSUPPORTED_VALUE_FUNCTIONS.count(colref.GetColumnName().GetIdentifierName()) > 0) { + return false; + } + } + return colref.ColumnNames().size() <= 3; + } + case ExpressionClass::CASE: + case ExpressionClass::CONJUNCTION: + return true; + case ExpressionClass::DEFAULT: + // DEFAULT in an UPDATE SET clause / INSERT VALUES list - anywhere else it is rejected by the binder + return true; + default: + // parameters, COLLATE, lambdas, positional references, ... - not supported + return false; + } +} + +bool PostgresCapabilities::SupportsPushdown(const TableRef &ref) { + if (ref.sample) { + // TABLESAMPLE semantics differ + return false; + } + switch (ref.type) { + case TableReferenceType::BASE_TABLE: { + auto &base = ref.Cast(); + if (base.at_clause) { + // AT (...) time travel clauses are DuckDB-specific + return false; + } + return true; + } + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + switch (join.ref_type) { + case JoinRefType::REGULAR: + case JoinRefType::NATURAL: + case JoinRefType::CROSS: + break; + default: + // ASOF / POSITIONAL / ... joins are DuckDB-specific + return false; + } + switch (join.type) { + case JoinType::INNER: + case JoinType::LEFT: + case JoinType::RIGHT: + case JoinType::OUTER: + break; + case JoinType::SEMI: + case JoinType::ANTI: + // serialized as a [NOT] EXISTS filter over the left side - which must be a single (named) relation + if (join.ref_type != JoinRefType::REGULAR || !join.condition || !join.using_columns.empty()) { + return false; + } + if (join.left->type == TableReferenceType::BASE_TABLE) { + break; + } + if (join.left->type == TableReferenceType::SUBQUERY && !join.left->alias.empty()) { + break; + } + return false; + default: + return false; + } + if (!join.duplicate_eliminated_columns.empty()) { + return false; + } + return true; + } + case TableReferenceType::SUBQUERY: + case TableReferenceType::EMPTY_FROM: + return true; + case TableReferenceType::EXPRESSION_LIST: { + auto &expression_list = ref.Cast(); + return !expression_list.values.empty(); + } + case TableReferenceType::TABLE_FUNCTION: { + auto &table_function = ref.Cast(); + if (table_function.subquery || table_function.function->GetExpressionClass() != ExpressionClass::FUNCTION) { + return false; + } + auto &func = table_function.function->Cast(); + if (func.GetQualifiedName().Path().size() > 1) { + return false; + } + auto &name = func.FunctionName(); + auto &children = func.GetArguments(); + for (auto &child : children) { + if (child.HasName()) { + return false; + } + } + if (name == "generate_series") { + return children.size() >= 2 && children.size() <= 3; + } + if (name == "range") { + // the exclusive upper bound is serialized with a literal step + if (children.size() == 3) { + int64_t step; + return PostgresTryGetIntegerLiteral(children[2].GetExpression(), step) && step != 0; + } + return !children.empty() && children.size() <= 2; + } + // other table functions are DuckDB-specific + return false; + } + default: + // pivots, ... - not supported + return false; + } +} + +static void PostgresCollectTableNames(QueryNode &node, identifier_set_t &cte_names, identifier_set_t &table_names, + bool all_tables); + +static void PostgresCollectTableNames(ParsedExpression &expr, identifier_set_t &cte_names, + identifier_set_t &table_names, bool all_tables) { + if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { + PostgresCollectTableNames(*expr.Cast().SubqueryMutable()->node, cte_names, table_names, + all_tables); + } + ParsedExpressionIterator::EnumerateChildren( + expr, [&](ParsedExpression &child) { PostgresCollectTableNames(child, cte_names, table_names, all_tables); }); +} + +//! Collect the names of all CTEs, and of all qualified table references (or of all table references) +static void PostgresCollectTableNames(QueryNode &node, identifier_set_t &cte_names, identifier_set_t &table_names, + bool all_tables) { + for (auto &cte : node.cte_map.map) { + cte_names.insert(cte.first); + } + ParsedExpressionIterator::EnumerateQueryNodeChildren( + node, + [&](unique_ptr &child) { + PostgresCollectTableNames(*child, cte_names, table_names, all_tables); + }, + [&](TableRef &ref) { + if (ref.type == TableReferenceType::BASE_TABLE) { + auto &base = ref.Cast(); + if (all_tables || base.GetQualifiedName().Path().size() > 1) { + table_names.insert(base.Table()); + } + } + }); +} + +//! The catalog qualification is stripped from table references before the query is serialized - a qualified +//! reference to a table with the same name as a CTE would then refer to the CTE +static bool PostgresHasCTETableNameConflict(const QueryNode &node) { + auto copy = node.Copy(); + identifier_set_t cte_names; + identifier_set_t qualified_tables; + PostgresCollectTableNames(*copy, cte_names, qualified_tables, false); + for (auto &name : cte_names) { + if (qualified_tables.count(name) > 0) { + return true; + } + } + return false; +} + +bool PostgresCapabilities::SupportsPushdown(const QueryNode &node) { + if (!SupportsCTEMap(node.cte_map)) { + return false; + } + if (!node.cte_map.map.empty() && PostgresHasCTETableNameConflict(node)) { + return false; + } + switch (node.type) { + case QueryNodeType::SELECT_NODE: { + auto &select = node.Cast(); + if (select.qualify) { + // Postgres has no QUALIFY + return false; + } + if (select.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { + // GROUP BY ALL is DuckDB-specific + return false; + } + if (select.sample) { + // Postgres has no USING SAMPLE + return false; + } + if (!SupportsSelectAliasUsage(select)) { + return false; + } + vector> expressions; + for (auto &expr : select.select_list) { + expressions.push_back(*expr); + } + for (auto &expr : select.groups.group_expressions) { + expressions.push_back(*expr); + } + if (select.where_clause) { + expressions.push_back(*select.where_clause); + } + if (select.having) { + expressions.push_back(*select.having); + } + for (auto &expr : expressions) { + if (PostgresContainsColumnsStar(expr.get())) { + // COLUMNS(*) is DuckDB-specific + return false; + } + } + if (select.from_table && PostgresJoinTreeHasCoalescingJoin(*select.from_table)) { + for (auto &expr : select.select_list) { + if (expr->GetExpressionClass() == ExpressionClass::STAR && + expr->Cast().RelationName().empty()) { + return false; + } + } + } + return SupportsResultModifiers(node, &select.select_list); + } + case QueryNodeType::SET_OPERATION_NODE: { + auto &setop = node.Cast(); + switch (setop.setop_type) { + case SetOperationType::UNION: + case SetOperationType::EXCEPT: + case SetOperationType::INTERSECT: + break; + default: + // UNION BY NAME is DuckDB-specific + return false; + } + for (auto &modifier : node.modifiers) { + if (modifier->type != ResultModifierType::ORDER_MODIFIER) { + continue; + } + for (auto &order : modifier->Cast().orders) { + // Postgres can only order the result of a set operation by output column names or positions + auto &expr = *order.expression; + bool is_name = expr.GetExpressionClass() == ExpressionClass::COLUMN_REF && + !expr.Cast().IsQualified(); + if (!is_name && !PostgresIsIntegerLiteral(expr)) { + return false; + } + } + } + return SupportsResultModifiers(node); + } + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &cte = node.Cast(); + if (!cte.key_targets.empty()) { + // USING KEY is DuckDB-specific + return false; + } + return SupportsResultModifiers(node); + } + case QueryNodeType::UPDATE_QUERY_NODE: { + auto &update = node.Cast(); + if (info.read_only) { + // the read-only check is performed by DuckDB when executing the statement locally + return false; + } + if (!update.set_info || !update.table || update.table->type != TableReferenceType::BASE_TABLE) { + return false; + } + // Postgres does not support ORDER BY / LIMIT in UPDATE statements + return node.modifiers.empty(); + } + case QueryNodeType::DELETE_QUERY_NODE: { + auto &del = node.Cast(); + if (info.read_only) { + return false; + } + if (!del.table || del.table->type != TableReferenceType::BASE_TABLE) { + return false; + } + return node.modifiers.empty(); + } + case QueryNodeType::INSERT_QUERY_NODE: { + auto &insert = node.Cast(); + if (info.read_only) { + return false; + } + if (insert.column_order != InsertColumnOrder::INSERT_BY_POSITION) { + // INSERT BY NAME is DuckDB-specific + return false; + } + if (!insert.select_statement && !insert.default_values) { + return false; + } + if (insert.on_conflict_info) { + auto &on_conflict = *insert.on_conflict_info; + if (on_conflict.condition) { + // conflict targets on partial indexes are not supported + return false; + } + switch (on_conflict.action_type) { + case OnConflictAction::THROW: + case OnConflictAction::NOTHING: + case OnConflictAction::REPLACE: + break; + case OnConflictAction::UPDATE: + if (!on_conflict.set_info) { + return false; + } + break; + default: + return false; + } + } + return node.modifiers.empty(); + } + case QueryNodeType::MERGE_QUERY_NODE: { + auto &merge = node.Cast(); + if (info.read_only || info.version < PostgresVersion(15, 0)) { + // MERGE requires Postgres 15 + return false; + } + if (!merge.target || merge.target->type != TableReferenceType::BASE_TABLE || !merge.source || + !node.modifiers.empty()) { + return false; + } + if (!merge.returning_list.empty() && info.version < PostgresVersion(17, 0)) { + // MERGE ... RETURNING requires Postgres 17 + return false; + } + for (auto &entry : merge.actions) { + if (entry.first == MergeActionCondition::WHEN_NOT_MATCHED_BY_SOURCE && + info.version < PostgresVersion(17, 0)) { + // WHEN NOT MATCHED BY SOURCE requires Postgres 17 + return false; + } + for (auto &action : entry.second) { + if (!action->exclude_columns.empty() || action->column_order != InsertColumnOrder::INSERT_BY_POSITION) { + return false; + } + switch (action->action_type) { + case MergeActionType::MERGE_UPDATE: + // UPDATE without a SET clause updates all columns by name in DuckDB + if (!action->update_info || action->update_info->columns.empty()) { + return false; + } + break; + case MergeActionType::MERGE_INSERT: + // INSERT without a column list inserts all columns by name in DuckDB + if (!action->default_values && action->expressions.empty()) { + return false; + } + break; + case MergeActionType::MERGE_DELETE: + case MergeActionType::MERGE_DO_NOTHING: + break; + default: + // THEN ERROR is DuckDB-specific + return false; + } + } + } + return true; + } + default: + return false; + } +} + +//! Functions whose serialization depends on the types of their arguments - these are verified when serializing, +//! and can therefore not be part of statements that cannot be executed locally instead +static bool PostgresIsTypeDependentFunction(const string &name) { + static const case_insensitive_set_t TYPE_DEPENDENT_FUNCTIONS = {"/", + "%", + "||", + "length", + "len", + "char_length", + "character_length", + "strlen", + "octet_length", + "concat", + "concat_ws", + "string_agg", + "group_concat", + "listagg", + "array_agg", + "list", + "bit_and", + "bit_or", + "bit_xor", + "median", + "quantile", + "quantile_cont", + "quantile_disc", + "arg_min", + "arg_max", + "min_by", + "max_by", + "lower", + "lcase", + "~~*", + "!~~*", + "contains", + "ends_with", + "suffix", + "^@", + "translate", + "unicode", + "chr", + "sha256", + "sqrt", + "ln", + "log", + "log10", + "exp", + "pow", + "power", + "^", + "**", + "isnan", + "isinf", + "isfinite", + "gcd", + "lcm", + "greatest_common_divisor", + "least_common_multiple", + "xor", + "&", + "|", + "~", + "round", + "round_even", + "roundbankers", + "date_part", + "datepart", + "year", + "month", + "day", + "dayofmonth", + "hour", + "minute", + "second", + "millisecond", + "microsecond", + "quarter", + "dayofweek", + "isodow", + "dayofyear", + "week", + "weekofyear", + "isoyear", + "decade", + "century", + "millennium", + "epoch", + "epoch_ms", + "date_diff", + "datediff", + "date_trunc", + "datetrunc", + "last_day", + "make_date", + "to_timestamp"}; + return TYPE_DEPENDENT_FUNCTIONS.count(name) > 0; +} + +bool PostgresCapabilities::SupportsTypeIndependent(ParsedExpression &expr) { + if (!SupportsPushdown(expr)) { + return false; + } + switch (expr.GetExpressionClass()) { + case ExpressionClass::CAST: { + Value value; + if (!TryGetConstantValue(expr, value)) { + // casts of non-constants depend on the source type + return false; + } + break; + } + case ExpressionClass::FUNCTION: { + interval_t interval; + if (TryGetIntervalLiteral(expr, interval)) { + return true; + } + if (PostgresIsTypeDependentFunction(expr.Cast().FunctionName().GetIdentifierName())) { + return false; + } + break; + } + case ExpressionClass::WINDOW: + if (PostgresIsTypeDependentFunction(expr.Cast().FunctionName().GetIdentifierName())) { + return false; + } + break; + case ExpressionClass::SUBQUERY: + if (!SupportsQueryTypeIndependent(*expr.Cast().SubqueryMutable()->node)) { + return false; + } + break; + default: + break; + } + bool result = true; + ParsedExpressionIterator::EnumerateChildren(expr, [&](ParsedExpression &child) { + if (result && !SupportsTypeIndependent(child)) { + result = false; + } + }); + return result; +} + +bool PostgresCapabilities::SupportsQueryTypeIndependent(QueryNode &node) { + if (!SupportsPushdown(node)) { + return false; + } + bool result = true; + ParsedExpressionIterator::EnumerateQueryNodeChildren( + node, + [&](unique_ptr &child) { + if (result && !SupportsTypeIndependent(*child)) { + result = false; + } + }, + [&](TableRef &ref) { + if (!result) { + return; + } + if (!SupportsPushdown(ref) || ref.type == TableReferenceType::TABLE_FUNCTION) { + // table functions are serialized depending on the types of their arguments + result = false; + } else if (ref.type == TableReferenceType::SUBQUERY && + !SupportsQueryTypeIndependent(*ref.Cast().subquery->node)) { + result = false; + } + }); + if (!result) { + return false; + } + for (auto &cte : node.cte_map.map) { + if (cte.second->query_node && !SupportsQueryTypeIndependent(*cte.second->query_node)) { + return false; + } + } + switch (node.type) { + case QueryNodeType::SET_OPERATION_NODE: + for (auto &child : node.Cast().children) { + if (!SupportsQueryTypeIndependent(*child)) { + return false; + } + } + break; + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &cte = node.Cast(); + if (!SupportsQueryTypeIndependent(*cte.left) || !SupportsQueryTypeIndependent(*cte.right)) { + return false; + } + break; + } + default: + break; + } + return true; +} + +PostgresCapabilities PostgresCapabilities::StatementCapabilities() const { + PostgresCapabilities result(*this); + result.writer_pins_collation = false; + return result; +} + +bool PostgresCapabilities::SupportsPushdown(const SQLStatement &statement) { + if (info.read_only) { + // the read-only check is performed by DuckDB when executing the statement locally + return false; + } + // note: statements cannot be executed locally instead when they turn out not to be translatable, so everything + // that is pushed down must be verified here + switch (statement.type) { + case StatementType::CREATE_STATEMENT: { + auto &create_info = *statement.Cast().info; + if (create_info.temporary) { + return false; + } + switch (create_info.type) { + case CatalogType::TABLE_ENTRY: { + auto &table_info = create_info.Cast(); + switch (table_info.on_conflict) { + case OnCreateConflict::ERROR_ON_CONFLICT: + case OnCreateConflict::REPLACE_ON_CONFLICT: + break; + case OnCreateConflict::IGNORE_ON_CONFLICT: + if (table_info.query) { + // the query must not be executed when the table exists + return false; + } + break; + default: + return false; + } + if (!table_info.partition_keys.empty() || !table_info.sort_keys.empty() || !table_info.options.empty()) { + return false; + } + if (!table_info.query) { + // a regular CREATE TABLE issues the same statement - and keeps the definition of the table (e.g. + // column defaults) in the catalog + return false; + } + // CREATE TABLE AS - the query is serialized into the INSERT that fills the table + auto query = table_info.query->node->Copy(); + if (table_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + // the existing table is dropped before the query is executed - the query must not read it + identifier_set_t cte_names; + identifier_set_t table_names; + PostgresCollectTableNames(*query, cte_names, table_names, true); + if (table_names.count(table_info.GetQualifiedName().Name()) > 0) { + return false; + } + } + return StatementCapabilities().SupportsQueryTypeIndependent(*query); + } + case CatalogType::VIEW_ENTRY: { + auto &view_info = create_info.Cast(); + if (view_info.on_conflict != OnCreateConflict::ERROR_ON_CONFLICT && + view_info.on_conflict != OnCreateConflict::REPLACE_ON_CONFLICT) { + // Postgres has no CREATE VIEW IF NOT EXISTS + return false; + } + if (!view_info.query) { + return false; + } + auto query = view_info.query->node->Copy(); + return StatementCapabilities().SupportsQueryTypeIndependent(*query); + } + default: + return false; + } + } + case StatementType::DROP_STATEMENT: { + auto &drop_info = *statement.Cast().info; + switch (drop_info.type) { + case CatalogType::TABLE_ENTRY: + case CatalogType::VIEW_ENTRY: + case CatalogType::INDEX_ENTRY: + case CatalogType::SEQUENCE_ENTRY: + case CatalogType::TYPE_ENTRY: + // note: schemas are not reloaded when they are removed from the cache, DROP SCHEMA is executed regularly + return drop_info.GetQualifiedName().Path().size() <= 2; + default: + return false; + } + } + case StatementType::ALTER_STATEMENT: { + auto &alter_info = *statement.Cast().info; + if (alter_info.type != AlterType::ALTER_TABLE) { + return false; + } + auto &alter_table_info = alter_info.Cast(); + switch (alter_table_info.alter_table_type) { + case AlterTableType::RENAME_TABLE: + case AlterTableType::REMOVE_COLUMN: + return true; + case AlterTableType::RENAME_COLUMN: + return true; + case AlterTableType::SET_NOT_NULL: + return alter_table_info.Cast().column_path.size() == 1; + case AlterTableType::DROP_NOT_NULL: + return alter_table_info.Cast().column_path.size() == 1; + case AlterTableType::ADD_COLUMN: { + auto &add_info = alter_table_info.Cast(); + auto &column = add_info.new_column; + if (column.HasDefaultValue() || column.Generated()) { + // DuckDB fills in the default values of the existing rows itself + return false; + } + LogicalType type = column.Type(); + if (type.id() == LogicalTypeId::UNBOUND) { + auto &type_expr = UnboundType::GetTypeExpression(type); + if (!type_expr || type_expr->GetExpressionClass() != ExpressionClass::TYPE) { + return false; + } + type = ResolveType(type_expr->Cast()); + } + return type.id() != LogicalTypeId::INVALID && SupportsCastType(type); + } + default: + return false; + } + } + default: + return false; + } +} + +} // namespace duckdb diff --git a/src/storage/postgres_catalog.cpp b/src/storage/postgres_catalog.cpp index 64c6305db..28571080a 100644 --- a/src/storage/postgres_catalog.cpp +++ b/src/storage/postgres_catalog.cpp @@ -1,32 +1,381 @@ #include "storage/postgres_catalog.hpp" -#include "storage/postgres_schema_entry.hpp" -#include "storage/postgres_transaction.hpp" -#include "postgres_oauth.hpp" -#include "postgres_connection.hpp" -#include "postgres_secrets.hpp" -#include "storage/postgres_secret_storage.hpp" -#include "postgres_utils.hpp" + +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/storage/database_size.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/comparison_expression.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" +#include "duckdb/parser/expression/conjunction_expression.hpp" +#include "duckdb/parser/expression/operator_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/type_expression.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/query_node/delete_query_node.hpp" +#include "duckdb/parser/query_node/insert_query_node.hpp" +#include "duckdb/parser/query_node/merge_query_node.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/update_query_node.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/planner/binder.hpp" #include "duckdb/main/attached_database.hpp" #include "duckdb/main/secret/secret_manager.hpp" +#include "postgres_connection_string.hpp" +#include "postgres_connection.hpp" +#include "postgres_oauth.hpp" +#include "postgres_secrets.hpp" +#include "postgres_sql_writer.hpp" +#include "postgres_utils.hpp" +#include "storage/postgres_schema_entry.hpp" +#include "storage/postgres_transaction.hpp" +#include "storage/postgres_secret_storage.hpp" + namespace duckdb { +void PostgresCatalog::InitializeRemotePushdown(ClientContext &context, PostgresConnection &connection) { + Value enabled_val; + if (context.TryGetCurrentSetting("pg_enable_remote_pushdown", enabled_val) && !enabled_val.IsNull()) { + remote_pushdown_enabled = BooleanValue::Get(enabled_val); + } + if (!remote_pushdown_enabled || version.type_v == PostgresInstanceType::REDSHIFT) { + remote_pushdown_enabled = false; + return; + } + // plain table scans are only executed locally when they can be parallelized - this mirrors the conditions of + // PostgresScanFunction::PrepareBind + Value ctid_scan_val; + bool use_ctid_scan = true; + if (context.TryGetCurrentSetting("pg_use_ctid_scan", ctid_scan_val) && !ctid_scan_val.IsNull()) { + use_ctid_scan = BooleanValue::Get(ctid_scan_val); + } + Value text_protocol_val; + bool text_protocol_setting = false; + if (context.TryGetCurrentSetting("pg_use_text_protocol", text_protocol_val) && !text_protocol_val.IsNull()) { + text_protocol_setting = BooleanValue::Get(text_protocol_val); + } + bool use_text_protocol = UseTextProtocol(PostgresUtils::ExtractQueryMode(context), text_protocol_setting); + parallel_scans_supported = use_ctid_scan && !use_text_protocol && version.major_v >= 14; + PostgresPushdownInfo info; + info.version = version; + info.read_only = access_mode == AccessMode::READ_ONLY; + // string ordering only matches DuckDB's (byte-wise) ordering with C-like collations, and case mapping only + // matches DuckDB's (Unicode) case mapping when the character classification is not plain ASCII + string provider_query = version >= PostgresVersion(15, 0) ? "datlocprovider::text" : "'c'"; + string locale_query = version >= PostgresVersion(17, 0) ? "datlocale" : "NULL"; + auto result = connection.TryQuery(context, "SELECT datcollate, datctype, " + provider_query + ", " + locale_query + + ", pg_encoding_to_char(encoding), current_setting('TimeZone') " + "FROM pg_database WHERE datname = current_database()"); + if (result && result->Count() > 0) { + auto collation = result->GetString(0, 0); + auto ctype = result->GetString(0, 1); + auto provider = result->GetString(0, 2); + auto locale = result->IsNull(0, 3) ? string() : result->GetString(0, 3); + auto encoding = result->GetString(0, 4); + static const case_insensitive_set_t BINARY_COLLATIONS = {"C", "POSIX", "C.UTF-8", "C.UTF8"}; + static const case_insensitive_set_t ASCII_CTYPES = {"C", "POSIX"}; + bool utf8 = StringUtil::CIEquals(encoding, "UTF8"); + if (provider == "b") { + // the builtin provider (C, C.UTF-8, PG_UNICODE_FAST) orders by code point + info.binary_collation = true; + info.unicode_case_mapping = utf8 && ASCII_CTYPES.count(locale) == 0; + } else if (provider == "c") { + info.binary_collation = BINARY_COLLATIONS.count(collation) > 0; + info.unicode_case_mapping = utf8 && ASCII_CTYPES.count(ctype) == 0; + } else { + // ICU + info.unicode_case_mapping = utf8; + } + remote_time_zone = result->GetString(0, 5); + } + capabilities = PostgresCapabilities(std::move(info)); +} + +template +static bool PostgresCheckPushdown(bool enabled, PostgresCapabilities &capabilities, const T &construct, + const char *construct_name) { + if (!enabled) { + return false; + } + auto result = capabilities.SupportsPushdown(construct); + if (!result && PostgresConnection::DebugPrintQueries()) { + Printer::Print("Remote pushdown: " + string(construct_name) + " not supported: " + construct.ToString()); + } + return result; +} + +bool PostgresCatalog::SupportsPushdown(const ParsedExpression &expression) { + return PostgresCheckPushdown(remote_pushdown_enabled, capabilities, expression, "expression"); +} + +bool PostgresCatalog::SupportsPushdown(const TableRef &ref) { + return PostgresCheckPushdown(remote_pushdown_enabled, capabilities, ref, "table reference"); +} + +bool PostgresCatalog::SupportsPushdown(const QueryNode &node) { + return PostgresCheckPushdown(remote_pushdown_enabled, capabilities, node, "query"); +} + +bool PostgresCatalog::SupportsPushdown(const SQLStatement &statement) { + return PostgresCheckPushdown(remote_pushdown_enabled, capabilities, statement, "statement"); +} + +vector PostgresCatalog::GetRemotePushdownPreliminaryQueries(ClientContext &context, bool requires_time_zone) { + vector result; + Value time_zone_val; + if (requires_time_zone && context.TryGetCurrentSetting("TimeZone", time_zone_val) && !time_zone_val.IsNull()) { + auto time_zone = time_zone_val.ToString(); + if (!time_zone.empty()) { + // time zone dependent operations (e.g. extracting date parts from a TIMESTAMP WITH TIME ZONE) + // must use DuckDB's time zone, the setting is reverted at the end of the transaction + result.push_back("SET LOCAL TimeZone = " + PostgresUtils::WriteLiteral(time_zone)); + } + } + return result; +} + +static bool PostgresIsSimpleFilter(const ParsedExpression &expr) { + switch (expr.GetExpressionClass()) { + case ExpressionClass::CONJUNCTION: + for (auto &child : expr.Cast().GetChildren()) { + if (!PostgresIsSimpleFilter(*child)) { + return false; + } + } + return true; + case ExpressionClass::COMPARISON: { + auto &comparison = expr.Cast(); + auto left_class = comparison.Left().GetExpressionClass(); + auto right_class = comparison.Right().GetExpressionClass(); + return (left_class == ExpressionClass::COLUMN_REF && right_class == ExpressionClass::CONSTANT) || + (left_class == ExpressionClass::CONSTANT && right_class == ExpressionClass::COLUMN_REF); + } + case ExpressionClass::OPERATOR: { + auto &children = expr.Cast().GetChildren(); + switch (expr.GetExpressionType()) { + case ExpressionType::OPERATOR_IS_NULL: + case ExpressionType::OPERATOR_IS_NOT_NULL: + return children[0]->GetExpressionClass() == ExpressionClass::COLUMN_REF; + case ExpressionType::COMPARE_IN: + if (children[0]->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + return false; + } + for (idx_t i = 1; i < children.size(); i++) { + if (children[i]->GetExpressionClass() != ExpressionClass::CONSTANT) { + return false; + } + } + return true; + default: + return false; + } + } + default: + return false; + } +} + +//! Whether a query is a plain scan of a single table - when parallel scans are possible, those are executed through +//! a regular (parallel) table scan, which pushes down projections and simple filters as well +static bool PostgresIsTrivialQuery(const QueryNode &node, const identifier_set_t &cte_names) { + if (node.type != QueryNodeType::SELECT_NODE || !node.cte_map.map.empty()) { + return false; + } + for (auto &modifier : node.modifiers) { + // a LIMIT is pushed into the table scan as well + if (modifier->type != ResultModifierType::LIMIT_MODIFIER || modifier->Cast().offset) { + return false; + } + } + auto &select = node.Cast(); + if (!select.from_table || select.from_table->type != TableReferenceType::BASE_TABLE || + !select.groups.grouping_sets.empty() || select.having || select.qualify || select.sample || + select.aggregate_handling != AggregateHandling::STANDARD_HANDLING) { + return false; + } + auto &base = select.from_table->Cast(); + if (base.sample || cte_names.count(base.Table()) > 0) { + return false; + } + for (auto &expr : select.select_list) { + auto expr_class = expr->GetExpressionClass(); + if (expr_class != ExpressionClass::COLUMN_REF && expr_class != ExpressionClass::STAR) { + return false; + } + } + return !select.where_clause || PostgresIsSimpleFilter(*select.where_clause); +} + +static bool PostgresIsDMLWithoutReturning(const QueryNode &node) { + switch (node.type) { + case QueryNodeType::UPDATE_QUERY_NODE: + return node.Cast().returning_list.empty(); + case QueryNodeType::DELETE_QUERY_NODE: + return node.Cast().returning_list.empty(); + case QueryNodeType::INSERT_QUERY_NODE: + return node.Cast().returning_list.empty(); + case QueryNodeType::MERGE_QUERY_NODE: + return node.Cast().returning_list.empty(); + default: + return false; + } +} + +static unique_ptr PostgresExecuteLocally(unique_ptr node) { + auto select = make_uniq(); + select->node = std::move(node); + return make_uniq(std::move(select)); +} + +unique_ptr PostgresCatalog::RemoteExecute(ClientContext &context, unique_ptr node) { + // bind the query locally to figure out the result types and names DuckDB would produce + identifier_set_t cte_names; + PostgresSQLWriter::CollectCTENames(*node, cte_names); + auto local_node = node->Copy(); + PostgresSQLWriter::QualifyTables(*local_node, GetName(), cte_names); + if (parallel_scans_supported && PostgresIsTrivialQuery(*node, cte_names)) { + return PostgresExecuteLocally(std::move(local_node)); + } + BoundStatement bound; + try { + auto bind_node = local_node->Copy(); + auto binder = Binder::CreateBinder(context); + bound = binder->Bind(*bind_node); + } catch (std::exception &ex) { + // the query cannot be bound - execute it locally so that DuckDB reports the error + if (PostgresConnection::DebugPrintQueries()) { + Printer::Print("Remote pushdown: executing locally, the query cannot be bound: " + string(ex.what())); + } + return PostgresExecuteLocally(std::move(local_node)); + } + + PostgresWriterResult query; + try { + query = PostgresSQLWriter::WriteQuery(context, *this, *node, &bound.types); + } catch (PostgresPushdownNotSupported &ex) { + // the query contains a construct that cannot be translated for the types it operates on + if (PostgresConnection::DebugPrintQueries()) { + Printer::Print("Remote pushdown: executing locally, " + string(ex.what())); + } + return PostgresExecuteLocally(std::move(local_node)); + } + auto preliminary_queries = GetRemotePushdownPreliminaryQueries(context, query.requires_time_zone); + for (auto &preliminary_query : query.preliminary_queries) { + preliminary_queries.push_back(std::move(preliminary_query)); + } + vector> args; + args.push_back(ConstantExpression::FromValue(Value(GetName()))); + args.push_back(ConstantExpression::FromValue(Value(query.sql))); + if (!preliminary_queries.empty()) { + vector preliminary_values; + for (auto &preliminary_query : preliminary_queries) { + preliminary_values.emplace_back(preliminary_query); + } + args.push_back(make_uniq( + ExpressionType::COMPARE_EQUAL, make_uniq("preliminary_queries"), + ConstantExpression::FromValue(Value::LIST(LogicalType::VARCHAR, std::move(preliminary_values))))); + } + if (PostgresIsDMLWithoutReturning(*node)) { + // the statement yields the number of changed rows + args.push_back(make_uniq(ExpressionType::COMPARE_EQUAL, + make_uniq("return_changed_rows"), + ConstantExpression::FromValue(Value::BOOLEAN(true)))); + } + auto func_ref = make_uniq(); + func_ref->function = make_uniq("postgres_query", std::move(args)); + func_ref->alias = Identifier("postgres_query"); + + // convert the result to the exact types and names DuckDB would produce + auto select_node = make_uniq(); + for (idx_t i = 0; i < bound.types.size(); i++) { + auto column_name = Identifier("c" + to_string(i)); + func_ref->column_name_alias.push_back(column_name); + unique_ptr expr = make_uniq(column_name, Identifier("postgres_query")); + if (TypeExpression::CanRepresent(bound.types[i])) { + expr = make_uniq(bound.types[i], std::move(expr)); + } + expr->SetAlias(bound.names[i]); + select_node->select_list.push_back(std::move(expr)); + } + select_node->from_table = std::move(func_ref); + auto select = make_uniq(); + select->node = std::move(select_node); + return make_uniq(std::move(select)); +} + +unique_ptr PostgresCatalog::RemoteExecute(ClientContext &context, unique_ptr statement) { + PostgresWriterResult query; + try { + query = PostgresSQLWriter::WriteStatement(context, *this, *statement); + } catch (PostgresPushdownNotSupported &ex) { + // statements cannot be executed locally instead + throw NotImplementedException("The statement cannot be executed in Postgres (%s) - use \"SET " + "pg_enable_remote_pushdown = false\" before attaching the database to execute " + "it without remote pushdown", + ex.what()); + } + auto preliminary_queries = GetRemotePushdownPreliminaryQueries(context, query.requires_time_zone); + for (auto &preliminary_query : query.preliminary_queries) { + preliminary_queries.push_back(std::move(preliminary_query)); + } + vector> args; + args.push_back(ConstantExpression::FromValue(Value(GetName()))); + args.push_back(ConstantExpression::FromValue(Value(query.sql))); + auto named_parameter = [&](const char *name, Value value) { + args.push_back(make_uniq(ExpressionType::COMPARE_EQUAL, + make_uniq(name), + ConstantExpression::FromValue(std::move(value)))); + }; + if (!preliminary_queries.empty()) { + vector preliminary_values; + for (auto &preliminary_query : preliminary_queries) { + preliminary_values.emplace_back(preliminary_query); + } + named_parameter("preliminary_queries", Value::LIST(LogicalType::VARCHAR, std::move(preliminary_values))); + } + // the statement can depend on the preliminary queries (e.g. INSERT into a table that is created by them) + named_parameter("prepare", Value::BOOLEAN(false)); + // remove the catalog entries that are modified by the statement from the cache + for (auto &invalidation : query.invalidations) { + InvalidateCacheEntry(context, invalidation); + } + if (query.changed_rows) { + named_parameter("return_changed_rows", Value::BOOLEAN(true)); + } else { + named_parameter("suppress_dml_output", Value::BOOLEAN(true)); + } + auto func_ref = make_uniq(); + func_ref->function = make_uniq("postgres_query", std::move(args)); + return std::move(func_ref); +} + unique_ptr PostgresCatalog::RemoteExecute(ClientContext &context, const string &sql) { vector> args; - args.push_back(make_uniq(Value(GetName()))); - args.push_back(make_uniq(Value(sql))); + args.push_back(ConstantExpression::FromValue(Value(GetName()))); + args.push_back(ConstantExpression::FromValue(Value(sql))); + args.push_back(make_uniq(ExpressionType::COMPARE_EQUAL, + make_uniq("suppress_dml_output"), + ConstantExpression::FromValue(Value::BOOLEAN(true)))); auto func_ref = make_uniq(); func_ref->function = make_uniq("postgres_query", std::move(args)); return func_ref; } string PostgresCatalog::GetConnectDisplay() { + if (!connect_display.empty()) { + // an explicit display string was provided in the ATTACH - this is used by extensions that attach + // Postgres compatible databases (such as AWS RDS or Redshift) through this extension, so that the + // prompt can show how the database was attached instead of the Postgres connection we ended up with + return connect_display; + } // `postgres://[:]` — standard postgres URI shorthand. dbname is already shown // via current_database in the prompt, so it's omitted here. string host, port; @@ -68,13 +417,17 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n } PostgresCatalog::PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, string attach_path_p, - AccessMode access_mode, string schema_to_load, PostgresIsolationLevel isolation_level, - const string &secret_name, SecretStorageTable secret_storage_table_p) + AccessMode access_mode, vector schemas_to_load, + PostgresIsolationLevel isolation_level, const string &secret_name, + SecretStorageTable secret_storage_table_p, PostgresTextProtocolMode text_protocol_mode, + string connect_display_p) : Catalog(db_p), attach_path(std::move(attach_path_p)), access_mode(access_mode), isolation_level(isolation_level), - schemas(*this, schema_to_load), connection_pool(make_shared_ptr(*this, ctx)), - default_schema(schema_to_load), secret_storage_table(std::move(secret_storage_table_p)) { + text_protocol_mode(text_protocol_mode), schemas(*this, schemas_to_load), + connection_pool(make_shared_ptr(*this, ctx)), + default_schema(schemas_to_load.size() > 0 ? Identifier(schemas_to_load[0]) : Identifier()), + secret_storage_table(std::move(secret_storage_table_p)), connect_display(std::move(connect_display_p)) { auto secret_entry = GetSecretEntry(ctx, secret_name); - this->rds_token_config = PostgresAws::ExtractTokenConfigFromSecret(secret_entry); + this->rds_token_config = PostgresAws::ExtractTokenConfigFromSecret(ctx, secret_entry); if (!rds_token_config.Enabled()) { this->connection_string = CreateConnectionString(secret_entry, attach_path); } @@ -86,9 +439,10 @@ PostgresCatalog::PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, str PostgresPoolConnection connection; { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(ctx); - connection = connection_pool->GetConnection(); + connection = connection_pool->Acquire(); } this->version = connection.GetConnection().GetPostgresVersion(ctx); + InitializeRemotePushdown(ctx, connection.GetConnection()); } unique_ptr PostgresCatalog::GetSecretEntry(ClientContext &ctx, const std::string &secret_name) { @@ -109,6 +463,9 @@ unique_ptr PostgresCatalog::GetSecretEntry(ClientContext &ctx, cons string PostgresCatalog::CreateConnectionString(optional_ptr secret_entry, const string &attach_path) { string connection_string = attach_path; + bool user_specified_app_name = PostgresConnectionString::ContainsApplicationName(connection_string); + bool secret_has_app_name = false; + string secret_connection_info; if (secret_entry) { // secret found - read data const auto &kv_secret = dynamic_cast(*secret_entry->secret); @@ -128,15 +485,35 @@ string PostgresCatalog::CreateConnectionString(optional_ptr secret_ throw BinderException("ATTACH path must be empty when 'URI' option is specified, was: \"%s\"", attach_path); } - return uri_val.ToString(); + string uri = uri_val.ToString(); + bool uri_has_app_name = PostgresConnectionString::ContainsApplicationName(uri); + if (!uri_has_app_name) { + PostgresConnectionString::AppendDefaultApplicationName(uri, PostgresConnectionStringFormat::URI); + } + return uri; } - string new_connection_info; + secret_has_app_name = !kv_secret.TryGetValue("application_name").IsNull() || + !kv_secret.TryGetValue("fallback_application_name").IsNull(); + for (const string &opt_name : PostgresSecrets::ConnectionOptionNames()) { - new_connection_info += PostgresUtils::ExtractConnectionOption(kv_secret, opt_name); + secret_connection_info += PostgresConnectionString::ExtractOption(kv_secret, opt_name); + } + } + + if (!(user_specified_app_name || secret_has_app_name)) { + if (PostgresConnectionString::IsURI(connection_string)) { + // libpq cannot read key/value pairs in front of a URI, so the name goes into the URI + PostgresConnectionString::AppendDefaultApplicationName(connection_string, + PostgresConnectionStringFormat::URI); + } else { + PostgresConnectionString::AppendDefaultApplicationName(secret_connection_info, + PostgresConnectionStringFormat::KEY_VALUE); + secret_connection_info.push_back(' '); } - connection_string = new_connection_info + connection_string; } + connection_string = secret_connection_info + connection_string; + return connection_string; } @@ -161,7 +538,7 @@ string PostgresCatalog::GetConnectionString() { this->rds_token = PostgresAws::GenerateRdsAuthToken(GetAttached(), rds_token_config); this->rds_token_last_refreshed = now; this->connection_string = rds_token_config.base_connection_string + - "password=" + PostgresUtils::EscapeConnectionString(rds_token) + " " + attach_path; + "password=" + PostgresConnectionString::Escape(rds_token) + " " + attach_path; } return std::string(connection_string.data(), connection_string.length()); } @@ -175,13 +552,14 @@ void PostgresCatalog::Initialize(bool load_builtin) { optional_ptr PostgresCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { auto &postgres_transaction = PostgresTransaction::Get(transaction.GetContext(), *this); - auto entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, info.schema.GetIdentifierName()); + auto entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, + info.GetQualifiedName().Schema().GetIdentifierName()); if (entry) { switch (info.on_conflict) { case OnCreateConflict::REPLACE_ON_CONFLICT: { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; - try_drop.name = info.schema; + try_drop.SetName(info.GetQualifiedName().Schema()); try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; try_drop.cascade = false; schemas.DropEntry(postgres_transaction, try_drop); @@ -191,7 +569,8 @@ optional_ptr PostgresCatalog::CreateSchema(CatalogTransaction tran return entry; case OnCreateConflict::ERROR_ON_CONFLICT: default: - throw BinderException("Failed to create schema \"%s\": schema already exists", info.schema); + throw BinderException("Failed to create schema \"%s\": schema already exists", + info.GetQualifiedName().Schema()); } } return schemas.CreateSchema(postgres_transaction, info); @@ -217,6 +596,11 @@ optional_ptr PostgresCatalog::LookupSchema(CatalogTransactio schema_name = postgres_transaction.GetTemporarySchema(); } auto entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, schema_name); + if (!entry && schema_name == DEFAULT_SCHEMA) { + // DuckDB's default schema name - resolve it to the default schema of the Postgres database, + // unless a schema with that name exists in Postgres + entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, default_schema.GetIdentifierName()); + } if (!entry && if_not_found != OnEntryNotFound::RETURN_NULL) { throw BinderException("Schema with name \"%s\" not found", schema_name); } @@ -244,6 +628,24 @@ DatabaseSize PostgresCatalog::GetDatabaseSize(ClientContext &context) { return size; } +void PostgresCatalog::InvalidateCacheEntry(ClientContext &context, const PostgresCacheInvalidation &invalidation) { + auto &transaction = PostgresTransaction::Get(context, *this); + if (invalidation.type == CatalogType::SCHEMA_ENTRY) { + schemas.InvalidateEntry(invalidation.name); + return; + } + auto schema = schemas.GetEntry(context, transaction, invalidation.schema); + if (!schema) { + return; + } + auto &postgres_schema = schema->Cast(); + if (invalidation.all_tables) { + postgres_schema.InvalidateTables(); + } else { + postgres_schema.InvalidateEntry(invalidation.type, invalidation.name); + } +} + void PostgresCatalog::ClearCache() { schemas.ClearEntries(); } @@ -256,7 +658,7 @@ void PostgresCatalog::RegisterSecretStorage() { } // Look up whether the table exists in DB - auto connection = connection_pool->GetConnection(); + auto connection = connection_pool->Acquire(); bool secret_storage_table_exists = secret_storage_table.Exists(connection.GetConnection()); string attached_database_name = GetAttached().GetName().GetIdentifierName(); @@ -306,4 +708,44 @@ void PostgresCatalog::RegisterSecretStorage() { } } +dbconnector::attached::AttachedCatalog PostgresCatalog::Lookup(ClientContext &ctx, const Identifier &name) { + using namespace dbconnector::attached; + + AttachedCatalog attached_catalog = AttachedCatalog::Lookup(ctx, "postgres", name); + if (!attached_catalog) { + throw InvalidInputException("Attached PostgreSQL database not found in the specified client session, name: %s", + name); + } + return attached_catalog; +} +//! Resolve whether scans against this database must use the text protocol instead of the binary COPY +//! protocol. This is the single place where that decision is made for an attached database, so every +//! scan path (table scans, postgres_query/CONNECT, ...) agrees. +bool PostgresCatalog::UseBinaryPlainQueryWithTextFallback(PostgresQueryMode query_mode, + bool use_text_protocol_setting) const { + return query_mode == PostgresQueryMode::AUTO && text_protocol_mode == PostgresTextProtocolMode::AUTO && + !use_text_protocol_setting && version.type_v == PostgresInstanceType::REDSHIFT; +} + +bool PostgresCatalog::UseTextProtocol(PostgresQueryMode query_mode, bool use_text_protocol_setting) const { + switch (query_mode) { + case PostgresQueryMode::COPY_BINARY: + case PostgresQueryMode::PLAIN_QUERY_BINARY: + return false; + case PostgresQueryMode::PLAIN_QUERY_TEXT: + return true; + default: { + switch (text_protocol_mode) { + case PostgresTextProtocolMode::TEXT: + return true; + case PostgresTextProtocolMode::BINARY: + return false; + default: + // Redshift does not support the binary COPY protocol + return use_text_protocol_setting || version.type_v == PostgresInstanceType::REDSHIFT; + } + } + } +} + } // namespace duckdb diff --git a/src/storage/postgres_catalog_set.cpp b/src/storage/postgres_catalog_set.cpp index 2243adb2a..c23eda443 100644 --- a/src/storage/postgres_catalog_set.cpp +++ b/src/storage/postgres_catalog_set.cpp @@ -1,19 +1,22 @@ #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "storage/postgres_schema_entry.hpp" namespace duckdb { -PostgresCatalogSet::PostgresCatalogSet(Catalog &catalog, bool is_loaded_p) : catalog(catalog), is_loaded(is_loaded_p) { +PostgresCatalogSet::PostgresCatalogSet(Catalog &catalog, bool is_loaded_p) + : catalog(catalog), is_loaded(is_loaded_p), loading_thread(thread_id()) { } optional_ptr PostgresCatalogSet::GetEntry(ClientContext &context, PostgresTransaction &transaction, const string &name) { - TryLoadEntries(context, transaction); { - lock_guard l(entry_lock); + auto l = LoadEntriesForRead(context, transaction); auto entry = entries.find(name); if (entry != entries.end()) { // entry found @@ -42,18 +45,115 @@ optional_ptr PostgresCatalogSet::GetEntry(ClientContext &context, return nullptr; } +static string ComputeStalenessSignature(PostgresResult &result) { + if (result.ColumnCount() < 3) { + throw InvalidInputException( + "pg_staleness_query must return at least 3 columns (identity, name, revision marker), got %llu", + result.ColumnCount()); + } + string signature; + auto rows = result.Count(); + for (idx_t row = 0; row < rows; row++) { + signature += result.GetString(row, 0); + signature += '\0'; + signature += result.GetString(row, 1); + signature += '\0'; + signature += result.GetString(row, 2); + signature += '\x01'; + } + return signature; +} + +//! Runs the staleness query on a short-lived pooled connection, not the caller's own. +//! Avoids pinning the caller's connection (and, under PgBouncer transaction-mode pooling, +//! a backend slot) for the round-trip. +static unique_ptr RunStalenessQuery(Catalog &catalog, ClientContext &context, + const string &staleness_query) { + auto connection = catalog.Cast().GetConnectionPool().Acquire(); + return connection.GetConnection().Query(context, staleness_query); +} + +void PostgresCatalogSet::RefreshStalenessSignature(PostgresTransaction &transaction, bool use_transaction_connection) { + auto &context = *transaction.GetContext(); + auto staleness_query = GetStalenessQuery(context); + if (staleness_query.empty()) { + return; + } + auto result = use_transaction_connection ? transaction.Query(staleness_query) + : RunStalenessQuery(catalog, context, staleness_query); + auto signature = ComputeStalenessSignature(*result); + if (transaction.GetTransactionState() == PostgresTransactionState::TRANSACTION_STARTED) { + // transaction still open - only valid on commit, since xmin reverts on rollback + transaction.StageStalenessSignature(*this, std::move(signature)); + return; + } + staleness_signature = std::move(signature); +} + +void PostgresCatalogSet::PromoteStalenessSignature(string signature) { + // called from Commit(), possibly a different thread than the one that staged it + lock_guard lock(load_lock); + staleness_signature = std::move(signature); +} + +void PostgresCatalogSet::LoadEntriesLocked(ClientContext &context, PostgresTransaction &transaction) { + loading_thread = ThreadUtil::GetThreadId(); + try { + LoadEntries(context, transaction); + } catch (...) { + loading_thread = thread_id(); + throw; + } + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/false); + is_loaded = true; + loading_thread = thread_id(); +} + +unique_lock PostgresCatalogSet::LoadEntriesForRead(ClientContext &context, PostgresTransaction &transaction) { + while (true) { + TryLoadEntries(context, transaction); + unique_lock l(entry_lock); + // the loading thread may read partially loaded entries + if (is_loaded || loading_thread == ThreadUtil::GetThreadId()) { + return l; + } + } +} + void PostgresCatalogSet::TryLoadEntries(ClientContext &context, PostgresTransaction &transaction) { if (HasInternalDependencies()) { - if (is_loaded) { + if (is_loaded || loading_thread == ThreadUtil::GetThreadId()) { return; } } + if (is_loaded) { + auto staleness_query = GetStalenessQuery(context); + if (staleness_query.empty()) { + return; + } + // runs outside load_lock - avoids blocking other threads on this thread's network round-trip + auto result = RunStalenessQuery(catalog, context, staleness_query); + auto signature = ComputeStalenessSignature(*result); + + lock_guard lock(load_lock); + if (is_loaded) { + if (signature == staleness_signature) { + return; + } + ClearEntriesLocked(); + } + // else: someone else cleared/reloaded while we were checking - fall through to the + // reload below, still holding load_lock. + LoadEntriesLocked(context, transaction); + return; + } + lock_guard lock(load_lock); if (is_loaded) { + // someone else already loaded while we were waiting for the lock. return; } - is_loaded = true; - LoadEntries(context, transaction); + LoadEntriesLocked(context, transaction); } optional_ptr PostgresCatalogSet::ReloadEntry(PostgresTransaction &transaction, const string &name) { @@ -66,25 +166,28 @@ void PostgresCatalogSet::DropEntry(PostgresTransaction &transaction, DropInfo &i if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { drop_query += " IF EXISTS "; } - if (!info.schema.empty()) { - drop_query += PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()) + "."; + if (!info.GetQualifiedName().Schema().empty() && info.type != CatalogType::SCHEMA_ENTRY) { + drop_query += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()) + "."; } - drop_query += PostgresUtils::WriteIdentifier(info.name.GetIdentifierName()); + drop_query += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Name().GetIdentifierName()); if (info.cascade) { - drop_query += "CASCADE"; + drop_query += " CASCADE"; } transaction.Query(drop_query); // erase the entry from the catalog set - lock_guard l(entry_lock); - entries.erase(info.name.GetIdentifierName()); + { + lock_guard l(entry_lock); + entries.erase(info.GetQualifiedName().Name().GetIdentifierName()); + } + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/true); } void PostgresCatalogSet::Scan(ClientContext &context, PostgresTransaction &transaction, const std::function &callback) { - TryLoadEntries(context, transaction); - lock_guard l(entry_lock); + auto l = LoadEntriesForRead(context, transaction); for (auto &entry : entries) { + transaction.ReferenceEntry(entry.second); callback(*entry.second); } } @@ -101,7 +204,28 @@ optional_ptr PostgresCatalogSet::CreateEntry(PostgresTransaction & return result; } +void PostgresCatalogSet::InvalidateEntry(const string &name) { + lock_guard l(entry_lock); + auto entry = entries.find(name); + if (entry == entries.end()) { + auto name_entry = entry_map.find(name); + if (name_entry == entry_map.end()) { + return; + } + entry = entries.find(name_entry->second); + if (entry == entries.end()) { + return; + } + } + entries.erase(entry); +} + void PostgresCatalogSet::ClearEntries() { + lock_guard load_guard(load_lock); + ClearEntriesLocked(); +} + +void PostgresCatalogSet::ClearEntriesLocked() { lock_guard entry_guard(entry_lock); entry_map.clear(); entries.clear(); diff --git a/src/storage/postgres_clear_cache.cpp b/src/storage/postgres_clear_cache.cpp index 058cbad30..b064d9770 100644 --- a/src/storage/postgres_clear_cache.cpp +++ b/src/storage/postgres_clear_cache.cpp @@ -1,4 +1,5 @@ #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "postgres_scanner.hpp" @@ -13,7 +14,7 @@ struct ClearCacheFunctionData : public TableFunctionData { }; static unique_ptr ClearCacheBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); return_types.push_back(LogicalType::BOOLEAN); names.emplace_back("Success"); @@ -46,6 +47,6 @@ void PostgresClearCacheFunction::ClearCacheOnSetting(ClientContext &context, Set } PostgresClearCacheFunction::PostgresClearCacheFunction() - : TableFunction("pg_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { + : TableFunction("postgres_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/postgres_configure_pool.cpp b/src/storage/postgres_configure_pool.cpp deleted file mode 100644 index 2032b2346..000000000 --- a/src/storage/postgres_configure_pool.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include "storage/postgres_connection_pool.hpp" - -#include - -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/database_manager.hpp" - -#include "storage/postgres_catalog.hpp" - -namespace duckdb { - -namespace { - -enum class ExecState { UNINITIALIZED, EXHAUSTED }; - -struct ConfigurePoolBindData : public TableFunctionData { - std::pair catalog_name; - std::pair acquire_mode; - std::pair max_connections; - std::pair wait_timeout_millis; - std::pair enable_thread_local_cache; - std::pair max_lifetime_millis; - std::pair idle_timeout_millis; - std::pair enable_reaper_thread; - std::pair health_check_query; - - static Value Lookup(const named_parameter_map_t &map, const std::string &key) { - auto it = map.find(Identifier(key)); - if (it == map.end()) { - return Value(); - } - return it->second; - } - - static std::pair LookupString(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair("", true); - } - std::string str = StringValue::Get(val); - return std::make_pair(std::move(str), false); - } - - static std::pair LookupUBigInt(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair(0, true); - } - uint64_t num = UBigIntValue::Get(val); - return std::make_pair(num, false); - } - - static std::pair LookupBool(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair(false, true); - } - bool flag = BooleanValue::Get(val); - return std::make_pair(flag, false); - } - - static std::pair LookupAcquireMode(const named_parameter_map_t &map, - const std::string &key) { - std::pair st_pair = LookupString(map, key); - if (st_pair.second) { - return std::make_pair(dbconnector::pool::AcquireMode::FORCE, true); - } - try { - dbconnector::pool::AcquireMode mode = dbconnector::pool::AcquireModeHelpers::FromString(st_pair.first); - return std::make_pair(mode, false); - } catch (const std::exception &e) { - throw BinderException(e.what()); - } - } - - ConfigurePoolBindData(const named_parameter_map_t &map) - : catalog_name(LookupString(map, "catalog_name")), acquire_mode(LookupAcquireMode(map, "acquire_mode")), - max_connections(LookupUBigInt(map, "max_connections")), - wait_timeout_millis(LookupUBigInt(map, "wait_timeout_millis")), - enable_thread_local_cache(LookupBool(map, "enable_thread_local_cache")), - max_lifetime_millis(LookupUBigInt(map, "max_lifetime_millis")), - idle_timeout_millis(LookupUBigInt(map, "idle_timeout_millis")), - enable_reaper_thread(LookupBool(map, "enable_reaper_thread")), - health_check_query(LookupString(map, "health_check_query")) { - if (catalog_name.second && - !(acquire_mode.second && max_connections.second && wait_timeout_millis.second && - enable_thread_local_cache.second && max_lifetime_millis.second && idle_timeout_millis.second && - enable_reaper_thread.second && health_check_query.second)) { - throw BinderException("'catalog_name' argument must be specified to change any option value on the " - "connection pool of this catalog"); - } - } -}; - -struct GlobalState : public GlobalTableFunctionState {}; - -struct LocalState : public LocalTableFunctionState { - ExecState exec_state = ExecState::UNINITIALIZED; -}; - -} // namespace - -static void AddColumn(vector &return_types, vector &names, const std::string &col_name, - LogicalType col_type) { - names.emplace_back(col_name); - return_types.emplace_back(col_type); -} - -static unique_ptr ConfigurePoolBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - AddColumn(return_types, names, "catalog_name", LogicalType::VARCHAR); - AddColumn(return_types, names, "acquire_mode", LogicalType::VARCHAR); - AddColumn(return_types, names, "available_connections", LogicalType::UBIGINT); - AddColumn(return_types, names, "max_connections", LogicalType::UBIGINT); - AddColumn(return_types, names, "wait_timeout_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "cache_hits", LogicalType::UBIGINT); - AddColumn(return_types, names, "cache_misses", LogicalType::UBIGINT); - AddColumn(return_types, names, "try_failures", LogicalType::UBIGINT); - AddColumn(return_types, names, "thread_local_cache_enabled", LogicalType::BOOLEAN); - AddColumn(return_types, names, "thread_local_cache_hits", LogicalType::UBIGINT); - AddColumn(return_types, names, "thread_local_cache_misses", LogicalType::UBIGINT); - AddColumn(return_types, names, "max_lifetime_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "idle_timeout_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "reaper_thread_running", LogicalType::BOOLEAN); - AddColumn(return_types, names, "reaper_thread_period_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "health_check_query", LogicalType::VARCHAR); - - return make_uniq(input.named_parameters); -} - -static unique_ptr ConfigurePoolInitGlobalState(ClientContext &, TableFunctionInitInput &) { - return make_uniq(); -} - -static unique_ptr ConfigurePoolInitLocalState(ExecutionContext &, TableFunctionInitInput &, - GlobalTableFunctionState *) { - return make_uniq(); -} - -static void ConfigurePoolFunction(ClientContext &context, TableFunctionInput &input, DataChunk &output) { - auto &bdata = input.bind_data->Cast(); - auto &lstate = input.local_state->Cast(); - - if (lstate.exec_state == ExecState::EXHAUSTED) { - output.SetChildCardinality(0); - return; - } - - // collect pools - std::vector cat_names; - std::vector> pools; - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { - auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - continue; - } - if (!bdata.catalog_name.second && catalog.GetName() != Identifier(bdata.catalog_name.first)) { - continue; - } - cat_names.push_back(catalog.GetName().GetIdentifierName()); - shared_ptr pool = catalog.Cast().GetConnectionPoolPtr(); - pools.emplace_back(std::move(pool)); - } - - if (!bdata.catalog_name.second && pools.size() == 0) { - throw InvalidInputException("Catalog not found, name: '%s'", bdata.catalog_name.first); - } - - // configure the single pool if specified - if (!bdata.catalog_name.second && pools.size() > 0) { - auto &pool = pools.at(0); - if (!bdata.acquire_mode.second) { - pool->SetAcquireMode(bdata.acquire_mode.first); - } - if (!bdata.max_connections.second) { - pool->SetMaxConnections(bdata.max_connections.first); - } - if (!bdata.wait_timeout_millis.second) { - pool->SetWaitTimeoutMillis(bdata.wait_timeout_millis.first); - } - if (!bdata.enable_thread_local_cache.second) { - pool->SetThreadLocalCacheEnabled(bdata.enable_thread_local_cache.first); - } - if (!bdata.max_lifetime_millis.second) { - pool->SetMaxLifetimeMillis(bdata.max_lifetime_millis.first); - } - if (!bdata.idle_timeout_millis.second) { - pool->SetIdleTimeoutMillis(bdata.idle_timeout_millis.first); - } - if (!bdata.enable_reaper_thread.second) { - if (bdata.enable_reaper_thread.first) { - pool->EnsureReaperRunning(); - } else { - pool->ShutdownReaper(); - } - } - if (!bdata.health_check_query.second) { - pool->SetHealthCheckQuery(bdata.health_check_query.first); - } - } - - // set results - idx_t row_idx = 0; - for (auto &pool : pools) { - idx_t col_idx = 0; - output.data[col_idx++].SetValue(row_idx, Value(cat_names.at(row_idx))); - output.data[col_idx++].SetValue(row_idx, - Value(dbconnector::pool::AcquireModeHelpers::ToString(pool->GetAcquireMode()))); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetAvailableConnections())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetMaxConnections())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetWaitTimeoutMillis())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetCacheHits())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetCacheMisses())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetTryFailures())); - output.data[col_idx++].SetValue(row_idx, Value::BOOLEAN(pool->IsThreadLocalCacheEnabled())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetThreadLocalCacheHits())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetThreadLocalCacheMisses())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetMaxLifetimeMillis())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetIdleTimeoutMillis())); - output.data[col_idx++].SetValue(row_idx, Value::BOOLEAN(pool->IsReaperRunning())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetReaperPeriodMillis())); - output.data[col_idx++].SetValue(row_idx, Value(pool->GetHealthCheckQuery())); - row_idx++; - } - - output.SetChildCardinality(row_idx); - lstate.exec_state = ExecState::EXHAUSTED; -} - -PostgresConfigurePoolFunction::PostgresConfigurePoolFunction() - : TableFunction("postgres_configure_pool", std::vector(), ConfigurePoolFunction, ConfigurePoolBind, - ConfigurePoolInitGlobalState, ConfigurePoolInitLocalState) { - named_parameters["catalog_name"] = LogicalType::VARCHAR; - named_parameters["acquire_mode"] = LogicalType::VARCHAR; - named_parameters["max_connections"] = LogicalType::UBIGINT; - named_parameters["wait_timeout_millis"] = LogicalType::UBIGINT; - named_parameters["enable_thread_local_cache"] = LogicalType::BOOLEAN; - named_parameters["max_lifetime_millis"] = LogicalType::UBIGINT; - named_parameters["idle_timeout_millis"] = LogicalType::UBIGINT; - named_parameters["enable_reaper_thread"] = LogicalType::BOOLEAN; - named_parameters["health_check_query"] = LogicalType::VARCHAR; -} - -} // namespace duckdb diff --git a/src/storage/postgres_connection_pool.cpp b/src/storage/postgres_connection_pool.cpp index 03518ec84..ea6ca9ee9 100644 --- a/src/storage/postgres_connection_pool.cpp +++ b/src/storage/postgres_connection_pool.cpp @@ -1,31 +1,22 @@ #include "storage/postgres_connection_pool.hpp" +#include #include +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/function/table_function.hpp" + +#include "dbconnector/functions/configure_pool.hpp" + #include "storage/postgres_catalog.hpp" namespace duckdb { -static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &context); - -static std::string GetHealthCheckQueryFromConfig(ClientContext &context) { - Value val; - if (context.TryGetCurrentSetting("pg_pool_health_check_query", val)) { - if (val.IsNull()) { - return std::string(); - } - return StringValue::Get(val); - } - return PostgresConnectionPool::DefaultHealthCheckQuery(); -} +using dbconnector::functions::ConfigurePool; PostgresConnectionPool::PostgresConnectionPool(PostgresCatalog &postgres_catalog, ClientContext &context) - : dbconnector::pool::ConnectionPool(CreateConfig(context)), postgres_catalog(postgres_catalog), - health_check_query(GetHealthCheckQueryFromConfig(context)) { -} - -PostgresPoolConnection PostgresConnectionPool::ForceGetConnection() { - return ForceAcquire(); + : dbconnector::pool::ConnectionPool(CreateConfig(context)), postgres_catalog(postgres_catalog) { } bool PostgresConnectionPool::TryGetConnection(PostgresPoolConnection &connection) { @@ -37,10 +28,6 @@ bool PostgresConnectionPool::TryGetConnection(PostgresPoolConnection &connection return true; } -PostgresPoolConnection PostgresConnectionPool::GetConnection() { - return Acquire(); -} - std::unique_ptr PostgresConnectionPool::CreateNewConnection() { auto conn = PostgresConnection::Open(postgres_catalog.GetConnectionString(), postgres_catalog.attach_path); return make_uniq(std::move(conn)); @@ -59,21 +46,7 @@ void PostgresConnectionPool::ResetConnection(PostgresConnection &conn) { conn.Reset(query); } -std::string PostgresConnectionPool::GetHealthCheckQuery() { - std::lock_guard guard(config_mutex); - return std::string(health_check_query.data(), health_check_query.length()); -} - -void PostgresConnectionPool::SetHealthCheckQuery(const std::string &query) { - std::lock_guard guard(config_mutex); - this->health_check_query = std::string(query.data(), query.length()); -} - -std::string PostgresConnectionPool::DefaultHealthCheckQuery() { - return "SELECT 1"; -} - -static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &ctx) { +dbconnector::pool::ConnectionPoolConfig PostgresConnectionPool::CreateConfig(ClientContext &ctx) { dbconnector::pool::ConnectionPoolConfig config; { @@ -118,6 +91,12 @@ static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &ctx) config.start_reaper_thread = BooleanValue::Get(val); } } + { + Value val; + if (ctx.TryGetCurrentSetting("pg_pool_health_check_query", val) && !val.IsNull()) { + config.health_check_query = StringValue::Get(val); + } + } return config; } @@ -141,4 +120,22 @@ void PostgresConnectionPool::ValidatePoolAcquireMode(ClientContext &context, Set } } +static shared_ptr GetConnnectionPoolFromCatalog(Catalog &catalog) { + if (catalog.GetCatalogType() != "postgres") { + return nullptr; + } + return catalog.Cast().GetConnectionPoolPtr(); +} + +PostgresConfigurePoolFunction::PostgresConfigurePoolFunction() + : TableFunction("postgres_configure_pool", std::vector(), + ConfigurePool::Function, ConfigurePool::Bind, + ConfigurePool::InitGlobalState, ConfigurePool::InitLocalState) { + GetSignature().WithTypedKwargs("options", [](TypedKwargs &options) { + for (auto &en : ConfigurePool::NamedParameters()) { + options.Add(en.first, en.second); + } + }); +} + } // namespace duckdb diff --git a/src/storage/postgres_delete.cpp b/src/storage/postgres_delete.cpp index 97b829fff..e47bab1bb 100644 --- a/src/storage/postgres_delete.cpp +++ b/src/storage/postgres_delete.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_delete.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_table_entry.hpp" #include "duckdb/planner/operator/logical_delete.hpp" diff --git a/src/storage/postgres_index.cpp b/src/storage/postgres_index.cpp index cf26c37bd..3b8bfcb63 100644 --- a/src/storage/postgres_index.cpp +++ b/src/storage/postgres_index.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_index.hpp" #include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/planner/operator/logical_extension_operator.hpp" @@ -25,19 +29,21 @@ SourceResultType PostgresCreateIndex::GetDataInternal(ExecutionContext &context, auto &catalog = table.catalog; auto &schema = table.schema; auto transaction = catalog.GetCatalogTransaction(context.client); - auto existing = schema.GetEntry(transaction, CatalogType::INDEX_ENTRY, info->index_name); + auto existing = schema.GetEntry(transaction, CatalogType::INDEX_ENTRY, info->GetIndexName()); if (existing) { switch (info->on_conflict) { case OnCreateConflict::IGNORE_ON_CONFLICT: return SourceResultType::FINISHED; case OnCreateConflict::ERROR_ON_CONFLICT: - throw BinderException("Index with name \"%s\" already exists in schema \"%s\"", info->index_name, + throw BinderException("Index with name \"%s\" already exists in schema \"%s\"", info->GetIndexName(), table.schema.name); case OnCreateConflict::REPLACE_ON_CONFLICT: { DropInfo drop_info; drop_info.type = CatalogType::INDEX_ENTRY; - drop_info.schema = info->schema; - drop_info.name = info->index_name; + drop_info.SetQualifiedName(QualifiedName(drop_info.GetQualifiedName().Catalog(), + info->GetQualifiedName().Schema(), + drop_info.GetQualifiedName().Name())); + drop_info.SetName(info->GetIndexName()); schema.DropEntry(context.client, drop_info); break; } @@ -66,6 +72,11 @@ class LogicalPostgresCreateIndex : public LogicalExtensionOperator { return planner.Make(std::move(info), table); } + // Optimizer hooks of other extensions call this on every extension operator; the default throws. + string GetExtensionName() const override { + return "postgres_scanner"; + } + void Serialize(Serializer &serializer) const override { throw NotImplementedException("Cannot serialize Postgres Create index"); } diff --git a/src/storage/postgres_index_entry.cpp b/src/storage/postgres_index_entry.cpp index 0f42ef32d..1c33b6108 100644 --- a/src/storage/postgres_index_entry.cpp +++ b/src/storage/postgres_index_entry.cpp @@ -1,4 +1,5 @@ #include "storage/postgres_index_entry.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" namespace duckdb { diff --git a/src/storage/postgres_index_set.cpp b/src/storage/postgres_index_set.cpp index e4f4c6e78..00cc881ba 100644 --- a/src/storage/postgres_index_set.cpp +++ b/src/storage/postgres_index_set.cpp @@ -1,4 +1,7 @@ #include "storage/postgres_index_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" @@ -15,7 +18,7 @@ PostgresIndexSet::PostgresIndexSet(PostgresSchemaEntry &schema, unique_ptr &schemas) { string base_query = R"( SELECT pg_namespace.oid, tablename, indexname FROM pg_indexes @@ -24,8 +27,8 @@ JOIN pg_namespace ON (schemaname=nspname) ORDER BY pg_namespace.oid; )"; string condition; - if (!schema.empty()) { - condition += "WHERE pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -39,9 +42,11 @@ void PostgresIndexSet::LoadEntries(ClientContext &context, PostgresTransaction & auto table_name = result.GetString(row, 1); auto index_name = result.GetString(row, 2); CreateIndexInfo info; - info.schema = schema.name; + info.constraint_type = IndexConstraintType::NONE; + info.SetQualifiedName( + QualifiedName(info.GetQualifiedName().Catalog(), schema.name, info.GetQualifiedName().Name())); info.table = Identifier(table_name); - info.index_name = Identifier(index_name); + info.SetIndexName(Identifier(index_name)); auto index_entry = make_shared_ptr(catalog, schema, info, table_name); CreateEntry(transaction, std::move(index_entry)); } @@ -65,7 +70,7 @@ string PGGetCreateIndexSQL(CreateIndexInfo &info, TableCatalogEntry &tbl) { sql += " UNIQUE"; } sql += " INDEX "; - sql += PostgresUtils::QuotePostgresIdentifier(info.index_name.GetIdentifierName()); + sql += PostgresUtils::QuotePostgresIdentifier(info.GetIndexName().GetIdentifierName()); sql += " ON "; sql += PostgresUtils::QuotePostgresIdentifier(tbl.schema.name.GetIdentifierName()) + "."; sql += PostgresUtils::QuotePostgresIdentifier(tbl.name.GetIdentifierName()); diff --git a/src/storage/postgres_insert.cpp b/src/storage/postgres_insert.cpp index 467b995b9..4142df9c2 100644 --- a/src/storage/postgres_insert.cpp +++ b/src/storage/postgres_insert.cpp @@ -1,4 +1,9 @@ #include "storage/postgres_insert.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/planner/operator/logical_insert.hpp" @@ -14,34 +19,96 @@ namespace duckdb { -PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, - physical_index_vector_t column_index_map_p) +PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, TableCatalogEntry &table) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), - column_index_map(std::move(column_index_map_p)) { + use_plain_inserts(UsePlainInserts(pg_catalog, context)) { } -PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, - unique_ptr info) +PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), - info(std::move(info)) { + info(std::move(info)), use_plain_inserts(UsePlainInserts(pg_catalog, context)) { +} + +//===--------------------------------------------------------------------===// +// Insert Columns +//===--------------------------------------------------------------------===// +//! The input of the insert contains every physical column of the table. Columns with a Postgres-side default receive +//! NULL when they were not specified by the user, so such NULL values are replaced with the Postgres-side default +struct PostgresInsertColumns { + explicit PostgresInsertColumns(const PostgresTableEntry &table) { + for (auto &col : table.GetColumns().Physical()) { + auto logical_index = col.Logical().index; + names.push_back(table.postgres_names[logical_index]); + defaults.push_back(table.column_defaults[logical_index]); + } + } + + //! Postgres names of the input columns + vector names; + //! How Postgres fills in each input column + vector defaults; + + void CheckGeneratedColumns(DataChunk &chunk) const { + if (chunk.ColumnCount() != defaults.size()) { + throw InternalException("PostgresInsert: expected %llu input columns, got %llu", defaults.size(), + chunk.ColumnCount()); + } + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + if (defaults[c] != PostgresColumnDefault::GENERATED) { + continue; + } + for (idx_t r = 0; r < chunk.size(); r++) { + if (!FlatVector::IsNull(chunk.data[c], r)) { + throw InvalidInputException("Cannot insert into a generated column \"%s\"", names[c]); + } + } + } + } + + //! Whether the value of column "c" in row "r" of the (flattened) input is filled in by Postgres + bool UsePostgresDefault(DataChunk &chunk, idx_t c, idx_t r) const { + switch (defaults[c]) { + case PostgresColumnDefault::GENERATED: + return true; + case PostgresColumnDefault::DEFAULT_VALUE: + return FlatVector::IsNull(chunk.data[c], r); + default: + return false; + } + } +}; + +//! Insert "count" rows that consist only of Postgres-side defaults +static void InsertDefaultRows(ClientContext &context, PostgresConnection &connection, PostgresTableEntry &table, + idx_t count) { + string query = "INSERT INTO "; + query += PostgresUtils::WriteIdentifier(table.schema.name.GetIdentifierName()); + query += "."; + query += PostgresUtils::WriteIdentifier(table.name.GetIdentifierName()); + query += " SELECT FROM generate_series(1, " + to_string(count) + ")"; + connection.Execute(context, query); } //===--------------------------------------------------------------------===// // States //===--------------------------------------------------------------------===// -class PostgresInsertGlobalState : public GlobalSinkState { +class PostgresInsertCopyGlobalState : public GlobalSinkState { public: - explicit PostgresInsertGlobalState(ClientContext &context, PostgresTableEntry &table, PostgresCopyFormat format) - : table(table), insert_count(0), format(format) { + explicit PostgresInsertCopyGlobalState(ClientContext &context, PostgresTableEntry &table, PostgresCopyFormat format) + : table(table), columns(table), insert_count(0), format(format) { } PostgresTableEntry &table; + PostgresInsertColumns columns; PostgresCopyState copy_state; DataChunk varchar_chunk; idx_t insert_count; PostgresCopyFormat format; - vector insert_column_names; bool copy_is_active = false; + //! For each input column, whether it is filled in by Postgres in the active COPY + vector copy_uses_default; void FinishCopyTo(PostgresConnection &connection) { if (!copy_is_active) { @@ -52,33 +119,23 @@ class PostgresInsertGlobalState : public GlobalSinkState { } }; -vector GetInsertColumns(const PostgresInsert &insert, PostgresTableEntry &entry) { - vector column_names; - auto &columns = entry.GetColumns(); - idx_t column_count; - if (!insert.column_index_map.empty()) { - column_count = 0; - vector column_indexes; - column_indexes.resize(columns.LogicalColumnCount(), PhysicalIndex(DConstants::INVALID_INDEX)); - for (idx_t c = 0; c < insert.column_index_map.size(); c++) { - auto column_index = PhysicalIndex(c); - auto mapped_index = insert.column_index_map[column_index]; - if (mapped_index == DConstants::INVALID_INDEX) { - // column not specified - continue; - } - column_indexes[mapped_index] = column_index; - column_count++; - } - for (idx_t c = 0; c < column_count; c++) { - auto &col = columns.GetColumn(column_indexes[c]); - column_names.push_back(col.GetName().GetIdentifierName()); - } +class PostgresInsertPlainGlobalState : public GlobalSinkState { +public: + explicit PostgresInsertPlainGlobalState(ClientContext &context, PostgresTableEntry &table) + : table(table), columns(table), insert_count(0) { + vector varchar_types(columns.names.size(), LogicalType::VARCHAR); + varchar_chunk.Initialize(context, varchar_types); } - return column_names; -} -unique_ptr PostgresInsert::GetGlobalSinkState(ClientContext &context) const { + PostgresTableEntry &table; + PostgresInsertColumns columns; + DataChunk varchar_chunk; + idx_t insert_count; + string base_insert_query; + string insert_values; +}; + +unique_ptr PostgresInsert::GetGlobalSinkStateCopy(ClientContext &context) const { optional_ptr insert_table; if (!table) { auto &schema_ref = *schema.get_mutable(); @@ -87,41 +144,120 @@ unique_ptr PostgresInsert::GetGlobalSinkState(ClientContext &co } else { insert_table = &table.get_mutable()->Cast(); } - auto &transaction = PostgresTransaction::Get(context, insert_table->catalog); - auto &connection = transaction.GetConnection(); - auto insert_columns = GetInsertColumns(*this, *insert_table); auto format = insert_table->GetCopyFormat(context); - auto result = make_uniq(context, *insert_table, format); - auto &insert_column_names = result->insert_column_names; - if (!insert_columns.empty()) { - for (auto &str : insert_columns) { - Identifier col_identifier(str); - auto index = insert_table->GetColumnIndex(col_identifier, true); - if (!index.IsValid()) { - insert_column_names.push_back(str); - } else { - insert_column_names.push_back(insert_table->postgres_names[index.index]); + return make_uniq(context, *insert_table, format); +} + +static string GetBaseInsertQuery(const PostgresTableEntry &table, const PostgresInsertColumns &columns) { + vector column_names; + for (idx_t c = 0; c < columns.names.size(); c++) { + if (columns.defaults[c] != PostgresColumnDefault::GENERATED) { + column_names.push_back(columns.names[c]); + } + } + string query; + query += "INSERT INTO "; + query += PostgresUtils::WriteIdentifier(table.schema.name.GetIdentifierName()); + query += "."; + query += PostgresUtils::WriteIdentifier(table.name.GetIdentifierName()); + query += " "; + if (!column_names.empty()) { + query += "("; + for (idx_t c = 0; c < column_names.size(); c++) { + if (c > 0) { + query += ", "; } + query += PostgresUtils::WriteIdentifier(column_names[c]); } + query += ")"; + } + query += " VALUES "; + return query; +} + +unique_ptr PostgresInsert::GetGlobalSinkStatePlain(ClientContext &context) const { + PostgresTableEntry *insert_table; + if (!table) { + auto &schema_ref = *schema.get_mutable(); + insert_table = + &schema_ref.CreateTable(schema_ref.GetCatalogTransaction(context), *info)->Cast(); + } else { + insert_table = &table.get_mutable()->Cast(); } + auto result = make_uniq(context, *insert_table); + result->base_insert_query = GetBaseInsertQuery(*insert_table, result->columns); return std::move(result); } //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// -SinkResultType PostgresInsert::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { - auto &gstate = input.global_state.Cast(); +SinkResultType PostgresInsert::SinkCopy(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + auto &gstate = input.global_state.Cast(); auto &transaction = PostgresTransaction::Get(context.client, gstate.table.catalog); auto &connection = transaction.GetConnection(); - if (!gstate.copy_is_active) { - // copy hasn't started yet - connection.BeginCopyTo(context.client, gstate.copy_state, gstate.format, - gstate.table.schema.name.GetIdentifierName(), gstate.table.name.GetIdentifierName(), - gstate.insert_column_names); - gstate.copy_is_active = true; + auto &columns = gstate.columns; + chunk.Flatten(); + columns.CheckGeneratedColumns(chunk); + + // COPY cannot use the Postgres-side default for individual rows - the columns that are filled in by Postgres have + // to be left out of the COPY column list instead + // split the chunk into runs of rows that leave out the same columns, and start a new COPY when this changes + vector uses_default(chunk.ColumnCount()); + idx_t start = 0; + while (start < chunk.size()) { + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + uses_default[c] = columns.UsePostgresDefault(chunk, c, start); + } + idx_t end = start + 1; + for (; end < chunk.size(); end++) { + bool same_defaults = true; + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + if (columns.UsePostgresDefault(chunk, c, end) != uses_default[c]) { + same_defaults = false; + break; + } + } + if (!same_defaults) { + break; + } + } + if (gstate.copy_is_active && gstate.copy_uses_default != uses_default) { + gstate.FinishCopyTo(connection); + } + + vector column_ids; + vector column_names; + vector column_types; + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + if (!uses_default[c]) { + column_ids.push_back(c); + column_names.push_back(columns.names[c]); + column_types.push_back(chunk.data[c].GetType()); + } + } + if (column_ids.empty()) { + // every column is filled in by Postgres - COPY needs at least one column + InsertDefaultRows(context.client, connection, gstate.table, end - start); + start = end; + continue; + } + if (!gstate.copy_is_active) { + connection.BeginCopyTo(context.client, gstate.copy_state, gstate.format, + gstate.table.schema.name.GetIdentifierName(), gstate.table.name.GetIdentifierName(), + column_names); + gstate.copy_is_active = true; + gstate.copy_uses_default = uses_default; + // the varchar chunk is initialized for the columns of the COPY + gstate.varchar_chunk.Destroy(); + } + DataChunk copy_chunk; + copy_chunk.InitializeEmpty(column_types); + copy_chunk.ReferenceColumns(chunk, column_ids); + copy_chunk.Slice(start, end - start); + connection.CopyChunk(context.client, gstate.copy_state, copy_chunk, gstate.varchar_chunk); + start = end; } - connection.CopyChunk(context.client, gstate.copy_state, chunk, gstate.varchar_chunk); gstate.insert_count += chunk.size(); if (!keep_copy_alive) { // if we are can't keep the copy alive we need to restart the copy during every sink @@ -130,12 +266,156 @@ SinkResultType PostgresInsert::Sink(ExecutionContext &context, DataChunk &chunk, return SinkResultType::NEED_MORE_INPUT; } +static void PostgresCastBlob(const Vector &input, Vector &result, idx_t count) { + static constexpr const char *HEX_TABLE = "0123456789ABCDEF"; + auto input_data = FlatVector::GetData(input); + auto result_data = FlatVector::GetDataMutable(result); + for (idx_t r = 0; r < count; r++) { + if (FlatVector::IsNull(input, r)) { + FlatVector::SetNull(result, r, true); + continue; + } + auto blob_data = const_data_ptr_cast(input_data[r].GetData()); + auto blob_size = input_data[r].GetSize(); + if (blob_size == 0) { + result_data[r] = StringVector::AddString(result, "''"); + continue; + } + string result_blob = "'\\x"; + for (idx_t b = 0; b < blob_size; b++) { + auto blob_entry = blob_data[b]; + auto byte_a = blob_entry >> 4; + auto byte_b = blob_entry & 0x0F; + result_blob += string(1, HEX_TABLE[byte_a]); + result_blob += string(1, HEX_TABLE[byte_b]); + } + result_blob += "'::BYTEA"; + result_data[r] = StringVector::AddString(result, result_blob); + } +} + +SinkResultType PostgresInsert::SinkPlain(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + static constexpr const idx_t INSERT_FLUSH_SIZE = 8000; + + auto &gstate = input.global_state.Cast(); + auto &transaction = PostgresTransaction::Get(context.client, gstate.table.catalog); + auto &con = transaction.GetConnection(); + auto &columns = gstate.columns; + // cast to varchar + if (chunk.ColumnCount() != gstate.varchar_chunk.ColumnCount()) { + throw InternalException("PostgresInsert: expected %llu input columns, got %llu", + gstate.varchar_chunk.ColumnCount(), chunk.ColumnCount()); + } + chunk.Flatten(); + columns.CheckGeneratedColumns(chunk); + bool has_insert_columns = false; + for (auto column_default : columns.defaults) { + if (column_default != PostgresColumnDefault::GENERATED) { + has_insert_columns = true; + } + } + if (!has_insert_columns) { + // every column is generated - VALUES needs at least one column + InsertDefaultRows(context.client, con, gstate.table, chunk.size()); + gstate.insert_count += chunk.size(); + return SinkResultType::NEED_MORE_INPUT; + } + gstate.varchar_chunk.Reset(); + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + switch (chunk.data[c].GetType().id()) { + case LogicalTypeId::BLOB: + PostgresCastBlob(chunk.data[c], gstate.varchar_chunk.data[c], chunk.size()); + break; + case LogicalTypeId::TIMESTAMP_TZ: { + Vector timestamp_vector(LogicalType::TIMESTAMP); + timestamp_vector.Reinterpret(chunk.data[c]); + VectorOperations::Cast(context.client, timestamp_vector, gstate.varchar_chunk.data[c], chunk.size()); + break; + } + default: + VectorOperations::Cast(context.client, chunk.data[c], gstate.varchar_chunk.data[c], chunk.size()); + break; + } + } + gstate.varchar_chunk.SetChildCardinality(chunk.size()); + // for each column type check if we need to add quotes or not + vector add_quotes; + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + bool add_quotes_for_type; + switch (chunk.data[c].GetType().id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::BLOB: + add_quotes_for_type = false; + break; + case LogicalTypeId::ARRAY: + case LogicalTypeId::LIST: + case LogicalTypeId::STRUCT: + throw InvalidInputException("Insertion of '%s' columns is not supported with compatibility mode enabled " + "('pg_use_text_protocol' option)", + chunk.data[c].GetType().ToString()); + default: + add_quotes_for_type = true; + break; + } + add_quotes.push_back(add_quotes_for_type); + } + + // generate INSERT INTO statements + for (idx_t r = 0; r < chunk.size(); r++) { + if (!gstate.insert_values.empty()) { + gstate.insert_values += ", "; + } + gstate.insert_values += "("; + bool first_value = true; + for (idx_t c = 0; c < gstate.varchar_chunk.ColumnCount(); c++) { + if (columns.defaults[c] == PostgresColumnDefault::GENERATED) { + continue; + } + if (!first_value) { + gstate.insert_values += ", "; + } + first_value = false; + if (columns.UsePostgresDefault(chunk, c, r)) { + gstate.insert_values += "DEFAULT"; + } else if (FlatVector::IsNull(gstate.varchar_chunk.data[c], r)) { + gstate.insert_values += "NULL"; + } else { + auto data = FlatVector::GetDataMutable(gstate.varchar_chunk.data[c]); + if (add_quotes[c]) { + gstate.insert_values += PostgresUtils::WriteLiteral(data[r].GetString()); + } else { + gstate.insert_values += data[r].GetString(); + } + } + } + gstate.insert_values += ")"; + if (gstate.insert_values.size() >= INSERT_FLUSH_SIZE) { + // perform the actual insert + con.Execute(context.client, gstate.base_insert_query + gstate.insert_values); + // reset the to-be-inserted values + gstate.insert_values = string(); + } + } + gstate.insert_count += chunk.size(); + return SinkResultType::NEED_MORE_INPUT; +} + //===--------------------------------------------------------------------===// // Finalize //===--------------------------------------------------------------------===// -SinkFinalizeType PostgresInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - auto &gstate = input.global_state.Cast(); +SinkFinalizeType PostgresInsert::FinalizeCopy(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); auto &transaction = PostgresTransaction::Get(context, gstate.table.catalog); auto &connection = transaction.GetConnection(); gstate.FinishCopyTo(connection); @@ -147,14 +427,33 @@ SinkFinalizeType PostgresInsert::Finalize(Pipeline &pipeline, Event &event, Clie return SinkFinalizeType::READY; } +SinkFinalizeType PostgresInsert::FinalizePlain(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); + if (!gstate.insert_values.empty()) { + // perform the final insert + auto &transaction = PostgresTransaction::Get(context, gstate.table.catalog); + auto &con = transaction.GetConnection(); + con.Execute(context, gstate.base_insert_query + gstate.insert_values); + } + return SinkFinalizeType::READY; +} + //===--------------------------------------------------------------------===// // GetData //===--------------------------------------------------------------------===// SourceResultType PostgresInsert::GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const { - auto &insert_gstate = sink_state->Cast(); + idx_t insert_count = 0; + if (use_plain_inserts) { + auto &insert_gstate = sink_state->Cast(); + insert_count = insert_gstate.insert_count; + } else { + auto &insert_gstate = sink_state->Cast(); + insert_count = insert_gstate.insert_count; + } chunk.SetChildCardinality(1); - chunk.data[0].SetValue(0, Value::BIGINT(insert_gstate.insert_count)); + chunk.data[0].SetValue(0, Value::BIGINT(insert_count)); return SourceResultType::FINISHED; } @@ -168,10 +467,20 @@ string PostgresInsert::GetName() const { InsertionOrderPreservingMap PostgresInsert::ParamsToString() const { InsertionOrderPreservingMap result; - result["Table Name"] = table ? table->name.GetIdentifierName() : info->Base().table.GetIdentifierName(); + result["Table Name"] = table ? table->name.GetIdentifierName() : info->Base().GetTableName().GetIdentifierName(); return result; } +bool PostgresInsert::UsePlainInserts(PostgresCatalog &pg_catalog, ClientContext &context) { + bool use_text_proto_user_option = false; + Value value; + if (context.TryGetCurrentSetting("pg_use_text_protocol", value) && !value.IsNull()) { + use_text_proto_user_option = BooleanValue::Get(value); + } + PostgresQueryMode query_mode = PostgresUtils::ExtractQueryMode(context); + return pg_catalog.UseTextProtocol(query_mode, use_text_proto_user_option); +} + //===--------------------------------------------------------------------===// // Plan //===--------------------------------------------------------------------===// @@ -215,13 +524,13 @@ PhysicalOperator &AddCastToPostgresTypes(ClientContext &context, PhysicalPlanGen } bool PostgresCatalog::IsPostgresScan(const string &name) { - return name == "postgres_scan" || name == "postgres_scan_pushdown" || name == "postgres_query"; + return name == "postgres_scan" || name == "postgres_query"; } void PostgresCatalog::MaterializePostgresScans(PhysicalOperator &op) { if (op.type == PhysicalOperatorType::TABLE_SCAN) { auto &table_scan = op.Cast(); - if (PostgresCatalog::IsPostgresScan(table_scan.function.name.GetIdentifierName())) { + if (PostgresCatalog::IsPostgresScan(table_scan.function.GetName().GetIdentifierName())) { auto &bind_data = table_scan.bind_data->Cast(); bind_data.requires_materialization = true; bind_data.max_threads = 1; @@ -243,11 +552,13 @@ PhysicalOperator &PostgresCatalog::PlanInsert(ClientContext &context, PhysicalPl throw BinderException("ON CONFLICT clause not yet supported for insertion into Postgres table"); } - D_ASSERT(plan); + if (!plan) { + throw InternalException("PostgresCatalog::PlanInsert: insert without a child plan"); + } MaterializePostgresScans(*plan); auto &inner_plan = AddCastToPostgresTypes(context, planner, *plan); - auto &insert = planner.Make(op, op.table, op.column_index_map); + auto &insert = planner.Make(*this, context, op, op.table); insert.children.push_back(inner_plan); return insert; } @@ -257,7 +568,7 @@ PhysicalOperator &PostgresCatalog::PlanCreateTableAs(ClientContext &context, Phy auto &inner_plan = AddCastToPostgresTypes(context, planner, plan); MaterializePostgresScans(inner_plan); - auto &insert = planner.Make(op, op.schema, std::move(op.info)); + auto &insert = planner.Make(*this, context, op, op.schema, std::move(op.info)); insert.children.push_back(inner_plan); return insert; } diff --git a/src/storage/postgres_merge_into.cpp b/src/storage/postgres_merge_into.cpp index 7bcab80df..16d1421b3 100644 --- a/src/storage/postgres_merge_into.cpp +++ b/src/storage/postgres_merge_into.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" #include "duckdb/execution/operator/persistent/physical_merge_into.hpp" #include "duckdb/planner/operator/logical_merge_into.hpp" #include "storage/postgres_update.hpp" @@ -98,6 +100,7 @@ unique_ptr PostgresPlanMergeIntoAction(PostgresCatalog &catal return result; } +// TODO: consider using generic PlanMergeInto from the base class PhysicalOperator &PostgresCatalog::PlanMergeInto(ClientContext &context, PhysicalPlanGenerator &planner, LogicalMergeInto &op, PhysicalOperator &plan) { if (op.return_chunk) { @@ -106,18 +109,27 @@ PhysicalOperator &PostgresCatalog::PlanMergeInto(ClientContext &context, Physica map>> actions; // plan the merge into clauses + // disable parallelism when we have multiple operations + idx_t append_count = 0; for (auto &entry : op.actions) { vector> planned_actions; for (auto &action : entry.second) { + if (action->action_type == MergeActionType::MERGE_INSERT || + action->action_type == MergeActionType::MERGE_UPDATE || + action->action_type == MergeActionType::MERGE_DELETE) { + append_count++; + } planned_actions.push_back(PostgresPlanMergeIntoAction(*this, context, op, planner, *action, plan)); } actions.emplace(entry.first, std::move(planned_actions)); } - auto &result = planner.Make(op.types, std::move(actions), op.row_id_start, op.source_marker, - false, op.return_chunk); - result.children.push_back(plan); - return result; + bool parallel = false; // append_count <= 1 && !op.return_chunk; + // multiple operators appending to the same table cannot run concurrently - run their actions one after the other + bool serialize_actions = append_count > 1; + + return planner.Make(op.types, plan, std::move(actions), op.row_id_start, op.source_marker, + parallel, op.return_chunk, serialize_actions); } } // namespace duckdb diff --git a/src/storage/postgres_optimizer.cpp b/src/storage/postgres_optimizer.cpp index a035516c4..afbb21978 100644 --- a/src/storage/postgres_optimizer.cpp +++ b/src/storage/postgres_optimizer.cpp @@ -1,9 +1,12 @@ #include "storage/postgres_optimizer.hpp" +#include + +#include "duckdb/planner/logical_operator.hpp" + #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_limit.hpp" -#include "dbconnector/optimizer/order_by_and_limit_optimizer.hpp" #include "dbconnector/optimizer/optimizer_util.hpp" #include "postgres_scanner.hpp" @@ -15,24 +18,19 @@ namespace duckdb { struct PostgresOperators { - reference_map_t>> scans; + std::map>> scans; }; static void GatherPostgresScans(LogicalOperator &op, PostgresOperators &result) { if (op.type == LogicalOperatorType::LOGICAL_GET) { auto &get = op.Cast(); auto &table_scan = get.function; - if (!PostgresCatalog::IsPostgresScan(table_scan.name.GetIdentifierName())) { + if (!PostgresCatalog::IsPostgresScan(table_scan.GetName().GetIdentifierName())) { // not a postgres scan - skip return; } auto &bind_data = get.bind_data->Cast(); - auto catalog = bind_data.GetCatalog(); - if (!catalog) { - // "postgres_scan" functions are fully independent - we can always stream them - return; - } - result.scans[*catalog].push_back(get); + result.scans[bind_data.catalog_name.GetIdentifierName()].push_back(get); } // recurse into children for (auto &child : op.children) { @@ -40,34 +38,77 @@ static void GatherPostgresScans(LogicalOperator &op, PostgresOperators &result) } } -static void DisableParallelLimit(LogicalOperator &op) { - LogicalGet *get = nullptr; - dbconnector::BindData *bind_data = nullptr; - if (dbconnector::optimizer::OptimizerUtil::FindExtensionGet("postgres_scan", op, get, bind_data)) { - auto &pg_bind_data = bind_data->Cast(); - if (!pg_bind_data.order_by_and_limit_bind_data.limit_clause.empty()) { +static void OptimizePostgresScanLimitPushdown(unique_ptr &op) { + if (op->type == LogicalOperatorType::LOGICAL_LIMIT) { + auto &limit = op->Cast(); + reference child = *op->children[0]; + + while (child.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + child = *child.get().children[0]; + } + + if (child.get().type != LogicalOperatorType::LOGICAL_GET) { + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + auto &get = child.get().Cast(); + if (!PostgresCatalog::IsPostgresScan(get.function.GetName().GetIdentifierName())) { + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + switch (limit.limit_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + // not a constant or unset limit + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + switch (limit.offset_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + // not a constant or unset offset + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + auto &bind_data = get.bind_data->Cast(); + + string generated_limit_clause = ""; + if (limit.limit_val.Type() != LimitNodeType::UNSET) { + generated_limit_clause += " LIMIT " + to_string(limit.limit_val.GetConstantValue()); + } + if (limit.offset_val.Type() != LimitNodeType::UNSET) { + generated_limit_clause += " OFFSET " + to_string(limit.offset_val.GetConstantValue()); + } + + if (!generated_limit_clause.empty()) { + bind_data.limit_clause = generated_limit_clause; // When LIMIT is pushed down to Postgres, we must ensure single-task execution // to avoid each task (whether parallel or sequential) applying the LIMIT independently. // Setting pages_approx = 0 disables CTID-based task splitting, ensuring a single query. - pg_bind_data.pages_approx = 0; - pg_bind_data.max_threads = 1; + bind_data.pages_approx = 0; + bind_data.max_threads = 1; + + op = std::move(op->children[0]); + return; } } - for (auto &child : op.children) { - DisableParallelLimit(*child); + for (auto &child : op->children) { + OptimizePostgresScanLimitPushdown(child); } } void PostgresOptimizer::Optimize(OptimizerExtensionInput &input, unique_ptr &plan) { using namespace dbconnector; // look at query plan and check if we can find LIMIT/OFFSET to pushdown - // OptimizePostgresScanLimitPushdown(plan); - - auto order_config = optimizer::OrderByAndLimitOptimizer::CreateConfig( - input.context, "pg_order_pushdown", '"', query::QuoteEscapeStyle::DOUBLE_QUOTE, "postgres_scan"); - optimizer::OrderByAndLimitOptimizer::Optimize(order_config, input, plan); - DisableParallelLimit(*plan); + OptimizePostgresScanLimitPushdown(plan); // look at the query plan and check if we can enable streaming query scans PostgresOperators operators; @@ -77,7 +118,6 @@ void PostgresOptimizer::Optimize(OptimizerExtensionInput &input, unique_ptr 1; for (auto &scan : entry.second) { auto &bind_data = scan.get().bind_data->Cast(); diff --git a/src/storage/postgres_schema_entry.cpp b/src/storage/postgres_schema_entry.cpp index f320e6f57..596ac6641 100644 --- a/src/storage/postgres_schema_entry.cpp +++ b/src/storage/postgres_schema_entry.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_schema_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "storage/postgres_table_entry.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" @@ -42,7 +46,7 @@ PostgresTransaction &GetPostgresTransaction(CatalogTransaction transaction) { void PostgresSchemaEntry::TryDropEntry(ClientContext &context, CatalogType catalog_type, const string &name) { DropInfo info; info.type = catalog_type; - info.name = Identifier(name); + info.SetName(Identifier(name)); info.cascade = false; info.if_not_found = OnEntryNotFound::RETURN_NULL; DropEntry(context, info); @@ -52,7 +56,7 @@ optional_ptr PostgresSchemaEntry::CreateTable(CatalogTransaction t BoundCreateTableInfo &info) { auto &postgres_transaction = GetPostgresTransaction(transaction); auto &base_info = info.Base(); - string table_name = base_info.table.GetIdentifierName(); + string table_name = base_info.GetTableName().GetIdentifierName(); if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { // CREATE OR REPLACE - drop any existing entries first (if any) TryDropEntry(transaction.GetContext(), CatalogType::TABLE_ENTRY, table_name); @@ -75,7 +79,7 @@ string PGGetCreateViewSQL(PostgresSchemaEntry &schema, CreateViewInfo &info) { string sql; sql = "CREATE VIEW "; sql += PostgresUtils::QuotePostgresIdentifier(schema.name.GetIdentifierName()) + "."; - sql += PostgresUtils::QuotePostgresIdentifier(info.view_name.GetIdentifierName()); + sql += PostgresUtils::QuotePostgresIdentifier(info.GetViewName().GetIdentifierName()); sql += " "; if (!info.aliases.empty()) { sql += "("; @@ -99,26 +103,26 @@ optional_ptr PostgresSchemaEntry::CreateView(CatalogTransaction tr } if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { - auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); + auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.GetViewName()); if (current_entry) { if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { return current_entry; } // CREATE OR REPLACE - drop any existing entries first (if any) - TryDropEntry(transaction.GetContext(), CatalogType::VIEW_ENTRY, info.view_name.GetIdentifierName()); + TryDropEntry(transaction.GetContext(), CatalogType::VIEW_ENTRY, info.GetViewName().GetIdentifierName()); } } auto &postgres_transaction = GetPostgresTransaction(transaction); postgres_transaction.Query(PGGetCreateViewSQL(*this, info)); - return tables.ReloadEntry(postgres_transaction, info.view_name.GetIdentifierName()); + return tables.ReloadEntry(postgres_transaction, info.GetViewName().GetIdentifierName()); } optional_ptr PostgresSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { auto &postgres_transaction = GetPostgresTransaction(transaction); - auto type_name = info.name; + auto type_name = info.GetTypeName(); if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { // CREATE OR REPLACE - drop any existing entries first (if any) - TryDropEntry(transaction.GetContext(), CatalogType::TYPE_ENTRY, info.name.GetIdentifierName()); + TryDropEntry(transaction.GetContext(), CatalogType::TYPE_ENTRY, info.GetTypeName().GetIdentifierName()); } return types.CreateType(postgres_transaction, info); } @@ -182,7 +186,7 @@ void PostgresSchemaEntry::Scan(CatalogType type, const std::function PostgresSchemaEntry::LookupEntry(CatalogTransaction t .GetEntry(transaction.GetContext(), postgres_transaction, lookup_info.GetEntryName()); } +void PostgresSchemaEntry::InvalidateEntry(CatalogType type, const string &name) { + GetCatalogSet(type).InvalidateEntry(name); +} + +void PostgresSchemaEntry::InvalidateTables() { + tables.ClearEntries(); +} + PostgresCatalogSet &PostgresSchemaEntry::GetCatalogSet(CatalogType type) { switch (type) { case CatalogType::TABLE_ENTRY: diff --git a/src/storage/postgres_schema_set.cpp b/src/storage/postgres_schema_set.cpp index d5613c773..01c779b71 100644 --- a/src/storage/postgres_schema_set.cpp +++ b/src/storage/postgres_schema_set.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_schema_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/common/shared_ptr.hpp" @@ -13,8 +15,8 @@ namespace duckdb { -PostgresSchemaSet::PostgresSchemaSet(Catalog &catalog, string schema_to_load_p) - : PostgresCatalogSet(catalog, false), schema_to_load(std::move(schema_to_load_p)) { +PostgresSchemaSet::PostgresSchemaSet(Catalog &catalog, vector schemas_to_load_p) + : PostgresCatalogSet(catalog, false), schemas_to_load(std::move(schemas_to_load_p)) { } vector> SliceResult(PostgresResult &schemas, unique_ptr to_slice_ptr) { @@ -37,7 +39,30 @@ vector> SliceResult(PostgresResult &schemas, uni return result; } -string PostgresSchemaSet::GetInitializeQuery(const string &schema) { +// Variant of SliceResult +// uses field name instead of an integer oid. +// Used by the information_schema introspection path where namespace oids are'nt available. +static vector> SliceResultByName(PostgresResult &schemas, + unique_ptr to_slice_ptr) { + auto shared_result = shared_ptr(to_slice_ptr.release()); + auto &to_slice = *shared_result; + + vector> result; + idx_t current_offset = 0; + for (idx_t schema_idx = 0; schema_idx < schemas.Count(); schema_idx++) { + auto name = schemas.GetString(schema_idx, 0); + idx_t start = current_offset; + for (; current_offset < to_slice.Count(); current_offset++) { + if (to_slice.GetString(current_offset, 0) != name) { + break; + } + } + result.push_back(make_uniq(shared_result, start, current_offset)); + } + return result; +} + +string PostgresSchemaSet::GetInitializeQuery(const vector &schemas) { string base_query = R"( SELECT oid, nspname FROM pg_namespace @@ -45,20 +70,71 @@ FROM pg_namespace ORDER BY oid; )"; string condition; - if (!schema.empty()) { - condition += "WHERE pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; + } + return StringUtil::Replace(base_query, "${CONDITION}", condition); +} + +string PostgresSchemaSet::GetInitializeQueryInformationSchema(const vector &schemas) { + string base_query = R"( +SELECT schema_name +FROM information_schema.schemata +WHERE schema_name NOT IN ('information_schema', 'pg_catalog', 'pg_toast') +${CONDITION} +ORDER BY schema_name; +)"; + string condition; + if (schemas.size() > 0) { + condition += "AND schema_name IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } +void PostgresSchemaSet::LoadEntriesInformationSchema(ClientContext &context, PostgresTransaction &transaction) { + string schema_query = GetInitializeQueryInformationSchema(schemas_to_load); + string tables_query = PostgresTableSet::GetInitializeQueryInformationSchema(schemas_to_load); + // Reading enums, composite types and indexes differs across DBs that emulate PG + // choice to either return those empty or have vendor specific queries for each. + // empty for now + string enum_types_query = "SELECT NULL, NULL, NULL, NULL LIMIT 0;\n"; + string composite_types_query = "SELECT NULL, NULL, NULL, NULL, NULL LIMIT 0;\n"; + string index_query = "SELECT NULL, NULL, NULL LIMIT 0;\n"; + + auto full_query = schema_query + tables_query + enum_types_query + composite_types_query + index_query; + + auto results = transaction.ExecuteQueries(context, full_query); + auto result = std::move(results[0]); + results.erase(results.begin()); + auto rows = result->Count(); + + auto tables = SliceResultByName(*result, std::move(results[0])); + auto enums = SliceResultByName(*result, std::move(results[1])); + auto composite_types = SliceResultByName(*result, std::move(results[2])); + auto indexes = SliceResultByName(*result, std::move(results[3])); + for (idx_t row = 0; row < rows; row++) { + auto schema_name = result->GetString(row, 0); + CreateSchemaInfo info; + info.SetSchema(Identifier(schema_name)); + info.internal = PostgresSchemaEntry::SchemaIsInternal(schema_name); + auto schema = make_shared_ptr(catalog, info, std::move(tables[row]), std::move(enums[row]), + std::move(composite_types[row]), std::move(indexes[row])); + CreateEntry(transaction, std::move(schema)); + } +} + void PostgresSchemaSet::LoadEntries(ClientContext &context, PostgresTransaction &transaction) { + if (PostgresUtils::UseInformationSchemaIntrospection(context)) { + LoadEntriesInformationSchema(context, transaction); + return; + } auto &pg_catalog = catalog.Cast(); auto pg_version = pg_catalog.GetPostgresVersion(); - string schema_query = PostgresSchemaSet::GetInitializeQuery(schema_to_load); - string tables_query = PostgresTableSet::GetInitializeQuery(schema_to_load); - string enum_types_query = PostgresTypeSet::GetInitializeEnumsQuery(pg_version, schema_to_load); - string composite_types_query = PostgresTypeSet::GetInitializeCompositesQuery(schema_to_load); - string index_query = PostgresIndexSet::GetInitializeQuery(schema_to_load); + string schema_query = PostgresSchemaSet::GetInitializeQuery(schemas_to_load); + string tables_query = PostgresTableSet::GetInitializeQuery(pg_version, schemas_to_load); + string enum_types_query = PostgresTypeSet::GetInitializeEnumsQuery(pg_version, schemas_to_load); + string composite_types_query = PostgresTypeSet::GetInitializeCompositesQuery(schemas_to_load); + string index_query = PostgresIndexSet::GetInitializeQuery(schemas_to_load); auto full_query = schema_query + tables_query + enum_types_query + composite_types_query + index_query; @@ -75,7 +151,8 @@ void PostgresSchemaSet::LoadEntries(ClientContext &context, PostgresTransaction auto oid = result->GetInt64(row, 0); auto schema_name = result->GetString(row, 1); CreateSchemaInfo info; - info.schema = Identifier(schema_name); + info.SetQualifiedName( + QualifiedName(info.GetQualifiedName().Catalog(), Identifier(schema_name), info.GetQualifiedName().Name())); info.internal = PostgresSchemaEntry::SchemaIsInternal(schema_name); auto schema = make_shared_ptr(catalog, info, std::move(tables[row]), std::move(enums[row]), std::move(composite_types[row]), std::move(indexes[row])); @@ -88,10 +165,10 @@ optional_ptr PostgresSchemaSet::CreateSchema(PostgresTransaction & if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { create_sql += " IF NOT EXISTS"; } - create_sql += PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()); + create_sql += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()); transaction.Query(create_sql); auto info_copy = info.Copy(); - info.internal = PostgresSchemaEntry::SchemaIsInternal(info_copy->schema.GetIdentifierName()); + info.internal = PostgresSchemaEntry::SchemaIsInternal(info_copy->GetQualifiedName().Schema().GetIdentifierName()); auto schema_entry = make_shared_ptr(catalog, info_copy->Cast()); return CreateEntry(transaction, std::move(schema_entry)); } diff --git a/src/storage/postgres_table_entry.cpp b/src/storage/postgres_table_entry.cpp index 8479e8bb9..409e09627 100644 --- a/src/storage/postgres_table_entry.cpp +++ b/src/storage/postgres_table_entry.cpp @@ -1,15 +1,20 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "storage/postgres_table_entry.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/storage/statistics/base_statistics.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/parser/constraints/unique_constraint.hpp" +#include "postgres_filter_pushdown.hpp" #include "postgres_scanner.hpp" namespace duckdb { PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) - : TableCatalogEntry(catalog, schema, info) { + : TableCatalogEntry(catalog, schema, info), columns(std::move(info.columns)) { for (idx_t c = 0; c < columns.LogicalColumnCount(); c++) { auto &col = columns.GetColumnMutable(LogicalIndex(c)); if (col.GetType().HasAlias()) { @@ -17,17 +22,33 @@ PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &sch } postgres_types.push_back(PostgresUtils::CreateEmptyPostgresType(col.GetType())); postgres_names.push_back(col.GetName().GetIdentifierName()); + if (col.Generated()) { + column_defaults.push_back(PostgresColumnDefault::GENERATED); + } else if (col.HasDefaultValue()) { + column_defaults.push_back(PostgresColumnDefault::DEFAULT_VALUE); + } else { + column_defaults.push_back(PostgresColumnDefault::NONE); + } } approx_num_pages.store(0, std::memory_order_release); } PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, PostgresTableInfo &info) - : TableCatalogEntry(catalog, schema, *info.create_info), postgres_types(std::move(info.postgres_types)), - postgres_names(std::move(info.postgres_names)) { - D_ASSERT(postgres_types.size() == columns.LogicalColumnCount()); + : TableCatalogEntry(catalog, schema, *info.create_info), columns(std::move(info.create_info->columns)), + postgres_types(std::move(info.postgres_types)), postgres_names(std::move(info.postgres_names)), + postgres_type_names(std::move(info.postgres_type_names)), column_defaults(std::move(info.column_defaults)) { + if (postgres_types.size() != columns.LogicalColumnCount() || + column_defaults.size() != columns.LogicalColumnCount()) { + throw InternalException("PostgresTableEntry: column count mismatch - %llu columns, %llu types, %llu defaults", + columns.LogicalColumnCount(), postgres_types.size(), column_defaults.size()); + } approx_num_pages.store(info.approx_num_pages, std::memory_order_release); } +const ColumnList &PostgresTableEntry::GetColumns() const { + return columns; +} + unique_ptr PostgresTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } @@ -40,29 +61,33 @@ TableFunction PostgresTableEntry::GetScanFunction(ClientContext &context, unique auto &pg_catalog = catalog.Cast(); auto &transaction = Transaction::Get(context, catalog).Cast(); - auto result = make_uniq(context); + auto result = make_uniq(context, pg_catalog.GetName()); result->schema_name = schema.name.GetIdentifierName(); result->table_name = name.GetIdentifierName(); result->dsn = transaction.GetDSN(); result->attach_path = pg_catalog.attach_path; - result->SetCatalog(pg_catalog); - result->SetTable(*this); + result->qualified_table_name = QualifiedName(ParentCatalog().GetName(), ParentSchema().name, name); + result->context_ptr = transaction.context; for (auto &col : columns.Logical()) { result->types.push_back(col.GetType()); } result->names = postgres_names; result->postgres_types = postgres_types; result->read_only = transaction.IsReadOnly(); + result->type_config = PostgresTypeConfig::FromContext(context); PostgresScanFunction::PrepareBind(pg_catalog.GetPostgresVersion(), context, *result, - approx_num_pages.load(std::memory_order_acquire)); + approx_num_pages.load(std::memory_order_acquire), pg_catalog); bind_data = std::move(result); auto function = PostgresScanFunction(); Value filter_pushdown; - if (context.TryGetCurrentSetting("pg_experimental_filter_pushdown", filter_pushdown)) { + if (context.TryGetCurrentSetting("pg_enable_filter_pushdown", filter_pushdown)) { function.filter_pushdown = BooleanValue::Get(filter_pushdown); } + if (function.filter_pushdown) { + function.pushdown_expression = PostgresFilterPushdown::CanPushExpressionDown; + } return function; } @@ -133,13 +158,32 @@ static bool CopyRequiresText(const LogicalType &type, const PostgresType &pg_typ } PostgresCopyFormat PostgresTableEntry::GetCopyFormat(ClientContext &context) { - Value use_binary_copy; - if (context.TryGetCurrentSetting("pg_use_binary_copy", use_binary_copy)) { - if (!BooleanValue::Get(use_binary_copy)) { - return PostgresCopyFormat::TEXT; + PostgresQueryMode query_mode = PostgresUtils::ExtractQueryMode(context); + switch (query_mode) { + case PostgresQueryMode::COPY_BINARY: + case PostgresQueryMode::PLAIN_QUERY_BINARY: + return GetBinaryCopyFormatIfTypesAllow(); + case PostgresQueryMode::PLAIN_QUERY_TEXT: + return PostgresCopyFormat::TEXT; + default: { + Value use_binary_copy; + if (context.TryGetCurrentSetting("pg_use_binary_copy", use_binary_copy)) { + if (!BooleanValue::Get(use_binary_copy)) { + return PostgresCopyFormat::TEXT; + } } + return GetBinaryCopyFormatIfTypesAllow(); + } + } +} + +PostgresCopyFormat PostgresTableEntry::GetBinaryCopyFormatIfTypesAllow() { + idx_t types_count = postgres_types.size(); + idx_t columns_count = columns.LogicalColumnCount(); + if (types_count != columns_count) { + throw IOException("Schema discrepancy detected, types count: %llu, columns count: %llu", types_count, + columns_count); } - D_ASSERT(postgres_types.size() == columns.LogicalColumnCount()); for (idx_t c = 0; c < postgres_types.size(); c++) { if (CopyRequiresText(columns.GetColumn(LogicalIndex(c)).GetType(), postgres_types[c])) { return PostgresCopyFormat::TEXT; @@ -148,4 +192,15 @@ PostgresCopyFormat PostgresTableEntry::GetCopyFormat(ClientContext &context) { return PostgresCopyFormat::BINARY; } +dbconnector::attached::AttachedTable PostgresTableEntry::Lookup(ClientContext &ctx, QualifiedName name) { + using namespace dbconnector::attached; + + AttachedTable table = AttachedTable::Lookup(ctx, "postgres", name); + if (!table) { + throw InvalidInputException("Attached PostgreSQL table, name: %s is not found in the specified client session", + name.ToString()); + } + return table; +} + } // namespace duckdb diff --git a/src/storage/postgres_table_set.cpp b/src/storage/postgres_table_set.cpp index 663e6d523..66a06619d 100644 --- a/src/storage/postgres_table_set.cpp +++ b/src/storage/postgres_table_set.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_table_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" #include "duckdb/parser/constraints/not_null_constraint.hpp" @@ -13,6 +15,7 @@ #include "postgres_conversion.hpp" #include "postgres_utils.hpp" +#include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" #include "storage/postgres_schema_entry.hpp" @@ -22,14 +25,72 @@ PostgresTableSet::PostgresTableSet(PostgresSchemaEntry &schema, unique_ptr vec; + vec.push_back(schema); + return GetInitializeQuery(version, vec, table); +} + +//! Expression returning 'g' for generated columns, 'd' for columns that have a default value (including SERIAL and +//! identity columns), and NULL otherwise +static string GetColumnDefaultExpression(PostgresVersion version) { + string result = "CASE"; + if (version >= PostgresVersion(12, 0)) { + result += " WHEN pg_attribute.attgenerated <> '' THEN 'g'"; + } + if (version >= PostgresVersion(10, 0)) { + result += " WHEN pg_attribute.attidentity <> '' THEN 'd'"; + } + result += " WHEN pg_attribute.atthasdef THEN 'd' END"; + return result; +} + +//! Redshift late-binding views (CREATE VIEW ... WITH NO SCHEMA BINDING) have no pg_attribute entries (relnatts = 0), +//! their columns are only exposed through pg_get_late_binding_view_cols(). The column type is returned formatted +//! (e.g. "numeric(10,2)" or "character varying(100)"), so it is split into the type name and the type modifier here. +static string GetRedshiftLateBindingViewQuery() { + return R"( +UNION ALL +SELECT pg_namespace.oid AS namespace_id, relname, relpages, lbv.col_name AS attname, + CASE WHEN lbv.paren > 0 THEN RTRIM(SUBSTRING(lbv.col_type, 1, lbv.paren - 1)) ELSE lbv.col_type END AS type_name, + CASE + WHEN lbv.paren = 0 THEN -1 + WHEN lbv.col_type LIKE 'numeric(%' OR lbv.col_type LIKE 'decimal(%' + THEN SPLIT_PART(lbv.type_args, ',', 1)::INTEGER * 65536 + + COALESCE(NULLIF(TRIM(SPLIT_PART(lbv.type_args, ',', 2)), ''), '0')::INTEGER + 4 + WHEN lbv.col_type LIKE 'character varying(%' OR lbv.col_type LIKE 'character(%' + THEN lbv.type_args::INTEGER + 4 + ELSE -1 + END AS type_modifier, + 0 AS ndim, lbv.col_num AS attnum, false AS notnull, NULL constraint_id, + NULL constraint_type, NULL constraint_key, NULL AS type_schema, + NULL AS column_comment, + tbl_desc.description AS table_comment, + NULL AS column_default +FROM ( + SELECT view_schema, view_name, col_name, col_type, col_num, STRPOS(col_type, '(') AS paren, + CASE WHEN STRPOS(col_type, '(') > 0 + THEN SUBSTRING(col_type, STRPOS(col_type, '(') + 1, STRPOS(col_type, ')') - STRPOS(col_type, '(') - 1) + END AS type_args + FROM pg_get_late_binding_view_cols() + cols(view_schema NAME, view_name NAME, col_name NAME, col_type VARCHAR, col_num INTEGER) +) lbv +JOIN pg_namespace ON pg_namespace.nspname = lbv.view_schema +JOIN pg_class ON pg_class.relnamespace = pg_namespace.oid AND pg_class.relname = lbv.view_name +LEFT JOIN pg_description tbl_desc ON tbl_desc.objoid=pg_class.oid AND tbl_desc.objsubid=0 +WHERE relkind = 'v' ${CONDITION})"; +} + +string PostgresTableSet::GetInitializeQuery(PostgresVersion version, const vector &schemas, + const string &table) { string base_query = R"( SELECT pg_namespace.oid AS namespace_id, relname, relpages, attname, pg_type.typname type_name, atttypmod type_modifier, pg_attribute.attndims ndim, attnum, pg_attribute.attnotnull AS notnull, NULL constraint_id, NULL constraint_type, NULL constraint_key, type_ns.nspname AS type_schema, col_desc.description AS column_comment, - tbl_desc.description AS table_comment + tbl_desc.description AS table_comment, + ${COLUMN_DEFAULT} AS column_default FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid JOIN pg_attribute ON pg_class.oid=pg_attribute.attrelid @@ -37,13 +98,13 @@ JOIN pg_type ON atttypid=pg_type.oid JOIN pg_namespace type_ns ON pg_type.typnamespace = type_ns.oid LEFT JOIN pg_description col_desc ON col_desc.objoid=pg_class.oid AND col_desc.objsubid=pg_attribute.attnum LEFT JOIN pg_description tbl_desc ON tbl_desc.objoid=pg_class.oid AND tbl_desc.objsubid=0 -WHERE attnum > 0 AND relkind IN ('r', 'v', 'm', 'f', 'p') ${CONDITION} +WHERE attnum > 0 AND relkind IN ('r', 'v', 'm', 'f', 'p') ${CONDITION}${LATE_BINDING_VIEWS} UNION ALL SELECT pg_namespace.oid AS namespace_id, relname, NULL relpages, NULL attname, NULL type_name, NULL type_modifier, NULL ndim, NULL attnum, NULL AS notnull, pg_constraint.oid AS constraint_id, contype AS constraint_type, conkey AS constraint_key, NULL AS type_schema, - NULL AS column_comment, NULL AS table_comment + NULL AS column_comment, NULL AS table_comment, NULL AS column_default FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid JOIN pg_constraint ON (pg_class.oid=pg_constraint.conrelid) @@ -51,22 +112,54 @@ WHERE relkind IN ('r', 'v', 'm', 'f', 'p') AND contype IN ('p', 'u') ${CONDITION ORDER BY namespace_id, relname, attnum, constraint_id; )"; string condition; - if (!schema.empty()) { - condition += "AND pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "AND pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } if (!table.empty()) { condition += "AND relname=" + PostgresUtils::WriteLiteral(table); } + base_query = StringUtil::Replace(base_query, "${COLUMN_DEFAULT}", GetColumnDefaultExpression(version)); + base_query = StringUtil::Replace( + base_query, "${LATE_BINDING_VIEWS}", + version.type_v == PostgresInstanceType::REDSHIFT ? GetRedshiftLateBindingViewQuery() : string()); + return StringUtil::Replace(base_query, "${CONDITION}", condition); +} + +string PostgresTableSet::GetInitializeQueryInformationSchema(const string &schema, const string &table) { + vector vec; + vec.push_back(schema); + return GetInitializeQueryInformationSchema(vec, table); +} + +string PostgresTableSet::GetInitializeQueryInformationSchema(const vector &schemas, const string &table) { + string base_query = R"( +SELECT table_schema AS namespace_id, table_name AS relname, 0 AS relpages, column_name AS attname, + data_type AS type_name, -1 AS type_modifier, 0 AS ndim, ordinal_position AS attnum, + CASE WHEN is_nullable = 'NO' THEN 't' ELSE 'f' END AS notnull, + NULL AS constraint_id, NULL AS constraint_type, NULL AS constraint_key, + NULL AS type_schema, NULL AS column_comment, NULL AS table_comment, + CASE WHEN column_default IS NOT NULL THEN 'd' END AS column_default +FROM information_schema.columns +WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'pg_toast') ${CONDITION} +ORDER BY table_schema, table_name, ordinal_position; +)"; + string condition; + if (schemas.size() > 0) { + condition += "AND table_schema IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; + } + if (!table.empty()) { + condition += " AND table_name=" + PostgresUtils::WriteLiteral(table); + } return StringUtil::Replace(base_query, "${CONDITION}", condition); } void PostgresTableSet::AddColumn(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, idx_t row, - PostgresTableInfo &table_info) { + optional_ptr schema, const PostgresTypeConfig &type_config, + PostgresResult &result, idx_t row, PostgresTableInfo &table_info) { PostgresTypeData type_info; idx_t column_index = 3; auto column_name = result.GetString(row, column_index); - type_info.type_name = result.GetString(row, column_index + 1); + type_info.type_name = PostgresUtils::DataTypeToTypeName(result.GetString(row, column_index + 1)); type_info.type_modifier = result.GetInt64(row, column_index + 2); type_info.array_dimensions = result.GetInt64(row, column_index + 3); bool is_not_null = result.GetBool(row, column_index + 5); @@ -75,24 +168,40 @@ void PostgresTableSet::AddColumn(optional_ptr transaction, if (!result.IsNull(row, type_schema_index)) { type_info.type_schema = result.GetString(row, type_schema_index); } + if (schema) { + type_info.instance_type = schema->catalog.Cast().GetPostgresVersion().type_v; + } string default_value; PostgresType postgres_type; - auto column_type = PostgresUtils::TypeToLogicalType(transaction, schema, type_info, postgres_type); + auto column_type = PostgresUtils::TypeToLogicalType(transaction, schema, type_config, type_info, postgres_type); table_info.postgres_types.push_back(std::move(postgres_type)); table_info.postgres_names.push_back(column_name); + string type_name = + type_info.type_schema.empty() ? string() : PostgresUtils::WriteIdentifier(type_info.type_schema) + "."; + table_info.postgres_type_names.push_back(type_name + PostgresUtils::WriteIdentifier(type_info.type_name)); + auto column_default = PostgresColumnDefault::NONE; + if (!result.IsNull(row, 15)) { + column_default = + result.GetString(row, 15) == "g" ? PostgresColumnDefault::GENERATED : PostgresColumnDefault::DEFAULT_VALUE; + } + table_info.column_defaults.push_back(column_default); ColumnDefinition column(Identifier(std::move(column_name)), std::move(column_type)); if (!column_comment.empty()) { column.SetComment(Value(column_comment)); } if (!default_value.empty()) { - auto expressions = Parser::ParseExpressionList(default_value); + auto expressions = Parser::GetBuiltinParser().ParseExpressionList(default_value); if (expressions.empty()) { throw InternalException("Expression list is empty"); } column.SetDefaultValue(std::move(expressions[0])); } auto &create_info = *table_info.create_info; + if (!result.IsNull(row, 7)) { + auto attnum = result.GetInt64(row, 7); + table_info.attnum_to_logical[attnum] = create_info.columns.LogicalColumnCount(); + } if (is_not_null) { create_info.constraints.push_back( make_uniq(LogicalIndex(create_info.columns.PhysicalColumnCount()))); @@ -114,24 +223,26 @@ void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, Postgres auto splits = StringUtil::Split(constraint_key.substr(1, constraint_key.size() - 2), ","); vector columns; for (auto &split : splits) { - auto index = std::stoull(split); - if (index <= 0 || index > create_info.columns.LogicalColumnCount()) { + auto attnum = std::stoll(split); + auto entry = table_info.attnum_to_logical.find(attnum); + if (entry == table_info.attnum_to_logical.end()) { return; } - columns.push_back(create_info.columns.GetColumn(LogicalIndex(index - 1)).Name()); + columns.push_back(create_info.columns.GetColumn(LogicalIndex(entry->second)).Name()); } create_info.constraints.push_back(make_uniq(std::move(columns), constraint_type == "p")); } void PostgresTableSet::AddColumnOrConstraint(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, - idx_t row, PostgresTableInfo &table_info) { + optional_ptr schema, + const PostgresTypeConfig &type_config, PostgresResult &result, idx_t row, + PostgresTableInfo &table_info) { if (result.IsNull(row, 3)) { // constraint AddConstraint(result, row, table_info); } else { - AddColumn(transaction, schema, result, row, table_info); + AddColumn(transaction, schema, type_config, result, row, table_info); } } @@ -139,6 +250,7 @@ void PostgresTableSet::CreateEntries(PostgresTransaction &transaction, PostgresR vector> tables; unique_ptr info; + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = start; row < end; row++) { auto table_name = result.GetString(row, 1); if (!info || info->GetTableName() != table_name) { @@ -152,7 +264,7 @@ void PostgresTableSet::CreateEntries(PostgresTransaction &transaction, PostgresR info->create_info->comment = Value(result.GetString(row, 14)); } } - AddColumnOrConstraint(&transaction, &schema, result, row, *info); + AddColumnOrConstraint(&transaction, &schema, type_config, result, row, *info); } if (info) { tables.push_back(std::move(info)); @@ -168,7 +280,10 @@ void PostgresTableSet::LoadEntries(ClientContext &context, PostgresTransaction & CreateEntries(transaction, table_result->GetResult(), table_result->start, table_result->end); table_result.reset(); } else { - auto query = GetInitializeQuery(schema.name.GetIdentifierName()); + auto query = PostgresUtils::UseInformationSchemaIntrospection(context) + ? GetInitializeQueryInformationSchema(schema.name.GetIdentifierName()) + : GetInitializeQuery(catalog.Cast().GetPostgresVersion(), + schema.name.GetIdentifierName()); auto result = transaction.Query(query); auto rows = result->Count(); @@ -177,17 +292,43 @@ void PostgresTableSet::LoadEntries(ClientContext &context, PostgresTransaction & } } +string PostgresTableSet::GetStalenessQuery(ClientContext &context) const { + if (!PostgresUtils::StalenessQueryEnabled(context)) { + return string(); + } + auto custom_query = PostgresUtils::GetCustomStalenessQuery(context); + if (!custom_query.empty()) { + if (custom_query.find("${SCHEMA}") == string::npos) { + throw InvalidInputException("pg_staleness_query must contain a ${SCHEMA} placeholder"); + } + return StringUtil::Replace(custom_query, "${SCHEMA}", + PostgresUtils::WriteLiteral(schema.name.GetIdentifierName())); + } + string base_query = R"( +SELECT pg_class.oid, relname, pg_class.xmin +FROM pg_class +JOIN pg_namespace ON relnamespace = pg_namespace.oid +WHERE relkind IN ('r', 'v', 'm', 'f', 'p') AND pg_namespace.nspname = ${SCHEMA} +ORDER BY pg_class.oid; +)"; + return StringUtil::Replace(base_query, "${SCHEMA}", PostgresUtils::WriteLiteral(schema.name.GetIdentifierName())); +} + unique_ptr PostgresTableSet::GetTableInfo(PostgresTransaction &transaction, PostgresSchemaEntry &schema, const string &table_name) { - auto query = PostgresTableSet::GetInitializeQuery(schema.name.GetIdentifierName(), table_name); + auto query = PostgresUtils::UseInformationSchemaIntrospection(transaction.GetContext()) + ? GetInitializeQueryInformationSchema(schema.name.GetIdentifierName(), table_name) + : GetInitializeQuery(schema.catalog.Cast().GetPostgresVersion(), + schema.name.GetIdentifierName(), table_name); auto result = transaction.Query(query); auto rows = result->Count(); if (rows == 0) { return nullptr; } auto table_info = make_uniq(schema, table_name); + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = 0; row < rows; row++) { - AddColumnOrConstraint(&transaction, &schema, *result, row, *table_info); + AddColumnOrConstraint(&transaction, &schema, type_config, *result, row, *table_info); } table_info->approx_num_pages = result->IsNull(0, 2) ? 0 : result->GetInt64(0, 2); // Read table-level comment from 14 @@ -199,15 +340,18 @@ unique_ptr PostgresTableSet::GetTableInfo(PostgresTransaction unique_ptr PostgresTableSet::GetTableInfo(ClientContext &context, PostgresConnection &connection, const string &schema_name, const string &table_name) { - auto query = PostgresTableSet::GetInitializeQuery(schema_name, table_name); + auto query = PostgresUtils::UseInformationSchemaIntrospection(context) + ? GetInitializeQueryInformationSchema(schema_name, table_name) + : GetInitializeQuery(connection.GetPostgresVersion(context), schema_name, table_name); auto result = connection.Query(context, query); auto rows = result->Count(); if (rows == 0) { throw InvalidInputException("Table %s does not contain any columns.", table_name); } auto table_info = make_uniq(schema_name, table_name); + auto type_config = PostgresTypeConfig::FromContext(context); for (idx_t row = 0; row < rows; row++) { - AddColumnOrConstraint(nullptr, nullptr, *result, row, *table_info); + AddColumnOrConstraint(nullptr, nullptr, type_config, *result, row, *table_info); } table_info->approx_num_pages = result->IsNull(0, 2) ? 0 : result->GetInt64(0, 2); // Read table-level comment @@ -223,7 +367,9 @@ optional_ptr PostgresTableSet::ReloadEntry(PostgresTransaction &tr return nullptr; } auto table_entry = make_shared_ptr(catalog, schema, *table_info); - return CreateEntry(transaction, std::move(table_entry)); + auto result = CreateEntry(transaction, std::move(table_entry)); + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/false); + return result; } // FIXME - this is almost entirely copied from TableCatalogEntry::ColumnsToSQL - should be unified @@ -331,11 +477,11 @@ string GetPostgresCreateTable(CreateTableInfo &info) { if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { ss << "IF NOT EXISTS "; } - if (!info.schema.empty()) { - ss << PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()); + if (!info.GetQualifiedName().Schema().empty()) { + ss << PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()); ss << "."; } - ss << PostgresUtils::WriteIdentifier(info.table.GetIdentifierName()); + ss << PostgresUtils::WriteIdentifier(info.GetTableName().GetIdentifierName()); ss << PostgresColumnsToSQL(info.columns, info.constraints); ss << ";"; return ss.str(); @@ -345,7 +491,9 @@ optional_ptr PostgresTableSet::CreateTable(PostgresTransaction &tr auto create_sql = GetPostgresCreateTable(info.Base()); transaction.Query(create_sql); auto tbl_entry = make_shared_ptr(catalog, schema, info.Base()); - return CreateEntry(transaction, std::move(tbl_entry)); + auto result = CreateEntry(transaction, std::move(tbl_entry)); + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/true); + return result; } string PostgresTableSet::GetAlterTablePrefix(const string &name, optional_ptr entry) { @@ -375,15 +523,15 @@ string PostgresTableSet::GetAlterTablePrefix(ClientContext &context, PostgresTra } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameTableInfo &info) { - string sql = GetAlterTablePrefix(context, transaction, info.name.GetIdentifierName()); + string sql = GetAlterTablePrefix(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); sql += " RENAME TO "; sql += PostgresUtils::WriteIdentifier(info.new_table_name.GetIdentifierName()); transaction.Query(sql); } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameColumnInfo &info) { - auto entry = GetEntry(context, transaction, info.name.GetIdentifierName()); - string sql = GetAlterTablePrefix(info.name.GetIdentifierName(), entry); + auto entry = GetEntry(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); + string sql = GetAlterTablePrefix(info.GetQualifiedName().Name().GetIdentifierName(), entry); sql += " RENAME COLUMN "; string column_name = GetAlterTableColumnName(info.old_name.GetIdentifierName(), entry); sql += PostgresUtils::WriteIdentifier(column_name); @@ -394,7 +542,7 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, AddColumnInfo &info) { - string sql = GetAlterTablePrefix(context, transaction, info.name.GetIdentifierName()); + string sql = GetAlterTablePrefix(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); sql += " ADD COLUMN "; if (info.if_column_not_exists) { sql += "IF NOT EXISTS "; @@ -410,7 +558,7 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t "only constant DEFAULT expressions are supported"); } const ConstantExpression &default_const_expr = default_expr.Cast(); - std::string sql_str = default_const_expr.GetValue().ToSQLString(); + std::string sql_str = default_const_expr.GetLiteral().ToValue().ToSQLString(); sql += " DEFAULT "; sql += sql_str; } @@ -419,8 +567,8 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RemoveColumnInfo &info) { - auto entry = GetEntry(context, transaction, info.name.GetIdentifierName()); - string sql = GetAlterTablePrefix(info.name.GetIdentifierName(), entry); + auto entry = GetEntry(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); + string sql = GetAlterTablePrefix(info.GetQualifiedName().Name().GetIdentifierName(), entry); sql += " DROP COLUMN "; if (info.if_column_exists) { sql += "IF EXISTS "; diff --git a/src/storage/postgres_transaction.cpp b/src/storage/postgres_transaction.cpp index 26c990bcf..d6b58a35b 100644 --- a/src/storage/postgres_transaction.cpp +++ b/src/storage/postgres_transaction.cpp @@ -1,5 +1,8 @@ #include "storage/postgres_transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "storage/postgres_catalog.hpp" +#include "storage/postgres_catalog_set.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" @@ -13,7 +16,7 @@ PostgresTransaction::PostgresTransaction(PostgresCatalog &postgres_catalog, Tran : Transaction(manager, context), access_mode(postgres_catalog.access_mode), isolation_level(postgres_catalog.isolation_level) { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - connection = postgres_catalog.GetConnectionPool().GetConnection(); + connection = postgres_catalog.GetConnectionPool().Acquire(); } PostgresTransaction::~PostgresTransaction() = default; @@ -30,12 +33,22 @@ void PostgresTransaction::Commit() { transaction_state = PostgresTransactionState::TRANSACTION_FINISHED; GetConnectionRaw().Execute(GetContext(), "COMMIT"); } + vector, string>> to_promote; + { + lock_guard l(pending_signatures_lock); + to_promote = std::move(pending_signatures); + } + for (auto &entry : to_promote) { + entry.first.get().PromoteStalenessSignature(std::move(entry.second)); + } } void PostgresTransaction::Rollback() { if (transaction_state == PostgresTransactionState::TRANSACTION_STARTED) { transaction_state = PostgresTransactionState::TRANSACTION_FINISHED; GetConnectionRaw().Execute(GetContext(), "ROLLBACK"); } + lock_guard l(pending_signatures_lock); + pending_signatures.clear(); } string PostgresTransaction::GetBeginTransactionQuery() { @@ -83,6 +96,10 @@ PostgresConnection &PostgresTransaction::GetConnection() { return con; } +uint64_t PostgresTransaction::GetPoolConnectionId() { + return connection.Id(); +} + PostgresConnection &PostgresTransaction::GetConnectionRaw() { return connection.GetConnection(); } @@ -126,10 +143,16 @@ vector> PostgresTransaction::ExecuteQueries(ClientCon optional_ptr PostgresTransaction::ReferenceEntry(shared_ptr &entry) { auto &ref = *entry; + lock_guard l(referenced_entries_lock); referenced_entries.emplace(ref, entry); return ref; } +void PostgresTransaction::StageStalenessSignature(PostgresCatalogSet &catalog_set, string signature) { + lock_guard l(pending_signatures_lock); + pending_signatures.emplace_back(catalog_set, std::move(signature)); +} + string PostgresTransaction::GetTemporarySchema() { if (temporary_schema.empty()) { auto result = Query("SELECT nspname FROM pg_namespace WHERE oid = pg_my_temp_schema();"); diff --git a/src/storage/postgres_transaction_manager.cpp b/src/storage/postgres_transaction_manager.cpp deleted file mode 100644 index b1347ba1e..000000000 --- a/src/storage/postgres_transaction_manager.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "storage/postgres_transaction_manager.hpp" -#include "duckdb/main/attached_database.hpp" - -namespace duckdb { - -PostgresTransactionManager::PostgresTransactionManager(AttachedDatabase &db_p, PostgresCatalog &postgres_catalog) - : TransactionManager(db_p), postgres_catalog(postgres_catalog) { -} - -Transaction &PostgresTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(postgres_catalog, *this, context); - transaction->Start(); - auto &result = *transaction; - lock_guard l(transaction_lock); - transactions[result] = std::move(transaction); - return result; -} - -ErrorData PostgresTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &postgres_transaction = transaction.Cast(); - postgres_transaction.Commit(); - lock_guard l(transaction_lock); - transactions.erase(transaction); - return ErrorData(); -} - -void PostgresTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &postgres_transaction = transaction.Cast(); - postgres_transaction.Rollback(); - lock_guard l(transaction_lock); - transactions.erase(transaction); -} - -void PostgresTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = PostgresTransaction::Get(context, db.GetCatalog()); - auto &db = transaction.GetConnection(); - db.Execute(context, "CHECKPOINT"); -} - -} // namespace duckdb diff --git a/src/storage/postgres_type_entry.cpp b/src/storage/postgres_type_entry.cpp index 5d9d621c4..ccc02821c 100644 --- a/src/storage/postgres_type_entry.cpp +++ b/src/storage/postgres_type_entry.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_type_entry.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" namespace duckdb { diff --git a/src/storage/postgres_type_set.cpp b/src/storage/postgres_type_set.cpp index cee52ac28..f956aa86f 100644 --- a/src/storage/postgres_type_set.cpp +++ b/src/storage/postgres_type_set.cpp @@ -1,4 +1,5 @@ #include "storage/postgres_type_set.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" #include "duckdb/parser/expression/constant_expression.hpp" @@ -23,7 +24,7 @@ PostgresTypeSet::PostgresTypeSet(PostgresSchemaEntry &schema, unique_ptr &schemas) { if (version.major_v < 8 || (version.major_v == 8 && version.minor_v < 3)) { // pg_enum support has been present since v8.3 - https://www.postgresql.org/docs/8.3/catalog-pg-enum.html // for older postgres versions we don't support enums instead @@ -41,8 +42,8 @@ JOIN pg_namespace AS n ON (typnamespace=n.oid) ORDER BY n.oid, enumtypid, enumsortorder; )"; string condition; - if (!schema.empty()) { - condition += "WHERE n.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE n.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -52,15 +53,14 @@ void PostgresTypeSet::CreateEnum(PostgresTransaction &transaction, PostgresResul PostgresType postgres_type; CreateTypeInfo info; postgres_type.oid = result.GetInt64(start_row, 1); - info.name = Identifier(result.GetString(start_row, 2)); + info.SetTypeName(Identifier(result.GetString(start_row, 2))); // construct the enum idx_t enum_count = end_row - start_row; Vector duckdb_levels(LogicalType::VARCHAR, enum_count); for (idx_t enum_idx = 0; enum_idx < enum_count; enum_idx++) { duckdb_levels.SetValue(enum_idx, result.GetString(start_row + enum_idx, 3)); } - info.type = LogicalType::ENUM(duckdb_levels, enum_count); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = LogicalType::ENUM(duckdb_levels, enum_count).WithAlias(info.GetTypeName().GetIdentifierName()); auto type_entry = make_shared_ptr(catalog, schema, info, postgres_type); CreateEntry(transaction, std::move(type_entry)); } @@ -85,7 +85,7 @@ void PostgresTypeSet::InitializeEnums(PostgresTransaction &transaction, Postgres } } -string PostgresTypeSet::GetInitializeCompositesQuery(const string &schema) { +string PostgresTypeSet::GetInitializeCompositesQuery(const vector &schemas) { string base_query = R"( SELECT n.oid, t.typrelid AS id, t.typname as type, pg_attribute.attname, sub_type.typname, sub_type_ns.nspname AS sub_type_schema @@ -103,8 +103,8 @@ AND NOT pg_attribute.attisdropped ORDER BY n.oid, t.oid, attrelid, attnum; )"; string condition; - if (!schema.empty()) { - condition += "AND n.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "AND n.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -114,21 +114,22 @@ void PostgresTypeSet::CreateCompositeType(PostgresTransaction &transaction, Post PostgresType postgres_type; CreateTypeInfo info; postgres_type.oid = result.GetInt64(start_row, 1); - info.name = Identifier(result.GetString(start_row, 2)); + info.SetTypeName(Identifier(result.GetString(start_row, 2))); child_list_t child_types; + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = start_row; row < end_row; row++) { auto type_name = result.GetString(row, 3); PostgresTypeData type_data; type_data.type_name = result.GetString(row, 4); type_data.type_schema = result.GetString(row, 5); PostgresType child_type; - child_types.push_back(make_pair( - Identifier(type_name), PostgresUtils::TypeToLogicalType(&transaction, &schema, type_data, child_type))); + child_types.push_back( + make_pair(Identifier(type_name), + PostgresUtils::TypeToLogicalType(&transaction, &schema, type_config, type_data, child_type))); postgres_type.children.push_back(std::move(child_type)); } - info.type = LogicalType::STRUCT(std::move(child_types)); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = LogicalType::STRUCT(std::move(child_types)).WithAlias(info.GetTypeName().GetIdentifierName()); auto type_entry = make_shared_ptr(catalog, schema, info, postgres_type); CreateEntry(transaction, std::move(type_entry)); } @@ -165,7 +166,7 @@ void PostgresTypeSet::LoadEntries(ClientContext &context, PostgresTransaction &t string GetCreateTypeSQL(CreateTypeInfo &info) { string sql = "CREATE TYPE "; - sql += PostgresUtils::WriteIdentifier(info.name.GetIdentifierName()); + sql += PostgresUtils::WriteIdentifier(info.GetTypeName().GetIdentifierName()); sql += " AS "; switch (info.type.id()) { case LogicalTypeId::ENUM: { @@ -206,7 +207,7 @@ optional_ptr PostgresTypeSet::CreateType(PostgresTransaction &tran auto create_sql = GetCreateTypeSQL(info); conn.Execute(transaction.GetContext(), create_sql); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = info.type.WithAlias(info.GetTypeName().GetIdentifierName()); auto pg_type = PostgresUtils::CreateEmptyPostgresType(info.type); auto type_entry = make_shared_ptr(catalog, schema, info, pg_type); return CreateEntry(transaction, std::move(type_entry)); diff --git a/src/storage/postgres_update.cpp b/src/storage/postgres_update.cpp index 49c47e00c..95ab2e81f 100644 --- a/src/storage/postgres_update.cpp +++ b/src/storage/postgres_update.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_update.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_table_entry.hpp" #include "duckdb/planner/operator/logical_update.hpp" diff --git a/test/sql/concurrent/concurrent_attach.test b/test/sql/concurrent/concurrent_attach.test new file mode 100644 index 000000000..1df69934b --- /dev/null +++ b/test/sql/concurrent/concurrent_attach.test @@ -0,0 +1,31 @@ +# name: test/sql/concurrent/concurrent_attach.test +# description: Test ATTACH called concurrently from multiple threads +# group: [concurrent] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s + +statement ok +SET threads = 4 + +concurrentloop i 0 16 + +statement ok +ATTACH 'dbname=postgresscanner' AS s{i} (TYPE POSTGRES) + +query I +SELECT count(*) FROM s{i}.pg_numtypes +---- +4 + +statement ok +DETACH s{i} + +endloop diff --git a/test/sql/concurrent/concurrent_clear_cache.test b/test/sql/concurrent/concurrent_clear_cache.test new file mode 100644 index 000000000..23b815c99 --- /dev/null +++ b/test/sql/concurrent/concurrent_clear_cache.test @@ -0,0 +1,36 @@ +# name: test/sql/concurrent/concurrent_clear_cache.test +# description: Test concurrent drop of the table being quried +# group: [concurrent] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.concurrent_drop_test AS +SELECT unnest(range(1<<12)) col1 + +loop i 0 16 + +concurrentloop j 0 16 + +statement ok +CALL postgres_clear_cache() + +query I +SELECT divide(count(*), 1) FROM s.concurrent_drop_test +---- +4096 + +endloop + +endloop + +statement ok +DROP TABLE s.concurrent_drop_test + +statement ok +DETACH s diff --git a/test/sql/storage/attach_concurrent_queries.test b/test/sql/concurrent/concurrent_queries.test similarity index 90% rename from test/sql/storage/attach_concurrent_queries.test rename to test/sql/concurrent/concurrent_queries.test index 852602f0f..890fad701 100644 --- a/test/sql/storage/attach_concurrent_queries.test +++ b/test/sql/concurrent/concurrent_queries.test @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_concurrent_queries.test +# name: test/sql/concurrent/concurrent_queries.test # description: Run queries concurrently over different connections -# group: [storage] +# group: [concurrent] require postgres_scanner diff --git a/test/sql/concurrent/concurrent_results.test b/test/sql/concurrent/concurrent_results.test new file mode 100644 index 000000000..98e3bb7a4 --- /dev/null +++ b/test/sql/concurrent/concurrent_results.test @@ -0,0 +1,32 @@ +# name: test/sql/concurrent/concurrent_results.test +# description: Test concurrent multiple results reading from multple scans +# group: [concurrent] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_a AS +SELECT '2020-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_b AS +SELECT '2021-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +SET threads = 4 + +concurrentloop i 0 16 + +query II +SELECT date(t.insert_date) AS col1, divide(sum(t.amount), 1) AS col2 +FROM s.aggr_union_test_a t +GROUP BY 1 +---- +2020-01-01 134209536 + +endloop diff --git a/test/sql/storage/attach_connection_pool.test_slow b/test/sql/connection_pool/connection_pool.test_slow similarity index 96% rename from test/sql/storage/attach_connection_pool.test_slow rename to test/sql/connection_pool/connection_pool.test_slow index 56f0d7f77..99f680322 100644 --- a/test/sql/storage/attach_connection_pool.test_slow +++ b/test/sql/connection_pool/connection_pool.test_slow @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_connection_pool.test_slow +# name: test/sql/connection_pool/connection_pool.test_slow # description: Test that the connection pool correctly limits the number of connections -# group: [storage] +# group: [connection_pool] require postgres_scanner diff --git a/test/sql/storage/attach_connection_pool_configure.test b/test/sql/connection_pool/connection_pool_configure.test similarity index 66% rename from test/sql/storage/attach_connection_pool_configure.test rename to test/sql/connection_pool/connection_pool_configure.test index 84c46e5ec..aebe2157b 100644 --- a/test/sql/storage/attach_connection_pool_configure.test +++ b/test/sql/connection_pool/connection_pool_configure.test @@ -1,17 +1,11 @@ -# name: test/sql/storage/attach_connection_pool_configure.test +# name: test/sql/connection_pool/connection_pool_configure.test # description: Test connection pool live configuration -# group: [storage] +# group: [connection_pool] require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -ATTACH 'dbname=postgresscanner' AS loader (TYPE POSTGRES); - -statement ok -DETACH loader; - statement ok SET pg_pool_max_connections = 8 @@ -24,13 +18,13 @@ ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES); statement ok RESET GLOBAL pg_pool_max_connections -query IIIIIIIIII -SELECT catalog_name, max_connections, /*cache_hits,*/ cache_misses, try_failures, thread_local_cache_enabled, max_lifetime_millis, idle_timeout_millis, reaper_thread_running, reaper_thread_period_millis, health_check_query +query IIIIIIIIIIII +SELECT catalog_name, max_connections, available_connections, total_connections, /*cache_hits,*/ cache_misses, try_failures, thread_local_cache_enabled, max_lifetime_millis, idle_timeout_millis, reaper_thread_running, reaper_thread_period_millis, health_check_query FROM postgres_configure_pool() ORDER BY catalog_name ---- -s 8 1 0 FALSE 0 60000 1 30000 SELECT 1 -s1 8 1 0 FALSE 0 60000 1 30000 SELECT 1 +s 8 1 1 1 0 FALSE 0 60000 1 30000 SELECT 1 +s1 8 1 1 1 0 FALSE 0 60000 1 30000 SELECT 1 query II SELECT catalog_name, acquire_mode @@ -157,3 +151,57 @@ DETACH s1 statement ok DETACH s + +# health check query + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query II +SELECT catalog_name, health_check_query +FROM postgres_configure_pool(catalog_name='s') +---- +s SELECT 1 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query II +SELECT catalog_name, health_check_query +FROM postgres_configure_pool(catalog_name='s', health_check_query='SELECT_FAIL') +---- +s SELECT_FAIL + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s diff --git a/test/sql/storage/attach_connection_pool_options.test b/test/sql/connection_pool/connection_pool_options.test similarity index 78% rename from test/sql/storage/attach_connection_pool_options.test rename to test/sql/connection_pool/connection_pool_options.test index 4c1841be4..988789ba0 100644 --- a/test/sql/storage/attach_connection_pool_options.test +++ b/test/sql/connection_pool/connection_pool_options.test @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_connection_pool_options.test +# name: test/sql/connection_pool/connection_pool_options.test # description: Test connection pool options -# group: [storage] +# group: [connection_pool] require postgres_scanner @@ -214,3 +214,56 @@ RESET GLOBAL pg_connection_cache statement ok DETACH s + +# health check query + +query I +SELECT current_setting('pg_pool_health_check_query') +---- +SELECT 1 + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +statement ok +DETACH s + +statement ok +SET GLOBAL pg_pool_health_check_query = 'SELECT_FAIL' + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query I +SELECT current_setting('pg_pool_health_check_query') +---- +SELECT_FAIL + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +RESET GLOBAL pg_pool_health_check_query + +statement ok +DETACH s diff --git a/test/sql/misc/postgres_binary.test b/test/sql/misc/postgres_binary.test index d855f19a8..e48bbcd86 100644 --- a/test/sql/misc/postgres_binary.test +++ b/test/sql/misc/postgres_binary.test @@ -6,12 +6,17 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); +require-env POSTGRES_TEST_SERVER_ON_HOST statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +# DuckDB supports timestamps below PostgreSQL's minimum. +statement error +COPY (SELECT TIMESTAMP '290309-12-22 (BC)') TO '{TEST_DIR}/pg_binary.bin' (FORMAT postgres_binary); +---- +out of range for Postgres + # straightforward integer copy statement ok COPY (SELECT i::INT AS i FROM range(100) t(i)) TO '{TEST_DIR}/pg_binary.bin' (FORMAT postgres_binary); @@ -33,7 +38,8 @@ CREATE TABLE all_types_tbl AS SELECT bool, smallint, int, bigint, float, double, dec_4_1, dec_9_4, dec_18_6, dec38_10, case when date < '1992-01-01' then '4712-01-01 (BC)' else '5874896-01-01' end as date, -- postgres has more constrained date ranges time, - timestamp, interval, uuid, blob, + case when timestamp < TIMESTAMP '4713-01-01 (BC)' then TIMESTAMP '4713-01-01 (BC)' else timestamp end as timestamp, -- postgres has a more constrained lower timestamp bound + interval, uuid, blob, replace(varchar, chr(0), '') as varchar, -- postgres does not support null bytes in varchar columns blob, int_array, varchar_array FROM test_all_types() diff --git a/test/sql/misc/postgres_binary_read_basic.test b/test/sql/misc/postgres_binary_read_basic.test index 34d659bd6..9bceef1b9 100644 --- a/test/sql/misc/postgres_binary_read_basic.test +++ b/test/sql/misc/postgres_binary_read_basic.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- BOOLEAN --- statement ok diff --git a/test/sql/misc/postgres_binary_read_buffer.test b/test/sql/misc/postgres_binary_read_buffer.test new file mode 100644 index 000000000..100d3bb75 --- /dev/null +++ b/test/sql/misc/postgres_binary_read_buffer.test @@ -0,0 +1,83 @@ +# name: test/sql/misc/postgres_binary_read_buffer.test +# description: Test reading postgres binary files that span multiple read buffers +# group: [misc] + +require postgres_scanner + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s + +# Each row here is 10 bytes (2 byte field count + 4 byte length + 4 byte payload) and the file +# header is 19 bytes, so these buffers hold only a handful of rows at a time. +statement ok +COPY (SELECT i::INTEGER AS a FROM range(1000) t(i)) TO '{TEST_DIR}/many_rows.bin' (FORMAT postgres_binary); + +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=64); +---- +1000 0 999 + +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=32); +---- +1000 0 999 + +# a buffer with no room for a row next to the header: the header is a boundary of its own, so the +# first row is carried over to the next fill rather than rejected +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=24); +---- +1000 0 999 + +# a buffer of exactly the header size +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=19); +---- +1000 0 999 + +# the values must round-trip exactly, not just in aggregate +query I +SELECT count(*) FROM ( + SELECT a FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=64) + EXCEPT + SELECT i::INTEGER FROM range(1000) t(i) +); +---- +0 + +# variable width columns and NULLs across buffer boundaries +statement ok +COPY (SELECT i::INTEGER AS a, ('str_' || i)::VARCHAR AS b, CASE WHEN i % 3 = 0 THEN NULL ELSE i::BIGINT END AS c FROM range(500) t(i)) TO '{TEST_DIR}/mixed_rows.bin' (FORMAT postgres_binary); + +query IIII +SELECT count(*), count(c), min(b), max(a) FROM read_postgres_binary('{TEST_DIR}/mixed_rows.bin', columns={a: 'INTEGER', b: 'VARCHAR', c: 'BIGINT'}, buffer_size=128); +---- +500 333 str_0 499 + +# rows of 56 bytes (2 byte field count + 4 byte length + 50 byte payload): larger than +# buffer_size - 19 but no larger than buffer_size, so they only fit once the header is out of the way +statement ok +COPY (SELECT repeat('x', 50) AS a FROM range(20) t(i)) TO '{TEST_DIR}/wide_rows.bin' (FORMAT postgres_binary); + +query III +SELECT count(*), min(length(a)), max(length(a)) FROM read_postgres_binary('{TEST_DIR}/wide_rows.bin', columns={a: 'VARCHAR'}, buffer_size=64); +---- +20 50 50 + +# the same rows in a buffer of exactly one row +query III +SELECT count(*), min(length(a)), max(length(a)) FROM read_postgres_binary('{TEST_DIR}/wide_rows.bin', columns={a: 'VARCHAR'}, buffer_size=56); +---- +20 50 50 + +# a row genuinely larger than the whole buffer is still rejected +statement ok +COPY (SELECT repeat('y', 100) AS a FROM range(5) t(i)) TO '{TEST_DIR}/huge_rows.bin' (FORMAT postgres_binary); + +statement error +SELECT count(*) FROM read_postgres_binary('{TEST_DIR}/huge_rows.bin', columns={a: 'VARCHAR'}, buffer_size=64); +---- +row larger than the read buffer diff --git a/test/sql/misc/postgres_binary_read_complex.test b/test/sql/misc/postgres_binary_read_complex.test index 65de94d70..b98121556 100644 --- a/test/sql/misc/postgres_binary_read_complex.test +++ b/test/sql/misc/postgres_binary_read_complex.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- INTEGER ARRAY --- statement ok diff --git a/test/sql/misc/postgres_binary_read_copy_from.test b/test/sql/misc/postgres_binary_read_copy_from.test index f4ec70eec..a230eb70e 100644 --- a/test/sql/misc/postgres_binary_read_copy_from.test +++ b/test/sql/misc/postgres_binary_read_copy_from.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- COPY FROM basic --- statement ok diff --git a/test/sql/misc/postgres_binary_read_errors.test b/test/sql/misc/postgres_binary_read_errors.test index f3ed80d7f..464c977a5 100644 --- a/test/sql/misc/postgres_binary_read_errors.test +++ b/test/sql/misc/postgres_binary_read_errors.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- Non-existent file --- statement error diff --git a/test/sql/misc/postgres_binary_read_schema_mismatch.test b/test/sql/misc/postgres_binary_read_schema_mismatch.test new file mode 100644 index 000000000..6d2cecc36 --- /dev/null +++ b/test/sql/misc/postgres_binary_read_schema_mismatch.test @@ -0,0 +1,65 @@ +# name: test/sql/misc/postgres_binary_read_schema_mismatch.test +# description: Reading a postgres binary file with a type list that does not match the file is an error, not wrong data +# group: [misc] + +require postgres_scanner + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s + +# --- field width mismatch: a BIGINT column read as INTEGER --- +statement ok +COPY (SELECT i::INTEGER AS id, 180388626432::BIGINT AS amount FROM range(1, 6) t(i)) TO '{TEST_DIR}/width_mismatch.bin' (FORMAT postgres_binary); + +query II +SELECT * FROM read_postgres_binary('{TEST_DIR}/width_mismatch.bin', columns={id: 'INTEGER', amount: 'BIGINT'}); +---- +1 180388626432 +2 180388626432 +3 180388626432 +4 180388626432 +5 180388626432 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/width_mismatch.bin', columns={id: 'INTEGER', amount: 'INTEGER'}); +---- +expected 4 bytes for a value of type INTEGER, got 8 + +# --- variable-length source: a DECIMAL column read as BIGINT --- +statement ok +COPY (SELECT 123.45::DECIMAL(10,2) AS amount) TO '{TEST_DIR}/decimal_as_bigint.bin' (FORMAT postgres_binary); + +query I +SELECT * FROM read_postgres_binary('{TEST_DIR}/decimal_as_bigint.bin', columns={amount: 'DECIMAL(10,2)'}); +---- +123.45 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/decimal_as_bigint.bin', columns={amount: 'BIGINT'}); +---- +expected 8 bytes for a value of type BIGINT, got 12 + +# --- field count mismatch: a two column file read as one column --- +statement ok +COPY (SELECT i::INTEGER AS a, (i * 10)::INTEGER AS b FROM range(1, 4) t(i)) TO '{TEST_DIR}/count_mismatch.bin' (FORMAT postgres_binary); + +query II +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER', b: 'INTEGER'}); +---- +1 10 +2 20 +3 30 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER'}); +---- +expected 1 field(s) per row, got 2 + +# --- array header too short: a scalar column read as a list --- +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER[]', b: 'INTEGER'}); +---- +expected at least 12 bytes for an array header of type INTEGER[], got 4 diff --git a/test/sql/misc/postgres_binary_read_temporal.test b/test/sql/misc/postgres_binary_read_temporal.test index 853041a51..ebba99ad9 100644 --- a/test/sql/misc/postgres_binary_read_temporal.test +++ b/test/sql/misc/postgres_binary_read_temporal.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- DATE --- statement ok diff --git a/test/sql/misc/postgres_hstore.test b/test/sql/misc/postgres_hstore.test index 441615cbc..53b802424 100644 --- a/test/sql/misc/postgres_hstore.test +++ b/test/sql/misc/postgres_hstore.test @@ -4,9 +4,6 @@ require postgres_scanner -statement ok -LOAD postgres_scanner; - # --- postgres_hstore_get: basic lookups --- query I diff --git a/test/sql/redshift/attach_redshift_late_binding_views.test b/test/sql/redshift/attach_redshift_late_binding_views.test new file mode 100644 index 000000000..1d5a3c074 --- /dev/null +++ b/test/sql/redshift/attach_redshift_late_binding_views.test @@ -0,0 +1,149 @@ +# name: test/sql/redshift/attach_redshift_late_binding_views.test +# description: Redshift late-binding views (WITH NO SCHEMA BINDING) are exposed in the attached catalog +# group: [redshift] + +require postgres_scanner + +# libpq connection string of a Redshift cluster, for example: +# REDSHIFT_TEST_DSN='host=localhost port=5439 dbname=dev user=awsuser sslmode=require' (password in PGPASSWORD) +require-env REDSHIFT_TEST_DSN + +statement ok +ATTACH '{REDSHIFT_TEST_DSN}' AS rs (TYPE POSTGRES) + +statement ok +CALL postgres_execute('rs', 'DROP SCHEMA IF EXISTS duckdb_lbv_test CASCADE') + +statement ok +CALL postgres_execute('rs', 'CREATE SCHEMA duckdb_lbv_test') + +statement ok +CALL postgres_execute('rs', ' +CREATE TABLE duckdb_lbv_test.base_tbl ( + c_small SMALLINT, c_int INTEGER NOT NULL, c_big BIGINT, c_dec DECIMAL(38,0), c_dec_scale DECIMAL(10,2), + c_dec_default DECIMAL, c_real REAL, c_dbl DOUBLE PRECISION, c_bool BOOLEAN, c_char CHAR(10), + c_varchar VARCHAR(100), c_varchar_max VARCHAR(MAX), c_date DATE, c_ts TIMESTAMP, c_time TIME +)') + +statement ok +CALL postgres_execute('rs', ' +INSERT INTO duckdb_lbv_test.base_tbl VALUES + (1, 10, 100, 12345678901234567890, 1.50, 42, 0.5, 0.25, true, ''abc'', ''hello'', ''world'', + ''2024-01-02'', ''2024-01-02 03:04:05'', ''03:04:05''), + (2, 20, NULL, NULL, NULL, NULL, NULL, NULL, false, NULL, ''second'', NULL, NULL, NULL, NULL) +') + +statement ok +CALL postgres_execute('rs', 'CREATE VIEW duckdb_lbv_test.regular_view AS SELECT c_int, c_varchar FROM duckdb_lbv_test.base_tbl') + +statement ok +CALL postgres_execute('rs', 'CREATE VIEW duckdb_lbv_test.late_view AS SELECT * FROM duckdb_lbv_test.base_tbl WITH NO SCHEMA BINDING') + +statement ok +CALL postgres_execute('rs', 'CREATE VIEW duckdb_lbv_test."late view quoted" AS SELECT c_int AS "my id", c_varchar AS "my name" FROM duckdb_lbv_test.base_tbl WITH NO SCHEMA BINDING') + +statement ok +CALL postgres_execute('rs', 'COMMENT ON VIEW duckdb_lbv_test.late_view IS ''late binding view comment''') + +# late-binding views have no pg_attribute entries +query II +SELECT * FROM postgres_query('rs', 'SELECT relname, relnatts FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid WHERE nspname = ''duckdb_lbv_test'' AND relkind = ''v''') ORDER BY 1 +---- +late view quoted 0 +late_view 0 +regular_view 2 + +statement ok +CALL postgres_clear_cache() + +query I +SELECT table_name FROM duckdb_tables() WHERE database_name = 'rs' AND schema_name = 'duckdb_lbv_test' ORDER BY 1 +---- +base_tbl +late view quoted +late_view +regular_view + +# column types of the late-binding view match the ones of the base table +query IIII +SELECT b.column_name, b.data_type, l.data_type, l.column_index +FROM duckdb_columns() b +FULL OUTER JOIN (SELECT * FROM duckdb_columns() WHERE database_name = 'rs' AND schema_name = 'duckdb_lbv_test' AND table_name = 'late_view') l USING (column_name) +WHERE b.database_name = 'rs' AND b.schema_name = 'duckdb_lbv_test' AND b.table_name = 'base_tbl' +ORDER BY l.column_index +---- +c_small SMALLINT SMALLINT 1 +c_int INTEGER INTEGER 2 +c_big BIGINT BIGINT 3 +c_dec DECIMAL(38,0) DECIMAL(38,0) 4 +c_dec_scale DECIMAL(10,2) DECIMAL(10,2) 5 +c_dec_default DECIMAL(18,0) DECIMAL(18,0) 6 +c_real FLOAT FLOAT 7 +c_dbl DOUBLE DOUBLE 8 +c_bool BOOLEAN BOOLEAN 9 +c_char VARCHAR VARCHAR 10 +c_varchar VARCHAR VARCHAR 11 +c_varchar_max VARCHAR VARCHAR 12 +c_date DATE DATE 13 +c_ts TIMESTAMP TIMESTAMP 14 +c_time TIME TIME 15 + +query I +SELECT comment FROM duckdb_tables() WHERE database_name = 'rs' AND schema_name = 'duckdb_lbv_test' AND table_name = 'late_view' +---- +late binding view comment + +# CHAR values are not blank-padded with the (default) binary results +query IIIIIIIIIIIIIII +SELECT * FROM rs.duckdb_lbv_test.late_view ORDER BY c_int +---- +1 10 100 12345678901234567890 1.50 42 0.5 0.25 true abc hello world 2024-01-02 2024-01-02 03:04:05 03:04:05 +2 20 NULL NULL NULL NULL NULL NULL false NULL second NULL NULL NULL NULL + +# projection and filter pushdown +query II +SELECT c_varchar, c_dec_scale FROM rs.duckdb_lbv_test.late_view WHERE c_int = 10 +---- +hello 1.50 + +query II +SELECT "my name", "my id" FROM rs.duckdb_lbv_test."late view quoted" ORDER BY "my id" +---- +hello 10 +second 20 + +# the late-binding view can be joined with regular tables and views +query III +SELECT r.c_varchar, l.c_big, b.c_bool +FROM rs.duckdb_lbv_test.regular_view r +JOIN rs.duckdb_lbv_test.late_view l USING (c_int) +JOIN rs.duckdb_lbv_test.base_tbl b USING (c_int) +ORDER BY c_int +---- +hello 100 true +second NULL false + +# a late-binding view created after the catalog has been loaded is found by the single table lookup +statement ok +CALL postgres_execute('rs', 'CREATE VIEW duckdb_lbv_test.late_view_2 AS SELECT c_int, c_dec_scale FROM duckdb_lbv_test.base_tbl WITH NO SCHEMA BINDING') + +query II +SELECT c_int, c_dec_scale FROM rs.duckdb_lbv_test.late_view_2 ORDER BY c_int +---- +10 1.50 +20 NULL + +# attaching with a schema filter +statement ok +ATTACH '{REDSHIFT_TEST_DSN}' AS rs_schema (TYPE POSTGRES, SCHEMA 'duckdb_lbv_test', READ_ONLY) + +query I +SELECT count(*) FROM rs_schema.duckdb_lbv_test.late_view +---- +2 + +statement ok +DETACH rs_schema + +statement ok +CALL postgres_execute('rs', 'DROP SCHEMA duckdb_lbv_test CASCADE') diff --git a/test/sql/redshift/attach_redshift_types.test b/test/sql/redshift/attach_redshift_types.test new file mode 100644 index 000000000..9493d58b6 --- /dev/null +++ b/test/sql/redshift/attach_redshift_types.test @@ -0,0 +1,308 @@ +# name: test/sql/redshift/attach_redshift_types.test +# description: Reading all Redshift data types through an attached Redshift database +# group: [redshift] + +require postgres_scanner + +# libpq connection string of a Redshift cluster, for example: +# REDSHIFT_TEST_DSN='host=localhost port=5439 dbname=dev user=awsuser sslmode=require' (password in PGPASSWORD) +require-env REDSHIFT_TEST_DSN + +statement ok +ATTACH '{REDSHIFT_TEST_DSN}' AS rs (TYPE POSTGRES) + +statement ok +CALL postgres_execute('rs', 'DROP SCHEMA IF EXISTS duckdb_types_test CASCADE') + +statement ok +CALL postgres_execute('rs', 'CREATE SCHEMA duckdb_types_test') + +# https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html +statement ok +CALL postgres_execute('rs', ' +CREATE TABLE duckdb_types_test.all_types ( + id INTEGER, + c_smallint SMALLINT, c_int2 INT2, c_integer INTEGER, c_int4 INT4, c_bigint BIGINT, c_int8 INT8, + c_decimal DECIMAL(38,10), c_numeric NUMERIC(5,2), c_decimal_default DECIMAL, c_decimal_precision DECIMAL(9), + c_real REAL, c_float4 FLOAT4, c_double DOUBLE PRECISION, c_float FLOAT, c_float8 FLOAT8, + c_boolean BOOLEAN, + c_char CHAR(5), c_nchar NCHAR(3), c_bpchar BPCHAR, + c_varchar VARCHAR(20), c_varchar_max VARCHAR(MAX), c_nvarchar NVARCHAR(10), c_text TEXT, + c_date DATE, c_time TIME, c_timetz TIMETZ, c_timestamp TIMESTAMP, c_timestamptz TIMESTAMPTZ, + c_interval_ym INTERVAL YEAR TO MONTH, c_interval_ds INTERVAL DAY TO SECOND, + c_varbyte VARBYTE(16), + c_super SUPER, + c_geometry GEOMETRY, c_geography GEOGRAPHY, + c_hllsketch HLLSKETCH +)') + +statement ok +CALL postgres_execute('rs', ' +INSERT INTO duckdb_types_test.all_types VALUES ( + 1, + -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, + 1234567890123456789012345678.0123456789, -123.45, 42, 123456789, + 1.5, -2.25, 3.125, -4.5, 1e100, + true, + ''ab'', ''xyz'', ''q'', + ''hello'', ''world wide'', ''nv'', ''some text'', + ''2024-02-29'', ''13:14:15.123456'', ''13:14:15+03'', ''2024-02-29 13:14:15.123456'', ''2024-02-29 13:14:15.123456+00'', + INTERVAL ''1 year 2 months'', INTERVAL ''3 days 04:05:06.789'', + FROM_HEX(''00ff61''), + JSON_PARSE(''{"a":[1,2,{"b":null}],"c":"d"}''), + ST_SetSRID(ST_GeomFromText(''LINESTRING(1 2,3 4)''), 3857), ST_GeogFromText(''POINT(-71.06 42.36)''), + NULL +)') + +statement ok +CALL postgres_execute('rs', ' +INSERT INTO duckdb_types_test.all_types (id, c_super, c_interval_ym, c_interval_ds) VALUES + (2, JSON_PARSE(''[1,"two",3.5]''), INTERVAL ''-1 year -2 months'', INTERVAL ''-3 days 04:05:06.789''), + (3, ''plain string'', NULL, NULL), + (4, NULL, NULL, NULL) +') + +statement ok +CALL postgres_execute('rs', 'CREATE TABLE duckdb_types_test.hll_source (i INTEGER)') + +statement ok +CALL postgres_execute('rs', 'INSERT INTO duckdb_types_test.hll_source VALUES (1), (2), (3), (3)') + +statement ok +CALL postgres_execute('rs', 'UPDATE duckdb_types_test.all_types SET c_hllsketch = (SELECT HLL_CREATE_SKETCH(i) FROM duckdb_types_test.hll_source) WHERE id = 1') + +statement ok +CALL postgres_execute('rs', 'CREATE VIEW duckdb_types_test.all_types_late_view AS SELECT * FROM duckdb_types_test.all_types WITH NO SCHEMA BINDING') + +statement ok +CALL postgres_clear_cache() + +# the type mapping is the same for regular tables (pg_attribute) and late-binding views +# (pg_get_late_binding_view_cols()) +foreach relation all_types all_types_late_view + +query II +SELECT column_name, data_type FROM duckdb_columns() +WHERE database_name = 'rs' AND schema_name = 'duckdb_types_test' AND table_name = '{relation}' +ORDER BY column_index +---- +id INTEGER +c_smallint SMALLINT +c_int2 SMALLINT +c_integer INTEGER +c_int4 INTEGER +c_bigint BIGINT +c_int8 BIGINT +c_decimal DECIMAL(38,10) +c_numeric DECIMAL(5,2) +c_decimal_default DECIMAL(18,0) +c_decimal_precision DECIMAL(9,0) +c_real FLOAT +c_float4 FLOAT +c_double DOUBLE +c_float DOUBLE +c_float8 DOUBLE +c_boolean BOOLEAN +c_char VARCHAR +c_nchar VARCHAR +c_bpchar VARCHAR +c_varchar VARCHAR +c_varchar_max VARCHAR +c_nvarchar VARCHAR +c_text VARCHAR +c_date DATE +c_time TIME +c_timetz TIME WITH TIME ZONE +c_timestamp TIMESTAMP +c_timestamptz TIMESTAMP WITH TIME ZONE +c_interval_ym INTERVAL +c_interval_ds INTERVAL +c_varbyte BLOB +c_super JSON +c_geometry GEOMETRY +c_geography GEOMETRY('EPSG:4326') +c_hllsketch JSON + +# AUTO uses plain queries with binary results on Redshift +foreach query_mode PLAIN_QUERY_TEXT AUTO PLAIN_QUERY_BINARY + +statement ok +SET pg_query_mode = '{query_mode}' + +# numeric types +query IIIIIIIIIIIIIII +SELECT c_smallint, c_int2, c_integer, c_int4, c_bigint, c_int8, c_decimal, c_numeric, c_decimal_default, + c_decimal_precision, c_real, c_float4, c_double, c_float, c_float8 +FROM rs.duckdb_types_test.{relation} ORDER BY id +---- +-32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1234567890123456789012345678.0123456789 -123.45 42 123456789 1.5 -2.25 3.125 -4.5 1e+100 +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL + +# boolean and character types +query IIIIIIII +SELECT c_boolean, rtrim(c_char), c_nchar, rtrim(c_bpchar), c_varchar, c_varchar_max, c_nvarchar, c_text +FROM rs.duckdb_types_test.{relation} ORDER BY id +---- +true ab xyz q hello world wide nv some text +NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL + +# datetime types (Redshift stores TIMETZ values in UTC) +query IIIIIII +SELECT c_date, c_time, c_timetz, c_timestamp, c_timestamptz = TIMESTAMPTZ '2024-02-29 13:14:15.123456+00', + c_interval_ym, c_interval_ds +FROM rs.duckdb_types_test.{relation} ORDER BY id +---- +2024-02-29 13:14:15.123456 10:14:15+00 2024-02-29 13:14:15.123456 true 1 year 2 months 3 days 04:05:06.789 +NULL NULL NULL NULL NULL -1 year -2 months -2 days -19:54:53.211 +NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL + +# VARBYTE, SUPER and spatial types +query IIII +SELECT c_varbyte, c_super, c_geometry, c_geography +FROM rs.duckdb_types_test.{relation} ORDER BY id +---- +\x00\xFFa {"a":[1,2,{"b":null}],"c":"d"} LINESTRING (1 2, 3 4) POINT (-71.06 42.36) +NULL [1,"two",3.5] NULL NULL +NULL "plain string" NULL NULL +NULL NULL NULL NULL + +# filters on the Redshift-specific types are pushed down +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_varbyte = '\x00\xFFa'::BLOB +---- +1 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_varbyte IN ('\x00'::BLOB, '\x00\xFFa'::BLOB) +---- +1 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_interval_ym = INTERVAL '14 months' +---- +1 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_interval_ds < INTERVAL '0 seconds' +---- +2 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_super IS NOT NULL ORDER BY id +---- +1 +2 +3 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_geography IS NOT NULL AND c_geometry IS NOT NULL +---- +1 + +query I +SELECT id FROM rs.duckdb_types_test.{relation} WHERE c_decimal > 1234567890123456789012345678 AND c_timetz IS NOT NULL +---- +1 + +endloop + +# Redshift returns CHAR values blank-padded, the padding is only preserved by the text protocol +statement ok +SET pg_query_mode = 'PLAIN_QUERY_TEXT' + +query III +SELECT c_char, length(c_bpchar), rtrim(c_bpchar) FROM rs.duckdb_types_test.{relation} WHERE id = 1 +---- +ab 256 q + +# HLLSKETCH has no binary output, it can only be read using the text protocol +query II +SELECT c_hllsketch::VARCHAR LIKE '{"version":1,%', count(*) FROM rs.duckdb_types_test.{relation} GROUP BY ALL ORDER BY ALL +---- +true 1 +NULL 3 + +# AUTO falls back to the text protocol (for the whole query) when HLLSKETCH is read, +# CHAR padding shows which protocol was used +statement ok +RESET pg_query_mode + +query II +SELECT c_char, c_hllsketch::VARCHAR LIKE '{"version":1,%' FROM rs.duckdb_types_test.{relation} WHERE id = 1 +---- +ab true + +query I +SELECT c_char FROM rs.duckdb_types_test.{relation} WHERE id = 1 +---- +ab + +query I +SELECT count(*) FROM rs.duckdb_types_test.{relation} WHERE c_hllsketch IS NOT NULL +---- +1 + +statement ok +SET pg_query_mode = 'PLAIN_QUERY_BINARY' + +statement error +SELECT c_hllsketch FROM rs.duckdb_types_test.{relation} +---- +column "c_hllsketch" can only be read using the text protocol, use pg_query_mode = 'PLAIN_QUERY_TEXT' + +statement ok +RESET pg_query_mode + +endloop + +# postgres_query maps the Redshift-specific types by their OIDs +foreach query_mode PLAIN_QUERY_TEXT AUTO PLAIN_QUERY_BINARY + +statement ok +SET pg_query_mode = '{query_mode}' + +query IIIIIIIII +SELECT typeof(c_interval_ym), c_interval_ym, typeof(c_interval_ds), c_interval_ds, typeof(c_varbyte), c_varbyte, + typeof(c_super), c_super, typeof(c_geography) +FROM postgres_query('rs', 'SELECT * FROM duckdb_types_test.all_types WHERE id = 1') +---- +INTERVAL 1 year 2 months INTERVAL 3 days 04:05:06.789 BLOB \x00\xFFa JSON {"a":[1,2,{"b":null}],"c":"d"} GEOMETRY('EPSG:4326') + +query IIII +SELECT typeof(c_geometry), c_geometry, c_geography, c_super +FROM postgres_query('rs', 'SELECT c_geometry, c_geography, c_super FROM duckdb_types_test.all_types ORDER BY id') +---- +GEOMETRY LINESTRING (1 2, 3 4) POINT (-71.06 42.36) {"a":[1,2,{"b":null}],"c":"d"} +GEOMETRY NULL NULL [1,"two",3.5] +GEOMETRY NULL NULL "plain string" +GEOMETRY NULL NULL NULL + +endloop + +statement ok +RESET pg_query_mode + +query II +SELECT typeof(c_hllsketch), c_hllsketch::VARCHAR LIKE '{"version":1,%' +FROM postgres_query('rs', 'SELECT c_hllsketch FROM duckdb_types_test.all_types WHERE id = 1') +---- +JSON true + +statement ok +SET pg_query_mode = 'PLAIN_QUERY_BINARY' + +statement error +SELECT * FROM postgres_query('rs', 'SELECT c_hllsketch FROM duckdb_types_test.all_types') +---- +column "c_hllsketch" can only be read using the text protocol + +statement ok +RESET pg_query_mode + +statement ok +CALL postgres_execute('rs', 'DROP SCHEMA duckdb_types_test CASCADE') diff --git a/test/sql/remote_pushdown/basic_remote_pushdown.test b/test/sql/remote_pushdown/basic_remote_pushdown.test new file mode 100644 index 000000000..b115c85dc --- /dev/null +++ b/test/sql/remote_pushdown/basic_remote_pushdown.test @@ -0,0 +1,268 @@ +# name: test/sql/remote_pushdown/basic_remote_pushdown.test +# description: Test basic remote pushdown of queries into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_basic AS SELECT i, i % 3 AS g, 'str' || i AS v FROM range(10) t(i) + +# aggregates are pushed down +query I +SELECT SUM(i) FROM s.rp_basic +---- +45 + +query II +EXPLAIN SELECT SUM(i) FROM s.rp_basic +---- +physical_plan :(?s).*Postgres Query.*SELECT CAST\(sum\("i"\) AS NUMERIC\(38,0\)\) AS "sum\(i\)" FROM "public"."rp_basic".* + +# the result types match the types DuckDB itself would produce +query IIII +SELECT typeof(SUM(i)), typeof(COUNT(*)), typeof(AVG(i)), typeof(MIN(v)) FROM s.rp_basic +---- +HUGEINT BIGINT DOUBLE VARCHAR + +# as do the result names +query IIII +SELECT * FROM (SELECT SUM(i), COUNT(*), max(V), i // 1 IS NULL FROM s.rp_basic GROUP BY i // 1 IS NULL) t(a, b, c, d) WHERE false +---- + +statement ok +CREATE OR REPLACE TABLE rp_names AS SELECT SUM(i), COUNT(*), max(V), I FROM s.rp_basic GROUP BY I LIMIT 0 + +query I +SELECT column_name FROM (DESCRIBE rp_names) +---- +sum(i) +count_star() +max(V) +i + +# unsupported aggregates are executed DuckDB side +query I +SELECT mode(i < 7) FROM s.rp_basic +---- +true + +query II +EXPLAIN SELECT mode(i < 7) FROM s.rp_basic +---- +physical_plan :(?s).*Postgres Query.* + +# group by + order by +query II +SELECT g, SUM(i) FROM s.rp_basic GROUP BY g ORDER BY g +---- +0 18 +1 12 +2 15 + +query II +EXPLAIN SELECT g, SUM(i) FROM s.rp_basic GROUP BY g ORDER BY g +---- +physical_plan :(?s).*GROUP BY "g" ORDER BY "g" ASC NULLS LAST.* + +# filters, LIKE and LIMIT +query I +SELECT v FROM s.rp_basic WHERE v LIKE 'str_' AND i > 5 ORDER BY i DESC LIMIT 2 OFFSET 1 +---- +str8 +str7 + +query II +EXPLAIN SELECT v FROM s.rp_basic WHERE v LIKE 'str_' AND i > 5 ORDER BY i DESC LIMIT 2 OFFSET 1 +---- +physical_plan :(?s).*Postgres Query.*LIKE 'str_' ESCAPE ''.*ORDER BY "i" DESC NULLS LAST LIMIT 2 OFFSET 1.* + +# DISTINCT +query I +SELECT DISTINCT g FROM s.rp_basic ORDER BY g +---- +0 +1 +2 + +query II +EXPLAIN SELECT DISTINCT g FROM s.rp_basic ORDER BY g +---- +physical_plan :(?s).*SELECT DISTINCT "g".* + +# identifiers are case insensitive in DuckDB, but case sensitive (when quoted) in Postgres +query II +SELECT G, Sum(I) FROM S.RP_BASIC GROUP BY G ORDER BY G +---- +0 18 +1 12 +2 15 + +query II +EXPLAIN SELECT G, Sum(I) FROM S.RP_BASIC GROUP BY G ORDER BY G +---- +physical_plan :(?s).*SELECT "g", CAST\(sum\("i"\) AS NUMERIC\(38,0\)\) AS "sum\(I\)" FROM "public"."rp_basic" GROUP BY "g".* + +# subqueries and joins +query I +SELECT COUNT(*) FROM s.rp_basic t1 JOIN s.rp_basic t2 ON t1.i = t2.g +---- +10 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_basic t1 JOIN s.rp_basic t2 ON t1.i = t2.g +---- +physical_plan :(?s).*Postgres Query.*INNER JOIN.* + +query I +SELECT SUM(x) FROM (SELECT i + 1 AS x FROM s.rp_basic WHERE i IN (SELECT g FROM s.rp_basic)) +---- +6 + +query II +EXPLAIN SELECT SUM(x) FROM (SELECT i + 1 AS x FROM s.rp_basic WHERE i IN (SELECT g FROM s.rp_basic)) +---- +physical_plan :(?s).*Postgres Query.* + +# mixing local and remote tables is not pushed down +statement ok +CREATE TABLE local_tbl AS SELECT 1 AS i + +query I +SELECT COUNT(*) FROM s.rp_basic JOIN local_tbl USING (i) +---- +1 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_basic JOIN local_tbl USING (i) +---- +physical_plan :(?s).*Postgres Query.* + +# remote pushdown can be disabled for newly attached databases +statement ok +SET pg_enable_remote_pushdown = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES, SCHEMA 'public') + +query I +SELECT SUM(i) FROM s2.rp_basic +---- +45 + +query II +EXPLAIN SELECT SUM(i) FROM s2.rp_basic +---- +physical_plan :(?s).*Postgres Query.* + +# the already attached database still pushes down +query II +EXPLAIN SELECT SUM(i) FROM s.rp_basic +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +RESET pg_enable_remote_pushdown + +# plain table scans are executed through the regular table scan, which can be parallelized with ctid ranges +query II +EXPLAIN SELECT i, v FROM s.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* + +# when the settings at attach time do not allow parallel table scans, plain table scans are pushed down as well +statement ok +SET pg_use_ctid_scan = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s_no_ctid (TYPE POSTGRES, SCHEMA 'public') + +statement ok +RESET pg_use_ctid_scan + +query II +EXPLAIN SELECT i, v FROM s_no_ctid.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.*SELECT "i", "v" FROM "public"."rp_basic" WHERE \("i" > 7\).* + +query II +SELECT i, v FROM s_no_ctid.rp_basic WHERE i > 7 ORDER BY i +---- +8 str8 +9 str9 + +statement ok +SET pg_use_text_protocol = true + +statement ok +ATTACH 'dbname=postgresscanner' AS s_text (TYPE POSTGRES, SCHEMA 'public') + +statement ok +RESET pg_use_text_protocol + +query II +EXPLAIN SELECT i, v FROM s_text.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +SET pg_query_mode = 'PLAIN_QUERY_TEXT' + +statement ok +ATTACH 'dbname=postgresscanner' AS s_plain_text (TYPE POSTGRES, SCHEMA 'public') + +statement ok +RESET pg_query_mode + +query II +EXPLAIN SELECT i, v FROM s_plain_text.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +ATTACH 'dbname=postgresscanner' AS s_text_option (TYPE POSTGRES, SCHEMA 'public', USE_TEXT_PROTOCOL TRUE) + +query II +EXPLAIN SELECT i, v FROM s_text_option.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, v FROM s_text_option.rp_basic WHERE i > 7 ORDER BY i +---- +8 str8 +9 str9 + +# the binary protocol without COPY still allows parallel table scans +statement ok +SET pg_query_mode = 'PLAIN_QUERY_BINARY' + +statement ok +ATTACH 'dbname=postgresscanner' AS s_plain_binary (TYPE POSTGRES, SCHEMA 'public') + +statement ok +RESET pg_query_mode + +query II +EXPLAIN SELECT i, v FROM s_plain_binary.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* + +# pg_use_binary_copy only affects writes +statement ok +SET pg_use_binary_copy = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s_no_binary_copy (TYPE POSTGRES, SCHEMA 'public') + +statement ok +RESET pg_use_binary_copy + +query II +EXPLAIN SELECT i, v FROM s_no_binary_copy.rp_basic WHERE i > 7 +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_aggregates.test b/test/sql/remote_pushdown/remote_pushdown_aggregates.test new file mode 100644 index 000000000..20012b432 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_aggregates.test @@ -0,0 +1,322 @@ +# name: test/sql/remote_pushdown/remote_pushdown_aggregates.test +# description: Test remote pushdown of aggregates into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_agg AS SELECT i, i % 3 AS g, CASE WHEN i % 4 = 0 THEN NULL ELSE 'v' || (i % 5) END AS v, i * 0.5::DOUBLE AS d, (i * 1.25)::DECIMAL(10,2) AS dec, DATE '2024-01-01' + (i * 10)::INT AS dt FROM range(12) t(i) + +# basic aggregates and their result types +query IIIIIIIII rowsort +SELECT COUNT(*), COUNT(v), SUM(i), SUM(d), SUM(dec), AVG(i), AVG(dec), MIN(v), MAX(dt) FROM s.rp_agg +---- +12 9 66 33.0 82.50 5.5 6.875 v0 2024-04-20 + +query II +EXPLAIN SELECT COUNT(*), COUNT(v), SUM(i), SUM(d), SUM(dec), AVG(i), AVG(dec), MIN(v), MAX(dt) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIIII rowsort +SELECT typeof(COUNT(*)), typeof(SUM(i)), typeof(SUM(d)), typeof(SUM(dec)), typeof(AVG(i)), typeof(AVG(dec)), typeof(MIN(v)), typeof(MAX(dt)) FROM s.rp_agg +---- +BIGINT HUGEINT DOUBLE DECIMAL(38,2) DOUBLE DOUBLE VARCHAR DATE + +query IIIIII rowsort +SELECT stddev(i), variance(i), stddev_pop(i), var_pop(i), stddev_samp(g), var_samp(g) FROM s.rp_agg +---- +3.605551275463989 13.0 3.452052529534663 11.916666666666666 0.8528028654224418 0.7272727272727273 + +query II +EXPLAIN SELECT stddev(i), variance(i), stddev_pop(i), var_pop(i), stddev_samp(g), var_samp(g) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query III rowsort +SELECT bool_and(i >= 0), bool_or(i > 10), bool_and(v IS NOT NULL) FROM s.rp_agg +---- +true true false + +query II +EXPLAIN SELECT bool_and(i >= 0), bool_or(i > 10), bool_and(v IS NOT NULL) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# DISTINCT aggregates +query IIII rowsort +SELECT COUNT(DISTINCT g), SUM(DISTINCT g), AVG(DISTINCT g), COUNT(DISTINCT v) FROM s.rp_agg +---- +3 3 1.0 5 + +query II +EXPLAIN SELECT COUNT(DISTINCT g), SUM(DISTINCT g), AVG(DISTINCT g), COUNT(DISTINCT v) FROM s.rp_agg +---- +physical_plan :(?s).*count\(DISTINCT "g"\).* + +# FILTER +query III rowsort +SELECT COUNT(*) FILTER (WHERE i > 5), SUM(i) FILTER (WHERE g = 0), MIN(v) FILTER (WHERE i > 3) FROM s.rp_agg +---- +6 18 v0 + +query II +EXPLAIN SELECT COUNT(*) FILTER (WHERE i > 5), SUM(i) FILTER (WHERE g = 0), MIN(v) FILTER (WHERE i > 3) FROM s.rp_agg +---- +physical_plan :(?s).*count\(\*\) FILTER \(WHERE \("i" > 5\)\).* + +query II +SELECT g, COUNT(*) FILTER (WHERE v IS NULL) FROM s.rp_agg GROUP BY g ORDER BY g +---- +0 1 +1 1 +2 1 + +query II +EXPLAIN SELECT g, COUNT(*) FILTER (WHERE v IS NULL) FROM s.rp_agg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# count_if is serialized as a conditional count +query III rowsort +SELECT count_if(i > 3), countif(v IS NULL), count_if(i > 3) FILTER (WHERE g = 1) FROM s.rp_agg +---- +8 3 3 + +query II +EXPLAIN SELECT count_if(i > 3), countif(v IS NULL), count_if(i > 3) FILTER (WHERE g = 1) FROM s.rp_agg +---- +physical_plan :(?s).*count\(CASE WHEN \("i" > 3\) THEN 1 END\).* + +# string aggregation - DuckDB's default separator is a comma +query II +SELECT g, string_agg(v ORDER BY i) FROM s.rp_agg GROUP BY g ORDER BY g +---- +0 v3,v1,v4 +1 v1,v2,v0 +2 v2,v0,v1 + +query II +EXPLAIN SELECT g, string_agg(v ORDER BY i) FROM s.rp_agg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*string_agg\("v", ',' ORDER BY "i" ASC NULLS LAST\).* + +query III +SELECT string_agg(v, ' | ' ORDER BY i DESC), group_concat(v, '' ORDER BY v DESC NULLS FIRST, i), listagg(DISTINCT v, '-' ORDER BY v) FROM s.rp_agg +---- +v1 | v0 | v4 | v2 | v1 | v0 | v3 | v2 | v1 v4v3v2v2v1v1v1v0v0 v0-v1-v2-v3-v4 + +query II +EXPLAIN SELECT string_agg(v, ' | ' ORDER BY i DESC), group_concat(v, '' ORDER BY v DESC NULLS FIRST, i) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# string aggregation of non-strings is executed locally +query I +SELECT string_agg(i, ',' ORDER BY i) FROM s.rp_agg +---- +0,1,2,3,4,5,6,7,8,9,10,11 + +query II +EXPLAIN SELECT string_agg(i, ',' ORDER BY i) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# lists +query III +SELECT array_agg(i ORDER BY i DESC), list(v ORDER BY i), array_agg(DISTINCT g ORDER BY g) FROM s.rp_agg +---- +[11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] [NULL, v1, v2, v3, NULL, v0, v1, v2, NULL, v4, v0, v1] [0, 1, 2] + +query II +EXPLAIN SELECT array_agg(i ORDER BY i DESC), list(v ORDER BY i), array_agg(DISTINCT g ORDER BY g) FROM s.rp_agg +---- +physical_plan :(?s).*array_agg\("i" ORDER BY "i" DESC NULLS LAST\).* + +query IIII +SELECT g, list(dt ORDER BY i), list(d ORDER BY i), list(dec ORDER BY i) FROM s.rp_agg GROUP BY g ORDER BY g +---- +0 [2024-01-01, 2024-01-31, 2024-03-01, 2024-03-31] [0.0, 1.5, 3.0, 4.5] [0.00, 3.75, 7.50, 11.25] +1 [2024-01-11, 2024-02-10, 2024-03-11, 2024-04-10] [0.5, 2.0, 3.5, 5.0] [1.25, 5.00, 8.75, 12.50] +2 [2024-01-21, 2024-02-20, 2024-03-21, 2024-04-20] [1.0, 2.5, 4.0, 5.5] [2.50, 6.25, 10.00, 13.75] + +query II +EXPLAIN SELECT g, list(dt ORDER BY i), list(d ORDER BY i), list(dec ORDER BY i) FROM s.rp_agg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT list(i) FILTER (WHERE false) FROM s.rp_agg +---- +NULL + +query II +EXPLAIN SELECT list(i) FILTER (WHERE false) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# bitwise aggregates +query IIII rowsort +SELECT bit_and(i), bit_or(i), bit_xor(i), bit_or(g) FILTER (WHERE i > 3) FROM s.rp_agg +---- +0 15 0 3 + +query II +EXPLAIN SELECT bit_and(i), bit_or(i), bit_xor(i), bit_or(g) FILTER (WHERE i > 3) FROM s.rp_agg +---- +physical_plan :(?s).*bit_and\("i"\).* + +# median / quantiles +query IIIII rowsort +SELECT median(i), quantile_cont(i, 0.25), quantile_disc(i, 0.75), quantile(g, 0.5), median(v) FROM s.rp_agg +---- +5.5 2.75 8 1 v1 + +query II +EXPLAIN SELECT median(i), quantile_cont(i, 0.25), quantile_disc(i, 0.75), quantile(g, 0.5), median(v) FROM s.rp_agg +---- +physical_plan :(?s).*percentile_cont\(0\.5\) WITHIN GROUP \(ORDER BY "i"\).* + +query III +SELECT g, median(i), quantile_disc(v, 0.2) FROM s.rp_agg GROUP BY g ORDER BY g +---- +0 4.5 v1 +1 5.5 v0 +2 6.5 v0 + +query II +EXPLAIN SELECT g, median(i), quantile_disc(v, 0.2) FROM s.rp_agg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# DuckDB interpolates decimals and temporal types, Postgres does not +query I rowsort +SELECT median(dec) FROM s.rp_agg +---- +6.87 + +query II +EXPLAIN SELECT median(dec) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT median(dt) FROM s.rp_agg +---- +2024-02-25 00:00:00 + +query II +EXPLAIN SELECT median(dt) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT quantile_cont(dt, 0.25) FROM s.rp_agg +---- +2024-01-28 12:00:00 + +query II +EXPLAIN SELECT quantile_cont(dt, 0.25) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# lists of quantiles are not supported +query I rowsort +SELECT quantile_disc(i, [0.25, 0.75]) FROM s.rp_agg +---- +[2, 8] + +query II +EXPLAIN SELECT quantile_disc(i, [0.25, 0.75]) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# arg_min / arg_max ignore rows where either argument is NULL +query IIII rowsort +SELECT arg_max(v, i), arg_min(v, i), max_by(i, d), min_by(dt, v) FROM s.rp_agg +---- +v1 v1 11 2024-02-20 + +query II +EXPLAIN SELECT arg_max(v, i), arg_min(v, i), max_by(i, d), min_by(dt, v) FROM s.rp_agg +---- +physical_plan :(?s).*\(array_agg\("v" ORDER BY "i" DESC\) FILTER \(WHERE \("v" IS NOT NULL\) AND \("i" IS NOT NULL\)\)\)\[1\].* + +query III +SELECT g, arg_max(v, i), arg_min(i, d) FILTER (WHERE i > 2) FROM s.rp_agg GROUP BY g ORDER BY g +---- +0 v4 3 +1 v0 4 +2 v1 5 + +query II +EXPLAIN SELECT g, arg_max(v, i), arg_min(i, d) FILTER (WHERE i > 2) FROM s.rp_agg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# statistical aggregates +query IIIIIII rowsort +SELECT regr_count(d, i), regr_avgx(d, i), regr_avgy(d, i), regr_sxx(d, i), regr_slope(d, i), regr_intercept(d, i), covar_pop(i, d) FROM s.rp_agg +---- +12 5.5 2.75 143.0 0.5 0.0 5.958333333333333 + +query II +EXPLAIN SELECT regr_count(d, i), regr_avgx(d, i), regr_avgy(d, i), regr_sxx(d, i), regr_slope(d, i), regr_intercept(d, i), covar_pop(i, d) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates that are not available in Postgres are executed locally +query I rowsort +SELECT mode(i < 9) FROM s.rp_agg +---- +true + +query II +EXPLAIN SELECT mode(i < 9) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT product(i) FROM s.rp_agg WHERE i > 0 +---- +39916800.0 + +query II +EXPLAIN SELECT product(i) FROM s.rp_agg WHERE i > 0 +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT approx_count_distinct(i) FROM s.rp_agg +---- +12 + +query II +EXPLAIN SELECT approx_count_distinct(i) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT first(i ORDER BY i) FROM s.rp_agg +---- +0 + +query II +EXPLAIN SELECT first(i ORDER BY i) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT entropy(g) FROM s.rp_agg +---- +1.584962500721156 + +query II +EXPLAIN SELECT entropy(g) FROM s.rp_agg +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_cast.test b/test/sql/remote_pushdown/remote_pushdown_cast.test new file mode 100644 index 000000000..b40ca97a5 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_cast.test @@ -0,0 +1,204 @@ +# name: test/sql/remote_pushdown/remote_pushdown_cast.test +# description: Test remote pushdown of casts into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_cast AS SELECT * FROM (VALUES (1, 2.5::DOUBLE, '12', 1.25::DECIMAL(5,2), true, DATE '2024-02-29', TIMESTAMP '2024-02-29 13:14:15.5', INTERVAL '1 day'), (2, -2.5, ' 7 ', -3.75, false, DATE '1999-12-31', TIMESTAMP '1999-12-31 23:59:59', INTERVAL '2 months'), (3, 0.5, '-0', 0, NULL, NULL, NULL, NULL), (4, 1e10, '2147483647', 999.99, true, DATE '2000-01-01', TIMESTAMP '2000-01-01 00:00:00', INTERVAL '3 hours')) t(id, d, s, dec, b, dt, ts, iv) + +# casts to integers - both DuckDB and Postgres round doubles to the nearest even integer +query IIIIIII +SELECT id, CAST(d AS INTEGER), CAST(d AS BIGINT), CAST(dec AS INTEGER), CAST(s AS INTEGER), CAST(b AS INTEGER), CAST(b AS BIGINT) FROM s.rp_cast WHERE id < 4 ORDER BY id +---- +1 2 2 1 12 1 1 +2 -2 -2 -4 7 0 0 +3 0 0 0 0 NULL NULL + +query II +EXPLAIN SELECT id, CAST(d AS INTEGER), CAST(d AS BIGINT), CAST(dec AS INTEGER), CAST(s AS INTEGER), CAST(b AS INTEGER), CAST(b AS BIGINT) FROM s.rp_cast WHERE id < 4 ORDER BY id +---- +physical_plan :(?s).*CAST\("d" AS INTEGER\).* + +query III +SELECT id, d::SMALLINT, dec::SMALLINT FROM s.rp_cast WHERE id < 4 ORDER BY id +---- +1 2 1 +2 -2 -4 +3 0 0 + +query II +EXPLAIN SELECT id, d::SMALLINT, dec::SMALLINT FROM s.rp_cast WHERE id < 4 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# casts to floating point / decimals +query IIIIIII +SELECT id, CAST(s AS DOUBLE), CAST(dec AS DOUBLE), CAST(d AS DECIMAL(12,1)), CAST(s AS DECIMAL(12,2)), d::FLOAT, id::REAL FROM s.rp_cast ORDER BY id +---- +1 12.0 1.25 2.5 12.00 2.5 1.0 +2 7.0 -3.75 -2.5 7.00 -2.5 2.0 +3 -0.0 0.0 0.5 0.00 0.5 3.0 +4 2147483647.0 999.99 10000000000.0 2147483647.00 10000000000.0 4.0 + +query II +EXPLAIN SELECT id, CAST(s AS DOUBLE), CAST(dec AS DOUBLE), CAST(d AS DECIMAL(12,1)), CAST(s AS DECIMAL(12,2)), d::FLOAT, id::REAL FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# casts to VARCHAR are only pushed for types that Postgres formats identically +query IIIIII +SELECT id, CAST(id AS VARCHAR), CAST(dec AS VARCHAR), CAST(b AS VARCHAR), CAST(dt AS VARCHAR), CAST(ts AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +1 1 1.25 true 2024-02-29 2024-02-29 13:14:15.5 +2 2 -3.75 false 1999-12-31 1999-12-31 23:59:59 +3 3 0.00 NULL NULL NULL +4 4 999.99 true 2000-01-01 2000-01-01 00:00:00 + +query II +EXPLAIN SELECT id, CAST(id AS VARCHAR), CAST(dec AS VARCHAR), CAST(b AS VARCHAR), CAST(dt AS VARCHAR), CAST(ts AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*SET LOCAL DateStyle.* + +query II +SELECT id, CAST(d AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +1 2.5 +2 -2.5 +3 0.5 +4 10000000000.0 + +query II +EXPLAIN SELECT id, CAST(d AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, CAST(iv AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +1 1 day +2 2 months +3 NULL +4 03:00:00 + +query II +EXPLAIN SELECT id, CAST(iv AS VARCHAR) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# date / time casts +query IIIII +SELECT id, CAST(ts AS DATE), CAST(dt AS TIMESTAMP), CAST(ts AS TIME), CAST('2024-01-01' AS DATE) < dt FROM s.rp_cast ORDER BY id +---- +1 2024-02-29 2024-02-29 00:00:00 13:14:15.5 true +2 1999-12-31 1999-12-31 00:00:00 23:59:59 false +3 NULL NULL NULL NULL +4 2000-01-01 2000-01-01 00:00:00 00:00:00 false + +query II +EXPLAIN SELECT id, CAST(ts AS DATE), CAST(dt AS TIMESTAMP), CAST(ts AS TIME), CAST('2024-01-01' AS DATE) < dt FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# casts to BOOLEAN +query III +SELECT id, CAST(id - 1 AS BOOLEAN), CAST(CASE WHEN id = 1 THEN 'true' ELSE 'false' END AS BOOLEAN) FROM s.rp_cast ORDER BY id +---- +1 false true +2 true false +3 true false +4 true false + +query II +EXPLAIN SELECT id, CAST(id - 1 AS BOOLEAN), CAST(CASE WHEN id = 1 THEN 'true' ELSE 'false' END AS BOOLEAN) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# casts of literals are evaluated locally +query I +SELECT id FROM s.rp_cast WHERE dt > CAST('2000-01-01' AS DATE) ORDER BY id +---- +1 + +query II +EXPLAIN SELECT id FROM s.rp_cast WHERE dt > CAST('2000-01-01' AS DATE) ORDER BY id +---- +physical_plan :(?s).*\("dt" > DATE '2000-01-01'\).* + +query III +SELECT id, d + '1.5'::DOUBLE, dec * 2::DECIMAL(3,1) FROM s.rp_cast ORDER BY id +---- +1 4.0 2.500 +2 -1.0 -7.500 +3 2.0 0.000 +4 10000000001.5 1999.980 + +query II +EXPLAIN SELECT id, d + '1.5'::DOUBLE, dec * 2::DECIMAL(3,1) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# TRY_CAST is DuckDB-specific +query II +SELECT id, TRY_CAST(s AS INTEGER) FROM s.rp_cast ORDER BY id +---- +1 12 +2 7 +3 0 +4 2147483647 + +query II +EXPLAIN SELECT id, TRY_CAST(s AS INTEGER) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# casts to unsigned / nested types are not supported by Postgres +query II +SELECT id, CAST(id AS UBIGINT) FROM s.rp_cast ORDER BY id +---- +1 1 +2 2 +3 3 +4 4 + +query II +EXPLAIN SELECT id, CAST(id AS UBIGINT) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, CAST(id AS TINYINT) FROM s.rp_cast ORDER BY id +---- +1 1 +2 2 +3 3 +4 4 + +query II +EXPLAIN SELECT id, CAST(id AS TINYINT) FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, [id]::BIGINT[] FROM s.rp_cast ORDER BY id +---- +1 [1] +2 [2] +3 [3] +4 [4] + +query II +EXPLAIN SELECT id, [id]::BIGINT[] FROM s.rp_cast ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# integer overflow raises an error in both systems +statement error +SELECT CAST(d AS INTEGER) FROM s.rp_cast WHERE id = 4 +---- +out of range diff --git a/test/sql/remote_pushdown/remote_pushdown_collation.test b/test/sql/remote_pushdown/remote_pushdown_collation.test new file mode 100644 index 000000000..673e18951 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_collation.test @@ -0,0 +1,393 @@ +# name: test/sql/remote_pushdown/remote_pushdown_collation.test +# description: Test that remote pushdown into Postgres orders strings like DuckDB, independent of the database collation +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test that remote pushdown into Postgres orders strings like DuckDB, independent of the database collation +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS rp_coll_list') + +statement ok +CREATE OR REPLACE TABLE s.rp_coll AS SELECT * FROM (VALUES (1, 'a', 1), (2, 'B', 2), (3, '_x', 1), (4, 'b', 2), (5, 'A', 1), (6, 'é', 2), (7, 'Z', 1), (8, NULL, 2), (9, 'a b', 1), (10, 'ab', 2)) t(id, v, g) + +# DuckDB orders strings byte-wise - if the database collation differs (e.g. en_US), the C collation is pinned +query I +SELECT v FROM s.rp_coll ORDER BY v +---- +A +B +Z +_x +a +a b +ab +b +é +NULL + +query II +EXPLAIN SELECT v FROM s.rp_coll ORDER BY v +---- +physical_plan :(?s).*ORDER BY ("v"|CAST\("v" AS TEXT\) COLLATE "C") ASC NULLS LAST.* + +query II +SELECT id, v FROM s.rp_coll ORDER BY v DESC NULLS FIRST, id +---- +8 NULL +6 é +4 b +10 ab +9 a b +1 a +3 _x +7 Z +2 B +5 A + +query II +EXPLAIN SELECT id, v FROM s.rp_coll ORDER BY v DESC NULLS FIRST, id +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT v || '!' AS w FROM s.rp_coll ORDER BY w +---- +A! +B! +Z! +_x! +a b! +a! +ab! +b! +é! +NULL + +query II +EXPLAIN SELECT v || '!' AS w FROM s.rp_coll ORDER BY w +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT v || '?' AS w, id FROM s.rp_coll ORDER BY 1, 2 +---- +A? 5 +B? 2 +Z? 7 +_x? 3 +a b? 9 +a? 1 +ab? 10 +b? 4 +é? 6 +NULL 8 + +query II +EXPLAIN SELECT v || '?' AS w, id FROM s.rp_coll ORDER BY 1, 2 +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT g, max(v) AS mx, min(v) AS mn FROM s.rp_coll GROUP BY g ORDER BY ALL +---- +1 a b A +2 é B + +query II +EXPLAIN SELECT g, max(v) AS mx, min(v) AS mn FROM s.rp_coll GROUP BY g ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, v FROM s.rp_coll WHERE v < 'b' ORDER BY id +---- +1 a +2 B +3 _x +5 A +7 Z +9 a b +10 ab + +query II +EXPLAIN SELECT id, v FROM s.rp_coll WHERE v < 'b' ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT id FROM s.rp_coll WHERE v BETWEEN 'B' AND 'a' ORDER BY id +---- +1 +2 +3 +7 + +query II +EXPLAIN SELECT id FROM s.rp_coll WHERE v BETWEEN 'B' AND 'a' ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT id FROM s.rp_coll WHERE v > ANY (SELECT v FROM s.rp_coll WHERE id IN (1, 2)) ORDER BY id +---- +1 +3 +4 +6 +7 +9 +10 + +query II +EXPLAIN SELECT id FROM s.rp_coll WHERE v > ANY (SELECT v FROM s.rp_coll WHERE id IN (1, 2)) ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT id, greatest(v, 'a'), least(v, 'Z') FROM s.rp_coll ORDER BY id +---- +1 a Z +2 a B +3 a Z +4 b Z +5 a A +6 é Z +7 a Z +8 a Z +9 a b Z +10 ab Z + +query II +EXPLAIN SELECT id, greatest(v, 'a'), least(v, 'Z') FROM s.rp_coll ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT g, string_agg(v, ',' ORDER BY v), arg_min(id, v), arg_max(id, v) FROM s.rp_coll GROUP BY g ORDER BY g +---- +1 A,Z,_x,a,a b 5 9 +2 B,ab,b,é 2 6 + +query II +EXPLAIN SELECT g, string_agg(v, ',' ORDER BY v), arg_min(id, v), arg_max(id, v) FROM s.rp_coll GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT median(v), quantile_disc(v, 0.25) FROM s.rp_coll +---- +a Z + +query II +EXPLAIN SELECT median(v), quantile_disc(v, 0.25) FROM s.rp_coll +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT id, v, row_number() OVER (ORDER BY v), max(v) OVER (PARTITION BY g ORDER BY id) FROM s.rp_coll ORDER BY id +---- +1 a 5 a +2 B 2 B +3 _x 4 a +4 b 8 b +5 A 1 a +6 é 9 é +7 Z 3 a +8 NULL 10 é +9 a b 6 a b +10 ab 7 é + +query II +EXPLAIN SELECT id, v, row_number() OVER (ORDER BY v), max(v) OVER (PARTITION BY g ORDER BY id) FROM s.rp_coll ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT DISTINCT ON (g) g, v FROM s.rp_coll ORDER BY g, v +---- +1 A +2 B + +query II +SELECT a.id, b.id FROM s.rp_coll a JOIN s.rp_coll b ON a.v < b.v WHERE a.id = 1 ORDER BY 1, 2 +---- +1 4 +1 6 +1 9 +1 10 + +query II +EXPLAIN SELECT a.id, b.id FROM s.rp_coll a JOIN s.rp_coll b ON a.v < b.v WHERE a.id = 1 ORDER BY 1, 2 +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, v FROM s.rp_coll WHERE CASE WHEN v >= 'a' THEN true ELSE false END ORDER BY id +---- +1 a +4 b +6 é +9 a b +10 ab + +query II +EXPLAIN SELECT id, v FROM s.rp_coll WHERE CASE WHEN v >= 'a' THEN true ELSE false END ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +CREATE OR REPLACE TABLE s.rp_coll_list AS SELECT id, [v, 'x'] AS l FROM (VALUES (1, 'a'), (2, 'B'), (3, '_x'), (4, NULL)) t(id, v) + +query II +SELECT id, l FROM s.rp_coll_list ORDER BY l, id +---- +2 [B, x] +3 [_x, x] +1 [a, x] +4 [NULL, x] + +query II +EXPLAIN SELECT id, l FROM s.rp_coll_list ORDER BY l, id +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT id FROM s.rp_coll_list WHERE l > ['B', 'x'] ORDER BY id +---- +1 +3 +4 + +query I +WITH c AS (SELECT v FROM s.rp_coll) SELECT v FROM c ORDER BY v +---- +A +B +Z +_x +a +a b +ab +b +é +NULL + +query II +EXPLAIN WITH c AS (SELECT v FROM s.rp_coll) SELECT v FROM c ORDER BY v +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT * FROM (SELECT v FROM s.rp_coll ORDER BY v LIMIT 4) ORDER BY v +---- +A +B +Z +_x + +query II +EXPLAIN SELECT * FROM (SELECT v FROM s.rp_coll ORDER BY v LIMIT 4) ORDER BY v +---- +physical_plan :(?s).*Postgres Query.* + +# set operations, DISTINCT and DISTINCT aggregates cannot pin the collation of string keys - with such a database +# collation they are executed locally +query I +SELECT v FROM s.rp_coll WHERE id < 5 UNION SELECT v FROM s.rp_coll WHERE id >= 5 ORDER BY v +---- +A +B +Z +_x +a +a b +ab +b +é +NULL + +query I +SELECT DISTINCT v FROM s.rp_coll ORDER BY v +---- +A +B +Z +_x +a +a b +ab +b +é +NULL + +query I +SELECT string_agg(DISTINCT v, ',' ORDER BY v) FROM s.rp_coll +---- +A,B,Z,_x,a,a b,ab,b,é + +query II +SELECT g, list(v ORDER BY v) FROM s.rp_coll GROUP BY g ORDER BY g +---- +1 [A, Z, _x, a, a b] +2 [B, ab, b, é, NULL] + +# DML with string comparisons +statement ok +CREATE OR REPLACE TABLE s.rp_coll_dml AS SELECT * FROM s.rp_coll + +statement ok +UPDATE s.rp_coll_dml SET g = 0 WHERE v < 'a' + +query I +SELECT id FROM s.rp_coll_dml WHERE g = 0 ORDER BY id +---- +2 +3 +5 +7 + +statement ok +DELETE FROM s.rp_coll_dml WHERE v BETWEEN 'a' AND 'b' + +query I +SELECT id FROM s.rp_coll_dml ORDER BY id +---- +2 +3 +5 +6 +7 +8 + +# CREATE TABLE AS with string ordering - with such a database collation only the query is pushed down +statement ok +CREATE OR REPLACE TABLE s.rp_coll_ctas AS SELECT id, v FROM s.rp_coll WHERE v < 'b' + +query I +SELECT id FROM s.rp_coll_ctas ORDER BY id +---- +1 +2 +3 +5 +7 +9 +10 + +statement ok +DROP TABLE s.rp_coll_ctas + +statement ok +DROP TABLE s.rp_coll_dml + +statement ok +DROP TABLE s.rp_coll_list + +statement ok +DROP TABLE s.rp_coll diff --git a/test/sql/remote_pushdown/remote_pushdown_constants.test b/test/sql/remote_pushdown/remote_pushdown_constants.test new file mode 100644 index 000000000..498832033 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_constants.test @@ -0,0 +1,228 @@ +# name: test/sql/remote_pushdown/remote_pushdown_constants.test +# description: Test serialization of constants in remote pushdown into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require icu + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# setting the time zone also loads ICU when the test runner autoloads extensions instead of loading them with +# "require" (e.g. with LOCAL_EXTENSION_REPO) - and makes the results independent of the time zone of the machine +statement ok +SET TimeZone = 'UTC' + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test serialization of constants in remote pushdown into Postgres +statement ok +CREATE OR REPLACE TABLE s.rp_const AS SELECT 1 AS i + +query IIIIIIIIIIII rowsort +SELECT i, true, false, NULL, 42, -42, 2147483648, -9223372036854775808, 99999999999999999999999999999999999999, 1.5, -0.001, 123456789012345678.123 FROM s.rp_const +---- +1 true false NULL 42 -42 2147483648 -9223372036854775808 99999999999999999999999999999999999999 1.5 -0.001 123456789012345678.123 + +query II +EXPLAIN SELECT i, true, false, NULL, 42, -42, 2147483648, -9223372036854775808, 99999999999999999999999999999999999999, 1.5, -0.001, 123456789012345678.123 FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIII rowsort +SELECT typeof(42), typeof(2147483648), typeof(1.5), typeof(99999999999999999999999999999999999999) FROM s.rp_const +---- +INTEGER BIGINT DECIMAL(2,1) HUGEINT + +query IIIIIIIII rowsort +SELECT i, 1.5::DOUBLE, -0.0::DOUBLE, 'inf'::DOUBLE, '-inf'::DOUBLE, 'nan'::DOUBLE, 1e300::DOUBLE, 1.5::FLOAT, 0.1::FLOAT FROM s.rp_const +---- +1 1.5 -0.0 inf -inf nan 1e+300 1.5 0.1 + +query II +EXPLAIN SELECT i, 1.5::DOUBLE, -0.0::DOUBLE, 'inf'::DOUBLE, '-inf'::DOUBLE, 'nan'::DOUBLE, 1e300::DOUBLE, 1.5::FLOAT, 0.1::FLOAT FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIII rowsort +SELECT i, 1.25::DECIMAL(4,2), 99999999999999999999999999999999999999::DECIMAL(38,0), -0.5::DECIMAL(3,2) FROM s.rp_const +---- +1 1.25 99999999999999999999999999999999999999 -0.50 + +query II +EXPLAIN SELECT i, 1.25::DECIMAL(4,2), 99999999999999999999999999999999999999::DECIMAL(38,0), -0.5::DECIMAL(3,2) FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII rowsort +SELECT i, 'text', '', 'it''s', 'back\slash', 'ünïcödé 🦆' FROM s.rp_const +---- +1 text (empty) it's back\slash ünïcödé 🦆 + +query II +EXPLAIN SELECT i, 'text', '', 'it''s', 'back\slash', 'ünïcödé 🦆' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII rowsort +SELECT i, DATE '2024-02-29', DATE '0001-01-01', DATE '9999-12-31', 'infinity'::DATE, '-infinity'::DATE FROM s.rp_const +---- +1 2024-02-29 0001-01-01 9999-12-31 infinity -infinity + +query II +EXPLAIN SELECT i, DATE '2024-02-29', DATE '0001-01-01', DATE '9999-12-31', 'infinity'::DATE, '-infinity'::DATE FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIII rowsort +SELECT i, TIMESTAMP '2024-02-29 12:34:56.789012', TIMESTAMP '2000-01-01', 'infinity'::TIMESTAMP FROM s.rp_const +---- +1 2024-02-29 12:34:56.789012 2000-01-01 00:00:00 infinity + +query II +EXPLAIN SELECT i, TIMESTAMP '2024-02-29 12:34:56.789012', TIMESTAMP '2000-01-01', 'infinity'::TIMESTAMP FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIII rowsort +SELECT i, TIME '12:34:56.789', TIME '00:00:00', TIME '23:59:59.999999' FROM s.rp_const +---- +1 12:34:56.789 00:00:00 23:59:59.999999 + +query II +EXPLAIN SELECT i, TIME '12:34:56.789', TIME '00:00:00', TIME '23:59:59.999999' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query III rowsort +SELECT i, TIMESTAMPTZ '2024-02-29 12:34:56+05:30', TIMESTAMPTZ '2000-01-01 00:00:00+00' FROM s.rp_const +---- +1 2024-02-29 07:04:56+00 2000-01-01 00:00:00+00 + +query II +EXPLAIN SELECT i, TIMESTAMPTZ '2024-02-29 12:34:56+05:30', TIMESTAMPTZ '2000-01-01 00:00:00+00' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII rowsort +SELECT i, INTERVAL '1 year 2 months 3 days 04:05:06.789', INTERVAL '-1 day', INTERVAL '36 hours', INTERVAL '0 seconds' FROM s.rp_const +---- +1 1 year 2 months 3 days 04:05:06.789 -1 day 36:00:00 00:00:00 + +query II +EXPLAIN SELECT i, INTERVAL '1 year 2 months 3 days 04:05:06.789', INTERVAL '-1 day', INTERVAL '36 hours', INTERVAL '0 seconds' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query IIII rowsort +SELECT i, '\x00\xFF'::BLOB, 'abc'::BLOB, ''::BLOB FROM s.rp_const +---- +1 \x00\xFF abc (empty) + +query II +EXPLAIN SELECT i, '\x00\xFF'::BLOB, 'abc'::BLOB, ''::BLOB FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, UUID '12345678-1234-5678-1234-567812345678' FROM s.rp_const +---- +1 12345678-1234-5678-1234-567812345678 + +query II +EXPLAIN SELECT i, UUID '12345678-1234-5678-1234-567812345678' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +# constants used in expressions +query IIIII rowsort +SELECT i + 1.5, i * 2.5::DOUBLE, i || 'x', i = 1, DATE '2024-01-01' + i FROM s.rp_const +---- +2.5 2.5 1x true 2024-01-02 + +query II +EXPLAIN SELECT i + 1.5, i * 2.5::DOUBLE, i || 'x', i = 1, DATE '2024-01-01' + i FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +# constants that cannot be represented in Postgres are not pushed down +query II rowsort +SELECT i, DATE '-0044-03-15' FROM s.rp_const +---- +1 0045-03-15 (BC) + +query II +EXPLAIN SELECT i, DATE '-0044-03-15' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, TIMESTAMP '10000-01-01 00:00:00' FROM s.rp_const +---- +1 10000-01-01 00:00:00 + +query II +EXPLAIN SELECT i, TIMESTAMP '10000-01-01 00:00:00' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, [1, 2, 3] FROM s.rp_const +---- +1 [1, 2, 3] + +query II +EXPLAIN SELECT i, [1, 2, 3] FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, {'a': 1} FROM s.rp_const +---- +1 {'a': 1} + +query II +EXPLAIN SELECT i, {'a': 1} FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query III rowsort +SELECT i, 255::UTINYINT, typeof(255::UTINYINT) FROM s.rp_const +---- +1 255 UTINYINT + +query II +EXPLAIN SELECT i, 255::UTINYINT, typeof(255::UTINYINT) FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, 'a\0b'::VARCHAR FROM s.rp_const +---- +1 a\0b + +query II +EXPLAIN SELECT i, 'a\0b'::VARCHAR FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, length('a' || chr(0) || 'b' || i) FROM s.rp_const +---- +1 4 + +query II +EXPLAIN SELECT i, length('a' || chr(0) || 'b' || i) FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT i, TIMETZ '12:00:00+01' FROM s.rp_const +---- +1 12:00:00+01 + +query II +EXPLAIN SELECT i, TIMETZ '12:00:00+01' FROM s.rp_const +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_cte.test b/test/sql/remote_pushdown/remote_pushdown_cte.test new file mode 100644 index 000000000..304abea69 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_cte.test @@ -0,0 +1,180 @@ +# name: test/sql/remote_pushdown/remote_pushdown_cte.test +# description: Test remote pushdown of common table expressions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_cte AS SELECT i, i % 3 AS g FROM range(10) t(i) + +# a CTE over a remote table is pushed down +query II +WITH c AS (SELECT g, SUM(i) AS s FROM s.rp_cte GROUP BY g) SELECT * FROM c ORDER BY g +---- +0 18 +1 12 +2 15 + +query II +EXPLAIN WITH c AS (SELECT g, SUM(i) AS s FROM s.rp_cte GROUP BY g) SELECT * FROM c ORDER BY g +---- +physical_plan :(?s).*WITH "c" AS \(SELECT "g", CAST\(sum\("i"\) AS NUMERIC\(38,0\)\) AS "s" FROM "public"\."rp_cte" GROUP BY "g"\) SELECT \* FROM "c" ORDER BY "g" ASC NULLS LAST.* + +# chained CTEs +query II rowsort +WITH c AS (SELECT g, SUM(i) AS s FROM s.rp_cte GROUP BY g), d AS (SELECT * FROM c WHERE s > 12) SELECT COUNT(*), SUM(s) FROM d +---- +2 33 + +query II +EXPLAIN WITH c AS (SELECT g, SUM(i) AS s FROM s.rp_cte GROUP BY g), d AS (SELECT * FROM c WHERE s > 12) SELECT COUNT(*), SUM(s) FROM d +---- +physical_plan :(?s).*Postgres Query.* + +# CTE names are case insensitive +query I rowsort +WITH MyCte AS (SELECT i FROM s.rp_cte) SELECT SUM(I) FROM mycte +---- +45 + +query II +EXPLAIN WITH MyCte AS (SELECT i FROM s.rp_cte) SELECT SUM(I) FROM mycte +---- +physical_plan :(?s).*Postgres Query.* + +# a CTE with the same name as a (qualified) remote table is not pushed down +query II rowsort +WITH rp_cte AS (SELECT 42 AS i) SELECT rp_cte.i, t.g FROM rp_cte, s.rp_cte AS t WHERE t.i = 1 +---- +42 1 + +query II +EXPLAIN WITH rp_cte AS (SELECT 42 AS i) SELECT rp_cte.i, t.g FROM rp_cte, s.rp_cte AS t WHERE t.i = 1 +---- +physical_plan :(?s).*Postgres Query.* + +# materialization hints +query I rowsort +WITH c AS MATERIALIZED (SELECT i FROM s.rp_cte) SELECT SUM(i) FROM c +---- +45 + +query II +EXPLAIN WITH c AS MATERIALIZED (SELECT i FROM s.rp_cte) SELECT SUM(i) FROM c +---- +physical_plan :(?s).*AS MATERIALIZED \(.* + +query I rowsort +WITH c AS NOT MATERIALIZED (SELECT i FROM s.rp_cte) SELECT SUM(i) FROM c +---- +45 + +query II +EXPLAIN WITH c AS NOT MATERIALIZED (SELECT i FROM s.rp_cte) SELECT SUM(i) FROM c +---- +physical_plan :(?s).*AS NOT MATERIALIZED \(.* + +# CTE column aliases +query II rowsort +WITH c(x, y) AS (SELECT i, g FROM s.rp_cte) SELECT SUM(x), MAX(y) FROM c +---- +45 2 + +query II +EXPLAIN WITH c(x, y) AS (SELECT i, g FROM s.rp_cte) SELECT SUM(x), MAX(y) FROM c +---- +physical_plan :(?s).*WITH "c" \("x", "y"\) AS.* + +# recursive CTEs +query I +WITH RECURSIVE r(n) AS (SELECT MIN(i) FROM s.rp_cte UNION ALL SELECT n + 1 FROM r WHERE n < 5) SELECT * FROM r ORDER BY n +---- +0 +1 +2 +3 +4 +5 + +query II +EXPLAIN WITH RECURSIVE r(n) AS (SELECT MIN(i) FROM s.rp_cte UNION ALL SELECT n + 1 FROM r WHERE n < 5) SELECT * FROM r ORDER BY n +---- +physical_plan :(?s).*WITH RECURSIVE "r" \("n"\) AS.* + +query I +WITH RECURSIVE r(n) AS (SELECT MIN(i) FROM s.rp_cte UNION SELECT (n + 1) % 4 FROM r) SELECT * FROM r ORDER BY n +---- +0 +1 +2 +3 + +query II +EXPLAIN WITH RECURSIVE r(n) AS (SELECT MIN(i) FROM s.rp_cte UNION SELECT (n + 1) % 4 FROM r) SELECT * FROM r ORDER BY n +---- +physical_plan :(?s).*Postgres Query.* + +query II +WITH RECURSIVE r(i, depth) AS (SELECT i, 0 FROM s.rp_cte WHERE i = 0 UNION ALL SELECT t.i, depth + 1 FROM r JOIN s.rp_cte t ON t.i = r.i + 3) SELECT * FROM r ORDER BY i +---- +0 0 +3 1 +6 2 +9 3 + +query II +EXPLAIN WITH RECURSIVE r(i, depth) AS (SELECT i, 0 FROM s.rp_cte WHERE i = 0 UNION ALL SELECT t.i, depth + 1 FROM r JOIN s.rp_cte t ON t.i = r.i + 3) SELECT * FROM r ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# a CTE in a subquery +query I rowsort +SELECT * FROM (WITH c AS (SELECT i FROM s.rp_cte WHERE i > 7) SELECT SUM(i) FROM c) +---- +17 + +query II +EXPLAIN SELECT * FROM (WITH c AS (SELECT i FROM s.rp_cte WHERE i > 7) SELECT SUM(i) FROM c) +---- +physical_plan :(?s).*Postgres Query.* + +# a recursive CTE without remote tables is not pushed down +query I rowsort +WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 10) SELECT SUM(n) FROM r +---- +55 + +query II +EXPLAIN WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 10) SELECT SUM(n) FROM r +---- +physical_plan :(?s).*Postgres Query.* + +# a local CTE prevents pushdown +statement ok +CREATE TABLE local_cte_tbl AS SELECT 1 AS i + +query I rowsort +WITH c AS (SELECT i FROM local_cte_tbl) SELECT COUNT(*) FROM s.rp_cte JOIN c USING (i) +---- +1 + +query II +EXPLAIN WITH c AS (SELECT i FROM local_cte_tbl) SELECT COUNT(*) FROM s.rp_cte JOIN c USING (i) +---- +physical_plan :(?s).*Postgres Query.* + +# USING KEY is DuckDB-specific +query II rowsort +WITH RECURSIVE r(n, v) USING KEY (n) AS (SELECT 1, 1 UNION SELECT n, v + 1 FROM r WHERE v < 3) SELECT r.* FROM r JOIN s.rp_cte t ON t.i = r.n +---- +1 3 + +query II +EXPLAIN WITH RECURSIVE r(n, v) USING KEY (n) AS (SELECT 1, 1 UNION SELECT n, v + 1 FROM r WHERE v < 3) SELECT r.* FROM r JOIN s.rp_cte t ON t.i = r.n +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_ddl.test b/test/sql/remote_pushdown/remote_pushdown_ddl.test new file mode 100644 index 000000000..68f2f47ec --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_ddl.test @@ -0,0 +1,288 @@ +# name: test/sql/remote_pushdown/remote_pushdown_ddl.test +# description: Test remote pushdown of CREATE / DROP / ALTER statements into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_ddl_src AS SELECT i, 'v' || i AS v FROM range(5) t(i) + +statement ok +DROP TABLE IF EXISTS s.rp_ddl_ctas + +statement ok +DROP TABLE IF EXISTS s.rp_ddl_altered + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS rp_ddl_schema CASCADE') + +# a regular CREATE TABLE is not pushed down - it issues the same statement, and keeps the table definition (e.g. the +# column defaults) in the catalog +query II +EXPLAIN CREATE TABLE s.rp_ddl (id INTEGER PRIMARY KEY, v VARCHAR NOT NULL, h HUGEINT, d DECIMAL(10,2) DEFAULT 42) +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +DROP TABLE IF EXISTS s.rp_ddl + +statement ok +CREATE TABLE s.rp_ddl (id INTEGER PRIMARY KEY, v VARCHAR NOT NULL, h HUGEINT, d DECIMAL(10,2) DEFAULT 42) + +query IIIIII +DESCRIBE s.rp_ddl +---- +id INTEGER NO PRI NULL NULL +v VARCHAR NO NULL NULL NULL +h DOUBLE YES NULL NULL NULL +d DECIMAL(10,2) YES NULL 42 NULL + +statement ok +CREATE OR REPLACE TABLE s.rp_ddl (id INTEGER) + +# CREATE TABLE AS over a remote table is executed in Postgres as a whole (statements with string ordering only with a +# byte-wise database collation - see remote_pushdown_collation.test) +query II +EXPLAIN CREATE TABLE s.rp_ddl_ctas AS SELECT i, i * 2 AS doubled, SUM(i) OVER () AS total, v FROM s.rp_ddl_src WHERE i IN (2, 3, 4) +---- +physical_plan :(?s).*Postgres Query.*CREATE TABLE "public"."rp_ddl_ctas"\("i" BIGINT, "doubled" BIGINT, "total" FLOAT, "v" VARCHAR\).*INSERT INTO "public"."rp_ddl_ctas".*FROM "public"."rp_ddl_src".* + +query I +CREATE TABLE s.rp_ddl_ctas AS SELECT i, i * 2 AS doubled, SUM(i) OVER () AS total, v FROM s.rp_ddl_src WHERE i IN (2, 3, 4) +---- +3 + +query IIII +SELECT * FROM s.rp_ddl_ctas ORDER BY i +---- +2 4 9.0 v2 +3 6 9.0 v3 +4 8 9.0 v4 + +query IIIIII +DESCRIBE s.rp_ddl_ctas +---- +i BIGINT YES NULL NULL NULL +doubled BIGINT YES NULL NULL NULL +total DOUBLE YES NULL NULL NULL +v VARCHAR YES NULL NULL NULL + +statement ok +CREATE OR REPLACE TABLE s.rp_ddl_ctas AS SELECT g, COUNT(*) AS cnt FROM (SELECT i % 2 AS g FROM s.rp_ddl_src) GROUP BY g + +query II +SELECT * FROM s.rp_ddl_ctas ORDER BY g +---- +0 3 +1 2 + +# CREATE TABLE AS with a query that depends on the types it operates on - only the query is pushed down +query II +EXPLAIN CREATE OR REPLACE TABLE s.rp_ddl_ctas AS SELECT i, v || '!' AS v FROM s.rp_ddl_src +---- +physical_plan :(?s).*Postgres Query.*CREATE TABLE.* + +statement ok +CREATE OR REPLACE TABLE s.rp_ddl_ctas AS SELECT i, v || '!' AS v FROM s.rp_ddl_src + +query II +SELECT * FROM s.rp_ddl_ctas ORDER BY i LIMIT 2 +---- +0 v0! +1 v1! + +# CREATE TABLE AS from a local query is not pushed down +statement ok +CREATE OR REPLACE TABLE s.rp_ddl_ctas AS SELECT 42 AS answer + +query I +SELECT * FROM s.rp_ddl_ctas +---- +42 + +# CREATE VIEW - the query of the view is translated like a pushed down query (views with string ordering only with a +# byte-wise database collation - see remote_pushdown_collation.test) +query II +EXPLAIN CREATE OR REPLACE VIEW s.rp_ddl_view AS SELECT i IN (3, 4) AS g, SUM(i) AS total, COUNT(V) AS cnt FROM s.rp_ddl_src WHERE v LIKE 'v_' GROUP BY g +---- +physical_plan :(?s).*Postgres Query.*CREATE VIEW "public"."rp_ddl_view" AS SELECT .*CAST\(sum\("i"\) AS NUMERIC\(38,0\)\) AS "total", count\("v"\) AS "cnt" FROM "public"."rp_ddl_src" WHERE \("v" LIKE 'v_' ESCAPE ''\) GROUP BY "g".* + +statement ok +CREATE OR REPLACE VIEW s.rp_ddl_view AS SELECT i IN (3, 4) AS g, SUM(i) AS total, COUNT(V) AS cnt FROM s.rp_ddl_src WHERE v LIKE 'v_' GROUP BY g + +query III +SELECT * FROM s.rp_ddl_view ORDER BY g +---- +false 3 3 +true 7 2 + +statement ok +CREATE OR REPLACE VIEW s.rp_ddl_view (a, b) AS SELECT i, v FROM s.rp_ddl_src WHERE i < 2 + +query II +SELECT * FROM s.rp_ddl_view ORDER BY a +---- +0 v0 +1 v1 + +statement ok +DROP VIEW s.rp_ddl_view + +# ALTER TABLE +statement ok +CREATE OR REPLACE TABLE s.rp_ddl_alter (id INTEGER, v VARCHAR) + +statement ok +INSERT INTO s.rp_ddl_alter VALUES (1, 'a') + +query II +EXPLAIN ALTER TABLE s.rp_ddl_alter ADD COLUMN d DATE +---- +physical_plan :(?s).*Postgres Query.*ALTER TABLE "public"."rp_ddl_alter" ADD COLUMN "d" DATE.* + +statement ok +ALTER TABLE s.rp_ddl_alter ADD COLUMN d DATE + +statement ok +ALTER TABLE s.rp_ddl_alter ADD COLUMN IF NOT EXISTS d DATE + +statement ok +ALTER TABLE s.rp_ddl_alter RENAME COLUMN V TO value + +statement ok +ALTER TABLE s.rp_ddl_alter ALTER COLUMN value SET NOT NULL + +statement error +INSERT INTO s.rp_ddl_alter VALUES (2, NULL, NULL) +---- +null value + +statement ok +ALTER TABLE s.rp_ddl_alter ALTER COLUMN value DROP NOT NULL + +statement ok +ALTER TABLE s.rp_ddl_alter DROP COLUMN id + +statement ok +ALTER TABLE s.rp_ddl_alter DROP COLUMN IF EXISTS id + +statement ok +ALTER TABLE s.rp_ddl_alter RENAME TO rp_ddl_altered + +query II +SELECT * FROM s.rp_ddl_altered +---- +a NULL + +# ADD COLUMN with a DEFAULT is not pushed down - DuckDB fills in the default values itself +query II +EXPLAIN ALTER TABLE s.rp_ddl_altered ADD COLUMN n INTEGER DEFAULT 42 +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +ALTER TABLE s.rp_ddl_altered ADD COLUMN n INTEGER DEFAULT 42 + +query III +SELECT * FROM s.rp_ddl_altered +---- +a NULL 42 + +statement ok +ALTER TABLE IF EXISTS s.rp_ddl_nonexistent RENAME TO rp_ddl_other + +# DROP +query II +EXPLAIN DROP TABLE s.rp_ddl_altered +---- +physical_plan :(?s).*Postgres Query.*DROP TABLE "public"."rp_ddl_altered".* + +statement ok +DROP TABLE s.rp_ddl_altered + +statement error +SELECT * FROM s.rp_ddl_altered +---- +does not exist + +statement ok +DROP TABLE IF EXISTS s.rp_ddl_altered + +# CREATE / DROP SCHEMA - schemas can only be referenced in databases that are attached without a SCHEMA filter, CREATE +# TABLE AS into another schema is pushed down +statement ok +ATTACH 'dbname=postgresscanner' AS s_all (TYPE POSTGRES) + +statement ok +CREATE SCHEMA s_all.rp_ddl_schema + +statement ok +CREATE SCHEMA IF NOT EXISTS s_all.rp_ddl_schema + +statement ok +CREATE TABLE s_all.rp_ddl_schema.tbl AS SELECT 1 AS i + +statement error +DROP SCHEMA s_all.rp_ddl_schema +---- +other objects depend on it + +# schema DDL is executed regularly (schemas are not reloaded when they are removed from the cache) +query II +EXPLAIN DROP SCHEMA s_all.rp_ddl_schema CASCADE +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +DROP SCHEMA s_all.rp_ddl_schema CASCADE + +statement ok +DROP SCHEMA IF EXISTS s_all.rp_ddl_schema + +# DDL is executed within the DuckDB transaction +statement ok +BEGIN + +statement ok +DROP TABLE s.rp_ddl + +statement ok +ROLLBACK + +query I +SELECT COUNT(*) FROM s.rp_ddl +---- +0 + +# DDL is never pushed down into read-only databases +statement ok +ATTACH 'dbname=postgresscanner' AS s_ro (TYPE POSTGRES, SCHEMA 'public', READ_ONLY) + +statement error +CREATE TABLE s_ro.rp_ddl_ro (i INTEGER) +---- +read-only + +statement error +DROP TABLE s_ro.rp_ddl +---- +read-only + +statement error +ALTER TABLE s_ro.rp_ddl ADD COLUMN j INTEGER +---- +read-only + +statement ok +DROP TABLE s.rp_ddl + +statement ok +DROP TABLE s.rp_ddl_ctas + +statement ok +DROP TABLE s.rp_ddl_src diff --git a/test/sql/remote_pushdown/remote_pushdown_dml.test b/test/sql/remote_pushdown/remote_pushdown_dml.test new file mode 100644 index 000000000..38b9d2079 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_dml.test @@ -0,0 +1,391 @@ +# name: test/sql/remote_pushdown/remote_pushdown_dml.test +# description: Test remote pushdown of UPDATE / DELETE / INSERT statements into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_dml (id INTEGER PRIMARY KEY, v VARCHAR, n DOUBLE, d DATE, dec DECIMAL(10,2) DEFAULT 42) + +statement ok +CREATE OR REPLACE TABLE s.rp_dml_src (id INTEGER, v VARCHAR) + +statement ok +INSERT INTO s.rp_dml_src VALUES (1, 'src1'), (2, 'src2'), (5, 'src5') + +# INSERT with a VALUES list - constants are cast to the column types locally, DEFAULT is supported +query II +EXPLAIN INSERT INTO s.rp_dml VALUES (1, 'a', 1.5, '2024-01-01', 1.255), (2, 'b', 2, NULL, DEFAULT) +---- +physical_plan :(?s).*Postgres Query.*INSERT INTO "public"."rp_dml" \("id", "v", "n", "d", "dec"\) VALUES \(1, 'a', CAST\('1.5' AS DOUBLE PRECISION\), DATE '2024-01-01', CAST\('1.26' AS NUMERIC\(10,2\)\)\), \(2, 'b', CAST\('2.0' AS DOUBLE PRECISION\), NULL, DEFAULT\).* + +# the statement returns the number of affected rows, like a regular INSERT +query I +INSERT INTO s.rp_dml VALUES (1, 'a', 1.5, '2024-01-01', 1.255), (2, 'b', 2, NULL, DEFAULT) +---- +2 + +query IIIII +SELECT * FROM s.rp_dml ORDER BY id +---- +1 a 1.5 2024-01-01 1.26 +2 b 2.0 NULL 42.00 + +# INSERT with a column list +statement ok +INSERT INTO s.rp_dml (v, id) VALUES ('c', 3), ('d', 4) + +# INSERT ... SELECT from a remote table is pushed down as a whole +query II +EXPLAIN INSERT INTO s.rp_dml (id, v) SELECT id + 10, v FROM s.rp_dml_src +---- +physical_plan :(?s).*Postgres Query.*INSERT INTO "public"."rp_dml" \("id", "v"\) SELECT.*FROM "public"."rp_dml_src".* + +query I +INSERT INTO s.rp_dml (id, v) SELECT id + 10, v FROM s.rp_dml_src +---- +3 + +# values of other types are cast like DuckDB casts them +statement ok +INSERT INTO s.rp_dml (id, v, n, d) SELECT 20, 1.5, '3', DATE '2024-02-02' + 1 FROM s.rp_dml_src LIMIT 1 + +query IIIII +SELECT * FROM s.rp_dml ORDER BY id +---- +1 a 1.5 2024-01-01 1.26 +2 b 2.0 NULL 42.00 +3 c NULL NULL 42.00 +4 d NULL NULL 42.00 +11 src1 NULL NULL 42.00 +12 src2 NULL NULL 42.00 +15 src5 NULL NULL 42.00 +20 1.5 3.0 2024-02-03 42.00 + +# UPDATE +query II +EXPLAIN UPDATE s.rp_dml SET n = id * 2, v = v || '!' WHERE id > 10 +---- +physical_plan :(?s).*Postgres Query.*UPDATE "public"."rp_dml" SET "n" = .*, "v" = \("v" \|\| '!'\) WHERE \("id" > 10\).* + +query I +UPDATE s.rp_dml SET n = id * 2, v = v || '!' WHERE id > 10 +---- +4 + +# UPDATE ... FROM +query I +UPDATE s.rp_dml SET v = src.v FROM s.rp_dml_src src WHERE rp_dml.id = src.id +---- +2 + +# UPDATE with DEFAULT +statement ok +UPDATE s.rp_dml SET dec = DEFAULT WHERE id = 1 + +query IIIII +SELECT * FROM s.rp_dml ORDER BY id +---- +1 src1 1.5 2024-01-01 42.00 +2 src2 2.0 NULL 42.00 +3 c NULL NULL 42.00 +4 d NULL NULL 42.00 +11 src1! 22.0 NULL 42.00 +12 src2! 24.0 NULL 42.00 +15 src5! 30.0 NULL 42.00 +20 1.5! 40.0 2024-02-03 42.00 + +# RETURNING is supported when the statement is pushed down +query III +UPDATE s.rp_dml SET n = n + 1 WHERE id IN (11, 12) RETURNING id, n, n * 2 AS doubled +---- +11 23.0 46.0 +12 25.0 50.0 + +query IIIIII +INSERT INTO s.rp_dml (id, v) VALUES (30, 'thirty') RETURNING *, dec + 1 +---- +30 thirty NULL NULL 42.00 43.00 + +query II +DELETE FROM s.rp_dml WHERE id = 30 RETURNING id, v +---- +30 thirty + +# DELETE +query II +EXPLAIN DELETE FROM s.rp_dml WHERE id >= 20 +---- +physical_plan :(?s).*Postgres Query.*DELETE FROM "public"."rp_dml" WHERE \("id" >= 20\).* + +query I +DELETE FROM s.rp_dml WHERE id >= 20 +---- +1 + +# DELETE ... USING +query I +DELETE FROM s.rp_dml USING s.rp_dml_src src WHERE rp_dml.id = src.id + 10 +---- +3 + +query I +SELECT id FROM s.rp_dml ORDER BY id +---- +1 +2 +3 +4 + +# ON CONFLICT +query II +EXPLAIN INSERT INTO s.rp_dml (id, v) VALUES (1, 'conflict') ON CONFLICT (id) DO UPDATE SET v = excluded.v || '_updated', n = rp_dml.n + 100 +---- +physical_plan :(?s).*Postgres Query.*ON CONFLICT \("id"\) DO UPDATE SET "v" = .* + +statement ok +INSERT INTO s.rp_dml (id, v) VALUES (1, 'conflict') ON CONFLICT (id) DO UPDATE SET v = excluded.v || '_updated', n = rp_dml.n + 100 + +statement ok +INSERT INTO s.rp_dml (id, v) VALUES (2, 'ignored') ON CONFLICT DO NOTHING + +statement ok +INSERT OR IGNORE INTO s.rp_dml (id, v) VALUES (3, 'ignored'), (5, 'five') + +# INSERT OR REPLACE updates the existing row using the primary key as the conflict target +query II +EXPLAIN INSERT OR REPLACE INTO s.rp_dml (id, v, n) VALUES (4, 'replaced', 0) +---- +physical_plan :(?s).*Postgres Query.*ON CONFLICT \("id"\) DO UPDATE SET "v" = "excluded"."v", "n" = "excluded"."n".* + +statement ok +INSERT OR REPLACE INTO s.rp_dml (id, v, n) VALUES (4, 'replaced', 0) + +query IIIII +SELECT * FROM s.rp_dml ORDER BY id +---- +1 conflict_updated 101.5 2024-01-01 42.00 +2 src2 2.0 NULL 42.00 +3 c NULL NULL 42.00 +4 replaced 0.0 NULL 42.00 +5 five NULL NULL 42.00 + +# DML is executed within the DuckDB transaction +statement ok +BEGIN + +statement ok +DELETE FROM s.rp_dml WHERE id > 1 + +query I +SELECT COUNT(*) FROM s.rp_dml +---- +1 + +statement ok +ROLLBACK + +query I +SELECT COUNT(*) FROM s.rp_dml +---- +5 + +# DML with a local table is not pushed down +statement ok +CREATE TABLE local_dml AS SELECT 1 AS id + +query II +EXPLAIN UPDATE s.rp_dml SET v = 'local' WHERE id IN (SELECT id FROM local_dml) +---- +physical_plan :(?s).*Postgres Query.* + +# but the query that is inserted can be pushed down on its own +query II +EXPLAIN INSERT INTO local_dml SELECT MAX(id) FROM s.rp_dml +---- +physical_plan :(?s).*Postgres Query.* + +# DML with constructs that cannot be translated is executed locally +query II +EXPLAIN UPDATE s.rp_dml SET n = n % 1.5::DOUBLE +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +UPDATE s.rp_dml SET n = n % 1.5::DOUBLE + +query II +SELECT id, n FROM s.rp_dml ORDER BY id +---- +1 1.0 +2 0.5 +3 NULL +4 0.0 +5 NULL + +# INSERT BY NAME is DuckDB-specific +query II +EXPLAIN INSERT INTO s.rp_dml BY NAME SELECT 6 AS id +---- +physical_plan :(?s).*Postgres Query.* + +# DML is never pushed down into read-only databases +statement ok +ATTACH 'dbname=postgresscanner' AS s_ro (TYPE POSTGRES, SCHEMA 'public', READ_ONLY) + +statement error +UPDATE s_ro.rp_dml SET v = 'x' +---- +read-only + +statement error +DELETE FROM s_ro.rp_dml +---- +read-only + +statement error +INSERT INTO s_ro.rp_dml (id) VALUES (100) +---- +read-only + +query I +SELECT COUNT(*) FROM s.rp_dml +---- +5 + +# columns with Postgres-side defaults - NULL values are replaced with the default, like in a regular INSERT +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS rp_dml_defaults') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE rp_dml_defaults (id SERIAL PRIMARY KEY, v INT, total INT GENERATED ALWAYS AS (v * 2) STORED, e TEXT DEFAULT ''dflt'')') + +statement ok +CALL postgres_clear_cache() + +query II +EXPLAIN INSERT INTO s.rp_dml_defaults VALUES (NULL, 1, NULL, NULL), (DEFAULT, 2, DEFAULT, 'x') +---- +physical_plan :(?s).*Postgres Query.*INSERT INTO "public"."rp_dml_defaults" \("id", "v", "e"\) VALUES \(DEFAULT, 1, DEFAULT\), \(DEFAULT, 2, 'x'\).* + +statement ok +INSERT INTO s.rp_dml_defaults VALUES (NULL, 1, NULL, NULL), (DEFAULT, 2, DEFAULT, 'x') + +statement ok +INSERT INTO s.rp_dml_defaults (v) VALUES (3) + +# values for generated columns are rejected +statement error +INSERT INTO s.rp_dml_defaults (v, total) VALUES (4, 8) +---- +generated column + +# INSERT ... SELECT into columns with defaults is executed by the regular INSERT +query II +EXPLAIN INSERT INTO s.rp_dml_defaults (v, e) SELECT v + 10, NULL FROM s.rp_dml_defaults +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +INSERT INTO s.rp_dml_defaults (v, e) SELECT v + 10, NULL FROM s.rp_dml_defaults + +query II +EXPLAIN INSERT INTO s.rp_dml_defaults (v) SELECT v + 10 FROM s.rp_dml_defaults +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT * FROM s.rp_dml_defaults ORDER BY id +---- +1 1 2 dflt +2 2 4 x +3 3 6 dflt +4 11 22 dflt +5 12 24 dflt +6 13 26 dflt + +# columns with types that DuckDB does not read natively +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS rp_dml_types') + +statement ok +CALL postgres_execute('s', 'DROP TYPE IF EXISTS rp_dml_mood') + +statement ok +CALL postgres_execute('s', 'CREATE TYPE rp_dml_mood AS ENUM (''sad'', ''ok'', ''happy'')') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE rp_dml_types (id INT PRIMARY KEY, j JSON, m rp_dml_mood)') + +statement ok +CALL postgres_clear_cache() + +statement ok +INSERT INTO s.rp_dml_types VALUES (1, '{"a": 1}', 'ok'), (2, NULL, NULL) + +query II +EXPLAIN UPDATE s.rp_dml_types SET j = '[' || id || ']', m = 'happy' WHERE id = 2 +---- +physical_plan :(?s).*Postgres Query.*SET "j" = CAST\(.* AS "pg_catalog"."json"\), "m" = 'happy'.* + +statement ok +UPDATE s.rp_dml_types SET j = '[' || id || ']', m = 'happy' WHERE id = 2 + +query III +SELECT id, j, m FROM s.rp_dml_types ORDER BY id +---- +1 {"a": 1} ok +2 [2] happy + +# invalid enum values are rejected by DuckDB +statement error +INSERT INTO s.rp_dml_types VALUES (3, NULL, 'unknown') +---- +Could not convert + +# Postgres raises an error when an upsert affects the same row twice - DuckDB does not, so these are executed by +# the regular INSERT +query II +EXPLAIN INSERT INTO s.rp_dml (id, v) VALUES (1, 'first'), (1, 'second') ON CONFLICT (id) DO UPDATE SET v = excluded.v +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +INSERT INTO s.rp_dml (id, v) VALUES (1, 'first'), (1, 'second') ON CONFLICT (id) DO UPDATE SET v = excluded.v + +query I +SELECT v FROM s.rp_dml WHERE id = 1 +---- +first + +statement ok +UPDATE s.rp_dml_types SET m = 'happy' WHERE id = 1 + +# unqualified columns in ON CONFLICT DO UPDATE refer to the existing row +query II +EXPLAIN INSERT INTO s.rp_dml_types VALUES (2, NULL, 'sad') ON CONFLICT (id) DO UPDATE SET m = excluded.m WHERE m = 'happy' +---- +physical_plan :(?s).*Postgres Query.*DO UPDATE SET "m" = "excluded"."m" WHERE \(CAST\("rp_dml_types"."m" AS TEXT\) = 'happy'\).* + +statement ok +INSERT INTO s.rp_dml_types VALUES (2, NULL, 'sad') ON CONFLICT (id) DO UPDATE SET m = excluded.m WHERE m = 'happy' + +query II +SELECT id, m FROM s.rp_dml_types ORDER BY id +---- +1 happy +2 sad + +statement ok +CALL postgres_execute('s', 'DROP TABLE rp_dml_types') + +statement ok +CALL postgres_execute('s', 'DROP TYPE rp_dml_mood') + +statement ok +CALL postgres_execute('s', 'DROP TABLE rp_dml_defaults') diff --git a/test/sql/remote_pushdown/remote_pushdown_edge_cases.test b/test/sql/remote_pushdown/remote_pushdown_edge_cases.test new file mode 100644 index 000000000..bc8da259b --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_edge_cases.test @@ -0,0 +1,304 @@ +# name: test/sql/remote_pushdown/remote_pushdown_edge_cases.test +# description: Test edge cases of remote pushdown into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require icu + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# setting the time zone also loads ICU when the test runner autoloads extensions instead of loading them with +# "require" (e.g. with LOCAL_EXTENSION_REPO) - and makes the results independent of the time zone of the machine +statement ok +SET TimeZone = 'UTC' + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test edge cases of remote pushdown into Postgres +statement ok +CREATE OR REPLACE TABLE s.rp_edge AS SELECT i, CASE WHEN i % 3 = 0 THEN NULL ELSE i END AS n, 'v' || i AS v FROM range(6) t(i) + +# empty results +query III rowsort +SELECT SUM(i), COUNT(*), MAX(v) FROM s.rp_edge WHERE i > 100 +---- +NULL 0 NULL + +query II +EXPLAIN SELECT SUM(i), COUNT(*), MAX(v) FROM s.rp_edge WHERE i > 100 +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT g, COUNT(*) FROM (SELECT i % 2 AS g FROM s.rp_edge) WHERE g > 5 GROUP BY g +---- + +query II +EXPLAIN SELECT g, COUNT(*) FROM (SELECT i % 2 AS g FROM s.rp_edge) WHERE g > 5 GROUP BY g +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates over NULLs +query IIIIII +SELECT SUM(n), AVG(n), MIN(n), COUNT(n), COUNT(*), string_agg(CAST(n AS VARCHAR), ',' ORDER BY n) FROM s.rp_edge +---- +12 3.0 1 4 6 1,2,4,5 + +query II +EXPLAIN SELECT SUM(n), AVG(n), MIN(n), COUNT(n), COUNT(*), string_agg(CAST(n AS VARCHAR), ',' ORDER BY n) FROM s.rp_edge +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT SUM(n), AVG(n) FROM s.rp_edge WHERE n IS NULL +---- +NULL NULL + +query II +EXPLAIN SELECT SUM(n), AVG(n) FROM s.rp_edge WHERE n IS NULL +---- +physical_plan :(?s).*Postgres Query.* + +# three-valued logic +query IIIIIII +SELECT i, n > 2, NOT (n > 2), n > 2 OR i > 3, n > 2 AND i > 3, n = NULL, n IS NULL FROM s.rp_edge ORDER BY i +---- +0 NULL NULL NULL false NULL true +1 false true false false NULL false +2 false true false false NULL false +3 NULL NULL NULL false NULL true +4 true false true true NULL false +5 true false true true NULL false + +query II +EXPLAIN SELECT i, n > 2, NOT (n > 2), n > 2 OR i > 3, n > 2 AND i > 3, n = NULL, n IS NULL FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_edge WHERE NOT (n IN (1, 2)) ORDER BY i +---- +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_edge WHERE NOT (n IN (1, 2)) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_edge WHERE n NOT IN (SELECT n FROM s.rp_edge WHERE n > 3) ORDER BY i +---- +1 +2 + +query II +EXPLAIN SELECT i FROM s.rp_edge WHERE n NOT IN (SELECT n FROM s.rp_edge WHERE n > 3) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# constant expressions are evaluated locally +query IIII +SELECT i + (2 * 3), CAST('2024-01-01' AS DATE) + 1, 'abc' || 'def', length('hello') FROM s.rp_edge ORDER BY i +---- +6 2024-01-02 abcdef 5 +7 2024-01-02 abcdef 5 +8 2024-01-02 abcdef 5 +9 2024-01-02 abcdef 5 +10 2024-01-02 abcdef 5 +11 2024-01-02 abcdef 5 + +query II +EXPLAIN SELECT i + (2 * 3), CAST('2024-01-01' AS DATE) + 1, 'abc' || 'def', length('hello') FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_edge WHERE i > abs(-2) ORDER BY i +---- +3 +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_edge WHERE i > abs(-2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# value functions are evaluated locally once +query I rowsort +SELECT COUNT(*) FROM s.rp_edge WHERE current_date > DATE '2020-01-01' +---- +6 + +query I rowsort +SELECT COUNT(*) FROM s.rp_edge WHERE now() > TIMESTAMPTZ '2020-01-01 00:00:00+00' +---- +6 + +# volatile functions are executed locally +query I rowsort +SELECT COUNT(*) FROM s.rp_edge WHERE random() < 2 +---- +6 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_edge WHERE random() < 2 +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT COUNT(gen_random_uuid()) FROM s.rp_edge +---- +6 + +query II +EXPLAIN SELECT COUNT(gen_random_uuid()) FROM s.rp_edge +---- +physical_plan :(?s).*Postgres Query.* + +# macros +statement ok +CREATE MACRO rp_double(x) AS x * 2 + +query I +SELECT rp_double(i) FROM s.rp_edge ORDER BY i +---- +0 +2 +4 +6 +8 +10 + +query II +EXPLAIN SELECT rp_double(i) FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i + rp_double(21) FROM s.rp_edge ORDER BY i +---- +42 +43 +44 +45 +46 +47 + +query II +EXPLAIN SELECT i + rp_double(21) FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# integer overflow raises an error in both systems +statement error +SELECT CAST(i AS INTEGER) * 2147483647 FROM s.rp_edge WHERE i = 5 +---- +out of range + +# division by zero yields inf / nan +query IIII +SELECT i, i / 0, n / 0, 0 / n FROM s.rp_edge ORDER BY i +---- +0 nan NULL NULL +1 inf inf 0.0 +2 inf inf 0.0 +3 inf NULL NULL +4 inf inf 0.0 +5 inf inf 0.0 + +query II +EXPLAIN SELECT i, i / 0, n / 0, 0 / n FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# integer division by zero raises an error in both systems +statement error +SELECT i % 0 FROM s.rp_edge +---- +zero + +# deeply nested expressions +query II +SELECT ((((i + 1) * 2) - 3) * (i + 4)) % 7, CASE WHEN i > 1 THEN CASE WHEN i > 3 THEN 'big' ELSE 'mid' END ELSE 'small' END FROM s.rp_edge ORDER BY i +---- +-4 small +5 small +4 mid +0 mid +0 big +4 big + +query II +EXPLAIN SELECT ((((i + 1) * 2) - 3) * (i + 4)) % 7, CASE WHEN i > 1 THEN CASE WHEN i > 3 THEN 'big' ELSE 'mid' END ELSE 'small' END FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# strings with quotes / backslashes / unicode +query IIII +SELECT i, v || '''quoted''', v || E'\\backslash\\', v || '🦆' FROM s.rp_edge WHERE v <> 'it''s' ORDER BY i +---- +0 v0'quoted' v0\backslash\ v0🦆 +1 v1'quoted' v1\backslash\ v1🦆 +2 v2'quoted' v2\backslash\ v2🦆 +3 v3'quoted' v3\backslash\ v3🦆 +4 v4'quoted' v4\backslash\ v4🦆 +5 v5'quoted' v5\backslash\ v5🦆 + +query II +EXPLAIN SELECT i, v || '''quoted''', v || E'\\backslash\\', v || '🦆' FROM s.rp_edge WHERE v <> 'it''s' ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT COUNT(*) FROM s.rp_edge WHERE v LIKE '%\\%' OR v LIKE 'v_' +---- +6 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_edge WHERE v LIKE '%\\%' OR v LIKE 'v_' +---- +physical_plan :(?s).*Postgres Query.* + +# DISTINCT on expressions +query II +SELECT DISTINCT i % 2, n IS NULL FROM s.rp_edge ORDER BY ALL +---- +0 false +0 true +1 false +1 true + +query II +EXPLAIN SELECT DISTINCT i % 2, n IS NULL FROM s.rp_edge ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +# LIMIT 0 +query I +SELECT i FROM s.rp_edge ORDER BY i LIMIT 0 +---- + +query II +EXPLAIN SELECT i FROM s.rp_edge ORDER BY i LIMIT 0 +---- +physical_plan :(?s).*Postgres Query.* + +# subquery in the SELECT list returning no rows +query II +SELECT i, (SELECT n FROM s.rp_edge e2 WHERE e2.i = rp_edge.i + 100) FROM s.rp_edge ORDER BY i +---- +0 NULL +1 NULL +2 NULL +3 NULL +4 NULL +5 NULL + +query II +EXPLAIN SELECT i, (SELECT n FROM s.rp_edge e2 WHERE e2.i = rp_edge.i + 100) FROM s.rp_edge ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_functions_date.test b/test/sql/remote_pushdown/remote_pushdown_functions_date.test new file mode 100644 index 000000000..e6a3a4a82 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_functions_date.test @@ -0,0 +1,425 @@ +# name: test/sql/remote_pushdown/remote_pushdown_functions_date.test +# description: Test remote pushdown of date / time functions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require icu + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# setting the time zone also loads ICU when the test runner autoloads extensions instead of loading them with +# "require" (e.g. with LOCAL_EXTENSION_REPO) - and makes the results independent of the time zone of the machine +statement ok +SET TimeZone = 'UTC' + +statement ok +CREATE OR REPLACE TABLE s.rp_date AS SELECT i, DATE '2023-12-25' + (i * 17)::INT AS dt, TIMESTAMP '1969-12-31 22:30:00.123456' + INTERVAL (i * 7) HOUR + INTERVAL (i * 13) MINUTE + INTERVAL (i * 1001) MILLISECOND AS ts, INTERVAL (i * 30) HOUR + INTERVAL (i) MONTH AS iv, TIME '10:15:30' + INTERVAL (i * 7) MINUTE AS tm FROM range(10) t(i) + +statement ok +CREATE OR REPLACE TABLE s.rp_date_tz AS SELECT i, TIMESTAMPTZ '2024-03-30 22:00:00+00' + INTERVAL (i * 5) HOUR AS tstz FROM range(10) t(i) + +# date parts are serialized as EXTRACT +query IIIIIIIIII +SELECT i, year(dt), month(dt), day(dt), quarter(dt), dayofweek(dt), isodow(dt), dayofyear(dt), week(dt), isoyear(dt) FROM s.rp_date ORDER BY i +---- +0 2023 12 25 4 1 1 359 52 2023 +1 2024 1 11 1 4 4 11 2 2024 +2 2024 1 28 1 0 7 28 4 2024 +3 2024 2 14 1 3 3 45 7 2024 +4 2024 3 2 1 6 6 62 9 2024 +5 2024 3 19 1 2 2 79 12 2024 +6 2024 4 5 2 5 5 96 14 2024 +7 2024 4 22 2 1 1 113 17 2024 +8 2024 5 9 2 4 4 130 19 2024 +9 2024 5 26 2 0 7 147 21 2024 + +query II +EXPLAIN SELECT i, year(dt), month(dt), day(dt), quarter(dt), dayofweek(dt), isodow(dt), dayofyear(dt), week(dt), isoyear(dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*CAST\(EXTRACT\(MONTH FROM "dt"\) AS BIGINT\).* + +query IIIIIIIII +SELECT i, hour(ts), minute(ts), second(ts), millisecond(ts), microsecond(ts), decade(ts), century(ts), millennium(dt) FROM s.rp_date ORDER BY i +---- +0 22 30 0 123 123456 196 20 3 +1 5 43 1 1124 1124456 197 20 3 +2 12 56 2 2125 2125456 197 20 3 +3 20 9 3 3126 3126456 197 20 3 +4 3 22 4 4127 4127456 197 20 3 +5 10 35 5 5128 5128456 197 20 3 +6 17 48 6 6129 6129456 197 20 3 +7 1 1 7 7130 7130456 197 20 3 +8 8 14 8 8131 8131456 197 20 3 +9 15 27 9 9132 9132456 197 20 3 + +query II +EXPLAIN SELECT i, hour(ts), minute(ts), second(ts), millisecond(ts), microsecond(ts), decade(ts), century(ts), millennium(dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIIIII +SELECT i, date_part('year', ts), date_part('month', dt), extract(day FROM ts), extract(hour FROM tm), extract(minute FROM tm), date_part('second', tm), date_part('doy', dt), date_part('dow', ts) FROM s.rp_date ORDER BY i +---- +0 1969 12 31 10 15 30 359 3 +1 1970 1 1 10 22 30 11 4 +2 1970 1 1 10 29 30 28 4 +3 1970 2 1 10 36 30 45 4 +4 1970 3 2 10 43 30 62 5 +5 1970 3 2 10 50 30 79 5 +6 1970 4 2 10 57 30 96 5 +7 1970 4 3 11 4 30 113 6 +8 1970 5 3 11 11 30 130 6 +9 1970 5 3 11 18 30 147 6 + +query II +EXPLAIN SELECT i, date_part('year', ts), date_part('month', dt), extract(day FROM ts), extract(hour FROM tm), extract(minute FROM tm), date_part('second', tm), date_part('doy', dt), date_part('dow', ts) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII +SELECT i, epoch(ts), epoch(dt), date_part('epoch', ts), epoch_ms(ts), date_part('julian', dt) FROM s.rp_date ORDER BY i +---- +0 -5399.876544 1703462400.0 -5399.876544 -5399877 2460304.0 +1 20581.124456 1704931200.0 20581.124456 20581124 2460321.0 +2 46562.125456 1706400000.0 46562.125456 46562125 2460338.0 +3 72543.126456 1707868800.0 72543.126456 72543126 2460355.0 +4 98524.127456 1709337600.0 98524.127456 98524127 2460372.0 +5 124505.128456 1710806400.0 124505.128456 124505128 2460389.0 +6 150486.129456 1712275200.0 150486.129456 150486129 2460406.0 +7 176467.130456 1713744000.0 176467.130456 176467130 2460423.0 +8 202448.131456 1715212800.0 202448.131456 202448131 2460440.0 +9 228429.132456 1716681600.0 228429.132456 228429132 2460457.0 + +query II +EXPLAIN SELECT i, epoch(ts), epoch(dt), date_part('epoch', ts), epoch_ms(ts), date_part('julian', dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIII +SELECT i, date_part('year', iv), date_part('month', iv), date_part('day', iv), date_part('hour', iv), date_part('minute', iv), date_part('epoch', iv) FROM s.rp_date ORDER BY i +---- +0 0 0 0 0 0 0.0 +1 0 1 0 30 0 2700000.0 +2 0 2 0 60 0 5400000.0 +3 0 3 0 90 0 8100000.0 +4 0 4 0 120 0 10800000.0 +5 0 5 0 150 0 13500000.0 +6 0 6 0 180 0 16200000.0 +7 0 7 0 210 0 18900000.0 +8 0 8 0 240 0 21600000.0 +9 0 9 0 270 0 24300000.0 + +query II +EXPLAIN SELECT i, date_part('year', iv), date_part('month', iv), date_part('day', iv), date_part('hour', iv), date_part('minute', iv), date_part('epoch', iv) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date parts of a TIMESTAMP WITH TIME ZONE use the DuckDB time zone +statement ok +SET TimeZone = 'America/New_York' + +query IIIIIIII +SELECT i, tstz, year(tstz), month(tstz), day(tstz), hour(tstz), dayofweek(tstz), epoch(tstz) FROM s.rp_date_tz ORDER BY i +---- +0 2024-03-30 18:00:00-04 2024 3 30 18 6 1711836000.0 +1 2024-03-30 23:00:00-04 2024 3 30 23 6 1711854000.0 +2 2024-03-31 04:00:00-04 2024 3 31 4 0 1711872000.0 +3 2024-03-31 09:00:00-04 2024 3 31 9 0 1711890000.0 +4 2024-03-31 14:00:00-04 2024 3 31 14 0 1711908000.0 +5 2024-03-31 19:00:00-04 2024 3 31 19 0 1711926000.0 +6 2024-04-01 00:00:00-04 2024 4 1 0 1 1711944000.0 +7 2024-04-01 05:00:00-04 2024 4 1 5 1 1711962000.0 +8 2024-04-01 10:00:00-04 2024 4 1 10 1 1711980000.0 +9 2024-04-01 15:00:00-04 2024 4 1 15 1 1711998000.0 + +query II +EXPLAIN SELECT i, tstz, year(tstz), month(tstz), day(tstz), hour(tstz), dayofweek(tstz), epoch(tstz) FROM s.rp_date_tz ORDER BY i +---- +physical_plan :(?s).*CAST\(EXTRACT\(HOUR FROM "tstz"\) AS BIGINT\).* + +query IIIII +SELECT i, date_trunc('day', tstz), CAST(tstz AS DATE), CAST(tstz AS TIMESTAMP), CAST(tstz AS VARCHAR) FROM s.rp_date_tz ORDER BY i +---- +0 2024-03-30 00:00:00-04 2024-03-30 2024-03-30 18:00:00 2024-03-30 18:00:00-04 +1 2024-03-30 00:00:00-04 2024-03-30 2024-03-30 23:00:00 2024-03-30 23:00:00-04 +2 2024-03-31 00:00:00-04 2024-03-31 2024-03-31 04:00:00 2024-03-31 04:00:00-04 +3 2024-03-31 00:00:00-04 2024-03-31 2024-03-31 09:00:00 2024-03-31 09:00:00-04 +4 2024-03-31 00:00:00-04 2024-03-31 2024-03-31 14:00:00 2024-03-31 14:00:00-04 +5 2024-03-31 00:00:00-04 2024-03-31 2024-03-31 19:00:00 2024-03-31 19:00:00-04 +6 2024-04-01 00:00:00-04 2024-04-01 2024-04-01 00:00:00 2024-04-01 00:00:00-04 +7 2024-04-01 00:00:00-04 2024-04-01 2024-04-01 05:00:00 2024-04-01 05:00:00-04 +8 2024-04-01 00:00:00-04 2024-04-01 2024-04-01 10:00:00 2024-04-01 10:00:00-04 +9 2024-04-01 00:00:00-04 2024-04-01 2024-04-01 15:00:00 2024-04-01 15:00:00-04 + +query II +EXPLAIN SELECT i, date_trunc('day', tstz), CAST(tstz AS DATE), CAST(tstz AS TIMESTAMP), CAST(tstz AS VARCHAR) FROM s.rp_date_tz ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +SET TimeZone = 'Asia/Kolkata' + +query IIIII +SELECT i, hour(tstz), minute(tstz), date_trunc('hour', tstz), CAST(tstz AS DATE) FROM s.rp_date_tz ORDER BY i +---- +0 3 30 2024-03-31 03:00:00+05:30 2024-03-31 +1 8 30 2024-03-31 08:00:00+05:30 2024-03-31 +2 13 30 2024-03-31 13:00:00+05:30 2024-03-31 +3 18 30 2024-03-31 18:00:00+05:30 2024-03-31 +4 23 30 2024-03-31 23:00:00+05:30 2024-03-31 +5 4 30 2024-04-01 04:00:00+05:30 2024-04-01 +6 9 30 2024-04-01 09:00:00+05:30 2024-04-01 +7 14 30 2024-04-01 14:00:00+05:30 2024-04-01 +8 19 30 2024-04-01 19:00:00+05:30 2024-04-01 +9 0 30 2024-04-02 00:00:00+05:30 2024-04-02 + +query II +EXPLAIN SELECT i, hour(tstz), minute(tstz), date_trunc('hour', tstz), CAST(tstz AS DATE) FROM s.rp_date_tz ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +SET TimeZone = 'UTC' + +# date_trunc - DuckDB truncates dates to a TIMESTAMP +query IIIIIIII +SELECT i, date_trunc('month', dt), date_trunc('week', dt), date_trunc('quarter', ts), date_trunc('hour', ts), date_trunc('decade', dt), date_trunc('second', ts), date_trunc('milliseconds', ts) FROM s.rp_date ORDER BY i +---- +0 2023-12-01 00:00:00 2023-12-25 00:00:00 1969-10-01 00:00:00 1969-12-31 22:00:00 2020-01-01 00:00:00 1969-12-31 22:30:00 1969-12-31 22:30:00.123 +1 2024-01-01 00:00:00 2024-01-08 00:00:00 1970-01-01 00:00:00 1970-01-01 05:00:00 2020-01-01 00:00:00 1970-01-01 05:43:01 1970-01-01 05:43:01.124 +2 2024-01-01 00:00:00 2024-01-22 00:00:00 1970-01-01 00:00:00 1970-01-01 12:00:00 2020-01-01 00:00:00 1970-01-01 12:56:02 1970-01-01 12:56:02.125 +3 2024-02-01 00:00:00 2024-02-12 00:00:00 1970-01-01 00:00:00 1970-01-01 20:00:00 2020-01-01 00:00:00 1970-01-01 20:09:03 1970-01-01 20:09:03.126 +4 2024-03-01 00:00:00 2024-02-26 00:00:00 1970-01-01 00:00:00 1970-01-02 03:00:00 2020-01-01 00:00:00 1970-01-02 03:22:04 1970-01-02 03:22:04.127 +5 2024-03-01 00:00:00 2024-03-18 00:00:00 1970-01-01 00:00:00 1970-01-02 10:00:00 2020-01-01 00:00:00 1970-01-02 10:35:05 1970-01-02 10:35:05.128 +6 2024-04-01 00:00:00 2024-04-01 00:00:00 1970-01-01 00:00:00 1970-01-02 17:00:00 2020-01-01 00:00:00 1970-01-02 17:48:06 1970-01-02 17:48:06.129 +7 2024-04-01 00:00:00 2024-04-22 00:00:00 1970-01-01 00:00:00 1970-01-03 01:00:00 2020-01-01 00:00:00 1970-01-03 01:01:07 1970-01-03 01:01:07.13 +8 2024-05-01 00:00:00 2024-05-06 00:00:00 1970-01-01 00:00:00 1970-01-03 08:00:00 2020-01-01 00:00:00 1970-01-03 08:14:08 1970-01-03 08:14:08.131 +9 2024-05-01 00:00:00 2024-05-20 00:00:00 1970-01-01 00:00:00 1970-01-03 15:00:00 2020-01-01 00:00:00 1970-01-03 15:27:09 1970-01-03 15:27:09.132 + +query II +EXPLAIN SELECT i, date_trunc('month', dt), date_trunc('week', dt), date_trunc('quarter', ts), date_trunc('hour', ts), date_trunc('decade', dt), date_trunc('second', ts), date_trunc('milliseconds', ts) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*date_trunc\('month', CAST\("dt" AS TIMESTAMP\)\).* + +query I rowsort +SELECT typeof(date_trunc('month', dt)) FROM s.rp_date LIMIT 1 +---- +TIMESTAMP + +query III +SELECT i, last_day(dt), last_day(ts) FROM s.rp_date ORDER BY i +---- +0 2023-12-31 1969-12-31 +1 2024-01-31 1970-01-31 +2 2024-01-31 1970-01-31 +3 2024-02-29 1970-01-31 +4 2024-03-31 1970-01-31 +5 2024-03-31 1970-01-31 +6 2024-04-30 1970-01-31 +7 2024-04-30 1970-01-31 +8 2024-05-31 1970-01-31 +9 2024-05-31 1970-01-31 + +query II +EXPLAIN SELECT i, last_day(dt), last_day(ts) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date_diff counts the boundaries crossed +query IIIIII +SELECT i, date_diff('day', dt, ts), date_diff('month', ts, dt), date_diff('year', dt, DATE '2030-01-01'), date_diff('quarter', dt, ts), date_diff('week', dt, DATE '2024-06-01') FROM s.rp_date ORDER BY i +---- +0 -19717 648 7 -216 22 +1 -19733 648 6 -216 20 +2 -19750 648 6 -216 17 +3 -19767 649 6 -216 15 +4 -19783 650 6 -216 13 +5 -19800 650 6 -216 10 +6 -19817 651 6 -217 8 +7 -19833 651 6 -217 5 +8 -19850 652 6 -217 3 +9 -19867 652 6 -217 0 + +query II +EXPLAIN SELECT i, date_diff('day', dt, ts), date_diff('month', ts, dt), date_diff('year', dt, DATE '2030-01-01'), date_diff('quarter', dt, ts), date_diff('week', dt, DATE '2024-06-01') FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*CAST\(\(CAST\("ts" AS DATE\) - CAST\("dt" AS DATE\)\) AS BIGINT\).* + +query IIIIII +SELECT i, date_diff('hour', ts, dt), date_diff('minute', dt, ts), date_diff('second', ts, TIMESTAMP '2024-01-01'), date_diff('millisecond', ts, TIMESTAMP '1960-01-01'), date_diff('microsecond', ts, dt) FROM s.rp_date ORDER BY i +---- +0 473186 -28391130 1704072600 -315613800123 1703467799876544 +1 473587 -28415177 1704046619 -315639781124 1704910618875544 +2 473988 -28439224 1704020638 -315665762125 1706353437874544 +3 474388 -28463271 1703994657 -315691743126 1707796256873544 +4 474789 -28487318 1703968676 -315717724127 1709239075872544 +5 475190 -28511365 1703942695 -315743705128 1710681894871544 +6 475591 -28535412 1703916714 -315769686129 1712124713870544 +7 475991 -28559459 1703890733 -315795667130 1713567532869544 +8 476392 -28583506 1703864752 -315821648131 1715010351868544 +9 476793 -28607553 1703838771 -315847629132 1716453170867544 + +query II +EXPLAIN SELECT i, date_diff('hour', ts, dt), date_diff('minute', dt, ts), date_diff('second', ts, TIMESTAMP '2024-01-01'), date_diff('millisecond', ts, TIMESTAMP '1960-01-01'), date_diff('microsecond', ts, dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT i, datediff('decade', dt, DATE '1999-01-01'), date_diff('century', ts, dt), date_diff('isoyear', dt, ts), datediff('millennium', ts, DATE '2001-01-01') FROM s.rp_date ORDER BY i +---- +0 -3 1 -53 1 +1 -3 1 -54 1 +2 -3 1 -54 1 +3 -3 1 -54 1 +4 -3 1 -54 1 +5 -3 1 -54 1 +6 -3 1 -54 1 +7 -3 1 -54 1 +8 -3 1 -54 1 +9 -3 1 -54 1 + +query II +EXPLAIN SELECT i, datediff('decade', dt, DATE '1999-01-01'), date_diff('century', ts, dt), date_diff('isoyear', dt, ts), datediff('millennium', ts, DATE '2001-01-01') FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date_diff of TIMESTAMP WITH TIME ZONE values uses calendar arithmetic in DuckDB +query II +SELECT i, date_diff('hour', tstz, TIMESTAMPTZ '2024-04-01 00:00:00+00') FROM s.rp_date_tz ORDER BY i +---- +0 26 +1 21 +2 16 +3 11 +4 6 +5 1 +6 -4 +7 -9 +8 -14 +9 -19 + +query II +EXPLAIN SELECT i, date_diff('hour', tstz, TIMESTAMPTZ '2024-04-01 00:00:00+00') FROM s.rp_date_tz ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# other date functions +query III +SELECT i, make_date(2024, i + 1, i + 10), to_timestamp(i * 1000000.5) FROM s.rp_date ORDER BY i +---- +0 2024-01-10 1970-01-01 00:00:00+00 +1 2024-02-11 1970-01-12 13:46:40.5+00 +2 2024-03-12 1970-01-24 03:33:21+00 +3 2024-04-13 1970-02-04 17:20:01.5+00 +4 2024-05-14 1970-02-16 07:06:42+00 +5 2024-06-15 1970-02-27 20:53:22.5+00 +6 2024-07-16 1970-03-11 10:40:03+00 +7 2024-08-17 1970-03-23 00:26:43.5+00 +8 2024-09-18 1970-04-03 14:13:24+00 +9 2024-10-19 1970-04-15 04:00:04.5+00 + +query II +EXPLAIN SELECT i, make_date(2024, i + 1, i + 10), to_timestamp(i * 1000000.5) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date arithmetic +query IIIIIIII +SELECT i, dt + 5, dt - 3, dt + INTERVAL 1 MONTH, ts - INTERVAL '90 minutes', ts - dt, dt - DATE '2024-01-01', tm + INTERVAL 1 HOUR FROM s.rp_date ORDER BY i +---- +0 2023-12-30 2023-12-22 2024-01-25 00:00:00 1969-12-31 21:00:00.123456 -19716 days -01:29:59.876544 -7 11:15:30 +1 2024-01-16 2024-01-08 2024-02-11 00:00:00 1970-01-01 04:13:01.124456 -19732 days -18:16:58.875544 10 11:22:30 +2 2024-02-02 2024-01-25 2024-02-28 00:00:00 1970-01-01 11:26:02.125456 -19749 days -11:03:57.874544 27 11:29:30 +3 2024-02-19 2024-02-11 2024-03-14 00:00:00 1970-01-01 18:39:03.126456 -19766 days -03:50:56.873544 44 11:36:30 +4 2024-03-07 2024-02-28 2024-04-02 00:00:00 1970-01-02 01:52:04.127456 -19782 days -20:37:55.872544 61 11:43:30 +5 2024-03-24 2024-03-16 2024-04-19 00:00:00 1970-01-02 09:05:05.128456 -19799 days -13:24:54.871544 78 11:50:30 +6 2024-04-10 2024-04-02 2024-05-05 00:00:00 1970-01-02 16:18:06.129456 -19816 days -06:11:53.870544 95 11:57:30 +7 2024-04-27 2024-04-19 2024-05-22 00:00:00 1970-01-02 23:31:07.130456 -19832 days -22:58:52.869544 112 12:04:30 +8 2024-05-14 2024-05-06 2024-06-09 00:00:00 1970-01-03 06:44:08.131456 -19849 days -15:45:51.868544 129 12:11:30 +9 2024-05-31 2024-05-23 2024-06-26 00:00:00 1970-01-03 13:57:09.132456 -19866 days -08:32:50.867544 146 12:18:30 + +query II +EXPLAIN SELECT i, dt + 5, dt - 3, dt + INTERVAL 1 MONTH, ts - INTERVAL '90 minutes', ts - dt, dt - DATE '2024-01-01', tm + INTERVAL 1 HOUR FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date / time functions that depend on the locale or are not available in Postgres +query I +SELECT monthname(dt) FROM s.rp_date ORDER BY i +---- +December +January +January +February +March +March +April +April +May +May + +query II +EXPLAIN SELECT monthname(dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT strftime(dt, '%Y/%m') FROM s.rp_date ORDER BY i +---- +2023/12 +2024/01 +2024/01 +2024/02 +2024/03 +2024/03 +2024/04 +2024/04 +2024/05 +2024/05 + +query II +EXPLAIN SELECT strftime(dt, '%Y/%m') FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT age(ts, TIMESTAMP '2020-01-01') FROM s.rp_date ORDER BY i +---- +-50 years -01:29:59.876544 +-49 years -11 months -30 days -18:16:58.875544 +-49 years -11 months -30 days -11:03:57.874544 +-49 years -11 months -30 days -03:50:56.873544 +-49 years -11 months -29 days -20:37:55.872544 +-49 years -11 months -29 days -13:24:54.871544 +-49 years -11 months -29 days -06:11:53.870544 +-49 years -11 months -28 days -22:58:52.869544 +-49 years -11 months -28 days -15:45:51.868544 +-49 years -11 months -28 days -08:32:50.867544 + +query II +EXPLAIN SELECT age(ts, TIMESTAMP '2020-01-01') FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT yearweek(dt) FROM s.rp_date ORDER BY i +---- +202352 +202402 +202404 +202407 +202409 +202412 +202414 +202417 +202419 +202421 + +query II +EXPLAIN SELECT yearweek(dt) FROM s.rp_date ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_functions_math.test b/test/sql/remote_pushdown/remote_pushdown_functions_math.test new file mode 100644 index 000000000..9f962f2ee --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_functions_math.test @@ -0,0 +1,461 @@ +# name: test/sql/remote_pushdown/remote_pushdown_functions_math.test +# description: Test remote pushdown of numeric functions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_math AS SELECT * FROM (VALUES (1, 0.49999999999999994::DOUBLE, 7, 1.255::DECIMAL(10,3)), (2, 2.675, -7, -2.5), (3, -2.5, 0, 0.5), (4, -0.4, 12, 125.125), (5, 'inf', 3, 1), (6, '-inf', -3, -1.5), (7, 'nan', 5, 2.675), (8, 0.5, 100, NULL), (9, 1.5, NULL, 0), (10, 0.0, 1, 9.999), (11, NULL, 2, 3), (12, 1234.5678, -12, -0.005)) t(id, x, n, dec) + +# arithmetic - DuckDB's division always yields a DOUBLE, and division by zero yields inf / nan +query IIIIIII +SELECT id, n + 1, n - 1, n * 2, n / 4, n % 5, -n FROM s.rp_math ORDER BY id +---- +1 8 6 14 1.75 2 -7 +2 -6 -8 -14 -1.75 -2 7 +3 1 -1 0 0.0 0 0 +4 13 11 24 3.0 2 -12 +5 4 2 6 0.75 3 -3 +6 -2 -4 -6 -0.75 -3 3 +7 6 4 10 1.25 0 -5 +8 101 99 200 25.0 0 -100 +9 NULL NULL NULL NULL NULL NULL +10 2 0 2 0.25 1 -1 +11 3 1 4 0.5 2 -2 +12 -11 -13 -24 -3.0 -2 12 + +query II +EXPLAIN SELECT id, n + 1, n - 1, n * 2, n / 4, n % 5, -n FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*\(CAST\("n" AS DOUBLE PRECISION\) / 4\).* + +query IIIIII +SELECT id, x / 0, n / 0, dec / 0, 0 / x, x / n FROM s.rp_math ORDER BY id +---- +1 inf inf inf 0.0 0.07142857142857142 +2 inf -inf -inf 0.0 -0.3821428571428571 +3 -inf nan inf -0.0 -inf +4 -inf inf inf -0.0 -0.03333333333333333 +5 inf inf inf 0.0 inf +6 -inf -inf -inf -0.0 inf +7 nan inf inf nan nan +8 inf inf NULL 0.0 0.005 +9 inf NULL nan 0.0 NULL +10 nan inf inf nan 0.0 +11 NULL inf inf NULL NULL +12 inf -inf -inf 0.0 -102.88065 + +query II +EXPLAIN SELECT id, x / 0, n / 0, dec / 0, 0 / x, x / n FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT id, n / dec, dec % 1, dec * dec, dec + n FROM s.rp_math WHERE dec <> 0 ORDER BY id +---- +1 5.577689243027889 0.255 1.575025 8.255 +2 2.8 -0.500 6.250000 -9.500 +3 0.0 0.500 0.250000 0.500 +4 0.0959040959040959 0.125 15656.265625 137.125 +5 3.0 0.000 1.000000 4.000 +6 2.0 -0.500 2.250000 -4.500 +7 1.869158878504673 0.675 7.155625 7.675 +10 0.1000100010001 0.999 99.980001 10.999 +11 0.6666666666666666 0.000 9.000000 5.000 +12 2400.0 -0.005 0.000025 -12.005 + +query II +EXPLAIN SELECT id, n / dec, dec % 1, dec * dec, dec + n FROM s.rp_math WHERE dec <> 0 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# Postgres has no modulo for doubles +query II +SELECT id, x % 2 FROM s.rp_math ORDER BY id +---- +1 0.49999999999999994 +2 0.6749999999999998 +3 -0.5 +4 -0.4 +5 nan +6 nan +7 nan +8 0.5 +9 1.5 +10 0.0 +11 NULL +12 0.5678000000000338 + +query II +EXPLAIN SELECT id, x % 2 FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# DuckDB's integer division +query II +SELECT id, n // 2 FROM s.rp_math ORDER BY id +---- +1 3 +2 -3 +3 0 +4 6 +5 1 +6 -1 +7 2 +8 50 +9 NULL +10 0 +11 1 +12 -6 + +query II +EXPLAIN SELECT id, n // 2 FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# rounding - DuckDB rounds doubles half away from zero, Postgres to the nearest even number +query IIIIII +SELECT id, round(x), round(x, 2), round(x, 1), round(x, -2), round(x, 20) FROM s.rp_math ORDER BY id +---- +1 0.0 0.5 0.5 0.0 0.49999999999999994 +2 3.0 2.68 2.7 0.0 2.675 +3 -3.0 -2.5 -2.5 -0.0 -2.5 +4 -0.0 -0.4 -0.4 -0.0 -0.4 +5 inf inf inf inf inf +6 -inf -inf -inf -inf -inf +7 nan nan nan nan nan +8 1.0 0.5 0.5 0.0 0.5 +9 2.0 1.5 1.5 0.0 1.5 +10 0.0 0.0 0.0 0.0 0.0 +11 NULL NULL NULL NULL NULL +12 1235.0 1234.57 1234.6 1200.0 1234.5678 + +query II +EXPLAIN SELECT id, round(x), round(x, 2), round(x, 1), round(x, -2), round(x, 20) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*CASE WHEN abs.* + +query IIII +SELECT id, round_even(x, 0), round_even(x, 1), roundbankers(x, 2) FROM s.rp_math ORDER BY id +---- +1 0.0 0.5 0.5 +2 3.0 2.7 2.68 +3 -2.0 -2.5 -2.5 +4 -0.0 -0.4 -0.4 +5 inf inf inf +6 -inf -inf -inf +7 nan nan nan +8 0.0 0.5 0.5 +9 2.0 1.5 1.5 +10 0.0 0.0 0.0 +11 NULL NULL NULL +12 1235.0 1234.6 1234.57 + +query II +EXPLAIN SELECT id, round_even(x, 0), round_even(x, 1), roundbankers(x, 2) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIII +SELECT id, round(dec), round(dec, 2), round(dec, -1), round(n, -1), round(n * 5, -1), round(n, 2) FROM s.rp_math ORDER BY id +---- +1 1 1.26 0 10 40 7 +2 -3 -2.50 0 -10 -40 -7 +3 1 0.50 0 0 0 0 +4 125 125.13 130 10 60 12 +5 1 1.00 0 0 20 3 +6 -2 -1.50 0 0 -20 -3 +7 3 2.68 0 10 30 5 +8 NULL NULL NULL 100 500 100 +9 0 0.00 0 NULL NULL NULL +10 10 10.00 10 0 10 1 +11 3 3.00 0 0 10 2 +12 0 -0.01 0 -10 -60 -12 + +query II +EXPLAIN SELECT id, round(dec), round(dec, 2), round(dec, -1), round(n, -1), round(n * 5, -1), round(n, 2) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query III rowsort +SELECT typeof(round(dec, 2)), typeof(round(n, -1)), typeof(round(x, 1)) FROM s.rp_math LIMIT 1 +---- +DECIMAL(13,2) INTEGER DOUBLE + +query IIIIIIIIIII +SELECT id, ceil(x), floor(x), trunc(x), ceil(dec), floor(dec), trunc(dec), abs(x), abs(n), sign(x), sign(n) FROM s.rp_math ORDER BY id +---- +1 1.0 0.0 0.0 2 1 1 0.49999999999999994 7 1 1 +2 3.0 2.0 2.0 -2 -3 -2 2.675 7 1 -1 +3 -2.0 -3.0 -2.0 1 0 0 2.5 0 -1 0 +4 -0.0 -1.0 -0.0 126 125 125 0.4 12 -1 1 +5 inf inf inf 1 1 1 inf 3 1 1 +6 -inf -inf -inf -1 -2 -1 inf 3 -1 -1 +7 nan nan nan 3 2 2 nan 5 0 1 +8 1.0 0.0 0.0 NULL NULL NULL 0.5 100 1 1 +9 2.0 1.0 1.0 0 0 0 1.5 NULL 1 NULL +10 0.0 0.0 0.0 10 9 9 0.0 1 0 1 +11 NULL NULL NULL 3 3 3 NULL 2 NULL 1 +12 1235.0 1234.0 1234.0 0 -1 0 1234.5678 12 1 -1 + +query II +EXPLAIN SELECT id, ceil(x), floor(x), trunc(x), ceil(dec), floor(dec), trunc(dec), abs(x), abs(n), sign(x), sign(n) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# functions where Postgres raises errors for values that yield nan / inf in DuckDB +query IIIIIIII +SELECT id, sqrt(x), ln(x), log(x), log10(x), sqrt(n), ln(dec), cbrt(x) FROM s.rp_math ORDER BY id +---- +1 0.7071067811865475 -0.6931471805599454 -0.30102999566398125 -0.30102999566398125 2.6457513110645907 0.22713557258374711 0.7937005259840997 +2 1.6355427233796125 0.9839493803479697 0.4273237863572472 0.4273237863572472 nan nan 1.3881655492727807 +3 nan nan nan nan 0.0 -0.6931471805599453 -1.3572088082974534 +4 nan nan nan nan 3.4641016151377544 4.829313237635384 -0.7368062997280773 +5 inf inf inf inf 1.7320508075688772 0.0 inf +6 nan nan nan nan nan nan -inf +7 nan nan nan nan 2.23606797749979 0.9839493803479697 nan +8 0.7071067811865476 -0.6931471805599453 -0.3010299956639812 -0.3010299956639812 10.0 NULL 0.7937005259840998 +9 1.224744871391589 0.4054651081081644 0.17609125905568124 0.17609125905568124 NULL -inf 1.1447142425533319 +10 0.0 -inf -inf -inf 1.0 2.302485087993712 0.0 +11 NULL NULL NULL NULL 1.4142135623730951 1.0986122886681098 NULL +12 35.13641700572214 7.118476228297786 3.0915149455092013 3.0915149455092013 nan nan 10.727659535728732 + +query II +EXPLAIN SELECT id, sqrt(x), ln(x), log(x), log10(x), sqrt(n), ln(dec), cbrt(x) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT id, exp(x), exp(n), exp(n * 100), exp(-1000.0) FROM s.rp_math ORDER BY id +---- +1 1.648721270700128 1096.6331584284585 1.0142320547350045e+304 0.0 +2 14.512349839590913 0.0009118819655545162 9.85967654375977e-305 0.0 +3 0.0820849986238988 1.0 1.0 0.0 +4 0.6703200460356393 162754.79141900392 inf 0.0 +5 inf 20.085536923187668 1.9424263952412558e+130 0.0 +6 0.0 0.049787068367863944 5.148200222412013e-131 0.0 +7 nan 148.4131591025766 1.4035922178528375e+217 0.0 +8 1.6487212707001282 2.6881171418161356e+43 inf 0.0 +9 4.4816890703380645 NULL NULL 0.0 +10 1.0 2.718281828459045 2.6881171418161356e+43 0.0 +11 NULL 7.38905609893065 7.225973768125749e+86 0.0 +12 inf 6.14421235332821e-06 0.0 0.0 + +query II +EXPLAIN SELECT id, exp(x), exp(n), exp(n * 100), exp(-1000.0) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIII +SELECT id, pow(x, 2), pow(x, 0.5), pow(x, -1), power(2, n), x ^ 3, n ** 2 FROM s.rp_math WHERE id <> 1 ORDER BY id +---- +2 7.155624999999999 1.6355427233796125 0.3738317757009346 0.0078125 19.141296874999995 49.0 +3 6.25 nan -0.4 1.0 -15.625 0.0 +4 0.16000000000000003 nan -2.5 4096.0 -0.06400000000000002 144.0 +5 inf inf 0.0 8.0 inf 9.0 +6 inf inf -0.0 0.125 -inf 9.0 +7 nan nan nan 32.0 nan 25.0 +8 0.25 0.7071067811865476 2.0 1.2676506002282294e+30 0.125 10000.0 +9 2.25 1.224744871391589 0.6666666666666666 NULL 3.375 NULL +10 0.0 0.0 inf 2.0 0.0 1.0 +11 NULL NULL NULL 4.0 NULL 4.0 +12 1524157.65279684 35.13641700572214 0.0008100000664200054 0.000244140625 1881675960.2665586 144.0 + +query II +EXPLAIN SELECT id, pow(x, 2), pow(x, 0.5), pow(x, -1), power(2, n), x ^ 3, n ** 2 FROM s.rp_math WHERE id <> 1 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII +SELECT id, isnan(x), isinf(x), isfinite(x), isnan(x / 0), isinf(n / 0) FROM s.rp_math ORDER BY id +---- +1 false false true false true +2 false false true false true +3 false false true false false +4 false false true false true +5 false true false false true +6 false true false false true +7 true false false true true +8 false false true false true +9 false false true false NULL +10 false false true true true +11 NULL NULL NULL NULL true +12 false false true false true + +query II +EXPLAIN SELECT id, isnan(x), isinf(x), isfinite(x), isnan(x / 0), isinf(n / 0) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIIIII +SELECT id, sin(x), cos(n), tan(dec), atan(x), atan2(n, 2), degrees(n), radians(n), pi() FROM s.rp_math WHERE isfinite(x) ORDER BY id +---- +1 0.47942553860420295 0.7539022543433046 3.060625934307019 0.4636476090008061 1.2924966677897853 401.07045659157626 0.12217304763960307 3.141592653589793 +2 0.44984578024662736 0.7539022543433046 0.7470222972386602 1.213050246329726 -1.2924966677897853 -401.07045659157626 -0.12217304763960307 3.141592653589793 +3 -0.5984721441039564 1.0 0.5463024898437905 -1.1902899496825317 0.0 0.0 0.0 3.141592653589793 +4 -0.3894183423086505 0.8438539587324921 -0.5976722256236242 -0.3805063771123649 1.4056476493802699 687.5493541569879 0.20943951023931956 3.141592653589793 +8 0.479425538604203 0.8623188722876839 NULL 0.46364760900080615 1.550798992821746 5729.5779513082325 1.7453292519943295 3.141592653589793 +9 0.9974949866040544 NULL 0.0 0.982793723247329 NULL NULL NULL 3.141592653589793 +10 0.0 0.5403023058681398 0.646941375540374 0.0 0.4636476090008061 57.29577951308232 0.017453292519943295 3.141592653589793 +12 0.07803344920002027 0.8438539587324921 -0.0050000416670833376 1.5699863269056236 -1.4056476493802699 -687.5493541569879 -0.20943951023931956 3.141592653589793 + +query II +EXPLAIN SELECT id, sin(x), cos(n), tan(dec), atan(x), atan2(n, 2), degrees(n), radians(n), pi() FROM s.rp_math WHERE isfinite(x) ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# integer functions +query IIIIIII +SELECT id, gcd(n, 6), lcm(n, 4), xor(n, 5), n & 3, n | 8, ~n FROM s.rp_math ORDER BY id +---- +1 1 28 2 3 15 -8 +2 1 28 -4 1 -7 6 +3 6 0 5 0 8 -1 +4 6 12 9 0 12 -13 +5 3 12 6 3 11 -4 +6 3 12 -8 1 -3 2 +7 1 20 0 1 13 -6 +8 2 100 97 0 108 -101 +9 NULL NULL NULL NULL NULL NULL +10 1 4 4 1 9 -2 +11 2 4 7 2 10 -3 +12 6 12 -15 0 -4 11 + +query II +EXPLAIN SELECT id, gcd(n, 6), lcm(n, 4), xor(n, 5), n & 3, n | 8, ~n FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT id, greatest(x, n, 1), least(n, dec), greatest(NULL, n) FROM s.rp_math ORDER BY id +---- +1 7.0 1.255 7 +2 2.675 -7.000 -7 +3 1.0 0.000 0 +4 12.0 12.000 12 +5 inf 1.000 3 +6 1.0 -3.000 -3 +7 nan 2.675 5 +8 100.0 100.000 100 +9 1.5 0.000 NULL +10 1.0 1.000 1 +11 2.0 2.000 2 +12 1234.5678 -12.000 -12 + +query II +EXPLAIN SELECT id, greatest(x, n, 1), least(n, dec), greatest(NULL, n) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# functions that are not available in Postgres +query II +SELECT id, log(2, n) FROM s.rp_math WHERE n > 0 ORDER BY id +---- +1 2.807354922057604 +4 3.5849625007211565 +5 1.584962500721156 +7 2.3219280948873626 +8 6.643856189774724 +10 0.0 +11 1.0 + +query II +EXPLAIN SELECT id, log(2, n) FROM s.rp_math WHERE n > 0 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, log2(n) FROM s.rp_math WHERE n > 0 ORDER BY id +---- +1 2.807354922057604 +4 3.584962500721156 +5 1.584962500721156 +7 2.321928094887362 +8 6.643856189774724 +10 0.0 +11 1.0 + +query II +EXPLAIN SELECT id, log2(n) FROM s.rp_math WHERE n > 0 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, factorial(n) FROM s.rp_math WHERE n BETWEEN 0 AND 10 ORDER BY id +---- +1 5040 +3 1 +5 6 +7 120 +10 1 +11 2 + +query II +EXPLAIN SELECT id, factorial(n) FROM s.rp_math WHERE n BETWEEN 0 AND 10 ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, even(x) FROM s.rp_math ORDER BY id +---- +1 2.0 +2 4.0 +3 -4.0 +4 -2.0 +5 inf +6 -inf +7 -nan +8 2.0 +9 2.0 +10 0.0 +11 NULL +12 1236.0 + +query II +EXPLAIN SELECT id, even(x) FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT id, random() > 2 FROM s.rp_math ORDER BY id +---- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false + +query II +EXPLAIN SELECT id, random() > 2 FROM s.rp_math ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# rounding of FLOAT values happens in single precision in DuckDB +query II +SELECT id, round(x::FLOAT, 1) FROM s.rp_math WHERE isfinite(x) ORDER BY id +---- +1 0.5 +2 2.7 +3 -2.5 +4 -0.4 +8 0.5 +9 1.5 +10 0.0 +12 1234.6 + +query II +EXPLAIN SELECT id, round(x::FLOAT, 1) FROM s.rp_math WHERE isfinite(x) ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_functions_string.test b/test/sql/remote_pushdown/remote_pushdown_functions_string.test new file mode 100644 index 000000000..2d944805c --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_functions_string.test @@ -0,0 +1,418 @@ +# name: test/sql/remote_pushdown/remote_pushdown_functions_string.test +# description: Test remote pushdown of string functions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_str AS SELECT i, CASE WHEN i % 5 = 0 THEN NULL ELSE 'Str_' || i || ' ÄÉ' END AS v, ' pad ' || i AS p FROM range(8) t(i) + +# string functions with identical semantics +query IIIIII +SELECT i, length(v), len(v), char_length(v), strlen(v), bit_length(v) FROM s.rp_str ORDER BY i +---- +0 NULL NULL NULL NULL NULL +1 8 8 8 10 80 +2 8 8 8 10 80 +3 8 8 8 10 80 +4 8 8 8 10 80 +5 NULL NULL NULL NULL NULL +6 8 8 8 10 80 +7 8 8 8 10 80 + +query II +EXPLAIN SELECT i, length(v), len(v), char_length(v), strlen(v), bit_length(v) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII +SELECT i, concat(v, '-', i), concat_ws('|', v, NULL, p), v || '!', replace(v, 'Str', 'X'), repeat(v, 2) FROM s.rp_str ORDER BY i +---- +0 -0 pad 0 NULL NULL NULL +1 Str_1 ÄÉ-1 Str_1 ÄÉ| pad 1 Str_1 ÄÉ! X_1 ÄÉ Str_1 ÄÉStr_1 ÄÉ +2 Str_2 ÄÉ-2 Str_2 ÄÉ| pad 2 Str_2 ÄÉ! X_2 ÄÉ Str_2 ÄÉStr_2 ÄÉ +3 Str_3 ÄÉ-3 Str_3 ÄÉ| pad 3 Str_3 ÄÉ! X_3 ÄÉ Str_3 ÄÉStr_3 ÄÉ +4 Str_4 ÄÉ-4 Str_4 ÄÉ| pad 4 Str_4 ÄÉ! X_4 ÄÉ Str_4 ÄÉStr_4 ÄÉ +5 -5 pad 5 NULL NULL NULL +6 Str_6 ÄÉ-6 Str_6 ÄÉ| pad 6 Str_6 ÄÉ! X_6 ÄÉ Str_6 ÄÉStr_6 ÄÉ +7 Str_7 ÄÉ-7 Str_7 ÄÉ| pad 7 Str_7 ÄÉ! X_7 ÄÉ Str_7 ÄÉStr_7 ÄÉ + +query II +EXPLAIN SELECT i, concat(v, '-', i), concat_ws('|', v, NULL, p), v || '!', replace(v, 'Str', 'X'), repeat(v, 2) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIIII +SELECT i, left(v, 3), right(v, -2), lpad(v, 10, '*'), rpad(v, 3, '*'), substring(v, 2, 3), substr(v, 0, 3), substring(v, 5) FROM s.rp_str ORDER BY i +---- +0 NULL NULL NULL NULL NULL NULL NULL +1 Str r_1 ÄÉ **Str_1 ÄÉ Str tr_ St 1 ÄÉ +2 Str r_2 ÄÉ **Str_2 ÄÉ Str tr_ St 2 ÄÉ +3 Str r_3 ÄÉ **Str_3 ÄÉ Str tr_ St 3 ÄÉ +4 Str r_4 ÄÉ **Str_4 ÄÉ Str tr_ St 4 ÄÉ +5 NULL NULL NULL NULL NULL NULL NULL +6 Str r_6 ÄÉ **Str_6 ÄÉ Str tr_ St 6 ÄÉ +7 Str r_7 ÄÉ **Str_7 ÄÉ Str tr_ St 7 ÄÉ + +query II +EXPLAIN SELECT i, left(v, 3), right(v, -2), lpad(v, 10, '*'), rpad(v, 3, '*'), substring(v, 2, 3), substr(v, 0, 3), substring(v, 5) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIII +SELECT i, trim(p), ltrim(p), rtrim(p), trim(v, 'S'), ltrim(v, 'St'), rtrim(v, 'ÄÉ ') FROM s.rp_str ORDER BY i +---- +0 pad 0 pad 0 pad 0 NULL NULL NULL +1 pad 1 pad 1 pad 1 tr_1 ÄÉ r_1 ÄÉ Str_1 +2 pad 2 pad 2 pad 2 tr_2 ÄÉ r_2 ÄÉ Str_2 +3 pad 3 pad 3 pad 3 tr_3 ÄÉ r_3 ÄÉ Str_3 +4 pad 4 pad 4 pad 4 tr_4 ÄÉ r_4 ÄÉ Str_4 +5 pad 5 pad 5 pad 5 NULL NULL NULL +6 pad 6 pad 6 pad 6 tr_6 ÄÉ r_6 ÄÉ Str_6 +7 pad 7 pad 7 pad 7 tr_7 ÄÉ r_7 ÄÉ Str_7 + +query II +EXPLAIN SELECT i, trim(p), ltrim(p), rtrim(p), trim(v, 'S'), ltrim(v, 'St'), rtrim(v, 'ÄÉ ') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIII +SELECT i, strpos(v, '_'), instr(v, 'r'), position('É' IN v), starts_with(v, 'Str'), prefix(v, 'X'), v ^@ 'S' FROM s.rp_str ORDER BY i +---- +0 NULL NULL NULL NULL NULL NULL +1 4 3 8 true false true +2 4 3 8 true false true +3 4 3 8 true false true +4 4 3 8 true false true +5 NULL NULL NULL NULL NULL NULL +6 4 3 8 true false true +7 4 3 8 true false true + +query II +EXPLAIN SELECT i, strpos(v, '_'), instr(v, 'r'), position('É' IN v), starts_with(v, 'Str'), prefix(v, 'X'), v ^@ 'S' FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIII +SELECT i, md5(v), sha256(v), ascii(v), unicode(v), translate(v, 'S_', 's-') FROM s.rp_str ORDER BY i +---- +0 NULL NULL NULL NULL NULL +1 9031616541952e6c6f970598c0bf2273 f418f1cf25b6e2ad36ef63238f8419b342365205887039c0d92f46efbe7a78ef 83 83 str-1 ÄÉ +2 3b0be2d574071eeb76ec889888ee94df 8b8c08963dbdbd93a72b16e64c18c82b1cc4b760d1e203280f486b2b9d8adc2c 83 83 str-2 ÄÉ +3 46b3ce9b293678957e542a621c5ec4af 36e8ba8f33c26f9a2db83153ed3dd9553b050cc42efea27fbfafdacbc54df68e 83 83 str-3 ÄÉ +4 b049e447cb91426e735baac1ae545fec 86e6d9f226884e80805536e6419288c3b89faaeafdeac8ecb3b665c26f2b2818 83 83 str-4 ÄÉ +5 NULL NULL NULL NULL NULL +6 e87796320c3c8edb9a9bfd4896748a6d 4eef919c909443e77e27272582bca6328ec179de4d8f10fbea64277d922cc5a1 83 83 str-6 ÄÉ +7 bdbcd94982144ef51d629787ed8a55e4 02474adc68efccbc2db0e33473c541afb897bdf310200ad4df5ec4e027d8b7c2 83 83 str-7 ÄÉ + +query II +EXPLAIN SELECT i, md5(v), sha256(v), ascii(v), unicode(v), translate(v, 'S_', 's-') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# rewritten functions +query II +SELECT i, contains(v, '1') FROM s.rp_str ORDER BY i +---- +0 NULL +1 true +2 false +3 false +4 false +5 NULL +6 false +7 false + +query II +EXPLAIN SELECT i, contains(v, '1') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*\(strpos\("v", '1'\) > 0\).* + +query III +SELECT i, ends_with(v, 'É'), suffix(v, '') FROM s.rp_str ORDER BY i +---- +0 NULL NULL +1 true true +2 true true +3 true true +4 true true +5 NULL NULL +6 true true +7 true true + +query II +EXPLAIN SELECT i, ends_with(v, 'É'), suffix(v, '') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*\(right\("v", length\('É'\)\) = 'É'\).* + +query II +SELECT i, lower(v) FROM s.rp_str ORDER BY i +---- +0 NULL +1 str_1 äé +2 str_2 äé +3 str_3 äé +4 str_4 äé +5 NULL +6 str_6 äé +7 str_7 äé + +query II +EXPLAIN SELECT i, lower(v) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*lower\("v"\).* + +# LIKE has no default escape character in DuckDB +query IIIII +SELECT i, v LIKE 'Str\_%', v NOT LIKE '%1%', v ILIKE 'str_%', v NOT ILIKE '%ä%' FROM s.rp_str ORDER BY i +---- +0 NULL NULL NULL NULL +1 false false true false +2 false true true false +3 false true true false +4 false true true false +5 NULL NULL NULL NULL +6 false true true false +7 false true true false + +query II +EXPLAIN SELECT i, v LIKE 'Str\_%', v NOT LIKE '%1%', v ILIKE 'str_%', v NOT ILIKE '%ä%' FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*\("v" LIKE E'Str\\\\_%' ESCAPE ''\).* + +query I rowsort +SELECT COUNT(*) FROM s.rp_str WHERE v LIKE '%\%' +---- +0 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_str WHERE v LIKE '%\%' +---- +physical_plan :(?s).*Postgres Query.* + +# string literals with special characters +query II +SELECT i, v || 'it''s' || E'\\back' FROM s.rp_str WHERE v <> 'O''Reilly' ORDER BY i +---- +1 Str_1 ÄÉit's\back +2 Str_2 ÄÉit's\back +3 Str_3 ÄÉit's\back +4 Str_4 ÄÉit's\back +6 Str_6 ÄÉit's\back +7 Str_7 ÄÉit's\back + +query II +EXPLAIN SELECT i, v || 'it''s' || E'\\back' FROM s.rp_str WHERE v <> 'O''Reilly' ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# functions with different semantics are executed locally +query I +SELECT upper(v) FROM s.rp_str ORDER BY i +---- +NULL +STR_1 ÄÉ +STR_2 ÄÉ +STR_3 ÄÉ +STR_4 ÄÉ +NULL +STR_6 ÄÉ +STR_7 ÄÉ + +query II +EXPLAIN SELECT upper(v) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT reverse(v) FROM s.rp_str ORDER BY i +---- +NULL +ÉÄ 1_rtS +ÉÄ 2_rtS +ÉÄ 3_rtS +ÉÄ 4_rtS +NULL +ÉÄ 6_rtS +ÉÄ 7_rtS + +query II +EXPLAIN SELECT reverse(v) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT split_part(v, '_', 2) FROM s.rp_str ORDER BY i +---- +NULL +1 ÄÉ +2 ÄÉ +3 ÄÉ +4 ÄÉ +NULL +6 ÄÉ +7 ÄÉ + +query II +EXPLAIN SELECT split_part(v, '_', 2) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT substring(v, -3, 2) FROM s.rp_str ORDER BY i +---- +NULL + Ä + Ä + Ä + Ä +NULL + Ä + Ä + +query II +EXPLAIN SELECT substring(v, -3, 2) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT regexp_matches(v, '\d') FROM s.rp_str ORDER BY i +---- +NULL +true +true +true +true +NULL +true +true + +query II +EXPLAIN SELECT regexp_matches(v, '\d') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT v SIMILAR TO 'S.*' FROM s.rp_str ORDER BY i +---- +NULL +true +true +true +true +NULL +true +true + +query II +EXPLAIN SELECT v SIMILAR TO 'S.*' FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT string_split(v, '_') FROM s.rp_str ORDER BY i +---- +NULL +[Str, 1 ÄÉ] +[Str, 2 ÄÉ] +[Str, 3 ÄÉ] +[Str, 4 ÄÉ] +NULL +[Str, 6 ÄÉ] +[Str, 7 ÄÉ] + +query II +EXPLAIN SELECT string_split(v, '_') FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# concatenating non-strings is only pushed for types with identical text representations +query III +SELECT i, concat(v, i, 1.5), v || i FROM s.rp_str ORDER BY i +---- +0 01.5 NULL +1 Str_1 ÄÉ11.5 Str_1 ÄÉ1 +2 Str_2 ÄÉ21.5 Str_2 ÄÉ2 +3 Str_3 ÄÉ31.5 Str_3 ÄÉ3 +4 Str_4 ÄÉ41.5 Str_4 ÄÉ4 +5 51.5 NULL +6 Str_6 ÄÉ61.5 Str_6 ÄÉ6 +7 Str_7 ÄÉ71.5 Str_7 ÄÉ7 + +query II +EXPLAIN SELECT i, concat(v, i, 1.5), v || i FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, concat(v, i > 3) FROM s.rp_str ORDER BY i +---- +0 false +1 Str_1 ÄÉfalse +2 Str_2 ÄÉfalse +3 Str_3 ÄÉfalse +4 Str_4 ÄÉtrue +5 true +6 Str_6 ÄÉtrue +7 Str_7 ÄÉtrue + +query II +EXPLAIN SELECT i, concat(v, i > 3) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, CAST(i > 3 AS VARCHAR) FROM s.rp_str ORDER BY i +---- +0 false +1 false +2 false +3 false +4 true +5 true +6 true +7 true + +query II +EXPLAIN SELECT i, CAST(i > 3 AS VARCHAR) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, concat(v, i * 1.5::DOUBLE) FROM s.rp_str ORDER BY i +---- +0 0.0 +1 Str_1 ÄÉ1.5 +2 Str_2 ÄÉ3.0 +3 Str_3 ÄÉ4.5 +4 Str_4 ÄÉ6.0 +5 7.5 +6 Str_6 ÄÉ9.0 +7 Str_7 ÄÉ10.5 + +query II +EXPLAIN SELECT i, concat(v, i * 1.5::DOUBLE) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, v || (i / 2) FROM s.rp_str ORDER BY i +---- +0 NULL +1 Str_1 ÄÉ0.5 +2 Str_2 ÄÉ1.0 +3 Str_3 ÄÉ1.5 +4 Str_4 ÄÉ2.0 +5 NULL +6 Str_6 ÄÉ3.0 +7 Str_7 ÄÉ3.5 + +query II +EXPLAIN SELECT i, v || (i / 2) FROM s.rp_str ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_groupby.test b/test/sql/remote_pushdown/remote_pushdown_groupby.test new file mode 100644 index 000000000..abb5ef9c7 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_groupby.test @@ -0,0 +1,277 @@ +# name: test/sql/remote_pushdown/remote_pushdown_groupby.test +# description: Test remote pushdown of GROUP BY / HAVING / grouping sets into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_groupby AS SELECT i, i % 3 AS g, i % 2 AS h, CASE WHEN i % 4 = 0 THEN NULL ELSE 'v' || (i % 4) END AS v FROM range(12) t(i) + +query IIIII +SELECT g, COUNT(*), SUM(i), MIN(v), MAX(v) FROM s.rp_groupby GROUP BY g ORDER BY g +---- +0 4 18 v1 v3 +1 4 22 v1 v3 +2 4 26 v1 v3 + +query II +EXPLAIN SELECT g, COUNT(*), SUM(i), MIN(v), MAX(v) FROM s.rp_groupby GROUP BY g ORDER BY g +---- +physical_plan :(?s).*GROUP BY "g" ORDER BY "g" ASC NULLS LAST.* + +# positional GROUP BY and ORDER BY +query III +SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY 1, 2 ORDER BY 2, 1 +---- +0 0 2 +1 0 2 +2 0 2 +0 1 2 +1 1 2 +2 1 2 + +query II +EXPLAIN SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY 1, 2 ORDER BY 2, 1 +---- +physical_plan :(?s).*GROUP BY 1, 2 ORDER BY 2 ASC NULLS LAST, 1 ASC NULLS LAST.* + +# grouping by an alias +query II +SELECT i % 3 AS grp, COUNT(*) FROM s.rp_groupby GROUP BY grp ORDER BY grp +---- +0 4 +1 4 +2 4 + +query II +EXPLAIN SELECT i % 3 AS grp, COUNT(*) FROM s.rp_groupby GROUP BY grp ORDER BY grp +---- +physical_plan :(?s).*Postgres Query.* + +# grouping by an expression +query II +SELECT i % 3 + 1, COUNT(*) FROM s.rp_groupby GROUP BY i % 3 + 1 ORDER BY 1 +---- +1 4 +2 4 +3 4 + +query II +EXPLAIN SELECT i % 3 + 1, COUNT(*) FROM s.rp_groupby GROUP BY i % 3 + 1 ORDER BY 1 +---- +physical_plan :(?s).*Postgres Query.* + +# NULL groups +query II +SELECT v, COUNT(*) FROM s.rp_groupby GROUP BY v ORDER BY v +---- +v1 3 +v2 3 +v3 3 +NULL 3 + +query II +EXPLAIN SELECT v, COUNT(*) FROM s.rp_groupby GROUP BY v ORDER BY v +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT v, COUNT(*) FROM s.rp_groupby GROUP BY v ORDER BY v DESC NULLS LAST +---- +v3 3 +v2 3 +v1 3 +NULL 3 + +query II +EXPLAIN SELECT v, COUNT(*) FROM s.rp_groupby GROUP BY v ORDER BY v DESC NULLS LAST +---- +physical_plan :(?s).*Postgres Query.* + +# HAVING +query II +SELECT g, SUM(i) FROM s.rp_groupby GROUP BY g HAVING SUM(i) > 15 ORDER BY g +---- +0 18 +1 22 +2 26 + +query II +EXPLAIN SELECT g, SUM(i) FROM s.rp_groupby GROUP BY g HAVING SUM(i) > 15 ORDER BY g +---- +physical_plan :(?s).*HAVING \(sum\("i"\) > 15\).* + +query I rowsort +SELECT COUNT(*) FROM s.rp_groupby HAVING COUNT(*) > 5 +---- +12 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_groupby HAVING COUNT(*) > 5 +---- +physical_plan :(?s).*Postgres Query.* + +# HAVING referencing an alias is not supported in Postgres +query II +SELECT g, SUM(i) AS total FROM s.rp_groupby GROUP BY g HAVING total > 15 ORDER BY g +---- +0 18 +1 22 +2 26 + +query II +EXPLAIN SELECT g, SUM(i) AS total FROM s.rp_groupby GROUP BY g HAVING total > 15 ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates with DISTINCT +query III rowsort +SELECT COUNT(DISTINCT g), SUM(DISTINCT h), COUNT(DISTINCT v) FROM s.rp_groupby +---- +3 1 3 + +query II +EXPLAIN SELECT COUNT(DISTINCT g), SUM(DISTINCT h), COUNT(DISTINCT v) FROM s.rp_groupby +---- +physical_plan :(?s).*count\(DISTINCT "g"\).* + +# GROUP BY ALL is DuckDB-specific +query III +SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY ALL ORDER BY ALL +---- +0 0 2 +0 1 2 +1 0 2 +1 1 2 +2 0 2 +2 1 2 + +query II +EXPLAIN SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY ALL ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +# ORDER BY ALL is serialized with positional references +query III +SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY g, h ORDER BY ALL DESC +---- +2 1 2 +2 0 2 +1 1 2 +1 0 2 +0 1 2 +0 0 2 + +query II +EXPLAIN SELECT g, h, COUNT(*) FROM s.rp_groupby GROUP BY g, h ORDER BY ALL DESC +---- +physical_plan :(?s).*ORDER BY 1 DESC NULLS LAST, 2 DESC NULLS LAST, 3 DESC NULLS LAST.* + +# grouping sets, ROLLUP and CUBE +query III +SELECT g, h, SUM(i) FROM s.rp_groupby GROUP BY GROUPING SETS ((g), (h), ()) ORDER BY ALL +---- +0 NULL 18 +1 NULL 22 +2 NULL 26 +NULL 0 30 +NULL 1 36 +NULL NULL 66 + +query II +EXPLAIN SELECT g, h, SUM(i) FROM s.rp_groupby GROUP BY GROUPING SETS ((g), (h), ()) ORDER BY ALL +---- +physical_plan :(?s).*GROUP BY GROUPING SETS \(\("g"\), \("h"\), \(\)\).* + +query III +SELECT g, h, SUM(i) FROM s.rp_groupby GROUP BY ROLLUP (g, h) ORDER BY ALL +---- +0 0 6 +0 1 12 +0 NULL 18 +1 0 14 +1 1 8 +1 NULL 22 +2 0 10 +2 1 16 +2 NULL 26 +NULL NULL 66 + +query II +EXPLAIN SELECT g, h, SUM(i) FROM s.rp_groupby GROUP BY ROLLUP (g, h) ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT g, h, SUM(i), GROUPING(g), GROUPING(g, h) FROM s.rp_groupby GROUP BY CUBE (g, h) ORDER BY ALL +---- +0 0 6 0 0 +0 1 12 0 0 +0 NULL 18 0 1 +1 0 14 0 0 +1 1 8 0 0 +1 NULL 22 0 1 +2 0 10 0 0 +2 1 16 0 0 +2 NULL 26 0 1 +NULL 0 30 1 2 +NULL 1 36 1 2 +NULL NULL 66 1 3 + +query II +EXPLAIN SELECT g, h, SUM(i), GROUPING(g), GROUPING(g, h) FROM s.rp_groupby GROUP BY CUBE (g, h) ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT g, h, COUNT(*), GROUPING_ID(g, h) FROM s.rp_groupby GROUP BY CUBE (g, h) ORDER BY ALL +---- +0 0 2 0 +0 1 2 0 +0 NULL 4 1 +1 0 2 0 +1 1 2 0 +1 NULL 4 1 +2 0 2 0 +2 1 2 0 +2 NULL 4 1 +NULL 0 6 2 +NULL 1 6 2 +NULL NULL 12 3 + +query II +EXPLAIN SELECT g, h, COUNT(*), GROUPING_ID(g, h) FROM s.rp_groupby GROUP BY CUBE (g, h) ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT g % 2 AS gg, h, COUNT(*) FROM s.rp_groupby GROUP BY ROLLUP (gg, h) ORDER BY ALL +---- +0 0 4 +0 1 4 +0 NULL 8 +1 0 2 +1 1 2 +1 NULL 4 +NULL NULL 12 + +query II +EXPLAIN SELECT g % 2 AS gg, h, COUNT(*) FROM s.rp_groupby GROUP BY ROLLUP (gg, h) ORDER BY ALL +---- +physical_plan :(?s).*Postgres Query.* + +# grouped aggregates in a subquery +query I rowsort +SELECT MAX(cnt) FROM (SELECT g, COUNT(*) AS cnt FROM s.rp_groupby GROUP BY g) +---- +4 + +query II +EXPLAIN SELECT MAX(cnt) FROM (SELECT g, COUNT(*) AS cnt FROM s.rp_groupby GROUP BY g) +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_identifiers.test b/test/sql/remote_pushdown/remote_pushdown_identifiers.test new file mode 100644 index 000000000..9e194fd9e --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_identifiers.test @@ -0,0 +1,141 @@ +# name: test/sql/remote_pushdown/remote_pushdown_identifiers.test +# description: Test identifier handling in remote pushdown into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test identifier handling in remote pushdown into Postgres +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS "RP Schema" CASCADE') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA "RP Schema"') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE "RP Schema"."Mixed Case" ("Id" INT, "select" TEXT, "quote""d" INT, lower_col INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO "RP Schema"."Mixed Case" VALUES (1, ''a'', 10, 100), (2, ''b'', 20, 200), (3, NULL, 30, 300)') + +statement ok +CALL postgres_clear_cache() + +statement ok +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +# identifiers are case insensitive in DuckDB - the Postgres spelling is used when serializing +query IIII +SELECT id, "SELECT", "QUOTE""D", LOWER_COL FROM s2."rp schema"."mixed case" ORDER BY ID +---- +1 a 10 100 +2 b 20 200 +3 NULL 30 300 + +query II +EXPLAIN SELECT id, "SELECT", "QUOTE""D", LOWER_COL FROM s2."rp schema"."mixed case" ORDER BY ID +---- +physical_plan :(?s).*SELECT "Id", "select", "quote""d", "lower_col" FROM "RP Schema"\."Mixed Case" ORDER BY "Id" ASC NULLS LAST.* + +query II +SELECT "Id", SUM("quote""d") FROM s2."RP Schema"."Mixed Case" GROUP BY "Id" ORDER BY "Id" +---- +1 10 +2 20 +3 30 + +query II +EXPLAIN SELECT "Id", SUM("quote""d") FROM s2."RP Schema"."Mixed Case" GROUP BY "Id" ORDER BY "Id" +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT t.id, t."select" FROM s2."RP Schema"."Mixed Case" AS T WHERE T.Id > 1 ORDER BY T.ID +---- +2 b +3 NULL + +query II +EXPLAIN SELECT t.id, t."select" FROM s2."RP Schema"."Mixed Case" AS T WHERE T.Id > 1 ORDER BY T.ID +---- +physical_plan :(?s).*Postgres Query.* + +# aliases are case insensitive as well +query II +SELECT id AS MyId, lower_col * 2 AS Doubled FROM s2."RP Schema"."Mixed Case" ORDER BY myid DESC, DOUBLED +---- +3 600 +2 400 +1 200 + +query II +EXPLAIN SELECT id AS MyId, lower_col * 2 AS Doubled FROM s2."RP Schema"."Mixed Case" ORDER BY myid DESC, DOUBLED +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +WITH MyCTE AS (SELECT id AS X FROM s2."RP Schema"."Mixed Case") SELECT SUM(x) FROM mycte +---- +6 + +query II +EXPLAIN WITH MyCTE AS (SELECT id AS X FROM s2."RP Schema"."Mixed Case") SELECT SUM(x) FROM mycte +---- +physical_plan :(?s).*Postgres Query.* + +# result column names are the names DuckDB would produce +statement ok +CREATE OR REPLACE TABLE ident_names AS SELECT ID, "SELECT", lower_col + 1, COUNT(*) AS Cnt FROM s2."RP Schema"."Mixed Case" GROUP BY ALL LIMIT 0 + +query I rowsort +SELECT column_name FROM (DESCRIBE ident_names) +---- +(lower_col + 1) +Cnt +Id +select + +# expressions are named the way DuckDB names them, also in subqueries +query I +SELECT "(lower_col + 1)" FROM (SELECT lower_col + 1 FROM s2."RP Schema"."Mixed Case") ORDER BY 1 +---- +101 +201 +301 + +query II +EXPLAIN SELECT "(lower_col + 1)" FROM (SELECT lower_col + 1 FROM s2."RP Schema"."Mixed Case") ORDER BY 1 +---- +physical_plan :(?s).*Postgres Query.* + +query I +WITH c AS (SELECT "select", COUNT(*) FROM s2."RP Schema"."Mixed Case" GROUP BY "select") SELECT "count_star()" FROM c ORDER BY 1 +---- +1 +1 +1 + +query II +EXPLAIN WITH c AS (SELECT "select", COUNT(*) FROM s2."RP Schema"."Mixed Case" GROUP BY "select") SELECT "count_star()" FROM c ORDER BY 1 +---- +physical_plan :(?s).*Postgres Query.* + +# long expression names exceed the Postgres identifier length +query II +SELECT lower_col + 1 + lower_col + 2 + lower_col + 3 + lower_col + 4 + lower_col + 5 + lower_col + 6, lower_col FROM s2."RP Schema"."Mixed Case" ORDER BY lower_col +---- +621 100 +1221 200 +1821 300 + +query II +EXPLAIN SELECT lower_col + 1 + lower_col + 2 + lower_col + 3 + lower_col + 4 + lower_col + 5 + lower_col + 6, lower_col FROM s2."RP Schema"."Mixed Case" ORDER BY lower_col +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA "RP Schema" CASCADE') diff --git a/test/sql/remote_pushdown/remote_pushdown_join.test b/test/sql/remote_pushdown/remote_pushdown_join.test new file mode 100644 index 000000000..c11a05802 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_join.test @@ -0,0 +1,329 @@ +# name: test/sql/remote_pushdown/remote_pushdown_join.test +# description: Test remote pushdown of joins into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_j1 AS SELECT i, i % 3 AS g, CASE WHEN i = 4 THEN NULL ELSE 'v' || i END AS v FROM range(6) t(i) + +statement ok +CREATE OR REPLACE TABLE s.rp_j2 AS SELECT i AS k, i * 10 AS w FROM range(2, 9) t(i) + +query III +SELECT i, v, w FROM s.rp_j1 JOIN s.rp_j2 ON i = k ORDER BY i +---- +2 v2 20 +3 v3 30 +4 NULL 40 +5 v5 50 + +query II +EXPLAIN SELECT i, v, w FROM s.rp_j1 JOIN s.rp_j2 ON i = k ORDER BY i +---- +physical_plan :(?s).*\("public"\."rp_j1" INNER JOIN "public"\."rp_j2" ON \("i" = "k"\)\).* + +query II +SELECT i, w FROM s.rp_j1 LEFT JOIN s.rp_j2 ON i = k ORDER BY i +---- +0 NULL +1 NULL +2 20 +3 30 +4 40 +5 50 + +query II +EXPLAIN SELECT i, w FROM s.rp_j1 LEFT JOIN s.rp_j2 ON i = k ORDER BY i +---- +physical_plan :(?s).*LEFT JOIN.* + +query III +SELECT i, k, w FROM s.rp_j1 RIGHT JOIN s.rp_j2 ON i = k ORDER BY k +---- +2 2 20 +3 3 30 +4 4 40 +5 5 50 +NULL 6 60 +NULL 7 70 +NULL 8 80 + +query II +EXPLAIN SELECT i, k, w FROM s.rp_j1 RIGHT JOIN s.rp_j2 ON i = k ORDER BY k +---- +physical_plan :(?s).*RIGHT JOIN.* + +query II +SELECT i, k FROM s.rp_j1 FULL OUTER JOIN s.rp_j2 ON i = k ORDER BY i, k +---- +0 NULL +1 NULL +2 2 +3 3 +4 4 +5 5 +NULL 6 +NULL 7 +NULL 8 + +query II +EXPLAIN SELECT i, k FROM s.rp_j1 FULL OUTER JOIN s.rp_j2 ON i = k ORDER BY i, k +---- +physical_plan :(?s).*FULL OUTER JOIN.* + +query II rowsort +SELECT COUNT(*), SUM(i * w) FROM s.rp_j1 CROSS JOIN s.rp_j2 +---- +42 5250 + +query II +EXPLAIN SELECT COUNT(*), SUM(i * w) FROM s.rp_j1 CROSS JOIN s.rp_j2 +---- +physical_plan :(?s).*CROSS JOIN.* + +query I rowsort +SELECT COUNT(*) FROM s.rp_j1, s.rp_j2 WHERE i < k +---- +32 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_j1, s.rp_j2 WHERE i < k +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT a.i, b.i FROM s.rp_j1 a JOIN s.rp_j1 b ON a.i = b.g ORDER BY a.i, b.i +---- +0 0 +0 3 +1 1 +1 4 +2 2 +2 5 + +query II +EXPLAIN SELECT a.i, b.i FROM s.rp_j1 a JOIN s.rp_j1 b ON a.i = b.g ORDER BY a.i, b.i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIIIIII +SELECT * FROM s.rp_j1 a JOIN s.rp_j2 b ON a.i = b.k JOIN s.rp_j1 c ON c.i = b.k - 2 ORDER BY a.i, c.i +---- +2 2 v2 2 20 0 0 v0 +3 0 v3 3 30 1 1 v1 +4 1 NULL 4 40 2 2 v2 +5 2 v5 5 50 3 0 v3 + +query II +EXPLAIN SELECT * FROM s.rp_j1 a JOIN s.rp_j2 b ON a.i = b.k JOIN s.rp_j1 c ON c.i = b.k - 2 ORDER BY a.i, c.i +---- +physical_plan :(?s).*Postgres Query.* + +# join conditions with complex expressions +query II +SELECT i, k FROM s.rp_j1 JOIN s.rp_j2 ON i + 2 = k OR (v IS NULL AND k = 8) ORDER BY i, k +---- +0 2 +1 3 +2 4 +3 5 +4 6 +4 8 +5 7 + +query II +EXPLAIN SELECT i, k FROM s.rp_j1 JOIN s.rp_j2 ON i + 2 = k OR (v IS NULL AND k = 8) ORDER BY i, k +---- +physical_plan :(?s).*Postgres Query.* + +# NATURAL / USING joins - an unqualified star is expanded differently by Postgres +query IIII +SELECT rp_j1.*, w FROM s.rp_j1 JOIN (SELECT k AS i, w FROM s.rp_j2) USING (i) ORDER BY i +---- +2 2 v2 20 +3 0 v3 30 +4 1 NULL 40 +5 2 v5 50 + +query II +EXPLAIN SELECT rp_j1.*, w FROM s.rp_j1 JOIN (SELECT k AS i, w FROM s.rp_j2) USING (i) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, w FROM s.rp_j1 NATURAL JOIN (SELECT k AS i, w FROM s.rp_j2) ORDER BY i +---- +2 20 +3 30 +4 40 +5 50 + +query II +EXPLAIN SELECT i, w FROM s.rp_j1 NATURAL JOIN (SELECT k AS i, w FROM s.rp_j2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT * FROM s.rp_j1 JOIN (SELECT k AS i, w FROM s.rp_j2) USING (i) ORDER BY i +---- +2 2 v2 20 +3 0 v3 30 +4 1 NULL 40 +5 2 v5 50 + +query II +EXPLAIN SELECT * FROM s.rp_j1 JOIN (SELECT k AS i, w FROM s.rp_j2) USING (i) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# SEMI / ANTI joins are serialized as [NOT] EXISTS +query III +SELECT * FROM s.rp_j1 SEMI JOIN s.rp_j2 ON i = k ORDER BY i +---- +2 2 v2 +3 0 v3 +4 1 NULL +5 2 v5 + +query II +EXPLAIN SELECT * FROM s.rp_j1 SEMI JOIN s.rp_j2 ON i = k ORDER BY i +---- +physical_plan :(?s).*\(SELECT \* FROM "public"\."rp_j1" WHERE EXISTS \(SELECT 1 FROM "public"\."rp_j2" WHERE \("i" = "k"\)\)\) AS "rp_j1".* + +query III +SELECT * FROM s.rp_j1 t ANTI JOIN s.rp_j2 ON t.i = k ORDER BY i +---- +0 0 v0 +1 1 v1 + +query II +EXPLAIN SELECT * FROM s.rp_j1 t ANTI JOIN s.rp_j2 ON t.i = k ORDER BY i +---- +physical_plan :(?s).*WHERE NOT EXISTS \(SELECT 1 FROM.* + +query II +SELECT q.* FROM (SELECT i, v FROM s.rp_j1 WHERE i > 0) q SEMI JOIN s.rp_j2 ON q.i = k ORDER BY i +---- +2 v2 +3 v3 +4 NULL +5 v5 + +query II +EXPLAIN SELECT q.* FROM (SELECT i, v FROM s.rp_j1 WHERE i > 0) q SEMI JOIN s.rp_j2 ON q.i = k ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT * FROM s.rp_j1 ANTI JOIN s.rp_j2 ON i = k AND v IS NOT NULL ORDER BY i +---- +0 0 v0 +1 1 v1 +4 1 NULL + +query II +EXPLAIN SELECT * FROM s.rp_j1 ANTI JOIN s.rp_j2 ON i = k AND v IS NOT NULL ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# ASOF / POSITIONAL joins are DuckDB-specific +query IIIII +SELECT * FROM s.rp_j1 POSITIONAL JOIN s.rp_j2 ORDER BY i +---- +0 0 v0 2 20 +1 1 v1 3 30 +2 2 v2 4 40 +3 0 v3 5 50 +4 1 NULL 6 60 +5 2 v5 7 70 +NULL NULL NULL 8 80 + +query II +EXPLAIN SELECT * FROM s.rp_j1 POSITIONAL JOIN s.rp_j2 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, k FROM s.rp_j1 ASOF JOIN s.rp_j2 ON i >= k ORDER BY i +---- +2 2 +3 3 +4 4 +5 5 + +query II +EXPLAIN SELECT i, k FROM s.rp_j1 ASOF JOIN s.rp_j2 ON i >= k ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# lateral joins - DuckDB detects lateral references automatically +query IIII +SELECT * FROM s.rp_j1, LATERAL (SELECT SUM(w) s FROM s.rp_j2 WHERE k <= i) ORDER BY i +---- +0 0 v0 NULL +1 1 v1 NULL +2 2 v2 20 +3 0 v3 50 +4 1 NULL 90 +5 2 v5 140 + +query II +EXPLAIN SELECT * FROM s.rp_j1, LATERAL (SELECT SUM(w) s FROM s.rp_j2 WHERE k <= i) ORDER BY i +---- +physical_plan :(?s).*CROSS JOIN LATERAL \(SELECT.* + +query IIII +SELECT * FROM s.rp_j1, (SELECT SUM(w) s FROM s.rp_j2 WHERE k <= i) ORDER BY i +---- +0 0 v0 NULL +1 1 v1 NULL +2 2 v2 20 +3 0 v3 50 +4 1 NULL 90 +5 2 v5 140 + +query II +EXPLAIN SELECT * FROM s.rp_j1, (SELECT SUM(w) s FROM s.rp_j2 WHERE k <= i) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, x FROM s.rp_j1 LEFT JOIN LATERAL (SELECT w AS x FROM s.rp_j2 WHERE k = i) ON true ORDER BY i +---- +0 NULL +1 NULL +2 20 +3 30 +4 40 +5 50 + +query II +EXPLAIN SELECT i, x FROM s.rp_j1 LEFT JOIN LATERAL (SELECT w AS x FROM s.rp_j2 WHERE k = i) ON true ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# joins with a local table are not pushed down +statement ok +CREATE TABLE local_join AS SELECT 1 AS i + +query I rowsort +SELECT COUNT(*) FROM s.rp_j1 JOIN local_join USING (i) +---- +1 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_j1 JOIN local_join USING (i) +---- +physical_plan :(?s).*Postgres Query.* + +# duplicate table names in a join are rejected by both DuckDB and Postgres +statement error +SELECT * FROM s.rp_j1 JOIN s.rp_j1 ON true +---- +duplicate alias diff --git a/test/sql/remote_pushdown/remote_pushdown_merge.test b/test/sql/remote_pushdown/remote_pushdown_merge.test new file mode 100644 index 000000000..ccc8d27ee --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_merge.test @@ -0,0 +1,93 @@ +# name: test/sql/remote_pushdown/remote_pushdown_merge.test +# description: Test remote pushdown of MERGE statements into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_merge_t (id INTEGER PRIMARY KEY, v VARCHAR, n INTEGER) + +statement ok +INSERT INTO s.rp_merge_t VALUES (1, 'a', 10), (2, 'b', 20), (3, 'c', 30) + +statement ok +CREATE OR REPLACE TABLE s.rp_merge_s (id INTEGER, v VARCHAR, del BOOLEAN) + +statement ok +INSERT INTO s.rp_merge_s VALUES (1, 'A', false), (2, 'B', true), (4, 'D', false) + +# MERGE requires Postgres 15 - on older versions the statement is executed by the regular MERGE +query II +EXPLAIN MERGE INTO s.rp_merge_t t USING s.rp_merge_s src ON t.id = src.id WHEN MATCHED AND src.del THEN DELETE WHEN MATCHED THEN UPDATE SET v = src.v, n = t.n + 1 WHEN NOT MATCHED THEN INSERT (id, v, n) VALUES (src.id, src.v, 0) +---- +physical_plan :(?s).*(Postgres Query.*MERGE INTO "public"."rp_merge_t" AS "t" USING "public"."rp_merge_s" AS "src" ON \("t"."id" = "src"."id"\) WHEN MATCHED AND "src"."del" THEN DELETE WHEN MATCHED THEN UPDATE SET "v" = "src"."v", "n" = \("t"."n" \+ 1\) WHEN NOT MATCHED THEN INSERT \("id", "v", "n"\) VALUES \("src"."id", "src"."v", 0\)|Merge Into.*Pg Delete.*Pg Update.*Pg Insert).* + +# the statement returns the number of changed rows +query I +MERGE INTO s.rp_merge_t t USING s.rp_merge_s src ON t.id = src.id WHEN MATCHED AND src.del THEN DELETE WHEN MATCHED THEN UPDATE SET v = src.v, n = t.n + 1 WHEN NOT MATCHED THEN INSERT (id, v, n) VALUES (src.id, src.v, 0) +---- +3 + +query III +SELECT * FROM s.rp_merge_t ORDER BY id +---- +1 A 11 +3 c 30 +4 D 0 + +# USING, DO NOTHING and WHEN NOT MATCHED BY SOURCE +statement ok +MERGE INTO s.rp_merge_t t USING (SELECT id, v FROM s.rp_merge_s WHERE id = 4) src USING (id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED BY SOURCE AND t.n > 20 THEN UPDATE SET n = -1 + +query III +SELECT * FROM s.rp_merge_t ORDER BY id +---- +1 A 11 +3 c -1 +4 D 0 + +# RETURNING requires Postgres 17 - it is not supported by the regular MERGE +statement maybe +MERGE INTO s.rp_merge_t t USING s.rp_merge_s src ON t.id = src.id WHEN MATCHED THEN UPDATE SET n = t.n * 2 RETURNING t.id, t.n, t.v || '!' +---- +RETURNING is not implemented for Postgres yet + +# MERGE with a local source is not pushed down +statement ok +CREATE TABLE local_merge AS SELECT 3 AS id + +query II +EXPLAIN MERGE INTO s.rp_merge_t t USING local_merge src ON t.id = src.id WHEN MATCHED THEN DELETE +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +MERGE INTO s.rp_merge_t t USING local_merge src ON t.id = src.id WHEN MATCHED THEN DELETE + +query I +SELECT id FROM s.rp_merge_t ORDER BY id +---- +1 +4 + +# THEN ERROR / UPDATE without SET / INSERT without values are DuckDB-specific +query II +EXPLAIN MERGE INTO s.rp_merge_t t USING (SELECT id, v, 0 AS n FROM s.rp_merge_s) src ON t.id = src.id WHEN MATCHED THEN UPDATE +---- +physical_plan :(?s).*Postgres Query.* + +query II +EXPLAIN MERGE INTO s.rp_merge_t t USING s.rp_merge_s src ON t.id = src.id WHEN MATCHED THEN ERROR +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +DROP TABLE s.rp_merge_t + +statement ok +DROP TABLE s.rp_merge_s diff --git a/test/sql/remote_pushdown/remote_pushdown_misc.test b/test/sql/remote_pushdown/remote_pushdown_misc.test new file mode 100644 index 000000000..1b0708ec5 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_misc.test @@ -0,0 +1,423 @@ +# name: test/sql/remote_pushdown/remote_pushdown_misc.test +# description: Test miscellaneous constructs in remote pushdown into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test miscellaneous constructs in remote pushdown into Postgres +statement ok +CREATE OR REPLACE TABLE s.rp_misc AS SELECT i, i % 3 AS g, CASE WHEN i % 4 = 0 THEN NULL ELSE 'v' || (i % 4) END AS v FROM range(10) t(i) + +# LIMIT / OFFSET +query I +SELECT i FROM s.rp_misc ORDER BY i LIMIT 3 OFFSET 2 +---- +2 +3 +4 + +query II +EXPLAIN SELECT i FROM s.rp_misc ORDER BY i LIMIT 3 OFFSET 2 +---- +physical_plan :(?s).*LIMIT 3 OFFSET 2.* + +query I +SELECT i FROM s.rp_misc ORDER BY i OFFSET 8 +---- +8 +9 + +query II +EXPLAIN SELECT i FROM s.rp_misc ORDER BY i OFFSET 8 +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_misc ORDER BY i LIMIT 20 PERCENT +---- +0 +1 + +query II +EXPLAIN SELECT i FROM s.rp_misc ORDER BY i LIMIT 20 PERCENT +---- +physical_plan :(?s).*Postgres Query.* + +# DISTINCT / DISTINCT ON +query II +SELECT DISTINCT g, v FROM s.rp_misc ORDER BY g, v +---- +0 v1 +0 v2 +0 v3 +0 NULL +1 v1 +1 v3 +1 NULL +2 v1 +2 v2 +2 NULL + +query II +EXPLAIN SELECT DISTINCT g, v FROM s.rp_misc ORDER BY g +---- +physical_plan :(?s).*SELECT DISTINCT "g", "v".* + +query I rowsort +SELECT COUNT(*) FROM (SELECT DISTINCT ON (g) g, i FROM s.rp_misc) +---- +3 + +query II +SELECT DISTINCT ON (g) g, i FROM s.rp_misc ORDER BY g, i DESC +---- +0 9 +1 7 +2 8 + +query II +EXPLAIN SELECT DISTINCT ON (g) g, i FROM s.rp_misc ORDER BY g, i DESC +---- +physical_plan :(?s).*Postgres Query.* + +# CASE expressions +query III +SELECT i, CASE WHEN i < 3 THEN 'low' WHEN i < 6 THEN 'mid' END, CASE g WHEN 0 THEN 'zero' ELSE 'other' END FROM s.rp_misc ORDER BY i +---- +0 low zero +1 low other +2 low other +3 mid zero +4 mid other +5 mid other +6 NULL zero +7 NULL other +8 NULL other +9 NULL zero + +query II +EXPLAIN SELECT i, CASE WHEN i < 3 THEN 'low' WHEN i < 6 THEN 'mid' END, CASE g WHEN 0 THEN 'zero' ELSE 'other' END FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# predicates +query I +SELECT i FROM s.rp_misc WHERE v IS DISTINCT FROM 'v1' AND v IS NOT DISTINCT FROM v ORDER BY i +---- +0 +2 +3 +4 +6 +7 +8 + +query II +EXPLAIN SELECT i FROM s.rp_misc WHERE v IS DISTINCT FROM 'v1' AND v IS NOT DISTINCT FROM v ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_misc WHERE i BETWEEN 2 AND 6 AND i NOT BETWEEN 3 AND 4 ORDER BY i +---- +2 +5 +6 + +query II +EXPLAIN SELECT i FROM s.rp_misc WHERE i BETWEEN 2 AND 6 AND i NOT BETWEEN 3 AND 4 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_misc WHERE (i > 7) IS TRUE OR (v IS NULL) IS NOT FALSE ORDER BY i +---- +0 +4 +8 +9 + +query II +EXPLAIN SELECT i FROM s.rp_misc WHERE (i > 7) IS TRUE OR (v IS NULL) IS NOT FALSE ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT i, NOT (i > 3 AND g = 1), i > 3 OR v IS NULL FROM s.rp_misc ORDER BY i +---- +0 true true +1 true false +2 true false +3 true false +4 false true +5 true true +6 true true +7 false true +8 true true +9 true true + +query II +EXPLAIN SELECT i, NOT (i > 3 AND g = 1), i > 3 OR v IS NULL FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT i, COALESCE(v, 'none'), ifnull(v, '-') FROM s.rp_misc ORDER BY i +---- +0 none - +1 v1 v1 +2 v2 v2 +3 v3 v3 +4 none - +5 v1 v1 +6 v2 v2 +7 v3 v3 +8 none - +9 v1 v1 + +query II +EXPLAIN SELECT i, COALESCE(v, 'none'), ifnull(v, '-') FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# NULLIF is a macro in DuckDB - macros from local catalogs prevent pushdown +query II +SELECT i, NULLIF(g, 0) FROM s.rp_misc ORDER BY i +---- +0 NULL +1 1 +2 2 +3 NULL +4 1 +5 2 +6 NULL +7 1 +8 2 +9 NULL + +query II +EXPLAIN SELECT i, NULLIF(g, 0) FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# select-list aliases can be used in GROUP BY / ORDER BY, but not in the WHERE clause in Postgres +query II +SELECT i % 2 AS parity, COUNT(*) FROM s.rp_misc GROUP BY parity ORDER BY parity +---- +0 5 +1 5 + +query II +EXPLAIN SELECT i % 2 AS parity, COUNT(*) FROM s.rp_misc GROUP BY parity ORDER BY parity +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i * 2 AS doubled FROM s.rp_misc WHERE doubled > 10 ORDER BY doubled +---- +12 +14 +16 +18 + +query II +EXPLAIN SELECT i * 2 AS doubled FROM s.rp_misc WHERE doubled > 10 ORDER BY doubled +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i * 2 AS doubled, doubled + 1 FROM s.rp_misc ORDER BY i +---- +0 1 +2 3 +4 5 +6 7 +8 9 +10 11 +12 13 +14 15 +16 17 +18 19 + +query II +EXPLAIN SELECT i * 2 AS doubled, doubled + 1 FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# an alias that shadows a real column - whether the WHERE clause references the alias or the column cannot be +# determined before binding, so the query is conservatively executed locally +query I +SELECT g AS i FROM s.rp_misc WHERE i > 5 ORDER BY 1 +---- +0 +0 +1 +2 + +query II +EXPLAIN SELECT g AS i FROM s.rp_misc WHERE i > 5 ORDER BY 1 +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, g AS i2 FROM s.rp_misc WHERE i > 5 ORDER BY 1 +---- +6 0 +7 1 +8 2 +9 0 + +query II +EXPLAIN SELECT i, g AS i2 FROM s.rp_misc WHERE i > 5 ORDER BY 1 +---- +physical_plan :(?s).*Postgres Query.* + +# queries without remote tables are not pushed down +query I rowsort +SELECT 42 +---- +42 + +query II +EXPLAIN SELECT 42 +---- +physical_plan :(?s).*Postgres Query.* + +# DuckDB-specific constructs are executed locally +query II +SELECT * EXCLUDE (v) FROM s.rp_misc ORDER BY i +---- +0 0 +1 1 +2 2 +3 0 +4 1 +5 2 +6 0 +7 1 +8 2 +9 0 + +query II +EXPLAIN SELECT * EXCLUDE (v) FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT COLUMNS('i|g') FROM s.rp_misc ORDER BY i +---- +0 0 +1 1 +2 2 +3 0 +4 1 +5 2 +6 0 +7 1 +8 2 +9 0 + +query II +EXPLAIN SELECT COLUMNS('i|g') FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT i FROM s.rp_misc USING SAMPLE 100% +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +query II +EXPLAIN SELECT i FROM s.rp_misc USING SAMPLE 100% +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT TRY(i // 0) FROM s.rp_misc +---- +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL + +query II +EXPLAIN SELECT TRY(i // 0) FROM s.rp_misc +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT [i, g] FROM s.rp_misc ORDER BY i +---- +[0, 0] +[1, 1] +[2, 2] +[3, 0] +[4, 1] +[5, 2] +[6, 0] +[7, 1] +[8, 2] +[9, 0] + +query II +EXPLAIN SELECT [i, g] FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT {'a': i} FROM s.rp_misc ORDER BY i +---- +{'a': 0} +{'a': 1} +{'a': 2} +{'a': 3} +{'a': 4} +{'a': 5} +{'a': 6} +{'a': 7} +{'a': 8} +{'a': 9} + +query II +EXPLAIN SELECT {'a': i} FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i::VARCHAR COLLATE nocase FROM s.rp_misc ORDER BY i +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +query II +EXPLAIN SELECT i::VARCHAR COLLATE nocase FROM s.rp_misc ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_order_nulls.test b/test/sql/remote_pushdown/remote_pushdown_order_nulls.test new file mode 100644 index 000000000..f75ca32d6 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_order_nulls.test @@ -0,0 +1,265 @@ +# name: test/sql/remote_pushdown/remote_pushdown_order_nulls.test +# description: Test NULL ordering in remote pushdown into Postgres - Postgres sorts NULLs as larger than any value +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test NULL ordering in remote pushdown into Postgres - Postgres sorts NULLs as larger than any value +statement ok +CREATE OR REPLACE TABLE s.rp_nulls AS SELECT i, CASE WHEN i % 3 = 0 THEN NULL ELSE i % 4 END AS n, CASE WHEN i % 4 = 1 THEN NULL ELSE 'v' || (i % 3) END AS v FROM range(9) t(i) + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +4 0 +8 0 +1 1 +5 1 +2 2 +7 3 +0 NULL +3 NULL +6 NULL + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +physical_plan :(?s).*ORDER BY "n" ASC NULLS LAST, "i" ASC NULLS LAST.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +7 3 +2 2 +1 1 +5 1 +4 0 +8 0 +0 NULL +3 NULL +6 NULL + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +physical_plan :(?s).*ORDER BY "n" DESC NULLS LAST.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n NULLS FIRST, i DESC +---- +6 NULL +3 NULL +0 NULL +8 0 +4 0 +5 1 +1 1 +2 2 +7 3 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n NULLS FIRST, i DESC +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n DESC NULLS FIRST, i +---- +0 NULL +3 NULL +6 NULL +7 3 +2 2 +1 1 +5 1 +4 0 +8 0 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n DESC NULLS FIRST, i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, v FROM s.rp_nulls ORDER BY v DESC, i +---- +2 v2 +8 v2 +4 v1 +7 v1 +0 v0 +3 v0 +6 v0 +1 NULL +5 NULL + +query II +EXPLAIN SELECT i, v FROM s.rp_nulls ORDER BY v DESC, i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY 2, 1 LIMIT 4 +---- +4 0 +8 0 +1 1 +5 1 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY 2, 1 LIMIT 4 +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT n, COUNT(*) FROM s.rp_nulls GROUP BY n ORDER BY n DESC +---- +3 1 +2 1 +1 2 +0 2 +NULL 3 + +query II +EXPLAIN SELECT n, COUNT(*) FROM s.rp_nulls GROUP BY n ORDER BY n DESC +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, row_number() OVER (ORDER BY n DESC, i) FROM s.rp_nulls ORDER BY i +---- +0 7 +1 3 +2 2 +3 8 +4 5 +5 4 +6 9 +7 1 +8 6 + +query II +EXPLAIN SELECT i, row_number() OVER (ORDER BY n DESC, i) FROM s.rp_nulls ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT string_agg(i, ',' ORDER BY n NULLS FIRST, i) FROM (SELECT i::VARCHAR i, n FROM s.rp_nulls) +---- +0,3,6,4,8,1,5,2,7 + +query II +EXPLAIN SELECT string_agg(i, ',' ORDER BY n NULLS FIRST, i) FROM (SELECT i::VARCHAR i, n FROM s.rp_nulls) +---- +physical_plan :(?s).*Postgres Query.* + +# the default NULL order setting is respected +statement ok +SET default_null_order = 'nulls_first' + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +0 NULL +3 NULL +6 NULL +4 0 +8 0 +1 1 +5 1 +2 2 +7 3 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +0 NULL +3 NULL +6 NULL +7 3 +2 2 +1 1 +5 1 +4 0 +8 0 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +SET default_null_order = 'nulls_first_on_asc_last_on_desc' + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +0 NULL +3 NULL +6 NULL +4 0 +8 0 +1 1 +5 1 +2 2 +7 3 + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +7 3 +2 2 +1 1 +5 1 +4 0 +8 0 +0 NULL +3 NULL +6 NULL + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n DESC, i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +SET default_order = 'DESC' + +query II +SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +7 3 +2 2 +5 1 +1 1 +8 0 +4 0 +6 NULL +3 NULL +0 NULL + +query II +EXPLAIN SELECT i, n FROM s.rp_nulls ORDER BY n, i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +RESET default_order + +statement ok +RESET default_null_order diff --git a/test/sql/remote_pushdown/remote_pushdown_regressions.test b/test/sql/remote_pushdown/remote_pushdown_regressions.test new file mode 100644 index 000000000..3ff87e0bb --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_regressions.test @@ -0,0 +1,262 @@ +# name: test/sql/remote_pushdown/remote_pushdown_regressions.test +# description: Regression tests for the translation of expressions in remote pushdown into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Regression tests for the translation of expressions in remote pushdown into Postgres +statement ok +CREATE OR REPLACE TABLE s.rp_reg AS SELECT i::INTEGER AS i, i % 3 AS g, CASE WHEN i % 4 = 0 THEN NULL ELSE 'v' || (i % 4) END AS v, DATE '2024-01-01' + i::INTEGER AS d, i % 2 = 0 AS b, (i * 1.5)::DOUBLE AS f FROM range(8) t(i) + +# typed NULLs +query III +SELECT i, CAST(NULL AS INTEGER) + i, COALESCE(NULL::VARCHAR, v) FROM s.rp_reg ORDER BY i +---- +0 NULL NULL +1 NULL v1 +2 NULL v2 +3 NULL v3 +4 NULL NULL +5 NULL v1 +6 NULL v2 +7 NULL v3 + +query II +EXPLAIN SELECT i, CAST(NULL AS INTEGER) + i, COALESCE(NULL::VARCHAR, v) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# comparisons of columns with constants of other types +query I +SELECT i FROM s.rp_reg WHERE d > '2024-01-03' ORDER BY i +---- +3 +4 +5 +6 +7 + +query II +EXPLAIN SELECT i FROM s.rp_reg WHERE d > '2024-01-03' ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_reg WHERE i = '3' OR f < 2 ORDER BY i +---- +0 +1 +3 + +query II +EXPLAIN SELECT i FROM s.rp_reg WHERE i = '3' OR f < 2 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates of booleans and floats +query IIII rowsort +SELECT min(b), max(b), sum(f), avg(i) FROM s.rp_reg +---- +false true 42.0 3.5 + +query II +EXPLAIN SELECT min(b), max(b), sum(f), avg(i) FROM s.rp_reg +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT g, max(b), min(v) FROM s.rp_reg GROUP BY g ORDER BY g +---- +0 true v2 +1 true v1 +2 true v1 + +query II +EXPLAIN SELECT g, max(b), min(v) FROM s.rp_reg GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# integer arguments of string functions +query IIIII +SELECT i, left(v, i), right(v, g), lpad(v, i, '*'), repeat(v, g) FROM s.rp_reg ORDER BY i +---- +0 NULL NULL NULL NULL +1 v 1 v v1 +2 v2 v2 v2 v2v2 +3 v3 (empty) *v3 (empty) +4 NULL NULL NULL NULL +5 v1 v1 ***v1 v1v1 +6 v2 (empty) ****v2 (empty) +7 v3 3 *****v3 v3 + +query II +EXPLAIN SELECT i, left(v, i), right(v, g), lpad(v, i, '*'), repeat(v, g) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, substr(v, g, 2) FROM s.rp_reg ORDER BY i +---- +0 NULL +1 v1 +2 2 +3 v +4 NULL +5 1 +6 v +7 v3 + +query II +EXPLAIN SELECT i, substr(v, g, 2) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, substr(v, 2, 2) FROM s.rp_reg ORDER BY i +---- +0 NULL +1 1 +2 2 +3 3 +4 NULL +5 1 +6 2 +7 3 + +query II +EXPLAIN SELECT i, substr(v, 2, 2) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# unicode of empty strings +query II +SELECT i, unicode(COALESCE(v, '')) FROM s.rp_reg ORDER BY i +---- +0 -1 +1 118 +2 118 +3 118 +4 -1 +5 118 +6 118 +7 118 + +query II +EXPLAIN SELECT i, unicode(COALESCE(v, '')) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# date parts of BC dates +query III +SELECT year(DATE '0044-03-15' - INTERVAL 100 YEAR), yearweek(d), isoyear(d) FROM s.rp_reg ORDER BY i LIMIT 2 +---- +-56 202401 2024 +-56 202401 2024 + +query II +SELECT i, year(d - INTERVAL 2100 YEAR) FROM s.rp_reg ORDER BY i +---- +0 -76 +1 -76 +2 -76 +3 -76 +4 -76 +5 -76 +6 -76 +7 -76 + +query II +EXPLAIN SELECT i, year(d - INTERVAL 2100 YEAR) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# DISTINCT with ORDER BY on columns of the select list +query II +SELECT DISTINCT g, v FROM s.rp_reg ORDER BY v NULLS FIRST, g +---- +0 NULL +1 NULL +1 v1 +2 v1 +0 v2 +2 v2 +0 v3 +1 v3 + +query II +EXPLAIN SELECT DISTINCT g, v FROM s.rp_reg ORDER BY g DESC +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT DISTINCT g FROM s.rp_reg ORDER BY i +---- +0 +1 +2 + +query II +EXPLAIN SELECT DISTINCT g FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# RANGE frame offsets +query II +SELECT i, sum(i) OVER (ORDER BY i RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM s.rp_reg ORDER BY i +---- +0 0 +1 1 +2 3 +3 6 +4 9 +5 12 +6 15 +7 18 + +query II +EXPLAIN SELECT i, sum(i) OVER (ORDER BY i RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, count(*) OVER (ORDER BY d RANGE BETWEEN INTERVAL 2 DAY PRECEDING AND INTERVAL 1 DAY FOLLOWING) FROM s.rp_reg ORDER BY i +---- +0 2 +1 3 +2 4 +3 4 +4 4 +5 4 +6 4 +7 3 + +query II +EXPLAIN SELECT i, count(*) OVER (ORDER BY d RANGE BETWEEN INTERVAL 2 DAY PRECEDING AND INTERVAL 1 DAY FOLLOWING) FROM s.rp_reg ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# ordering by expressions +query II +SELECT i, v FROM s.rp_reg ORDER BY v DESC NULLS LAST, i % 3, i +---- +3 v3 +7 v3 +6 v2 +2 v2 +1 v1 +5 v1 +0 NULL +4 NULL + +query II +EXPLAIN SELECT i, v FROM s.rp_reg ORDER BY v DESC NULLS LAST, i % 3, i +---- +physical_plan :(?s).*Postgres Query.* + +statement ok +DROP TABLE s.rp_reg diff --git a/test/sql/remote_pushdown/remote_pushdown_set_operations.test b/test/sql/remote_pushdown/remote_pushdown_set_operations.test new file mode 100644 index 000000000..0d5d89145 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_set_operations.test @@ -0,0 +1,242 @@ +# name: test/sql/remote_pushdown/remote_pushdown_set_operations.test +# description: Test remote pushdown of set operations into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_setop1 AS SELECT i, i % 3 AS g FROM range(6) t(i) + +statement ok +CREATE OR REPLACE TABLE s.rp_setop2 AS SELECT i FROM range(3, 9) t(i) + +query I +SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i +---- +0 +1 +2 +3 +3 +4 +4 +5 +5 +6 +7 +8 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i +---- +physical_plan :(?s).*\(SELECT "i" FROM "public"\."rp_setop1"\) UNION ALL \(SELECT "i" FROM "public"\."rp_setop2"\) ORDER BY "i" ASC NULLS LAST.* + +query I +SELECT i FROM s.rp_setop1 UNION SELECT i FROM s.rp_setop2 ORDER BY 1 DESC +---- +8 +7 +6 +5 +4 +3 +2 +1 +0 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 UNION SELECT i FROM s.rp_setop2 ORDER BY 1 DESC +---- +physical_plan :(?s).*UNION \(SELECT.* + +query I +SELECT i FROM s.rp_setop1 EXCEPT SELECT i FROM s.rp_setop2 ORDER BY i +---- +0 +1 +2 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 EXCEPT SELECT i FROM s.rp_setop2 ORDER BY i +---- +physical_plan :(?s).*EXCEPT \(SELECT.* + +query I +SELECT g FROM s.rp_setop1 EXCEPT ALL SELECT i FROM s.rp_setop2 ORDER BY g +---- +0 +0 +1 +1 +2 +2 + +query II +EXPLAIN SELECT g FROM s.rp_setop1 EXCEPT ALL SELECT i FROM s.rp_setop2 ORDER BY g +---- +physical_plan :(?s).*EXCEPT ALL \(SELECT.* + +query I +SELECT i FROM s.rp_setop1 INTERSECT SELECT i FROM s.rp_setop2 ORDER BY i +---- +3 +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 INTERSECT SELECT i FROM s.rp_setop2 ORDER BY i +---- +physical_plan :(?s).*INTERSECT \(SELECT.* + +query I +SELECT g FROM s.rp_setop1 INTERSECT ALL SELECT g FROM s.rp_setop1 WHERE i > 1 ORDER BY g +---- +0 +1 +2 +2 + +query II +EXPLAIN SELECT g FROM s.rp_setop1 INTERSECT ALL SELECT g FROM s.rp_setop1 WHERE i > 1 ORDER BY g +---- +physical_plan :(?s).*INTERSECT ALL \(SELECT.* + +# nested set operations +query I +(SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2) EXCEPT SELECT g FROM s.rp_setop1 ORDER BY i +---- +3 +4 +5 +6 +7 +8 + +query II +EXPLAIN (SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2) EXCEPT SELECT g FROM s.rp_setop1 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# set operations with a LIMIT +query I +SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i DESC LIMIT 3 +---- +8 +7 +6 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i DESC LIMIT 3 +---- +physical_plan :(?s).*Postgres Query.* + +query I +(SELECT i FROM s.rp_setop1 ORDER BY i DESC LIMIT 2) UNION ALL (SELECT i FROM s.rp_setop2 ORDER BY i LIMIT 2) ORDER BY i +---- +3 +4 +4 +5 + +query II +EXPLAIN (SELECT i FROM s.rp_setop1 ORDER BY i DESC LIMIT 2) UNION ALL (SELECT i FROM s.rp_setop2 ORDER BY i LIMIT 2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates in set operations keep their exact types +query I +SELECT SUM(i) AS s FROM s.rp_setop1 UNION ALL SELECT SUM(i) FROM s.rp_setop2 ORDER BY s +---- +15 +33 + +query II +EXPLAIN SELECT SUM(i) AS s FROM s.rp_setop1 UNION ALL SELECT SUM(i) FROM s.rp_setop2 ORDER BY s +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT typeof(s) FROM (SELECT SUM(i) AS s FROM s.rp_setop1 UNION ALL SELECT SUM(i) FROM s.rp_setop2) LIMIT 1 +---- +HUGEINT + +# a set operation over constants and a remote table +query I +SELECT i FROM s.rp_setop1 UNION SELECT 42 ORDER BY i +---- +0 +1 +2 +3 +4 +5 +42 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 UNION SELECT 42 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# ordering the result of a set operation by an expression is not supported in Postgres +query I +SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i % 4, i +---- +0 +4 +4 +8 +1 +5 +5 +2 +6 +3 +3 +7 + +query II +EXPLAIN SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM s.rp_setop2 ORDER BY i % 4, i +---- +physical_plan :(?s).*Postgres Query.* + +# UNION BY NAME is DuckDB-specific +query II +SELECT i, g FROM s.rp_setop1 UNION BY NAME SELECT i FROM s.rp_setop2 ORDER BY i, g +---- +0 0 +1 1 +2 2 +3 0 +3 NULL +4 1 +4 NULL +5 2 +5 NULL +6 NULL +7 NULL +8 NULL + +query II +EXPLAIN SELECT i, g FROM s.rp_setop1 UNION BY NAME SELECT i FROM s.rp_setop2 ORDER BY i, g +---- +physical_plan :(?s).*Postgres Query.* + +# a set operation with a local table is not pushed down as a whole - but the remote side is +statement ok +CREATE TABLE local_setop AS SELECT 100 AS i + +query I +SELECT i FROM s.rp_setop1 UNION ALL SELECT i FROM local_setop ORDER BY i +---- +0 +1 +2 +3 +4 +5 +100 diff --git a/test/sql/remote_pushdown/remote_pushdown_subquery.test b/test/sql/remote_pushdown/remote_pushdown_subquery.test new file mode 100644 index 000000000..d326b02ab --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_subquery.test @@ -0,0 +1,236 @@ +# name: test/sql/remote_pushdown/remote_pushdown_subquery.test +# description: Test remote pushdown of subqueries into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_sq1 AS SELECT i, i % 3 AS g, CASE WHEN i = 4 THEN NULL ELSE i END AS n FROM range(6) t(i) + +statement ok +CREATE OR REPLACE TABLE s.rp_sq2 AS SELECT i AS k, i * 10 AS w FROM range(2, 9) t(i) + +# scalar subqueries +query II +SELECT i, (SELECT MAX(w) FROM s.rp_sq2 WHERE k = i) FROM s.rp_sq1 ORDER BY i +---- +0 NULL +1 NULL +2 20 +3 30 +4 40 +5 50 + +query II +EXPLAIN SELECT i, (SELECT MAX(w) FROM s.rp_sq2 WHERE k = i) FROM s.rp_sq1 ORDER BY i +---- +physical_plan :(?s).*\(SELECT max\("w"\) AS "max\(w\)" FROM "public"\."rp_sq2" WHERE \("k" = "i"\)\).* + +query I +SELECT i FROM s.rp_sq1 WHERE i > (SELECT AVG(k) FROM s.rp_sq2) ORDER BY i +---- + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i > (SELECT AVG(k) FROM s.rp_sq2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT (SELECT COUNT(*) FROM s.rp_sq1), (SELECT SUM(w) FROM s.rp_sq2) +---- +6 350 + +query II +EXPLAIN SELECT (SELECT COUNT(*) FROM s.rp_sq1), (SELECT SUM(w) FROM s.rp_sq2) +---- +physical_plan :(?s).*Postgres Query.* + +# EXISTS +query I +SELECT i FROM s.rp_sq1 WHERE EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i) ORDER BY i +---- +2 +3 +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i) ORDER BY i +---- +physical_plan :(?s).*EXISTS\(SELECT 1.* + +query I +SELECT i FROM s.rp_sq1 WHERE NOT EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i) ORDER BY i +---- +0 +1 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE NOT EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i) ORDER BY i +---- +physical_plan :(?s).*NOT EXISTS\(SELECT 1.* + +query II +SELECT i, EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i * 2) FROM s.rp_sq1 ORDER BY i +---- +0 false +1 true +2 true +3 true +4 true +5 false + +query II +EXPLAIN SELECT i, EXISTS (SELECT 1 FROM s.rp_sq2 WHERE k = i * 2) FROM s.rp_sq1 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# IN / ANY / ALL +query I +SELECT i FROM s.rp_sq1 WHERE i IN (SELECT k FROM s.rp_sq2) ORDER BY i +---- +2 +3 +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i IN (SELECT k FROM s.rp_sq2) ORDER BY i +---- +physical_plan :(?s).*\("i" = ANY\(SELECT "k" FROM "public"\."rp_sq2"\)\).* + +query I +SELECT i FROM s.rp_sq1 WHERE i NOT IN (SELECT k FROM s.rp_sq2) ORDER BY i +---- +0 +1 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i NOT IN (SELECT k FROM s.rp_sq2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT i, n IN (SELECT k FROM s.rp_sq2), n NOT IN (SELECT k FROM s.rp_sq2 WHERE k < 4) FROM s.rp_sq1 ORDER BY i +---- +0 false true +1 false true +2 true false +3 true false +4 NULL NULL +5 true true + +query II +EXPLAIN SELECT i, n IN (SELECT k FROM s.rp_sq2), n NOT IN (SELECT k FROM s.rp_sq2 WHERE k < 4) FROM s.rp_sq1 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_sq1 WHERE i > ALL (SELECT g FROM s.rp_sq1) ORDER BY i +---- +3 +4 +5 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i > ALL (SELECT g FROM s.rp_sq1) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_sq1 WHERE i < ANY (SELECT k FROM s.rp_sq2 WHERE k < 4) ORDER BY i +---- +0 +1 +2 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i < ANY (SELECT k FROM s.rp_sq2 WHERE k < 4) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_sq1 WHERE i IN (SELECT k FROM s.rp_sq2 ORDER BY k LIMIT 2) ORDER BY i +---- +2 +3 + +query II +EXPLAIN SELECT i FROM s.rp_sq1 WHERE i IN (SELECT k FROM s.rp_sq2 ORDER BY k LIMIT 2) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# NULL semantics of IN lists +query IIII +SELECT i, n IN (1, 2, NULL), n NOT IN (3, 4), n NOT IN (3, NULL) FROM s.rp_sq1 ORDER BY i +---- +0 NULL true NULL +1 true true NULL +2 true true NULL +3 NULL false false +4 NULL NULL NULL +5 NULL true NULL + +query II +EXPLAIN SELECT i, n IN (1, 2, NULL), n NOT IN (3, 4), n NOT IN (3, NULL) FROM s.rp_sq1 ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# subqueries in FROM +query II +SELECT g, MAX(s) FROM (SELECT g, i + n AS s FROM s.rp_sq1) GROUP BY g ORDER BY g +---- +0 6 +1 2 +2 10 + +query II +EXPLAIN SELECT g, MAX(s) FROM (SELECT g, i + n AS s FROM s.rp_sq1) GROUP BY g ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT * FROM (SELECT * FROM (SELECT i, n FROM s.rp_sq1 WHERE i > 1) WHERE n IS NOT NULL) t ORDER BY i +---- +2 2 +3 3 +5 5 + +query II +EXPLAIN SELECT * FROM (SELECT * FROM (SELECT i, n FROM s.rp_sq1 WHERE i > 1) WHERE n IS NOT NULL) t ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT t.x, t.y FROM (SELECT i, g FROM s.rp_sq1) t(x, y) ORDER BY x +---- +0 0 +1 1 +2 2 +3 0 +4 1 +5 2 + +query II +EXPLAIN SELECT t.x, t.y FROM (SELECT i, g FROM s.rp_sq1) t(x, y) ORDER BY x +---- +physical_plan :(?s).*Postgres Query.* + +# correlated subqueries referencing a local table are not pushed down +statement ok +CREATE TABLE local_sq AS SELECT 3 AS i + +query II rowsort +SELECT i, (SELECT MAX(w) FROM s.rp_sq2 WHERE k = local_sq.i) FROM local_sq +---- +3 30 + +query II +EXPLAIN SELECT i, (SELECT MAX(w) FROM s.rp_sq2 WHERE k = local_sq.i) FROM local_sq +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_types.test b/test/sql/remote_pushdown/remote_pushdown_types.test new file mode 100644 index 000000000..f27ebe3e3 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_types.test @@ -0,0 +1,113 @@ +# name: test/sql/remote_pushdown/remote_pushdown_types.test +# description: Test remote pushdown of queries over Postgres types that DuckDB reads as different types +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +# DESC: Test remote pushdown of queries over Postgres types that DuckDB reads as different types +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS rp_types') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE rp_types (id INT, ip INET, m MONEY, j JSONB, js JSON, n NUMERIC, b BIT(3), c CHAR(4))') + +statement ok +CALL postgres_execute('s', 'INSERT INTO rp_types VALUES (1, ''10.0.0.1'', 12.5, ''{"a": 1}'', ''{"b": 2}'', 1.5, B''101'', ''ab''), (2, ''::1'', 3, ''[1]'', ''[]'', 0.1, B''001'', ''abcd''), (3, ''10.0.0.1/8'', 100, ''{}'', ''{}'', 1e20, NULL, NULL)') + +statement ok +CALL postgres_clear_cache() + +# references to these columns are cast to the type DuckDB reads them as - so they are compared / ordered the same +query II +SELECT id, m FROM s.rp_types ORDER BY m +---- +3 $100.00 +1 $12.50 +2 $3.00 + +query II +EXPLAIN SELECT id, m FROM s.rp_types ORDER BY m +---- +physical_plan :(?s).*ORDER BY (CAST\("m" AS TEXT\)|CAST\(CAST\("m" AS TEXT\) AS TEXT\) COLLATE "C") ASC NULLS LAST.* + +query I rowsort +SELECT COUNT(*) FROM s.rp_types WHERE ip = '10.0.0.1/32' +---- +1 + +query II +EXPLAIN SELECT COUNT(*) FROM s.rp_types WHERE ip = '10.0.0.1/32' +---- +physical_plan :(?s).*WHERE \(CAST\("ip" AS TEXT\) = '10\.0\.0\.1/32'\).* + +query II +SELECT ip, COUNT(*) FROM s.rp_types GROUP BY ip ORDER BY ip +---- +10.0.0.1/32 1 +10.0.0.1/8 1 +::1/128 1 + +query II +EXPLAIN SELECT ip, COUNT(*) FROM s.rp_types GROUP BY ip ORDER BY ip +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT MIN(j), MAX(js), MAX(b), MIN(c), string_agg(ip, ',' ORDER BY ip) FROM s.rp_types +---- +[1] {} 101 ab 10.0.0.1/32,10.0.0.1/8,::1/128 + +query II +EXPLAIN SELECT MIN(j), MAX(js), MAX(b), MIN(c), string_agg(ip, ',' ORDER BY ip) FROM s.rp_types +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT id, length(ip), j || '!', lower(js) FROM s.rp_types ORDER BY id +---- +1 11 {"a": 1}! {"b": 2} +2 7 [1]! [] +3 10 {}! {} + +query II +EXPLAIN SELECT id, length(ip), j || '!', lower(js) FROM s.rp_types ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT id, c, length(c), c || '!', c = 'ab' FROM s.rp_types ORDER BY id +---- +1 ab 2 ab! true +2 abcd 4 abcd! false +3 NULL NULL NULL NULL + +query II +EXPLAIN SELECT id, c, length(c), c || '!', c = 'ab' FROM s.rp_types ORDER BY id +---- +physical_plan :(?s).*Postgres Query.* + +# unconstrained NUMERIC values are read as DOUBLE +query III +SELECT id, n * 3, SUM(n) OVER () FROM s.rp_types ORDER BY id +---- +1 4.5 1e+20 +2 0.30000000000000004 1e+20 +3 3e+20 1e+20 + +query II +EXPLAIN SELECT id, n * 3, SUM(n) OVER () FROM s.rp_types ORDER BY id +---- +physical_plan :(?s).*\(CAST\("n" AS DOUBLE PRECISION\) \* 3\).* + +query II rowsort +SELECT typeof(n), typeof(SUM(n)) FROM s.rp_types GROUP BY ALL +---- +DOUBLE DOUBLE + +statement ok +CALL postgres_execute('s', 'DROP TABLE rp_types') diff --git a/test/sql/remote_pushdown/remote_pushdown_values.test b/test/sql/remote_pushdown/remote_pushdown_values.test new file mode 100644 index 000000000..05889fae8 --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_values.test @@ -0,0 +1,275 @@ +# name: test/sql/remote_pushdown/remote_pushdown_values.test +# description: Test remote pushdown of VALUES lists and table functions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_vals AS SELECT i, 'v' || i AS v FROM range(5) t(i) + +# VALUES lists are named the way DuckDB names them +query IIII +SELECT * FROM s.rp_vals JOIN (VALUES (1, 'a'), (3, 'c')) t(x, y) ON i = x ORDER BY i +---- +1 v1 1 a +3 v3 3 c + +query II +EXPLAIN SELECT * FROM s.rp_vals JOIN (VALUES (1, 'a'), (3, 'c')) t(x, y) ON i = x ORDER BY i +---- +physical_plan :(?s).*\(VALUES \(1, 'a'\), \(3, 'c'\)\) AS "valueslist"\("col0", "col1"\).* + +query III +SELECT i, col0, col1 FROM s.rp_vals, (VALUES (10, 'x'), (NULL, NULL)) ORDER BY i, col0 +---- +0 10 x +0 NULL NULL +1 10 x +1 NULL NULL +2 10 x +2 NULL NULL +3 10 x +3 NULL NULL +4 10 x +4 NULL NULL + +query II +EXPLAIN SELECT i, col0, col1 FROM s.rp_vals, (VALUES (10, 'x'), (NULL, NULL)) ORDER BY i, col0 +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT * FROM (VALUES (1), (3), (NULL)) t(x) LEFT JOIN s.rp_vals ON x = i ORDER BY x +---- +1 1 v1 +3 3 v3 +NULL NULL NULL + +query II +EXPLAIN SELECT * FROM (VALUES (1), (3), (NULL)) t(x) LEFT JOIN s.rp_vals ON x = i ORDER BY x +---- +physical_plan :(?s).*Postgres Query.* + +query I +SELECT i FROM s.rp_vals WHERE i IN (SELECT * FROM (VALUES (2), (4))) ORDER BY i +---- +2 +4 + +query II +EXPLAIN SELECT i FROM s.rp_vals WHERE i IN (SELECT * FROM (VALUES (2), (4))) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIIII +SELECT * FROM s.rp_vals, (VALUES (DATE '2024-01-01', 1.5, true)) v(d, n, b) ORDER BY i +---- +0 v0 2024-01-01 1.5 true +1 v1 2024-01-01 1.5 true +2 v2 2024-01-01 1.5 true +3 v3 2024-01-01 1.5 true +4 v4 2024-01-01 1.5 true + +query II +EXPLAIN SELECT * FROM s.rp_vals, (VALUES (DATE '2024-01-01', 1.5, true)) v(d, n, b) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# table functions - DuckDB's range excludes the upper bound +query II +SELECT i, r FROM s.rp_vals, range(3) t(r) ORDER BY i, r +---- +0 0 +0 1 +0 2 +1 0 +1 1 +1 2 +2 0 +2 1 +2 2 +3 0 +3 1 +3 2 +4 0 +4 1 +4 2 + +query II +EXPLAIN SELECT i, r FROM s.rp_vals, range(3) t(r) ORDER BY i, r +---- +physical_plan :(?s).*generate_series\(CAST\(0 AS BIGINT\), CAST\(3 AS BIGINT\) - 1\) AS "t"\("r"\).* + +query II +SELECT i, range FROM s.rp_vals, range(1, 3) ORDER BY i, range +---- +0 1 +0 2 +1 1 +1 2 +2 1 +2 2 +3 1 +3 2 +4 1 +4 2 + +query II +EXPLAIN SELECT i, range FROM s.rp_vals, range(1, 3) ORDER BY i, range +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, t FROM s.rp_vals, range(10, 2, -3) t ORDER BY i, t +---- +0 4 +0 7 +0 10 +1 4 +1 7 +1 10 +2 4 +2 7 +2 10 +3 4 +3 7 +3 10 +4 4 +4 7 +4 10 + +query II +EXPLAIN SELECT i, t FROM s.rp_vals, range(10, 2, -3) t ORDER BY i, t +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, r FROM s.rp_vals, range(0, 7, 3) t(r) ORDER BY i, r +---- +0 0 +0 3 +0 6 +1 0 +1 3 +1 6 +2 0 +2 3 +2 6 +3 0 +3 3 +3 6 +4 0 +4 3 +4 6 + +query II +EXPLAIN SELECT i, r FROM s.rp_vals, range(0, 7, 3) t(r) ORDER BY i, r +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, generate_series FROM s.rp_vals, generate_series(1, 2) ORDER BY ALL +---- +0 1 +0 2 +1 1 +1 2 +2 1 +2 2 +3 1 +3 2 +4 1 +4 2 + +query II +EXPLAIN SELECT i, generate_series FROM s.rp_vals, generate_series(1, 2) ORDER BY ALL +---- +physical_plan :(?s).*generate_series\(CAST\(1 AS BIGINT\), CAST\(2 AS BIGINT\)\) AS "generate_series".* + +query II +SELECT i, x FROM s.rp_vals, generate_series(5, 1, -2) t(x) ORDER BY i, x +---- +0 1 +0 3 +0 5 +1 1 +1 3 +1 5 +2 1 +2 3 +2 5 +3 1 +3 3 +3 5 +4 1 +4 3 +4 5 + +query II +EXPLAIN SELECT i, x FROM s.rp_vals, generate_series(5, 1, -2) t(x) ORDER BY i, x +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT * FROM s.rp_vals, generate_series(1, i) t(x) ORDER BY i, x +---- +1 v1 1 +2 v2 1 +2 v2 2 +3 v3 1 +3 v3 2 +3 v3 3 +4 v4 1 +4 v4 2 +4 v4 3 +4 v4 4 + +query II +EXPLAIN SELECT * FROM s.rp_vals, generate_series(1, i) t(x) ORDER BY i, x +---- +physical_plan :(?s).*Postgres Query.* + +query III +SELECT i, x, ordinality FROM s.rp_vals, generate_series(i, i + 1) WITH ORDINALITY t(x, ordinality) ORDER BY i, x +---- +0 0 1 +0 1 2 +1 1 1 +1 2 2 +2 2 1 +2 3 2 +3 3 1 +3 4 2 +4 4 1 +4 5 2 + +query II +EXPLAIN SELECT i, x, ordinality FROM s.rp_vals, generate_series(i, i + 1) WITH ORDINALITY t(x, ordinality) ORDER BY i, x +---- +physical_plan :(?s).*Postgres Query.* + +query I rowsort +SELECT typeof(r) FROM s.rp_vals, range(1) t(r) LIMIT 1 +---- +BIGINT + +# table functions that are not available in Postgres (note: lists / timestamps as arguments of table functions are not +# tested, they trigger a use-after-free in the remote pushdown optimizer of DuckDB v2.0.0) +query II +SELECT i, x FROM s.rp_vals, unnest(string_split(v, '_')) t(x) ORDER BY i, x +---- +0 v0 +1 v1 +2 v2 +3 v3 +4 v4 + +query II +EXPLAIN SELECT i, x FROM s.rp_vals, unnest(string_split(v, '_')) t(x) ORDER BY i, x +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/remote_pushdown/remote_pushdown_window.test b/test/sql/remote_pushdown/remote_pushdown_window.test new file mode 100644 index 000000000..b7240935d --- /dev/null +++ b/test/sql/remote_pushdown/remote_pushdown_window.test @@ -0,0 +1,470 @@ +# name: test/sql/remote_pushdown/remote_pushdown_window.test +# description: Test remote pushdown of window functions into Postgres +# group: [remote_pushdown] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 'public') + +statement ok +CREATE OR REPLACE TABLE s.rp_window AS SELECT i, i % 3 AS g, CASE WHEN i % 4 = 0 THEN NULL ELSE i * 10 END AS v, (i * 1.25)::DECIMAL(10,2) AS d FROM range(10) t(i) + +query II +SELECT i, row_number() OVER (ORDER BY i DESC) FROM s.rp_window ORDER BY i +---- +0 10 +1 9 +2 8 +3 7 +4 6 +5 5 +6 4 +7 3 +8 2 +9 1 + +query II +EXPLAIN SELECT i, row_number() OVER (ORDER BY i DESC) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*row_number\(\) OVER \(ORDER BY "i" DESC NULLS LAST\).* + +query III +SELECT i, rank() OVER (PARTITION BY g ORDER BY v), dense_rank() OVER (ORDER BY g) FROM s.rp_window ORDER BY i +---- +0 4 1 +1 1 2 +2 1 3 +3 1 1 +4 3 2 +5 2 3 +6 2 1 +7 2 2 +8 3 3 +9 3 1 + +query II +EXPLAIN SELECT i, rank() OVER (PARTITION BY g ORDER BY v), dense_rank() OVER (ORDER BY g) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*rank\(\) OVER \(PARTITION BY "g" ORDER BY "v" ASC NULLS LAST\).* + +query IIII +SELECT i, percent_rank() OVER (ORDER BY g), cume_dist() OVER (ORDER BY g), ntile(3) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 0.0 0.4 1 +1 0.4444444444444444 0.7 1 +2 0.7777777777777778 1.0 1 +3 0.0 0.4 1 +4 0.4444444444444444 0.7 2 +5 0.7777777777777778 1.0 2 +6 0.0 0.4 2 +7 0.4444444444444444 0.7 3 +8 0.7777777777777778 1.0 3 +9 0.0 0.4 3 + +query II +EXPLAIN SELECT i, percent_rank() OVER (ORDER BY g), cume_dist() OVER (ORDER BY g), ntile(3) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# value functions +query IIII +SELECT i, lag(v) OVER (ORDER BY i), lead(v, 2) OVER (ORDER BY i), lag(i, 1, -1) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 NULL 20 -1 +1 NULL 30 -1 +2 10 NULL -1 +3 20 50 0 +4 30 60 1 +5 NULL 70 2 +6 50 NULL 3 +7 60 90 4 +8 70 NULL 5 +9 NULL NULL 6 + +query II +EXPLAIN SELECT i, lag(v) OVER (ORDER BY i), lead(v, 2) OVER (ORDER BY i), lag(i, 1, -1) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query IIII +SELECT i, first_value(v) OVER (PARTITION BY g ORDER BY i), last_value(v) OVER (PARTITION BY g ORDER BY i ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), nth_value(v, 2) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 NULL 90 NULL +1 10 70 NULL +2 20 NULL NULL +3 NULL 90 30 +4 10 70 NULL +5 20 NULL 50 +6 NULL 90 30 +7 10 70 NULL +8 20 NULL 50 +9 NULL 90 30 + +query II +EXPLAIN SELECT i, first_value(v) OVER (PARTITION BY g ORDER BY i), last_value(v) OVER (PARTITION BY g ORDER BY i ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), nth_value(v, 2) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# aggregates as window functions +query IIIIII +SELECT i, SUM(v) OVER (PARTITION BY g), COUNT(*) OVER (), AVG(i) OVER (ORDER BY i), MIN(v) OVER (ORDER BY i DESC), MAX(d) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 180 10 0.0 10 0.00 +1 80 10 0.5 10 1.25 +2 70 10 1.0 20 2.50 +3 180 10 1.5 30 3.75 +4 80 10 2.0 50 5.00 +5 70 10 2.5 50 6.25 +6 180 10 3.0 60 7.50 +7 80 10 3.5 70 8.75 +8 70 10 4.0 90 10.00 +9 180 10 4.5 90 11.25 + +query II +EXPLAIN SELECT i, SUM(v) OVER (PARTITION BY g), COUNT(*) OVER (), AVG(i) OVER (ORDER BY i), MIN(v) OVER (ORDER BY i DESC), MAX(d) OVER (PARTITION BY g ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II rowsort +SELECT typeof(SUM(v) OVER ()), typeof(SUM(d) OVER ()) FROM s.rp_window LIMIT 1 +---- +HUGEINT DECIMAL(38,2) + +# window frames +query II +SELECT i, SUM(i) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM s.rp_window ORDER BY i +---- +0 1 +1 3 +2 6 +3 9 +4 12 +5 15 +6 18 +7 21 +8 24 +9 17 + +query II +EXPLAIN SELECT i, SUM(i) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING.* + +query II +SELECT i, SUM(i) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM s.rp_window ORDER BY i +---- +0 45 +1 45 +2 44 +3 42 +4 39 +5 35 +6 30 +7 24 +8 17 +9 9 + +query II +EXPLAIN SELECT i, SUM(i) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING.* + +query II +SELECT i, SUM(i) OVER (ORDER BY i RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM s.rp_window ORDER BY i +---- +0 0 +1 1 +2 3 +3 6 +4 9 +5 12 +6 15 +7 18 +8 21 +9 24 + +query II +EXPLAIN SELECT i, SUM(i) OVER (ORDER BY i RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*RANGE BETWEEN 2 PRECEDING AND CURRENT ROW.* + +query II +SELECT i, COUNT(*) OVER (ORDER BY g GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM s.rp_window ORDER BY i +---- +0 4 +1 7 +2 6 +3 4 +4 7 +5 6 +6 4 +7 7 +8 6 +9 4 + +query II +EXPLAIN SELECT i, COUNT(*) OVER (ORDER BY g GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW.* + +query II +SELECT i, COUNT(*) OVER (ORDER BY g ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW) FROM s.rp_window ORDER BY i +---- +0 9 +1 9 +2 9 +3 9 +4 9 +5 9 +6 9 +7 9 +8 9 +9 9 + +query II +EXPLAIN SELECT i, COUNT(*) OVER (ORDER BY g ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*EXCLUDE CURRENT ROW.* + +query II +SELECT i, COUNT(*) OVER (ORDER BY g RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) FROM s.rp_window ORDER BY i +---- +0 0 +1 4 +2 7 +3 0 +4 4 +5 7 +6 0 +7 4 +8 7 +9 0 + +query II +EXPLAIN SELECT i, COUNT(*) OVER (ORDER BY g RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, COUNT(*) OVER (ORDER BY g RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) FROM s.rp_window ORDER BY i +---- +0 1 +1 5 +2 8 +3 1 +4 5 +5 8 +6 1 +7 5 +8 8 +9 1 + +query II +EXPLAIN SELECT i, COUNT(*) OVER (ORDER BY g RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# FILTER on aggregate window functions +query II +SELECT i, COUNT(*) FILTER (WHERE v IS NULL) OVER (PARTITION BY g) FROM s.rp_window ORDER BY i +---- +0 1 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +8 1 +9 1 + +query II +EXPLAIN SELECT i, COUNT(*) FILTER (WHERE v IS NULL) OVER (PARTITION BY g) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*count\(\*\) FILTER \(WHERE \("v" IS NULL\)\) OVER.* + +# NULL ordering within the window +query II +SELECT i, SUM(i) OVER (ORDER BY v NULLS FIRST, i) FROM s.rp_window ORDER BY i +---- +0 0 +1 13 +2 15 +3 18 +4 4 +5 23 +6 29 +7 36 +8 12 +9 45 + +query II +EXPLAIN SELECT i, SUM(i) OVER (ORDER BY v NULLS FIRST, i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, SUM(i) OVER (ORDER BY v DESC, i) FROM s.rp_window ORDER BY i +---- +0 33 +1 33 +2 32 +3 30 +4 37 +5 27 +6 22 +7 16 +8 45 +9 9 + +query II +EXPLAIN SELECT i, SUM(i) OVER (ORDER BY v DESC, i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# named windows +query III +SELECT i, MIN(i) OVER w, MAX(i) OVER w FROM s.rp_window WINDOW w AS (PARTITION BY g ORDER BY i) ORDER BY i +---- +0 0 0 +1 1 1 +2 2 2 +3 0 3 +4 1 4 +5 2 5 +6 0 6 +7 1 7 +8 2 8 +9 0 9 + +query II +EXPLAIN SELECT i, MIN(i) OVER w, MAX(i) OVER w FROM s.rp_window WINDOW w AS (PARTITION BY g ORDER BY i) ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# filtering on window results in a subquery +query III +SELECT * FROM (SELECT i, g, row_number() OVER (PARTITION BY g ORDER BY i DESC) AS rn FROM s.rp_window) WHERE rn = 1 ORDER BY g +---- +9 0 1 +7 1 1 +8 2 1 + +query II +EXPLAIN SELECT * FROM (SELECT i, g, row_number() OVER (PARTITION BY g ORDER BY i DESC) AS rn FROM s.rp_window) WHERE rn = 1 ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# QUALIFY is not supported in Postgres +query II +SELECT i, g FROM s.rp_window QUALIFY row_number() OVER (PARTITION BY g ORDER BY i DESC) = 1 ORDER BY g +---- +9 0 +7 1 +8 2 + +query II +EXPLAIN SELECT i, g FROM s.rp_window QUALIFY row_number() OVER (PARTITION BY g ORDER BY i DESC) = 1 ORDER BY g +---- +physical_plan :(?s).*Postgres Query.* + +# IGNORE NULLS / DISTINCT / ORDER BY arguments are not pushed down +query II +SELECT i, last_value(v IGNORE NULLS) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 NULL +1 10 +2 20 +3 30 +4 30 +5 50 +6 60 +7 70 +8 70 +9 90 + +query II +EXPLAIN SELECT i, last_value(v IGNORE NULLS) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, COUNT(DISTINCT g) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 1 +1 2 +2 3 +3 3 +4 3 +5 3 +6 3 +7 3 +8 3 +9 3 + +query II +EXPLAIN SELECT i, COUNT(DISTINCT g) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, first_value(v ORDER BY i DESC) OVER (PARTITION BY g) FROM s.rp_window ORDER BY i +---- +0 90 +1 70 +2 NULL +3 90 +4 70 +5 NULL +6 90 +7 70 +8 NULL +9 90 + +query II +EXPLAIN SELECT i, first_value(v ORDER BY i DESC) OVER (PARTITION BY g) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +# negative / non-constant offsets are not pushed down +query II +SELECT i, lag(i, -1) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 1 +1 2 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 NULL + +query II +EXPLAIN SELECT i, lag(i, -1) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* + +query II +SELECT i, lag(i, g) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +0 0 +1 0 +2 0 +3 3 +4 3 +5 3 +6 6 +7 6 +8 6 +9 9 + +query II +EXPLAIN SELECT i, lag(i, g) OVER (ORDER BY i) FROM s.rp_window ORDER BY i +---- +physical_plan :(?s).*Postgres Query.* diff --git a/test/sql/scanner/application_name.test b/test/sql/scanner/application_name.test new file mode 100644 index 000000000..0a8ea9f6d --- /dev/null +++ b/test/sql/scanner/application_name.test @@ -0,0 +1,386 @@ +# name: test/sql/scanner/application_name.test +# description: Check default and custom application_name +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +require-env POSTGRES_CONNECTION_IS_DIRECT + +# default key-value + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# name in conn_string + +statement ok +ATTACH 'dbname=postgresscanner application_name=custom1' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom1'' +', +connection=getvariable('conn')) +---- +custom1 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# name in secret + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + APPLICATION_NAME custom2 +) + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom2'' +', +connection=getvariable('conn')) +---- +custom2 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# default URI + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# default URI 2 + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner?sslmode=allow' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# custom URI + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner?sslmode=allow&application_name=custom2' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom2'' +', +connection=getvariable('conn')) +---- +custom2 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# a connection error fails the URI cases instead of skipping the test + +set ignore_error_messages + +# default in URI ATTACH path + +foreach uri postgresql://127.0.0.1:/postgresscanner postgres://postgres@127.0.0.1:/postgresscanner postgresql://127.0.0.1:/postgresscanner? postgresql://127.0.0.1:/postgresscanner?sslmode=allow postgresql://127.0.0.1:/postgresscanner?sslmode=allow& postgresql://127.0.0.1:/postgresscanner?passfile=x? + +statement ok +ATTACH '{uri}' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +endloop + +# name in URI ATTACH path + +statement ok +ATTACH 'postgres://127.0.0.1:/postgresscanner?sslmode=allow&application_name=custom3' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom3'' +', +connection=getvariable('conn')) +---- +custom3 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# fallback name in conn_string + +statement ok +ATTACH 'dbname=postgresscanner fallback_application_name=custom4' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom4'' +', +connection=getvariable('conn')) +---- +custom4 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# fallback name in URI ATTACH path + +statement ok +ATTACH 'postgresql://127.0.0.1:/postgresscanner?fallback_application_name=custom5' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom5'' +', +connection=getvariable('conn')) +---- +custom5 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# fallback name in secret + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + FALLBACK_APPLICATION_NAME custom6 +) + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom6'' +', +connection=getvariable('conn')) +---- +custom6 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 diff --git a/test/sql/scanner/arrays.test b/test/sql/scanner/arrays.test index 657a481c4..a7840573a 100644 --- a/test/sql/scanner/arrays.test +++ b/test/sql/scanner/arrays.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query IIIIII select * exclude (float_col, double_col) from pg_numarraytypes; @@ -45,4 +40,6 @@ NULL NULL NULL NULL NULL statement ok USE memory -endloop +statement ok +DETACH s + diff --git a/test/sql/scanner/bug63.test b/test/sql/scanner/bug63.test index 28d275e4e..a0e2bbeae 100644 --- a/test/sql/scanner/bug63.test +++ b/test/sql/scanner/bug63.test @@ -7,9 +7,21 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner', source_schema='some_schema'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +SET schema 'some_schema' query I -SELECT * FROM some_table; +SELECT * FROM some_table ---- two + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/bug69.test b/test/sql/scanner/bug69.test index 40104ba74..cf7969a70 100644 --- a/test/sql/scanner/bug69.test +++ b/test/sql/scanner/bug69.test @@ -6,19 +6,14 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query I -SELECT * FROM fail; +SELECT * FROM fail ---- 42.8875000 42.0000000 @@ -33,4 +28,5 @@ SELECT * FROM fail; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug71.test b/test/sql/scanner/bug71.test index a2d612915..8f1959b9d 100644 --- a/test/sql/scanner/bug71.test +++ b/test/sql/scanner/bug71.test @@ -6,23 +6,19 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query III -SELECT * FROM my_table; +SELECT * FROM my_table ---- 42 something something else statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug74.test b/test/sql/scanner/bug74.test index 26385162f..88ff49189 100644 --- a/test/sql/scanner/bug74.test +++ b/test/sql/scanner/bug74.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query I SELECT * from oids @@ -26,4 +21,5 @@ SELECT * from oids statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug76.test b/test/sql/scanner/bug76.test index 3f053d621..fb36cbe26 100644 --- a/test/sql/scanner/bug76.test +++ b/test/sql/scanner/bug76.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query II SELECT * from daterange; @@ -25,4 +20,5 @@ SELECT * from daterange; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug77.test b/test/sql/scanner/bug77.test index 146765f61..4b0698af9 100644 --- a/test/sql/scanner/bug77.test +++ b/test/sql/scanner/bug77.test @@ -6,17 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE - -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query III SELECT id, my_ints,e FROM duckdb_arr_test; @@ -28,4 +22,5 @@ SELECT id, my_ints,e FROM duckdb_arr_test; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/count_star.test b/test/sql/scanner/count_star.test index 91f93f688..83b888fd7 100644 --- a/test/sql/scanner/count_star.test +++ b/test/sql/scanner/count_star.test @@ -7,9 +7,18 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I -select count(*) from cars; +select count(*) from cars ---- -4 \ No newline at end of file +4 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/ctid_scan.test b/test/sql/scanner/ctid_scan.test new file mode 100644 index 000000000..55dd74c70 --- /dev/null +++ b/test/sql/scanner/ctid_scan.test @@ -0,0 +1,43 @@ +# name: test/sql/scanner/ctid_scan.test +# description: Test parallel ctid scan with different readers +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +CREATE OR REPLACE TABLE tab1 AS FROM range(10000) + +statement ok +SET pg_pages_per_task=1 + +foreach mode COPY_BINARY PLAIN_QUERY_BINARY AUTO + +statement ok +SET pg_query_mode = '{mode}' + +query I +SELECT divide(COUNT(*),1) FROM tab1 +---- +10000 + +endloop + +statement ok +use memory + +statement ok +DETACH s + +statement ok +RESET pg_pages_per_task + +statement ok +RESET pg_query_mode diff --git a/test/sql/scanner/daterange_array.test b/test/sql/scanner/daterange_array.test index c0620bc27..631d3e447 100644 --- a/test/sql/scanner/daterange_array.test +++ b/test/sql/scanner/daterange_array.test @@ -6,23 +6,19 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query II -SELECT * from daterange_array; +SELECT * from daterange_array ---- 1108 ['["2010-01-01 14:30:00","2010-01-01 15:30:00")'] statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/decimals.test_slow b/test/sql/scanner/decimals.test_slow index cab435845..40a815d52 100644 --- a/test/sql/scanner/decimals.test_slow +++ b/test/sql/scanner/decimals.test_slow @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I SELECT * FROM dec_0_1 @@ -132488,3 +132491,9 @@ SELECT * FROM dec_18_18 100000000000000000.000000000000000001 -100000000000000000.000000000000000001 0.0 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/interval.test b/test/sql/scanner/interval.test index a05437a7c..d6bab6931 100644 --- a/test/sql/scanner/interval.test +++ b/test/sql/scanner/interval.test @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I select * from intervals @@ -20,3 +23,9 @@ select * from intervals 00:00:00.42 -42 days NULL + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/missing_table.test b/test/sql/scanner/missing_table.test deleted file mode 100644 index 7c044864a..000000000 --- a/test/sql/scanner/missing_table.test +++ /dev/null @@ -1,11 +0,0 @@ -# name: test/sql/scanner/missing_table.test -# description: Missing table -# group: [scanner] - -require postgres_scanner - -require-env POSTGRES_TEST_DATABASE_AVAILABLE - -statement error -select * from postgres_scan('dbname=postgresscanner', 'public', 'does_not_exist') ----- \ No newline at end of file diff --git a/test/sql/scanner/nulls.test b/test/sql/scanner/nulls.test index c951df86e..b108b8b6a 100644 --- a/test/sql/scanner/nulls.test +++ b/test/sql/scanner/nulls.test @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query IIIIIIIIII select * from nulltest; @@ -15,4 +18,10 @@ select * from nulltest; 1 2 3 4 5 6 7 8 9 10 1 NULL 3 4 NULL 6 7 8 NULL 10 NULL NULL 3 4 5 6 7 NULL NULL NULL -NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL \ No newline at end of file +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/numeric_double_rounding.test b/test/sql/scanner/numeric_double_rounding.test new file mode 100644 index 000000000..8a77a1207 --- /dev/null +++ b/test/sql/scanner/numeric_double_rounding.test @@ -0,0 +1,88 @@ +# name: test/sql/scanner/numeric_double_rounding.test +# description: Binary NUMERIC conversion rounds once and agrees with PostgreSQL float8 +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +foreach query_mode COPY_BINARY PLAIN_QUERY_BINARY PLAIN_QUERY_TEXT + +statement ok +SET pg_query_mode = '{query_mode}' + +# Materialize the decoded values so the comparison cannot be pushed into PostgreSQL. +# Include the examples from issue #604 and exact halfway-rounding cases. +statement ok +CREATE OR REPLACE TEMP TABLE numeric_values AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8 AS expected FROM (VALUES + (8629433::numeric / 1000), + ((275.38::numeric + 232.58 + 785.33) / 3), + (1e32::numeric), + (-8629433::numeric / 1000), + (-1e32::numeric), + (0.1::numeric), + (0.00000000000123456789::numeric), + (12345678901234567890.1234567890123456789::numeric), + (1.00000000000000011102230246251565404236316680908203125::numeric), + (1.00000000000000033306690738754696212708950042724609375::numeric), + (-1.00000000000000011102230246251565404236316680908203125::numeric), + (1e308::numeric), + (1.7976931348623157e308::numeric), + (-1.7976931348623157e308::numeric), + (1e-300::numeric), + (2.2250738585072014e-308::numeric), + (4.9406564584124654e-324::numeric), + (-4.9406564584124654e-324::numeric), + (1e-323::numeric), + (0::numeric), + (NULL::numeric), + ('NaN'::numeric), + ('Infinity'::numeric), + ('-Infinity'::numeric) + ) AS values_to_check(n) +$$) + +query I +SELECT count(*) FROM numeric_values WHERE n IS DISTINCT FROM expected +---- +0 + +# The display scale must not cause a finite value to overflow during decoding. +statement ok +CREATE OR REPLACE TEMP TABLE high_scale_values AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8 AS expected FROM (VALUES + (1.5::numeric(1000,500)), + (-1.5::numeric(1000,500)), + (0.125::numeric(1000,500)), + (1e300::numeric(1000,500)), + (0::numeric(1000,500)) + ) AS values_to_check(n) +$$) + +query I +SELECT count(*) FROM high_scale_values WHERE n IS DISTINCT FROM expected +---- +0 + +# NUMERIC array elements use the same binary conversion. +statement ok +CREATE OR REPLACE TEMP TABLE numeric_arrays AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8[] AS expected FROM ( + SELECT ARRAY[8629.433::numeric, 1e32::numeric, -1e32::numeric, + 2.2250738585072014e-308::numeric, NULL::numeric] AS n + ) AS values_to_check +$$) + +query I +SELECT n IS NOT DISTINCT FROM expected FROM numeric_arrays +---- +true + +endloop + +statement ok +RESET pg_query_mode diff --git a/test/sql/scanner/numerics_nan_inf.test b/test/sql/scanner/numerics_nan_inf.test new file mode 100644 index 000000000..a06c97cd2 --- /dev/null +++ b/test/sql/scanner/numerics_nan_inf.test @@ -0,0 +1,360 @@ +# name: test/sql/scanner/numerics_nan_inf.test +# description: Test for out-of-bound values for numeric types +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# FLOAT + +query I +FROM postgres_query('s', 'SELECT ''NaN''::REAL') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Inf''::REAL') +---- +Inf + +query I +FROM postgres_query('s', 'SELECT ''-Inf''::REAL') +---- +-Inf + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::REAL') +) +---- +FLOAT + +# DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::DOUBLE PRECISION') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Inf''::DOUBLE PRECISION') +---- +Inf + +query I +FROM postgres_query('s', 'SELECT ''-Inf''::DOUBLE PRECISION') +---- +-Inf + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::DOUBLE PRECISION') +) +---- +DOUBLE + +# NUMERIC + +# int16_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +) +---- +DECIMAL(4,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(4,3)') +---- +A field with precision 4, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(4,3)') +---- +A field with precision 4, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# int32_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +) +---- +DECIMAL(9,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(9,3)') +---- +A field with precision 9, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(9,3)') +---- +A field with precision 9, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# int64_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +) +---- +DECIMAL(18,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(18,3)') +---- +A field with precision 18, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(18,3)') +---- +A field with precision 18, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# huge_int + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +) +---- +DECIMAL(38,19) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(38,19)') +---- +A field with precision 38, scale 19 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(38,19)') +---- +A field with precision 38, scale 19 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +---- +Unsupported NUMERIC value: NaN + +# plain scan + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS out_of_bounds_numeric_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE out_of_bounds_numeric_test(col1 INT, col2 NUMERIC(38,19))') + +statement ok +CALL postgres_execute('s', 'INSERT INTO out_of_bounds_numeric_test values(42, ''NaN'')') + +statement error +SELECT divide(col1, 1), col2 FROM s.out_of_bounds_numeric_test +---- +Unsupported NUMERIC value: NaN + +statement ok +CALL postgres_execute('s', 'DROP TABLE out_of_bounds_numeric_test') + +statement ok +RESET pg_numeric_nan_as_null + +# NUMERIC as DOUBLE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +SET pg_numeric_nan_as_null = FALSE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +RESET pg_numeric_nan_as_null + +# NUMERIC as VARCHAR + +statement ok +SET pg_numeric_as_varchar = TRUE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +VARCHAR + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +SET pg_numeric_nan_as_null = FALSE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +VARCHAR + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +RESET pg_numeric_nan_as_null + +statement ok +RESET pg_numeric_as_varchar + +statement ok +DETACH s diff --git a/test/sql/scanner/postgres_query.test b/test/sql/scanner/postgres_query.test index 7d589ba89..8a1b12c3e 100644 --- a/test/sql/scanner/postgres_query.test +++ b/test/sql/scanner/postgres_query.test @@ -10,7 +10,7 @@ statement ok ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) query III -select * from postgres_query('s1', 'SELECT * FROM cars'); +FROM postgres_query('s1', 'SELECT * FROM cars'); ---- ferari testarosa red aston martin db2 blue @@ -19,12 +19,12 @@ ford T black # filters query III -select * from postgres_query('s1', 'SELECT * FROM cars WHERE color=''red'''); +FROM postgres_query('s1', 'SELECT * FROM cars WHERE color=''red'''); ---- ferari testarosa red query III -select * from postgres_query('s1', 'SELECT * FROM cars') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars') WHERE color='red'; ---- ferari testarosa red @@ -77,24 +77,24 @@ DATE[] TIME[] TIME WITH TIME ZONE[] TIMESTAMP[] TIMESTAMP WITH TIME ZONE[] # semicolons query III -select * from postgres_query('s1', 'SELECT * FROM cars;') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars;') WHERE color='red'; ---- ferari testarosa red query III -select * from postgres_query('s1', 'SELECT * FROM cars; ; ') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars; ; ') WHERE color='red'; ---- ferari testarosa red # incorrect usage statement error -select * from postgres_query('s1', 'CREATE TABLE my_table(i INT)'); +FROM postgres_query('s1', 'CREATE TABLE my_table(i INT)'); ---- Invalid Error: Failed to execute query # database not found statement error -select * from postgres_query('xx', 'SELECT * FROM cars'); +FROM postgres_query('xx', 'SELECT * FROM cars'); ---- Binder Error: Failed to find attached database @@ -102,12 +102,12 @@ mode skip # empty statements statement error -select * from postgres_query('s1', '') WHERE color='red'; +FROM postgres_query('s1', '') WHERE color='red'; ---- Invalid Error: Failed to execute query statement error -select * from postgres_query('s1', ';;') WHERE color='red'; +FROM postgres_query('s1', ';;') WHERE color='red'; ---- Invalid Error: Failed to execute query @@ -115,12 +115,12 @@ mode unskip # NULL parameters statement error -select * from postgres_query(NULL, 'SELECT 42'); +FROM postgres_query(NULL, 'SELECT 42'); ---- cannot be NULL statement error -select * from postgres_query('s1', NULL); +FROM postgres_query('s1', NULL); ---- cannot be NULL @@ -129,7 +129,7 @@ statement ok ATTACH ':memory:' AS ddb statement error -select * from postgres_query('ddb', 'SELECT * FROM cars'); +FROM postgres_query('ddb', 'SELECT * FROM cars'); ---- does not refer to a Postgres database @@ -144,3 +144,272 @@ statement error select count(*) from postgres_query('s1', 'SELECT * FROM nonexistent_table'); ---- does not exist + +# DML + +query I +FROM postgres_query('s1', 'DROP TABLE IF EXISTS query_dml_test') +---- +-1 + +query I +FROM postgres_query('s1', 'CREATE TABLE query_dml_test(col1 INT)') +---- +-1 + +query I +FROM postgres_query('s1', 'INSERT INTO query_dml_test values(41), (42)') +---- +2 + +query I +FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 43') +---- +2 + +query I +FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +---- +0 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +) +---- +rowcount BIGINT + +statement ok +CALL postgres_query('s1', 'DROP TABLE query_dml_test') + +# suppress DML output + +statement ok +CALL postgres_query('s1', 'DROP TABLE IF EXISTS dml_output_test') + +statement ok +CALL postgres_query('s1', 'CREATE TABLE dml_output_test (col1 INT)') + +statement ok +CALL postgres_query('s1', 'INSERT INTO dml_output_test values(41),(42)') + +# unittester ignores StatementReturnType::NOTHING +query I +CALL postgres_query('s1', 'UPDATE dml_output_test SET col1 = 43', suppress_dml_output=TRUE) +---- +2 + +statement ok +CALL postgres_query('s1', 'DROP TABLE dml_output_test') + +# non-preparable statement + +statement error +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;') +---- +Failed to prepare + +statement error +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;', prepare=FALSE, params=row(42)) +---- +query parameters cannot be used with 'prepare=FALSE' + +# use the same connection + +statement ok +BEGIN TRANSACTION + +statement ok +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;', prepare=FALSE) + +query I +FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +---- +1 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +) +---- +rowcount BIGINT + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_execute('s1', 'EXECUTE p1', prepare=FALSE) +) +---- +rowcount BIGINT + +statement ok +CALL postgres_query('s1', 'DEALLOCATE p1', prepare=FALSE) + +statement error +FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +---- +does not exist + +statement ok +ROLLBACK + +# wrapping rules + +statement ok +CALL enable_logging('PostgresQueryLog') + +# no wrap copy + +query II +FROM postgres_query('s1', 'SELECT brand, model FROM cars', query_mode='COPY_BINARY'); +---- +ferari testarosa +aston martin db2 +bentley mulsanne +ford T + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'COPY (SELECT brand, model FROM cars) TO STDOUT (FORMAT "binary");' +---- +1 + +# no wrap binary + +query II +FROM postgres_query('s1', 'SELECT brand, model FROM cars', query_mode='PLAIN_QUERY_BINARY'); +---- +ferari testarosa +aston martin db2 +bentley mulsanne +ford T + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT brand, model FROM cars' +---- +1 + +# no wrap text + +query II +FROM postgres_query('s1', 'SELECT model, brand FROM cars', query_mode='PLAIN_QUERY_TEXT'); +---- +testarosa ferari +db2 aston martin +mulsanne bentley +T ford + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT model, brand FROM cars' +---- +1 + +# wrap color_t + +query II +FROM postgres_query('s1', 'SELECT brand, color FROM cars', query_mode='PLAIN_QUERY_BINARY'); +---- +ferari red +aston martin blue +bentley gray +ford black + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT "brand", "color"::VARCHAR FROM (SELECT brand, color FROM cars) AS __unnamed_subquery ' +---- +1 + +# wrap projection + +query I +select count(*) from postgres_query('s1', 'SELECT * FROM cars WHERE color=''red''', query_mode='PLAIN_QUERY_BINARY'); +---- +1 + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT "brand" FROM (SELECT * FROM cars WHERE color=''red'') AS __unnamed_subquery ' +---- +1 + +statement ok +CALL disable_logging() + +# duplicate column names + +query III +FROM postgres_query('s1', 'SELECT 41, 42, 43') +---- +41 42 43 + +query I +SELECT column_name FROM (DESCRIBE FROM postgres_query('s1', 'SELECT 41, 42, 43')) +---- +?column? +?column?_1 +?column?_2 + +# outer query reorders all the columns of the remote query + +statement ok +CALL postgres_execute('s1', 'DROP TABLE IF EXISTS query_reorder_tbl') + +statement ok +CALL postgres_execute('s1', 'CREATE TABLE query_reorder_tbl (a varchar, b char(11), c integer, n numeric(18,2))') + +statement ok +CALL postgres_execute('s1', 'INSERT INTO query_reorder_tbl VALUES (''a'', ''b'', 3, 1.5)') + +query III +SELECT a, c, b FROM postgres_query('s1', 'SELECT a, b, c FROM query_reorder_tbl') +---- +a 3 b + +query III +SELECT n, a, trim(b) FROM postgres_query('s1', 'SELECT a, b, n FROM query_reorder_tbl') +---- +1.50 a b + +query IIII +SELECT n, c, trim(b), a FROM postgres_query('s1', 'SELECT * FROM query_reorder_tbl') +---- +1.50 3 b a + +query II +SELECT c, a FROM postgres_query('s1', 'SELECT a, c FROM query_reorder_tbl') WHERE c = 3 +---- +3 a + +statement ok +SET pg_use_text_protocol=true + +query III +SELECT a, c, trim(b) FROM postgres_query('s1', 'SELECT a, b, c FROM query_reorder_tbl') +---- +a 3 b + +query IIII +SELECT n, c, trim(b), a FROM postgres_query('s1', 'SELECT * FROM query_reorder_tbl') +---- +1.50 3 b a + +statement ok +RESET pg_use_text_protocol + +query IIII +SELECT n, c, trim(b), a FROM postgres_query('s1', 'SELECT * FROM query_reorder_tbl', query_mode='plain_query_binary') +---- +1.50 3 b a + +statement ok +CALL postgres_execute('s1', 'DROP TABLE query_reorder_tbl') + +statement ok +DETACH s1 diff --git a/test/sql/scanner/postgres_query_clear_catalog_cache.test b/test/sql/scanner/postgres_query_clear_catalog_cache.test new file mode 100644 index 000000000..d3993f844 --- /dev/null +++ b/test/sql/scanner/postgres_query_clear_catalog_cache.test @@ -0,0 +1,126 @@ +# name: test/sql/scanner/postgres_query_clear_catalog_cache.test +# description: Test clear_catalog_cache option of postgres_query +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS clear_catalog_cache_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE clear_catalog_cache_test(col1 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO clear_catalog_cache_test VALUES (1)') + +query I +SELECT * FROM s.clear_catalog_cache_test +---- +1 + +# cache is not cleared by default +statement ok +CALL postgres_execute('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col2 INT DEFAULT 2') + +query I +SELECT * FROM s.clear_catalog_cache_test +---- +1 + +statement ok +CALL postgres_execute('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col3 INT DEFAULT 3', clear_catalog_cache=FALSE) + +query I +SELECT * FROM s.clear_catalog_cache_test +---- +1 + +# command +statement ok +CALL postgres_execute('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col4 INT DEFAULT 4', clear_catalog_cache=TRUE) + +query IIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 + +# command with prepare=FALSE +statement ok +CALL postgres_query('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col5 INT DEFAULT 5', prepare=FALSE, clear_catalog_cache=TRUE) + +query IIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 + +# streaming query returning rows, DDL is run as a preliminary query +query I +FROM postgres_query('s', 'SELECT 42', clear_catalog_cache=TRUE, preliminary_queries=[ + 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col6 INT DEFAULT 6' +]) +---- +42 + +query IIIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 6 + +# materialized query, multiple scans in the plan +query II +SELECT * FROM postgres_query('s', 'SELECT 42 AS a', clear_catalog_cache=TRUE, preliminary_queries=[ + 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col7 INT DEFAULT 7' +]), postgres_query('s', 'SELECT 43 AS b') +---- +42 43 + +query IIIIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 6 7 + +# EXPLAIN does not execute the query and does not clear the cache +statement ok +CALL postgres_execute('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col8 INT DEFAULT 8') + +statement ok +EXPLAIN FROM postgres_query('s', 'SELECT 42', clear_catalog_cache=TRUE) + +query IIIIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 6 7 + +# cache is not cleared when a pinned connection is used +statement ok +SET VARIABLE conn_id = postgres_pin_connection('s') + +statement ok +CALL postgres_execute('s', 'ALTER TABLE clear_catalog_cache_test ADD COLUMN col9 INT DEFAULT 9', connection=getvariable('conn_id'), clear_catalog_cache=TRUE) + +query IIIIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 6 7 + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn_id')) + +statement ok +CALL postgres_execute('s', 'SELECT 1', clear_catalog_cache=TRUE) + +query IIIIIIIII +SELECT * FROM s.clear_catalog_cache_test +---- +1 2 3 4 5 6 7 8 9 + +statement ok +CALL postgres_execute('s', 'DROP TABLE clear_catalog_cache_test') + +statement ok +DETACH s diff --git a/test/sql/scanner/postgres_query_error.test b/test/sql/scanner/postgres_query_error.test index e752fe384..7f6b186af 100644 --- a/test/sql/scanner/postgres_query_error.test +++ b/test/sql/scanner/postgres_query_error.test @@ -13,7 +13,7 @@ statement ok BEGIN statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query III select * from postgres_query('s1', 'SELECT * FROM cars'); diff --git a/test/sql/scanner/postgres_query_params.test b/test/sql/scanner/postgres_query_params.test index e83d041e9..3dd82bb10 100644 --- a/test/sql/scanner/postgres_query_params.test +++ b/test/sql/scanner/postgres_query_params.test @@ -6,10 +6,6 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -# COPY does not support parameters: ERROR: there is no parameter $1 -statement ok -SET pg_use_text_protocol=true - statement ok ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) @@ -108,3 +104,64 @@ query I FROM postgres_query('s1', 'SELECT $1::DOUBLE PRECISION', params=row(42.123::DOUBLE)) ---- 42.123 + +statement error +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='COPY_BINARY') +---- +Failed to prepare COPY + +statement ok +SET pg_query_mode = 'COPY_BINARY' + +statement error +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo')) +---- +Failed to prepare COPY + +query II +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='PLAIN_QUERY_BINARY') +---- +42 foo + +query II +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='PLAIN_QUERY_TEXT') +---- +42 foo + +statement ok +RESET pg_query_mode + +# statements without result columns +statement ok +CALL postgres_execute('s1', 'DROP TABLE IF EXISTS execute_params_tbl') + +statement ok +CALL postgres_execute('s1', 'CREATE TABLE execute_params_tbl (i INTEGER)') + +query I +FROM postgres_execute('s1', 'INSERT INTO execute_params_tbl VALUES ($1)', params=row(42)) +---- +1 + +query I +FROM postgres_query('s1', 'SELECT i FROM execute_params_tbl') +---- +42 + +statement error +CALL postgres_execute('s1', 'INSERT INTO execute_params_tbl VALUES ($1)', params=row(42, 43)) +---- +Incorrect number of parameters specified, expected: 1, actual: 2 + +statement ok +CALL postgres_execute('s1', 'DROP TABLE execute_params_tbl') + +statement ok +DETACH s1 \ No newline at end of file diff --git a/test/sql/scanner/postgres_query_preliminary.test b/test/sql/scanner/postgres_query_preliminary.test new file mode 100644 index 000000000..8fdb59fa5 --- /dev/null +++ b/test/sql/scanner/postgres_query_preliminary.test @@ -0,0 +1,101 @@ +# name: test/sql/scanner/postgres_query_preliminary.test +# description: Test preliminary_queries option of postgres_query +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# command, the table is created by the preliminary queries +statement ok +CALL postgres_query('s', 'DROP TABLE preliminary_queries_test', preliminary_queries=[ + 'DROP TABLE IF EXISTS preliminary_queries_test', + 'CREATE TABLE preliminary_queries_test(col1 INT)' +]) + +query I +SELECT count(*) FROM postgres_query('s', 'SELECT * FROM pg_tables WHERE tablename = ''preliminary_queries_test''') +---- +0 + +# command with prepare=FALSE +statement ok +CALL postgres_execute('s', 'DROP TABLE preliminary_queries_test', prepare=FALSE, preliminary_queries=[ + 'CREATE TABLE preliminary_queries_test(col1 INT)' +]) + +query I +SELECT count(*) FROM postgres_query('s', 'SELECT * FROM pg_tables WHERE tablename = ''preliminary_queries_test''') +---- +0 + +# query, preliminary queries are run on the same connection as the main query +query I +FROM postgres_query('s', 'SELECT current_setting(''duckdb.prelim_test'', true)', preliminary_queries=[ + 'SET LOCAL duckdb.prelim_test = ''foo''' +]) +---- +foo + +query I +FROM postgres_query('s', 'SELECT current_setting(''duckdb.prelim_test'', true) || current_setting(''duckdb.prelim_test2'', true)', preliminary_queries=[ + 'SET LOCAL duckdb.prelim_test = ''foo''', + 'SET LOCAL duckdb.prelim_test2 = ''bar''' +]) +---- +foobar + +# text protocol +query I +FROM postgres_query('s', 'SELECT current_setting(''duckdb.prelim_test'', true)', query_mode='plain_query_text', preliminary_queries=[ + 'SET LOCAL duckdb.prelim_test = ''foo''' +]) +---- +foo + +# query with parameters +query I +FROM postgres_query('s', 'SELECT current_setting(''duckdb.prelim_test'', true) || $1', params=row('bar'), preliminary_queries=[ + 'SET LOCAL duckdb.prelim_test = ''foo''' +]) +---- +foobar + +# empty list +query I +FROM postgres_query('s', 'SELECT 42', preliminary_queries=[]) +---- +42 + +# preliminary queries are not run on EXPLAIN +statement ok +EXPLAIN FROM postgres_query('s', 'SELECT 42', preliminary_queries=[ + 'CREATE TABLE preliminary_queries_test(col1 INT)' +]) + +query I +SELECT count(*) FROM postgres_query('s', 'SELECT * FROM pg_tables WHERE tablename = ''preliminary_queries_test''') +---- +0 + +# errors +statement error +FROM postgres_query('s', 'SELECT 42', preliminary_queries=NULL) +---- +Specified preliminary queries LIST must be not NULL + +statement error +FROM postgres_query('s', 'SELECT 42', preliminary_queries=['SELECT 1', NULL]) +---- +All preliminary queries must be not NULL + +statement error +FROM postgres_query('s', 'SELECT 42', preliminary_queries=['SELECT * FROM preliminary_queries_nonexistent']) +---- +preliminary_queries_nonexistent + +statement ok +DETACH s diff --git a/test/sql/scanner/postgres_query_prepared.test b/test/sql/scanner/postgres_query_prepared.test new file mode 100644 index 000000000..2ffda19ee --- /dev/null +++ b/test/sql/scanner/postgres_query_prepared.test @@ -0,0 +1,310 @@ +# name: test/sql/scanner/postgres_query_prepared.test +# description: Test that postgres_query in plain query modes executes the statement prepared in bind +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +require-env POSTGRES_CONNECTION_IS_DIRECT + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL enable_logging('PostgresQueryLog') + +foreach mode PLAIN_QUERY_TEXT PLAIN_QUERY_BINARY + +# The queries below look up the prepared statements of the current session that contain a marker from their own +# text. Executions are counted before the query runs, so "executions" is 1 when the query was run through the +# statement prepared in bind, and 0 when it was run as a plain query. The statement is closed after it is executed, +# the checks that count the remaining statements build the marker with || so they do not match themselves. + +# transaction connection: prepared statements are only visible within the same session, so use an explicit +# transaction to have bind, execution and the checks on the same connection + +statement ok +BEGIN + +query I +FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_plain_{mode}%''', query_mode='{mode}') +---- +1 + +# the query sent to Postgres is the one specified by the user +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_plain_{mode}%''' +---- +1 + +# the statement was closed after it was executed +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_tx'' || ''_plain_{mode}%''', + query_mode='{mode}') +---- +0 + +# with parameters + +query II +FROM postgres_query('s', 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_params_{mode}%''', params=row(43), query_mode='{mode}') +---- +44 1 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_params_{mode}%''' +---- +1 + +# a projection modifies the query, the prepared statement is not used, but it is closed + +query I +SELECT executions FROM postgres_query('s', 'SELECT 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_projection_{mode}%''', query_mode='{mode}') +---- +0 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query LIKE 'SELECT "executions" FROM (%m_tx_projection_{mode}%' +---- +1 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_tx'' || ''_projection_{mode}%''', + query_mode='{mode}') +---- +0 + +# COPY mode does not use the prepared statement, but closes it + +query I +FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_tx_binary_{mode}%''', query_mode='COPY_BINARY') +---- +0 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_tx'' || ''_binary_{mode}%''', + query_mode='{mode}') +---- +0 + +# a command is not executed through the prepared statement, the statement is closed + +statement ok +CALL postgres_execute('s', 'CREATE TEMPORARY TABLE m_tx_command_{mode} (i INTEGER)') + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_tx'' || ''_command_{mode}%''', + query_mode='{mode}') +---- +0 + +statement ok +COMMIT + +# PREPARE and EXECUTE in different (auto-commit) transactions: the statement prepared in bind belongs to the +# connection of the PREPARE transaction, so EXECUTE must not use it + +statement ok +PREPARE p1 AS FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_autocommit_{mode}%''', query_mode='{mode}') + +query I +EXECUTE p1 +---- +0 + +query I +EXECUTE p1 +---- +0 + +# the query sent to Postgres is the one specified by the user +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_autocommit_{mode}%''' +---- +2 + +statement ok +DEALLOCATE p1 + +# PREPARE and EXECUTE in the same transaction: the first EXECUTE uses the statement prepared in bind and closes it, +# the second one runs the query as a plain query + +statement ok +BEGIN + +statement ok +PREPARE p2 AS FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_same_tx_{mode}%''', query_mode='{mode}') + +query I +EXECUTE p2 +---- +1 + +query I +EXECUTE p2 +---- +0 + +statement ok +COMMIT + +statement ok +DEALLOCATE p2 + +# pinned connection + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query I +FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned_plain_{mode}%''', query_mode='{mode}', + connection=getvariable('conn')) +---- +1 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned_plain_{mode}%''' +---- +1 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned'' || ''_plain_{mode}%''', + query_mode='{mode}', connection=getvariable('conn')) +---- +0 + +query II +FROM postgres_query('s', 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned_params_{mode}%''', params=row(48), query_mode='{mode}', + connection=getvariable('conn')) +---- +49 1 + +query I +SELECT executions FROM postgres_query('s', 'SELECT 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned_projection_{mode}%''', query_mode='{mode}', + connection=getvariable('conn')) +---- +0 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned'' || ''_projection_{mode}%''', + query_mode='{mode}', connection=getvariable('conn')) +---- +0 + +query I +FROM postgres_query('s', 'SELECT 1 AS m_pinned_copy_{mode}', query_mode='COPY_BINARY', connection=getvariable('conn')) +---- +1 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned'' || ''_copy_{mode}%''', + query_mode='{mode}', connection=getvariable('conn')) +---- +0 + +statement ok +CALL postgres_execute('s', 'CREATE TEMPORARY TABLE m_pinned_command_{mode} (i INTEGER)', connection=getvariable('conn')) + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned'' || ''_command_{mode}%''', + query_mode='{mode}', connection=getvariable('conn')) +---- +0 + +# reordering all the columns modifies the query, the prepared statement is not used even on the connection +# the statement was prepared on + +query III +SELECT executions, b, a FROM postgres_query('s', 'SELECT 51 AS a, 52 AS b, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_pinned_reorder_{mode}%''', query_mode='{mode}', + connection=getvariable('conn')) +---- +0 52 51 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query LIKE 'SELECT "executions", "b", "a" FROM (SELECT 51 AS a, 52 AS b, %m_pinned_reorder_{mode}%' +---- +1 + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +endloop + +# AUTO query mode (the default): with parameters it resolves to PLAIN_QUERY_BINARY and uses the prepared statement, +# without parameters it resolves to COPY and does not + +statement ok +BEGIN + +query II +FROM postgres_query('s', 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_tx_params%''', params=row(43)) +---- +44 1 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_tx_params%''' +---- +1 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_tx'' || ''_params%''') +---- +0 + +query I +FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_tx_no_params%''') +---- +0 + +statement ok +COMMIT + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_pinned_params%''', params=row(48), + connection=getvariable('conn')) +---- +49 1 + +query I +SELECT COUNT(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'SELECT $1::INTEGER + 1 AS a, coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_pinned_params%''' +---- +1 + +query I +FROM postgres_query('s', 'SELECT count(*) FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_pinned'' || ''_params%''', + connection=getvariable('conn')) +---- +0 + +query I +FROM postgres_query('s', 'SELECT coalesce(max(generic_plans + custom_plans), 0) AS executions + FROM pg_prepared_statements WHERE statement LIKE ''%m_auto_pinned_no_params%''', connection=getvariable('conn')) +---- +0 + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) diff --git a/test/sql/scanner/query_pin_connection.test b/test/sql/scanner/query_pin_connection.test new file mode 100644 index 000000000..196417567 --- /dev/null +++ b/test/sql/scanner/query_pin_connection.test @@ -0,0 +1,96 @@ +# name: test/sql/scanner/query_pin_connection.test +# description: Test connection pinning with postgres_query +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement error +SELECT postgres_pin_connection() +---- +No function matches the given name and argument types + +statement error +SELECT postgres_pin_connection(NULL) +---- +Specified attached database name must be not null + +statement error +SELECT postgres_pin_connection('fail') +---- +Failed to find attached database + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +statement error +SELECT postgres_close_pinned_connection('fail', 0) +---- +Failed to find attached database + +statement error +SELECT postgres_close_pinned_connection('s', 0) +---- +not found in the specified attached database + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement error +SELECT postgres_close_pinned_connection('s', getvariable('conn')) +---- +not found in the specified attached database + +statement ok +RESET VARIABLE conn + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +statement ok +CALL postgres_execute('s', 'CREATE TEMPORARY TABLE temp1(col1 INT)', connection=getvariable('conn')) + +statement ok +CALL postgres_execute('s', 'INSERT INTO temp1 values(42)', connection=getvariable('conn')) + +query I +FROM postgres_query('s', 'SELECT * FROM temp1', connection=getvariable('conn')) +---- +42 + +statement error +FROM postgres_query('s', 'SELECT 42', connection=getvariable('conn')) +UNION ALL +FROM postgres_query('s', 'SELECT 43', connection=getvariable('conn')) +---- +not found in the specified attached database + +statement ok +SET VARIABLE conn2 = postgres_pin_connection('s') + +query I +FROM postgres_query('s', 'SELECT 42', connection=getvariable('conn')) +UNION ALL +FROM postgres_query('s', 'SELECT 43', connection=getvariable('conn2')) +---- +42 +43 + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn2')) + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn1 + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s \ No newline at end of file diff --git a/test/sql/scanner/remote_connect.test b/test/sql/scanner/remote_connect.test new file mode 100644 index 000000000..3aa2fe156 --- /dev/null +++ b/test/sql/scanner/remote_connect.test @@ -0,0 +1,57 @@ +# name: test/sql/scanner/remote_connect.test +# description: Test CONNECT statement +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +query I +SELECT current_database() +---- +memory + +statement ok +CONNECT 'dbname=postgresscanner' (TYPE POSTGRES) + +query I +SELECT current_database() +---- +postgresscanner + +statement ok +DROP TABLE IF EXISTS connect_test + +statement ok +CREATE TABLE connect_test(col1 INT) + +statement ok +INSERT INTO connect_test values (41), (42) + +statement error +FROM connect_test +---- +Failed to prepare query + +query I +SELECT col1::regclass FROM connect_test ORDER BY col1 +---- +41 +42 + +# unittester ignores StatementReturnType::NOTHING +query I +UPDATE connect_test SET col1 = 43 +---- +2 + +statement ok +DROP TABLE connect_test + +statement ok +DISCONNECT + +query I +SELECT current_database() +---- +memory diff --git a/test/sql/scanner/timestamp_range.test b/test/sql/scanner/timestamp_range.test new file mode 100644 index 000000000..4e05f0170 --- /dev/null +++ b/test/sql/scanner/timestamp_range.test @@ -0,0 +1,49 @@ +# name: test/sql/scanner/timestamp_range.test +# description: Reject Postgres timestamps outside DuckDB's supported range +# group: [scanner] + +require postgres_scanner + +require parquet + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS timestamp_range_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE timestamp_range_test (id INTEGER PRIMARY KEY, ts TIMESTAMP, tstz TIMESTAMPTZ)') + +# postgres_execute sends this directly to PostgreSQL, whose timestamp range is wider than DuckDB's. +statement ok +CALL postgres_execute('s', 'INSERT INTO timestamp_range_test VALUES + (1, ''294276-12-31 23:59:59'', ''294276-12-31 23:59:59+00''), + (2, ''10000-01-01 00:00:00'', ''10000-01-01 00:00:00+00''), + (3, ''2026-01-15 12:00:00'', ''2026-01-15 12:00:00+00''), + (4, ''4713-01-01 BC'', ''4713-01-01 BC'')') + +# TODO: investigate me +#4 4713-01-01 (BC) 00:00:00 4713-01-01 (BC) 00:25:21+00 +query III +SELECT id, ts::VARCHAR, tstz::VARCHAR FROM s.timestamp_range_test WHERE id > 1 AND id < 4 ORDER BY id +---- +2 10000-01-01 00:00:00 10000-01-01 00:00:00+00 +3 2026-01-15 12:00:00 2026-01-15 12:00:00+00 + +statement error +SELECT ts FROM s.timestamp_range_test WHERE id = 1 +---- +timestamp out of range + +statement error +SELECT tstz FROM s.timestamp_range_test WHERE id = 1 +---- +timestamp out of range + +statement error +COPY (SELECT ts, tstz FROM s.timestamp_range_test WHERE id = 1) TO '{TEST_DIR}/timestamp_range.parquet' (FORMAT PARQUET) +---- +timestamp out of range diff --git a/test/sql/scanner/tpch.test_slow b/test/sql/scanner/tpch.test_slow index 865c164d6..3ae9b233e 100644 --- a/test/sql/scanner/tpch.test_slow +++ b/test/sql/scanner/tpch.test_slow @@ -9,7 +9,13 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner', source_schema='tpch', filter_pushdown=true); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +SET schema 'tpch' loop i 1 9 @@ -28,3 +34,9 @@ PRAGMA tpch({i}) :duckdb/extension/tpch/dbgen/answers/sf0.01/q{i}.csv endloop + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/types.test b/test/sql/scanner/types.test index df8aac10d..c25bce164 100644 --- a/test/sql/scanner/types.test +++ b/test/sql/scanner/types.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query IIIIIIII select * from pg_numtypes; @@ -50,4 +45,5 @@ ford T black statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/storage/attach_alter.test b/test/sql/storage/attach_alter.test index 5a5043055..2fff7cff1 100644 --- a/test/sql/storage/attach_alter.test +++ b/test/sql/storage/attach_alter.test @@ -85,12 +85,27 @@ ALTER TABLE s1.test2 ADD COLUMN i INTEGER ---- already exists -# unsupported alter table type -statement error +# DROP NOT NULL is only supported when the statement is pushed down into Postgres +statement ok ALTER TABLE s1.test2 ALTER COLUMN i DROP NOT NULL + +statement ok +SET pg_enable_remote_pushdown = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s1_local (TYPE POSTGRES) + +statement error +ALTER TABLE s1_local.test2 ALTER COLUMN i DROP NOT NULL ---- Unsupported ALTER TABLE type +statement ok +DETACH s1_local + +statement ok +RESET pg_enable_remote_pushdown + # rename column does not exist statement error ALTER TABLE s1.test2 RENAME zz TO i diff --git a/test/sql/storage/attach_comments.test b/test/sql/storage/attach_comments.test index 8f4723766..f720a3c71 100644 --- a/test/sql/storage/attach_comments.test +++ b/test/sql/storage/attach_comments.test @@ -30,7 +30,7 @@ CALL postgres_execute('s', $$COMMENT ON COLUMN comment_test.name IS 'this is a c # Clear DuckDB's metadata cache so fresh comments are loaded statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); # Verify the table comment appears in duckdb_tables() query I diff --git a/test/sql/storage/attach_connect_display.test b/test/sql/storage/attach_connect_display.test new file mode 100644 index 000000000..3b44b9a57 --- /dev/null +++ b/test/sql/storage/attach_connect_display.test @@ -0,0 +1,162 @@ +# name: test/sql/storage/attach_connect_display.test +# description: Test validation of the CONNECT_DISPLAY attach option +# group: [storage] + +require postgres_scanner + +# CONNECT_DISPLAY is validated while binding the ATTACH, before any connection is made, so none of this +# needs a Postgres to connect to. Values that pass validation get as far as the connection attempt, which +# is why the accepted cases below expect "Unable to connect to Postgres at". + +# NULL is rejected by DuckDB itself, before the option reaches the Postgres extension +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY NULL) +---- +NULL is not supported as a valid option for ATTACH option + +# the value ends up in the shell prompt verbatim, so escape sequences must not survive +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://x' || chr(27) || '[31m') +---- +must not contain control characters + +# newline, carriage return and tab would all break up the prompt +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'line1' || chr(10) || 'line2') +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(13)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(9) || 'b') +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(127)) +---- +must not contain control characters + +# C1 control characters - 0xC2 0x9B is CSI, which some terminals honour. These are multi-byte in UTF-8, +# so a check that only looks at bytes below 0x20 would let them through. +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY chr(194) || chr(155)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY chr(194) || chr(128)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('x', 1025)) +---- +must be at most 1024 columns wide, was 1025 + +# the limit is on rendered width, so wide characters count double +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('日', 513)) +---- +must be at most 1024 columns wide, was 1026 + +# unrecognized options are still rejected +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY_NAME 'rds://my-instance') +---- +Unrecognized option for Postgres attach: connect_display_name + +# --- values that pass validation --- + +# a realistic endpoint is nowhere near the width limit +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'redshift://my-cluster.abc123xyz789.eu-west-1.redshift.amazonaws.com:5439/analytics') +---- +Unable to connect to Postgres at + +# exactly at the width limit, in both narrow and wide characters +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('x', 1024)) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('日', 512)) +---- +Unable to connect to Postgres at + +# printable non-ASCII is fine - only control characters are rejected +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://café-日本-🚀') +---- +Unable to connect to Postgres at + +# an empty display string is accepted and falls back to the derived postgres://host:port +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY '') +---- +Unable to connect to Postgres at + +# the option name is matched case insensitively +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, connect_display 'rds://x') +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CoNnEcT_DiSpLaY 'rds://x') +---- +Unable to connect to Postgres at + +# whitespace and zero width characters are not control characters, so they are accepted +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY ' ') +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(8203) || 'b') +---- +Unable to connect to Postgres at + +# there is no test for invalid UTF-8 because it cannot be reached from SQL - DuckDB VARCHAR is always +# valid UTF-8, and casting a BLOB to VARCHAR yields the escaped text form rather than the raw bytes. +# The UTF-8 check in ExtractConnectDisplay guards the C++ path, where a Value is built directly. +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY CAST('\xC3\x28'::BLOB AS VARCHAR)) +---- +Unable to connect to Postgres at + +# non-VARCHAR values are accepted and converted with ToString(), matching how SECRET is handled +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 42) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY ['a','b']) +---- +Unable to connect to Postgres at + +# --- the option must not disturb the rest of the attach --- + +# omitting it entirely still binds - it is an optional constructor argument +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://x', SCHEMA 'public', READ_ONLY) +---- +Unable to connect to Postgres at + +# repeating the option is resolved by DuckDB before the extension sees it +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a', CONNECT_DISPLAY 'b') +---- +Parser Error: Unexpected duplicate option "connect_display" diff --git a/test/sql/storage/attach_connection_init.test b/test/sql/storage/attach_connection_init.test index ac79959f3..55b463d9d 100644 --- a/test/sql/storage/attach_connection_init.test +++ b/test/sql/storage/attach_connection_init.test @@ -9,6 +9,16 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); +statement ok +DETACH s + +# this test inspects the connections used by parallel table scans +statement ok +SET pg_enable_remote_pushdown = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + statement ok CREATE OR REPLACE TABLE s.large_connection_init AS FROM range(1000000) t(i) diff --git a/test/sql/storage/attach_delete.test b/test/sql/storage/attach_delete.test index ea2607157..bc1d1a820 100644 --- a/test/sql/storage/attach_delete.test +++ b/test/sql/storage/attach_delete.test @@ -63,12 +63,24 @@ DELETE FROM s1.test ---- 0 -# RETURNING statement +# RETURNING statement - only supported when the DELETE is pushed down into Postgres +statement ok +SET pg_enable_remote_pushdown = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s1_local (TYPE POSTGRES) + statement error -DELETE FROM s1.test RETURNING *; +DELETE FROM s1_local.test RETURNING *; ---- not yet supported +statement ok +DETACH s1_local + +statement ok +RESET pg_enable_remote_pushdown + statement ok INSERT INTO s1.test VALUES (1), (2), (3), (NULL); diff --git a/test/sql/storage/attach_describe.test b/test/sql/storage/attach_describe.test index 2c478b0f4..94603b635 100644 --- a/test/sql/storage/attach_describe.test +++ b/test/sql/storage/attach_describe.test @@ -48,7 +48,7 @@ hello, world mode skip statement ok -CALL pg_clear_cache() +CALL postgres_clear_cache() query IIIIII DESCRIBE s1.test diff --git a/test/sql/storage/attach_duckdb_indexes_constraint_type.test b/test/sql/storage/attach_duckdb_indexes_constraint_type.test new file mode 100644 index 000000000..43ed05244 --- /dev/null +++ b/test/sql/storage/attach_duckdb_indexes_constraint_type.test @@ -0,0 +1,35 @@ +# name: test/sql/storage/attach_duckdb_indexes_constraint_type.test +# description: duckdb_indexes() must not crash scanning Postgres indexes (constraint_type uninitialized read) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.constraint_type_tbl + +statement ok +CREATE TABLE s.constraint_type_tbl(i INTEGER) + +statement ok +CREATE INDEX constraint_type_idx ON s.constraint_type_tbl(i) + +query I +SELECT count(*) FROM duckdb_indexes() WHERE database_name = 's' AND index_name = 'constraint_type_idx' +---- +1 + +query II +SELECT is_unique, is_primary FROM duckdb_indexes() WHERE database_name = 's' AND index_name = 'constraint_type_idx' +---- +false false + +statement ok +DROP INDEX s.constraint_type_idx + +statement ok +DROP TABLE s.constraint_type_tbl diff --git a/test/sql/storage/attach_existing_multidimensional_array.test b/test/sql/storage/attach_existing_multidimensional_array.test index 5fc34f775..cc2f9451f 100644 --- a/test/sql/storage/attach_existing_multidimensional_array.test +++ b/test/sql/storage/attach_existing_multidimensional_array.test @@ -47,10 +47,17 @@ statement ok SET pg_array_as_varchar=true statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query I SELECT * FROM mixed_arrays ---- {{{1,2,3}},{{4,5,6}},{{7,8,9}}} {1,2,3} + +# pg_array_as_varchar also applies to postgres_query +query I +SELECT * FROM postgres_query('s', 'SELECT * FROM mixed_arrays') +---- +{{{1,2,3}},{{4,5,6}},{{7,8,9}}} +{1,2,3} diff --git a/test/sql/storage/attach_filter_pushdown.test b/test/sql/storage/attach_filter_pushdown.test index 4d01031ba..e603dcf60 100644 --- a/test/sql/storage/attach_filter_pushdown.test +++ b/test/sql/storage/attach_filter_pushdown.test @@ -1,5 +1,5 @@ # name: test/sql/storage/attach_filter_pushdown.test -# description: Test experimental filter pushdown +# description: Test filter pushdown for expressions when remote execute is not supported # group: [storage] require postgres_scanner @@ -7,58 +7,358 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -CREATE OR REPLACE TABLE s1.filter_pushdown(i INTEGER) +CREATE OR REPLACE TABLE s.filter_pushdown_test(col1 INTEGER, col2 VARCHAR) statement ok -INSERT INTO s1.filter_pushdown FROM range(1000000) +INSERT INTO s.filter_pushdown_test values (41, 'foo'), (42, 'bar'), (44, NULL) -statement ok -SET pg_experimental_filter_pushdown=true +# ExpressionClass::BOUND_CONJUNCTION + +# ExpressionType::CONJUNCTION_AND + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 AND col2 = 'foo' +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 AND col2 = 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col1 = 41.*col2 = 'foo'.* + +# ExpressionType::CONJUNCTION_OR + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE (col1 = 41 AND col2 = 'foo') OR col2 = 'bar' +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE (col1 = 41 AND col2 = 'foo') OR col2 = 'bar' +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 'foo'\) OR \(col2 = 'bar'\)\).* + +# ExpressionClass::BOUND_OPERATOR + +# ExpressionType::OPERATOR_IS_NULL + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NULL +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NULL +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NULL.* + +# ExpressionType::OPERATOR_IS_NOT_NULL + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT NULL +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT NULL +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NOT NULL.* + +# ExpressionType::COMPARE_IN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 IN (41, 44) +---- +41 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 IN (41, 44) +---- +physical_plan :.*Postgres Scan.*Filters: optional: \(col1 IN \(41, 44\)\).* + +# indirect reference + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE len(col2) IN (3, 4) +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE len(col2) IN (3, 4) +---- +physical_plan :.*Postgres Scan.*Filters:.* + +# ExpressionClass::BOUND_FUNCTION + +# ExpressionType::COMPARE_EQUAL: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 = 41.* + +# ExpressionType::COMPARE_NOTEQUAL: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 != 41 +---- +42 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 != 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 != 41.* + +# ExpressionType::COMPARE_LESSTHAN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 < 42 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 < 42 +---- +physical_plan :.*Postgres Scan.*Filters: col1 < 42.* + +# ExpressionType::COMPARE_GREATERTHAN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 > 42 +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 > 42 +---- +physical_plan :.*Postgres Scan.*Filters: col1 > 42.* + +# ExpressionType::COMPARE_LESSTHANOREQUALTO: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 <= 41 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 <= 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 <= 41.* + +# ExpressionType::COMPARE_GREATERTHANOREQUALTO: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 >= 44 +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 >= 44 +---- +physical_plan :.*Postgres Scan.*Filters: col1 >= 44.* + +# ExpressionType::COMPARE_DISTINCT_FROM: query I -SELECT * FROM s1.filter_pushdown WHERE i=52525 +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS DISTINCT FROM 'foo' ---- -52525 +42 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS DISTINCT FROM 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS DISTINCT FROM 'foo'.* + +# ExpressionType::COMPARE_NOT_DISTINCT_FROM: query I -SELECT * FROM s1.filter_pushdown WHERE i IN (32, 52525, 777777) ORDER BY ALL +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT DISTINCT FROM 'foo' ---- -32 -52525 -777777 +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT DISTINCT FROM 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NOT DISTINCT FROM 'foo'.* + +# VARCHAR comparison logic + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'foo' +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 = 'foo'.* + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'FOO' +---- + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'FOO' +---- +physical_plan :.*Postgres Scan.*Filters: col2 = 'FOO'.* statement ok -CREATE OR REPLACE TABLE s1.filter_pushdown_types(v VARCHAR, d DATE); +DROP TABLE s.filter_pushdown_test + +# VARCHAR statement ok -INSERT INTO s1.filter_pushdown_types VALUES ('hello world', DATE '1992-01-01'), (NULL, NULL), ('I''m here', DATE '2020-01-01'); +CREATE OR REPLACE TABLE s.filter_pushdown_string(i STRING) + +statement ok +INSERT INTO s.filter_pushdown_string SELECT CAST(range AS STRING) FROM range(10000); + +query II +SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i = '5252' +---- +5252 4 + +query II +EXPLAIN SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i = '5252' +---- +physical_plan :.*Postgres Scan.*Filters: i = '5252'.* + +query II +SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i > '10' ORDER BY i LIMIT 10 +---- +100 3 +1000 4 +1001 4 +1002 4 +1003 4 +1004 4 +1005 4 +1006 4 +1007 4 +1008 4 + +query II +EXPLAIN SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i > '10' ORDER BY i LIMIT 10 +---- +physical_plan :.*Postgres Scan.*Filters:.*i > '10'.* + +statement ok +DROP TABLE s.filter_pushdown_string + +# timestamp pushdown + +statement ok +CREATE OR REPLACE TABLE s.datetime_tbl(id INTEGER, d DATE, ts TIMESTAMP) + +statement ok +INSERT INTO s.datetime_tbl VALUES (1, '2020-02-03', '2029-02-14 08:47:23'),(2, '1000-01-01', '1970-01-01 00:00:01.000000'), (3, '9999-12-31', '2038-01-18 22:14:07.499999'), (4, NULL, NULL) + +query I +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE d >= DATE '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE d >= DATE '2000-01-01' +---- +physical_plan :.*Postgres Scan.*Filters: d >= '2000-01-01'::DATE.* query I -SELECT v FROM s1.filter_pushdown_types WHERE v IN ('hello world', 'I''m here') ORDER BY ALL +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' ---- -I'm here -hello world +physical_plan :.*Postgres Scan.*Filters:.*ts >= '2000-01-01 00:00:00'::TIMESTAMP.* query I -SELECT v FROM s1.filter_pushdown_types WHERE v='I''m here' +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' ---- -I'm here +physical_plan :.*Postgres Scan.*Filters:.*ts >= '2000-01-01 00:00:00'::TIMESTAMP.* + +statement ok +DROP TABLE s.datetime_tbl + +# boolean pushdown + +statement ok +CREATE OR REPLACE TABLE s.booleans_tbl(b BOOLEAN); + +statement ok +INSERT INTO s.booleans_tbl VALUES (false), (true), (NULL); query I -SELECT d FROM s1.filter_pushdown_types WHERE d=DATE '1992-01-01' +SELECT divide(b::INT, 1) FROM s.booleans_tbl WHERE b = true ---- -1992-01-01 +1 + +query II +EXPLAIN SELECT divide(b::INT, 1) FROM s.booleans_tbl WHERE b = true +---- +physical_plan :.*Postgres Scan.*Filters: b = true.* + +statement ok +DROP TABLE s.booleans_tbl + +# string pushdown + +statement ok +CREATE OR REPLACE TABLE s.text_tbl(v VARCHAR(4), c CHAR(4), t TEXT); + +statement ok +INSERT INTO s.text_tbl VALUES ('ab ', 'ab ', 'thisisalongstring'), ('', '', ''), ('🦆', '🦆', '🦆🦆🦆🦆'), (NULL, NULL, NULL) query I -SELECT * FROM s1.composites_of_composites WHERE b.k < 8 +SELECT divide(COUNT(*), 1) FROM s.text_tbl WHERE v= '🦆' ---- -{'a': {'i': 1, 'j': 2}, 'k': 3} -{'a': {'i': 4, 'j': 5}, 'k': 6} +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.text_tbl WHERE v= '🦆' +---- +physical_plan :.*Postgres Scan.*Filters: v = '🦆'.* + +statement ok +DROP TABLE s.text_tbl + +# blob pushdown + +statement ok +CREATE OR REPLACE TABLE s.blob_tbl(bl BLOB); + +statement ok +INSERT INTO s.blob_tbl VALUES (''), ('\x80'), (NULL) query I -SELECT * FROM s1.composites_of_composites WHERE b.a.j = 5 +SELECT divide(COUNT(*), 1) FROM s.blob_tbl WHERE bl= BLOB '\x80' ---- -{'a': {'i': 4, 'j': 5}, 'k': 6} +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.blob_tbl WHERE bl= BLOB '\x80' +---- +physical_plan :.*Postgres Scan.*Filters: bl = '\\x80'::BLOB.* + +statement ok +DROP TABLE s.blob_tbl diff --git a/test/sql/storage/attach_filter_pushdown_nonfinite.test b/test/sql/storage/attach_filter_pushdown_nonfinite.test new file mode 100644 index 000000000..3cb2b8e0d --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_nonfinite.test @@ -0,0 +1,286 @@ +# name: test/sql/storage/attach_filter_pushdown_nonfinite.test +# description: Test filter pushdown with non-finite input for expressions when remote execute is not supported +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.filter_pushdown_test_infinite(col1 FLOAT, col2 DOUBLE, col3 DATE, col4 TIMESTAMP) + +statement ok +INSERT INTO s.filter_pushdown_test_infinite values (41.0, 42.0, '2020-12-31', '2020-12-31 23:59:59') + +# ExpressionType::COMPARE_IN: + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT, 41.0::FLOAT) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT, 41.0::FLOAT) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col1 IN \('inf'::FLOAT, 'nan'::FLOAT, 41.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col1 IN \('inf'::FLOAT, 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE, 42.0::DOUBLE) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE, 42.0::DOUBLE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col2 IN \(1e1000, 'nan'::DOUBLE, 42.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col2 IN \(1e1000, 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE, '2020-12-31'::DATE) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE, '2020-12-31'::DATE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col3 IN \('infinity'::DATE, '2020-12-31'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE) +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 = 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col4 IN \('infinity'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP) +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 = 'infinity'::TIMESTAMP.* + +# ExpressionType::COMPARE_EQUAL + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT OR col1 = 41.0::FLOAT +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT OR col1 = 41.0::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 = 'inf'::FLOAT\) OR \(col1 = 'nan'::FLOAT\) OR \(col1 = 41.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 = 'inf'::FLOAT\) OR \(col1 = 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE OR col2 = 42.0::DOUBLE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE OR col2 = 42.0::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 1e1000\) OR \(col2 = 'nan'::DOUBLE\) OR \(col2 = 42.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 1e1000\) OR \(col2 = 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE OR col3 = '2020-12-31'::DATE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE OR col3 = '2020-12-31'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col3 = 'infinity'::DATE\) OR \(col3 = '2020-12-31'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 = 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP OR col4 = '2020-12-31 23:59:59'::TIMESTAMP +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP OR col4 = '2020-12-31 23:59:59'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col4 = 'infinity'::TIMESTAMP\) OR \(col4 = '2020-12-31 23:59:59'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 = 'infinity'::TIMESTAMP.* + +# ExpressionType::COMPARE_GREATERTHAN + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT OR col1 > 40.0::FLOAT +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT OR col1 > 40.0::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 > 'inf'::FLOAT\) OR \(col1 > 'nan'::FLOAT\) OR \(col1 > 40.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 > 'inf'::FLOAT\) OR \(col1 > 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE OR col2 > 40.0::DOUBLE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE OR col2 > 40.0::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 > 1e1000\) OR \(col2 > 'nan'::DOUBLE\) OR \(col2 > 40.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 > 1e1000\) OR \(col2 > 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE OR col3 > '2020-12-30'::DATE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE OR col3 > '2020-12-30'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col3 > 'infinity'::DATE\) OR \(col3 > '2020-12-30'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 > 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP OR col4 > '2020-12-31 23:59:58'::TIMESTAMP +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP OR col4 > '2020-12-31 23:59:58'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col4 > 'infinity'::TIMESTAMP\) OR \(col4 > '2020-12-31 23:59:58'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 > 'infinity'::TIMESTAMP.* diff --git a/test/sql/storage/attach_filter_pushdown_or_enum.test b/test/sql/storage/attach_filter_pushdown_or_enum.test new file mode 100644 index 000000000..76c1f7a3b --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_or_enum.test @@ -0,0 +1,49 @@ +# name: test/sql/storage/attach_filter_pushdown_or_enum.test +# description: Test filter pushdown of an OR filter with a child that cannot be pushed down +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.filter_pushdown_or_enum + +statement ok +DROP TYPE IF EXISTS s.filter_pushdown_mood + +statement ok +CREATE TYPE s.filter_pushdown_mood AS ENUM ('sad', 'happy') + +statement ok +CREATE TABLE s.filter_pushdown_or_enum(id INTEGER, m s.filter_pushdown_mood) + +statement ok +INSERT INTO s.filter_pushdown_or_enum VALUES (1, 'sad'), (2, 'happy'), (3, NULL) + +# ENUM comparisons are not pushed down, so neither is the OR +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE m = 'happy' OR m IS NULL ORDER BY id +---- +2 +3 + +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE (m = 'happy' AND m IS NOT NULL) OR m IS NULL ORDER BY id +---- +2 +3 + +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE id = 1 OR id = 3 ORDER BY id +---- +1 +3 + +query II +EXPLAIN SELECT id FROM s.filter_pushdown_or_enum WHERE id = 1 OR id = 3 +---- +physical_plan :.*Postgres Scan.*Filters:.*id = 1.*id = 3.* diff --git a/test/sql/storage/attach_filter_pushdown_timestamptz.test b/test/sql/storage/attach_filter_pushdown_timestamptz.test new file mode 100644 index 000000000..eeed7ed97 --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_timestamptz.test @@ -0,0 +1,36 @@ +# name: test/sql/storage/attach_filter_pushdown_timestamptz.test +# description: Test filter pushdown of TIMESTAMPTZ constants when the session time zone of the server is not UTC +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner options=-ctimezone=America/New_York' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.filter_pushdown_timestamptz(id INTEGER, ts TIMESTAMPTZ) + +statement ok +INSERT INTO s.filter_pushdown_timestamptz VALUES (1, '2024-01-01 12:00:00+00'), (2, '2024-01-01 13:00:00+00') + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts < TIMESTAMPTZ '2024-01-01 11:30:00+00' +---- + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts > TIMESTAMPTZ '2024-01-01 12:30:00+00' +---- +2 + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts = TIMESTAMPTZ '2024-01-01 12:00:00+00' +---- +1 + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts < 'infinity'::TIMESTAMPTZ ORDER BY id +---- +1 +2 diff --git a/test/sql/storage/attach_generated_columns.test b/test/sql/storage/attach_generated_columns.test new file mode 100644 index 000000000..e36180291 --- /dev/null +++ b/test/sql/storage/attach_generated_columns.test @@ -0,0 +1,75 @@ +# name: test/sql/storage/attach_generated_columns.test +# description: Test inserting into Postgres tables with generated columns +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# COPY and plain INSERT statements are generated differently +foreach use_text_protocol false true + +statement ok +SET pg_use_text_protocol={use_text_protocol} + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS generated_columns_test') + +# VIRTUAL generated columns require Postgres 18, older versions use STORED instead +statement ok +CALL postgres_execute('s', 'DO $$ BEGIN EXECUTE format($q$CREATE TABLE generated_columns_test ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + price NUMERIC(10,2), + qty INTEGER, + total NUMERIC(12,2) GENERATED ALWAYS AS (price * qty) STORED, + total_virtual NUMERIC(12,2) GENERATED ALWAYS AS (price * qty * 2) %s)$q$, + CASE WHEN current_setting(''server_version_num'')::INTEGER >= 180000 THEN ''VIRTUAL'' ELSE ''STORED'' END); +END $$') + +statement ok +CALL postgres_clear_cache() + +# generated and identity columns not specified +statement ok +INSERT INTO s.generated_columns_test (price, qty) VALUES (2.50, 4) + +# columns specified in a different order +statement ok +INSERT INTO s.generated_columns_test (qty, price) VALUES (3, 1.00) + +# all columns, generated columns set to NULL +statement ok +INSERT INTO s.generated_columns_test VALUES (100, 5.00, 2, NULL, NULL) + +# explicit values for generated columns are not allowed +statement error +INSERT INTO s.generated_columns_test (price, qty, total) VALUES (1.00, 1, 42) +---- +generated column + +statement error +INSERT INTO s.generated_columns_test (price, qty, total_virtual) VALUES (1.00, 1, 42) +---- +generated column + +# insert from a query +statement ok +INSERT INTO s.generated_columns_test (price, qty) SELECT 1.5, i FROM range(3) t(i) + +query IIIII +SELECT * FROM s.generated_columns_test ORDER BY id +---- +1 2.50 4 10.00 20.00 +2 1.00 3 3.00 6.00 +3 1.50 0 0.00 0.00 +4 1.50 1 1.50 3.00 +5 1.50 2 3.00 6.00 +100 5.00 2 10.00 20.00 + +statement ok +CALL postgres_execute('s', 'DROP TABLE generated_columns_test') + +endloop diff --git a/test/sql/storage/attach_giant_numeric.test b/test/sql/storage/attach_giant_numeric.test index 1f85debfc..4317ebfdc 100644 --- a/test/sql/storage/attach_giant_numeric.test +++ b/test/sql/storage/attach_giant_numeric.test @@ -14,13 +14,13 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 query I SELECT * FROM s.pg_giant_numeric_array ---- -[0.0, 1.0000000000000002e+32, -1.234567891234568e+26] +[0.0, 1e+32, -1.2345678912345679e+26] [] # insert into a NUMERIC field @@ -35,8 +35,8 @@ query I SELECT * FROM s.pg_numeric_empty ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DELETE FROM s.pg_numeric_array_empty @@ -49,5 +49,5 @@ query I SELECT UNNEST(n) FROM s.pg_numeric_array_empty ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 diff --git a/test/sql/storage/attach_insert_postgres_defaults.test b/test/sql/storage/attach_insert_postgres_defaults.test new file mode 100644 index 000000000..69d0e823b --- /dev/null +++ b/test/sql/storage/attach_insert_postgres_defaults.test @@ -0,0 +1,174 @@ +# name: test/sql/storage/attach_insert_postgres_defaults.test +# description: Test inserting into Postgres tables with SERIAL, identity and DEFAULT columns +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# COPY and plain INSERT statements are generated differently +foreach use_text_protocol false true + +statement ok +SET pg_use_text_protocol={use_text_protocol} + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS insert_defaults_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE insert_defaults_test ( + id SERIAL PRIMARY KEY, + v INTEGER, + d INTEGER DEFAULT 7, + ts TIMESTAMP DEFAULT now())') + +statement ok +CALL postgres_clear_cache() + +# SERIAL and DEFAULT columns not specified +statement ok +INSERT INTO s.insert_defaults_test (v) VALUES (1) + +# explicit values +statement ok +INSERT INTO s.insert_defaults_test (id, v, d) VALUES (100, 2, 8) + +# NULL values in DEFAULT columns are replaced with the default +statement ok +INSERT INTO s.insert_defaults_test VALUES (NULL, 3, NULL, NULL) + +statement ok +INSERT INTO s.insert_defaults_test DEFAULT VALUES + +query IIII +SELECT id, v, d, ts IS NULL FROM s.insert_defaults_test ORDER BY id +---- +1 1 7 false +2 3 7 false +3 NULL 7 false +100 2 8 false + +statement ok +DELETE FROM s.insert_defaults_test + +# NULL and non-NULL values in the same input +statement ok +INSERT INTO s.insert_defaults_test (v, d) SELECT i, CASE WHEN i % 3 = 0 THEN NULL ELSE i END FROM range(5000) t(i) + +query IIII +SELECT COUNT(*), COUNT(DISTINCT id), SUM(v), SUM(d) FROM s.insert_defaults_test +---- +5000 5000 12497500 8343336 + +query I +SELECT COUNT(*) FROM s.insert_defaults_test WHERE d IS DISTINCT FROM (CASE WHEN v % 3 = 0 THEN 7 ELSE v END) +---- +0 + +statement ok +CALL postgres_execute('s', 'DROP TABLE insert_defaults_test') + +# identity columns +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS insert_identity_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE insert_identity_test ( + id_always INTEGER GENERATED ALWAYS AS IDENTITY, + id_default INTEGER GENERATED BY DEFAULT AS IDENTITY, + v INTEGER)') + +statement ok +CALL postgres_clear_cache() + +statement ok +INSERT INTO s.insert_identity_test (v) VALUES (1), (2) + +statement ok +INSERT INTO s.insert_identity_test (id_default, v) VALUES (100, 3) + +query III +SELECT * FROM s.insert_identity_test ORDER BY v +---- +1 1 1 +2 2 2 +3 100 3 + +statement ok +CALL postgres_execute('s', 'DROP TABLE insert_identity_test') + +# every column has a default +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS insert_all_defaults_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE insert_all_defaults_test (id SERIAL, d INTEGER DEFAULT 7)') + +statement ok +CALL postgres_clear_cache() + +statement ok +INSERT INTO s.insert_all_defaults_test DEFAULT VALUES + +statement ok +INSERT INTO s.insert_all_defaults_test SELECT NULL, NULL FROM range(3) + +query II +SELECT * FROM s.insert_all_defaults_test ORDER BY id +---- +1 7 +2 7 +3 7 +4 7 + +statement ok +CALL postgres_execute('s', 'DROP TABLE insert_all_defaults_test') + +# table created through DuckDB keeps its default after the catalog is reloaded +statement ok +CREATE OR REPLACE TABLE s.insert_duckdb_defaults_test (i INTEGER DEFAULT 42, j INTEGER) + +statement ok +INSERT INTO s.insert_duckdb_defaults_test (j) VALUES (1) + +statement ok +CALL postgres_clear_cache() + +statement ok +INSERT INTO s.insert_duckdb_defaults_test (j) VALUES (2) + +query II +SELECT * FROM s.insert_duckdb_defaults_test ORDER BY j +---- +42 1 +42 2 + +statement ok +DROP TABLE s.insert_duckdb_defaults_test + +# MERGE INTO inserts +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS insert_merge_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE insert_merge_test (id SERIAL PRIMARY KEY, k INTEGER, d INTEGER DEFAULT 7)') + +statement ok +CALL postgres_clear_cache() + +statement ok +MERGE INTO s.insert_merge_test USING (SELECT 1 AS k) src ON insert_merge_test.k = src.k +WHEN NOT MATCHED THEN INSERT (k) VALUES (src.k) + +query III +SELECT * FROM s.insert_merge_test +---- +1 1 7 + +statement ok +CALL postgres_execute('s', 'DROP TABLE insert_merge_test') + +endloop diff --git a/test/sql/storage/attach_isolation_level.test b/test/sql/storage/attach_isolation_level.test index b660fbde1..5db6051c4 100644 --- a/test/sql/storage/attach_isolation_level.test +++ b/test/sql/storage/attach_isolation_level.test @@ -14,8 +14,8 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DETACH s @@ -28,8 +28,8 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DETACH s diff --git a/test/sql/storage/attach_multi_join_large.test_slow b/test/sql/storage/attach_multi_join_large.test_slow index 8b0525c85..762898028 100644 --- a/test/sql/storage/attach_multi_join_large.test_slow +++ b/test/sql/storage/attach_multi_join_large.test_slow @@ -30,14 +30,3 @@ left join test3_large c on a.id = c.id; ---- 1000000 - -# same query but with postgres_scan -query I -select count(*) -from postgres_scan('dbname=postgresscanner', 'public', 'test1_large') a -left join postgres_scan('dbname=postgresscanner', 'public', 'test2_large') b - on a.id = b.id -left join postgres_scan('dbname=postgresscanner', 'public', 'test3_large') c - on a.id = c.id; ----- -1000000 diff --git a/test/sql/storage/attach_numeric_as_varchar.test b/test/sql/storage/attach_numeric_as_varchar.test new file mode 100644 index 000000000..235adba9d --- /dev/null +++ b/test/sql/storage/attach_numeric_as_varchar.test @@ -0,0 +1,116 @@ +# name: test/sql/storage/attach_numeric_as_varchar.test +# description: Test reading unbound numerics as varchar with pg_numeric_as_varchar +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS unbound_numerics') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE unbound_numerics(n NUMERIC)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO unbound_numerics VALUES (12345678901234567890123456789012345678901234567890), (0.12345678901234567890123456789012345678901234567890), (NULL)') + +# by default unbound numerics are read as double +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +DOUBLE + +statement ok +SET pg_numeric_as_varchar=true + +statement ok +CALL postgres_clear_cache(); + +# with pg_numeric_as_varchar the values are read as varchar without precision loss +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +VARCHAR + +query I +SELECT n FROM s.unbound_numerics +---- +12345678901234567890123456789012345678901234567890 +0.12345678901234567890123456789012345678901234567890 +NULL + +# the setting also applies to postgres_query +query I +SELECT * FROM postgres_query('s', 'SELECT n FROM unbound_numerics WHERE n > 1') +---- +12345678901234567890123456789012345678901234567890 + +# unbound numerics inside arrays +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS unbound_numeric_arrays') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE unbound_numeric_arrays(n NUMERIC[])') + +statement ok +CALL postgres_execute('s', 'INSERT INTO unbound_numeric_arrays VALUES (ARRAY[12345678901234567890123456789012345678901234567890::NUMERIC, 2])') + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT n FROM s.unbound_numeric_arrays +---- +[12345678901234567890123456789012345678901234567890, 2] + +# bounded numerics with a precision above DuckDB's decimal maximum of 38 take the same fallback +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS oversized_numerics') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE oversized_numerics(n NUMERIC(50,10))') + +statement ok +CALL postgres_execute('s', 'INSERT INTO oversized_numerics VALUES (1234567890123456789012345678901234567890.0123456789)') + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT typeof(n) FROM s.oversized_numerics +---- +VARCHAR + +query I +SELECT n FROM s.oversized_numerics +---- +1234567890123456789012345678901234567890.0123456789 + +statement ok +SET pg_numeric_as_varchar=false + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +DOUBLE + +query I +SELECT typeof(n) FROM s.oversized_numerics +---- +DOUBLE + +statement ok +CALL postgres_execute('s', 'DROP TABLE unbound_numerics') + +statement ok +CALL postgres_execute('s', 'DROP TABLE oversized_numerics') + +statement ok +CALL postgres_execute('s', 'DROP TABLE unbound_numeric_arrays') diff --git a/test/sql/storage/attach_oauth_token.test b/test/sql/storage/attach_oauth_token.test index c1c4ca6a0..3d6e198b5 100644 --- a/test/sql/storage/attach_oauth_token.test +++ b/test/sql/storage/attach_oauth_token.test @@ -10,10 +10,6 @@ require-env PGOAUTHTOKEN require-env PG_OAUTH_DSN -# Explicitly load the extension so settings are registered -statement ok -LOAD postgres_scanner; - # Test 1: Connect using the pg_oauth_token DuckDB setting statement ok SET pg_oauth_token = '${PGOAUTHTOKEN}'; diff --git a/test/sql/storage/attach_on_conflict_dropped_column.test b/test/sql/storage/attach_on_conflict_dropped_column.test new file mode 100644 index 000000000..d97b99fd8 --- /dev/null +++ b/test/sql/storage/attach_on_conflict_dropped_column.test @@ -0,0 +1,107 @@ +# name: test/sql/storage/attach_on_conflict_dropped_column.test +# description: ON CONFLICT must resolve UNIQUE/PRIMARY KEY constraints correctly after DROP COLUMN (issue #454) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) + +# Postgres does not renumber attnums after DROP COLUMN, so a column dropped +# physically before the key columns leaves a gap between the constraint's +# attnums and the remaining columns. UNIQUE(a, b, c) has attnums {3, 4, 5}, and +# dropping 'e' (attnum 2) must not break resolution of the conflict target. +statement ok +CREATE OR REPLACE TABLE s1.tbl(d INTEGER, e INTEGER, a INTEGER, b INTEGER, c INTEGER, UNIQUE(a, b, c)); + +statement ok +ALTER TABLE s1.tbl DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 4); + +# DO NOTHING leaves the existing row untouched +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 99) ON CONFLICT (a, b, c) DO NOTHING; + +query IIII +SELECT d, a, b, c FROM s1.tbl +---- +4 1 2 3 + +# DO UPDATE targets the conflicting row +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 7) ON CONFLICT (a, b, c) DO UPDATE SET d = excluded.d; + +query IIII +SELECT d, a, b, c FROM s1.tbl +---- +7 1 2 3 + +# The same attnum gap must not silently shift the key onto the wrong in-range +# columns: the real UNIQUE key is (a, b) (attnums {3, 4}), so (b, c) is not a +# valid conflict target while (a, b) is. +statement ok +CREATE OR REPLACE TABLE s1.tbl2(d INTEGER, e INTEGER, a INTEGER, b INTEGER, c INTEGER, f INTEGER, UNIQUE(a, b)); + +statement ok +ALTER TABLE s1.tbl2 DROP COLUMN e; + +statement error +INSERT INTO s1.tbl2(a, b, c) VALUES (1, 2, 3) ON CONFLICT (b, c) DO NOTHING; +---- +The specified columns as conflict target are not referenced by a UNIQUE/PRIMARY KEY CONSTRAINT + +statement ok +INSERT INTO s1.tbl2(a, b, c) VALUES (1, 2, 3) ON CONFLICT (a, b) DO NOTHING; + +# PRIMARY KEY decodes conkey the same way; verify the gap case there too. +statement ok +CREATE OR REPLACE TABLE s1.tbl3(d INTEGER, e INTEGER, a INTEGER, b INTEGER, PRIMARY KEY(a, b)); + +statement ok +ALTER TABLE s1.tbl3 DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl3(a, b, d) VALUES (1, 2, 10); + +statement ok +INSERT INTO s1.tbl3(a, b, d) VALUES (1, 2, 20) ON CONFLICT (a, b) DO UPDATE SET d = excluded.d; + +query III +SELECT a, b, d FROM s1.tbl3 +---- +1 2 20 + +# Dropping a column that comes physically *after* the key columns leaves no gap +# in the key's attnums, but must still resolve correctly. +statement ok +CREATE OR REPLACE TABLE s1.tbl4(a INTEGER, b INTEGER, e INTEGER, c INTEGER, UNIQUE(a, b)); + +statement ok +ALTER TABLE s1.tbl4 DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl4(a, b, c) VALUES (1, 2, 3); + +statement ok +INSERT INTO s1.tbl4(a, b, c) VALUES (1, 2, 99) ON CONFLICT (a, b) DO NOTHING; + +query III +SELECT a, b, c FROM s1.tbl4 +---- +1 2 3 + +statement ok +DROP TABLE s1.tbl + +statement ok +DROP TABLE s1.tbl2 + +statement ok +DROP TABLE s1.tbl3 + +statement ok +DROP TABLE s1.tbl4 diff --git a/test/sql/storage/attach_order_pushdown.test b/test/sql/storage/attach_order_pushdown.test index ed0aef328..7545c7a6b 100644 --- a/test/sql/storage/attach_order_pushdown.test +++ b/test/sql/storage/attach_order_pushdown.test @@ -16,7 +16,7 @@ statement ok CALL postgres_execute('s1', 'CREATE TABLE order_test (id INT NOT NULL PRIMARY KEY, val INT, name TEXT)'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); statement ok INSERT INTO s1.order_test VALUES @@ -146,13 +146,11 @@ SELECT id FROM s1.order_test ORDER BY name; # WHERE + ORDER BY on non-output column + LIMIT query I -SELECT name FROM s1.order_test WHERE val IS NOT NULL ORDER BY val LIMIT 5; +SELECT name FROM s1.order_test WHERE val IS NOT NULL ORDER BY val LIMIT 3; ---- heidi alice eve -bob -judy # Output multiple columns, sort by non-output column query II diff --git a/test/sql/storage/attach_parallel_isolation_level.test b/test/sql/storage/attach_parallel_isolation_level.test index 0b6e6a40f..c50b487ee 100644 --- a/test/sql/storage/attach_parallel_isolation_level.test +++ b/test/sql/storage/attach_parallel_isolation_level.test @@ -9,6 +9,16 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); +statement ok +DETACH s + +# this test inspects the connections used by parallel table scans +statement ok +SET pg_enable_remote_pushdown = false + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + statement ok CREATE OR REPLACE TABLE s.large_parallel_isolation AS FROM range(1000000) t(i) diff --git a/test/sql/storage/attach_postgres_execute.test b/test/sql/storage/attach_postgres_execute.test index 32b25cb88..c38a29b87 100644 --- a/test/sql/storage/attach_postgres_execute.test +++ b/test/sql/storage/attach_postgres_execute.test @@ -27,3 +27,40 @@ statement error CALL postgres_execute('s', 'CREATE TABLE public.my_unlogged_table(i INTEGER)') ---- already exists + +# row count + +query I +FROM postgres_execute('s', 'DROP TABLE IF EXISTS query_dml_test') +---- +-1 + +query I +FROM postgres_execute('s', 'CREATE TABLE query_dml_test(col1 INT)') +---- +-1 + +query I +FROM postgres_execute('s', 'INSERT INTO query_dml_test values(41), (42)') +---- +2 + +query I +FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 43') +---- +2 + +query I +FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +---- +0 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +) +---- +rowcount BIGINT + +statement ok +CALL postgres_execute('s', 'DROP TABLE query_dml_test') diff --git a/test/sql/storage/attach_rds_iam.test b/test/sql/storage/attach_rds_iam.test index e969ee98c..d9ffc83a1 100644 --- a/test/sql/storage/attach_rds_iam.test +++ b/test/sql/storage/attach_rds_iam.test @@ -16,8 +16,8 @@ statement ok CREATE SECRET rds_template_secret ( TYPE rds, PROVIDER credential_chain, - CHAIN 'env;sso;', - PROFILE 'DatabaseAdministrator-xxx', + CHAIN 'env;sso;process;', + PROFILE 'admin_service', REGION 'eu-west-1', RDS_USER 'postgres', RDS_HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', @@ -43,6 +43,55 @@ FROM postgres_query('rds_attached', 'SELECT current_user') ---- postgres +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +') +---- +DuckDB active + +statement ok +DROP SECRET rds_secret + +statement ok +DETACH rds_attached + +# custom app name + +statement ok +CREATE SECRET rds_secret ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + PORT '5432', + USER 'postgres', + DATABASE 'postgres', + SSLMODE require, + AWS_RDS_SECRET rds_template_secret, + APPLICATION_NAME custom1 +); + +statement ok +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret) + +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +') +---- + +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom1'' +') +---- +custom1 active + statement ok DROP SECRET rds_secret diff --git a/test/sql/storage/attach_rds_iam_token_refresh.test b/test/sql/storage/attach_rds_iam_token_refresh.test new file mode 100644 index 000000000..e25b7e619 --- /dev/null +++ b/test/sql/storage/attach_rds_iam_token_refresh.test @@ -0,0 +1,158 @@ +# name: test/sql/storage/attach_rds_iam_token_refresh.test +# description: Test re-generating AWS RDS IAM token when the attached DB is also a secret storage +# group: [storage] + +require postgres_scanner + +require-env AWS_RDS_DATABASE_AVAILABLE + +statement ok +LOAD postgres + +statement ok +LOAD aws + +statement ok +CREATE SECRET rds_template_secret ( + TYPE rds, + PROVIDER credential_chain, + CHAIN 'env;sso;process;', + PROFILE 'admin_service', + REGION 'eu-west-1', + RDS_USER 'postgres', + RDS_HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + RDS_PORT '5432' +); + +statement ok +CREATE SECRET rds_secret ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + PORT '5432', + USER 'postgres', + DATABASE 'postgres', + SSLMODE require, + AWS_RDS_SECRET rds_template_secret +); + +# clean up leftovers from previous runs +statement ok +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret) + +statement ok +CALL postgres_execute('rds_attached', 'DROP TABLE IF EXISTS duckdb_rds_iam_secrets') + +statement ok +DETACH rds_attached + +# invalid max token age + +statement error +CREATE SECRET rds_secret_invalid ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + AWS_RDS_SECRET rds_template_secret, + AWS_RDS_TOKEN_MAX_AGE_SECONDS -1 +); +---- +Failed to cast option "aws_rds_token_max_age_seconds" to type 'UBIGINT' + +statement ok +CREATE OR REPLACE SECRET rds_secret_invalid ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + AWS_RDS_SECRET rds_template_secret, + AWS_RDS_TOKEN_MAX_AGE_SECONDS 841 +); + +statement error +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret_invalid) +---- +'AWS_RDS_TOKEN_MAX_AGE_SECONDS' must be between 0 and 840, specified: 841 + +statement ok +CREATE OR REPLACE SECRET rds_secret_invalid ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + AWS_RDS_TOKEN_MAX_AGE_SECONDS 1 +); + +statement error +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret_invalid) +---- +'AWS_RDS_TOKEN_MAX_AGE_SECONDS' parameter can only be specified along with 'AWS_RDS_SECRET' parameter + +statement ok +DROP SECRET rds_secret_invalid + +# make the token expire after 1 second +statement ok +CREATE OR REPLACE SECRET rds_secret ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + PORT '5432', + USER 'postgres', + DATABASE 'postgres', + SSLMODE require, + AWS_RDS_SECRET rds_template_secret, + AWS_RDS_TOKEN_MAX_AGE_SECONDS 1 +); + +statement ok +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret, SECRET_STORAGE_TABLE duckdb_rds_iam_secrets) + +statement ok +CREATE SECRET rds_stored_secret IN postgres_rds_attached ( + TYPE postgres, + HOST 'localhost' +) + +query I +FROM postgres_query('rds_attached', 'SELECT current_user') +---- +postgres + +# do not keep connections in pool, so every query opens a new one +query I +SELECT catalog_name FROM postgres_configure_pool(catalog_name='rds_attached', max_connections=0) +---- +rds_attached + +sleep 3 seconds + +# token is expired, it is re-generated when opening a new connection, +# that requires a secret lookup that reaches the secret storage in this DB +query I +FROM postgres_query('rds_attached', 'SELECT current_user') +---- +postgres + +sleep 3 seconds + +query I +FROM postgres_query('rds_attached', 'SELECT current_user') +---- +postgres + +query II +SELECT name, storage FROM duckdb_secrets() WHERE name = 'rds_stored_secret' +---- +rds_stored_secret postgres_rds_attached + +# temporary secret used for token generation is cleaned up +query I +SELECT count(*) FROM duckdb_secrets() WHERE name LIKE '%_postgres_temp' +---- +0 + +statement ok +DROP SECRET rds_stored_secret + +statement ok +CALL postgres_execute('rds_attached', 'DROP TABLE duckdb_rds_iam_secrets') + +statement ok +DETACH rds_attached + +statement ok +DROP SECRET rds_secret diff --git a/test/sql/storage/attach_rollback_throws.test b/test/sql/storage/attach_rollback_throws.test new file mode 100644 index 000000000..d81d6ae38 --- /dev/null +++ b/test/sql/storage/attach_rollback_throws.test @@ -0,0 +1,90 @@ +# name: test/sql/storage/attach_rollback_throws.test +# description: Test that a throw on a server ROLLBACK or COMMIT does not leak a pool connection +# group: [storage] + +# see: https://github.com/duckdb/duckdb-postgres/issues/545 + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# ROLLBACK throws + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# a connection is opened on ATTACH and kept in the pool +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +# kill Postgres backend of our connection, ROLLBACK is attempted and fails +statement error +CALL postgres_execute('s', 'SELECT pg_terminate_backend(pg_backend_pid())') +---- +terminating connection due to administrator command + +# connection must have been discarded, as it cannot pass the health check on returning it to the pool +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s + +# COMMIT throws + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) + +statement ok +SET VARIABLE s1_pid = (SELECT pid FROM postgres_query('s', 'SELECT pg_backend_pid() AS pid')) + +statement ok +USE s + +statement ok +BEGIN TRANSACTION + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +statement ok +SET pg_use_text_protocol=TRUE + +statement ok +CALL postgres_query('s1', 'SELECT pg_terminate_backend($1)', params=row(getvariable(s1_pid))) + +statement ok +RESET pg_use_text_protocol + +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +statement error +COMMIT +---- +closed + +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s1 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_schema_param_list.test b/test/sql/storage/attach_schema_param_list.test new file mode 100644 index 000000000..85815302c --- /dev/null +++ b/test/sql/storage/attach_schema_param_list.test @@ -0,0 +1,106 @@ +# name: test/sql/storage/attach_schema_param_list.test +# description: Test attaching multiple specific schemas +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# prepare schemas + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch1 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch2 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch3 CASCADE') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch1') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch2') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch3') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch1.tab1(col1 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch1.tab1 VALUES(42)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch2.tab2(col2 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch2.tab2 VALUES(43)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch3.tab3(col3 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch3.tab3 VALUES(44)') + +statement ok +DETACH s + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA NULL) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 42) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA [41, 42]) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA ['foo', NULL]) +---- +Binder Error + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA ['duckdb_test_sch1', 'duckdb_test_sch2']) + +query I +FROM s.duckdb_test_sch1.tab1 +---- +42 + +query I +FROM s.duckdb_test_sch2.tab2 +---- +43 + +statement error +FROM s.duckdb_test_sch3.tab3 +---- +Catalog Error: Table with name "duckdb_test_sch3.tab3" does not exist because schema "duckdb_test_sch3" does not exist. + +# cleanup + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch1 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch2 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch3 CASCADE') + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_secret.test b/test/sql/storage/attach_secret.test index 0d479a982..4dd83bbc7 100644 --- a/test/sql/storage/attach_secret.test +++ b/test/sql/storage/attach_secret.test @@ -152,7 +152,7 @@ CREATE OR REPLACE SECRET postgres_db ( ); statement error -ATTACH 'bar' AS secret_attach (TYPE POSTGRES, SECRET postgres_db) +ATTACH 'sslmode=require' AS secret_attach (TYPE POSTGRES, SECRET postgres_db) ---- 'URI' option diff --git a/test/sql/storage/attach_text_protocol.test b/test/sql/storage/attach_text_protocol.test new file mode 100644 index 000000000..ab0a573a9 --- /dev/null +++ b/test/sql/storage/attach_text_protocol.test @@ -0,0 +1,110 @@ +# name: test/sql/storage/attach_text_protocol.test +# description: Test the use_text_protocol option of ATTACH +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.text_protocol + +statement ok +CREATE TABLE s.text_protocol(i INTEGER, s VARCHAR) + +statement ok +INSERT INTO s.text_protocol VALUES (42, 'hello'), (NULL, NULL) + +statement ok +DETACH s + +# the protocol is resolved at ATTACH time and applies to every scan of the database: +# both regular table scans and postgres_query (which is what CONNECT runs) + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL true) + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL false) + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL 'auto') + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +# an explicit option at ATTACH time overrides the pg_use_text_protocol setting + +statement ok +SET pg_use_text_protocol=true + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL false) + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +RESET pg_use_text_protocol + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL 'maybe') +---- +Invalid value "maybe" for use_text_protocol, expected TRUE, FALSE or AUTO + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.text_protocol diff --git a/test/sql/storage/attach_timeout_error.test b/test/sql/storage/attach_timeout_error.test index 156f78fc6..033f65485 100644 --- a/test/sql/storage/attach_timeout_error.test +++ b/test/sql/storage/attach_timeout_error.test @@ -13,3 +13,24 @@ statement error SELECT * FROM postgres_query(s, 'select count(*) from tpch.lineitem, tpch.lineitem t2, tpch.lineitem t3'); ---- canceling statement due to statement timeout + +# The server closes the connection while the local transaction is doing +# unrelated work, then the failure is surfaced by its rollback. +statement ok +ATTACH 'dbname=postgresscanner port=5432 sslmode=allow options=''-c idle_in_transaction_session_timeout=100''' AS idle_timeout (TYPE POSTGRES, READ_ONLY); + +statement ok +BEGIN; + +query I +CALL postgres_query('idle_timeout', 'SELECT 1'); +---- +1 + +statement ok +SELECT sum(i) FROM range(1000000000) t(i); + +statement error +ROLLBACK; +---- +server closed the connection unexpectedly diff --git a/test/sql/storage/attach_types.test b/test/sql/storage/attach_types.test index b495ffaab..dde2a0849 100644 --- a/test/sql/storage/attach_types.test +++ b/test/sql/storage/attach_types.test @@ -9,6 +9,11 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +foreach mode COPY_BINARY PLAIN_QUERY_BINARY AUTO + +statement ok +SET pg_query_mode = '{mode}' + statement ok DROP TABLE IF EXISTS s.types @@ -32,8 +37,8 @@ NULL NULL NULL NULL # test all types statement ok -CREATE TABLE all_types_tbl AS SELECT * -EXCLUDE (float, double, ubigint, hugeint, uhugeint, nested_int_array, struct, struct_of_arrays, array_of_structs, map, "union", fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array, bignum, geometry, timestamp_tz_ns, empty_struct) +CREATE OR REPLACE TABLE all_types_tbl AS SELECT * +EXCLUDE (float, double, ubigint, hugeint, uhugeint, nested_int_array, struct, struct_of_arrays, array_of_structs, map, "union", fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array, bignum, geometry, timestamp_tz_ns, empty_struct, tuple) REPLACE( CASE WHEN int IS NOT NULL THEN '2000-01-01' ELSE NULL END AS date, CASE WHEN int IS NOT NULL THEN '2000-01-01 01:02:03' ELSE NULL END AS timestamp, @@ -68,7 +73,10 @@ true 127 32767 2147483647 9223372036854775807 255 65535 4294967295 2000-01-01 24 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL # filter pushdown -foreach column_name bool tinyint smallint int bigint utinyint usmallint uint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array varchar_array +# TODO: re-add varchar_array. Since duckdb#23539 (nested comparison semantics) an equality +# comparison on a list containing NULL evaluates to NULL, so the value cannot be round-tripped +# through `WHERE col = ` anymore. The test_all_types varchar_array holds a NULL element. +foreach column_name bool tinyint smallint int bigint utinyint usmallint uint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array statement ok SET VARIABLE minimum_value=(SELECT MIN({column_name}) min_val FROM s.all_types); @@ -80,6 +88,8 @@ true endloop +endloop + # text protocol statement ok SET pg_use_text_protocol=true; diff --git a/test/sql/storage/attach_types_blob.test b/test/sql/storage/attach_types_blob.test index faad34505..3a602bb63 100644 --- a/test/sql/storage/attach_types_blob.test +++ b/test/sql/storage/attach_types_blob.test @@ -9,6 +9,11 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE s.blobs(b BLOB); @@ -27,3 +32,8 @@ SELECT * FROM s.blobs \x00\x00\x00 NULL thisisalongstring\x00\x00withnullbytes + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_char.test b/test/sql/storage/attach_types_char.test index 9160b6b9d..e198532b9 100644 --- a/test/sql/storage/attach_types_char.test +++ b/test/sql/storage/attach_types_char.test @@ -22,6 +22,11 @@ hello 5 (empty) 0 NULL NULL +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement error INSERT INTO chars VALUES ('this string is too long') ---- @@ -39,6 +44,9 @@ INSERT INTO chars VALUES ('hello'), ('world'), ('maxlength1'), ('hello '), ( statement ok COMMIT +statement ok +SET pg_use_text_protocol = FALSE + query II SELECT c, LENGTH(c) FROM chars ---- @@ -71,7 +79,15 @@ hello 10 5 NULL NULL +statement ok +SET pg_use_text_protocol = {plain} + statement error INSERT INTO varchars_fixed_len VALUES ('this string is too long') ---- value too long + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_date.test b/test/sql/storage/attach_types_date.test index 629940539..99464f12e 100644 --- a/test/sql/storage/attach_types_date.test +++ b/test/sql/storage/attach_types_date.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE dates(d DATE); @@ -59,6 +64,13 @@ INSERT INTO dates VALUES (DATE '5881580-07-10'); ---- out of range +endloop + +statement ok +RESET pg_use_text_protocol + +# TODO: BC with text proto + query I SELECT * FROM dates ---- diff --git a/test/sql/storage/attach_types_decimal.test b/test/sql/storage/attach_types_decimal.test index 1d8844f10..0b20f90f3 100644 --- a/test/sql/storage/attach_types_decimal.test +++ b/test/sql/storage/attach_types_decimal.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + # no scale (integers) statement ok CREATE OR REPLACE TABLE decimals(d DECIMAL(4,0)); @@ -183,3 +188,8 @@ SELECT * FROM decimals 1234567891234567891.1234567891234567891 9999999999999999999.9999999999999999999 -9999999999999999999.9999999999999999999 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_enum.test b/test/sql/storage/attach_types_enum.test index fdae4a705..c352aa8f4 100644 --- a/test/sql/storage/attach_types_enum.test +++ b/test/sql/storage/attach_types_enum.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok DROP TABLE IF EXISTS enums @@ -52,6 +57,11 @@ sad happy NULL +endloop + +statement ok +RESET pg_use_text_protocol + # reconnect statement ok ATTACH ':memory:' AS x @@ -68,6 +78,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + query I SELECT UNNEST(enum_range(NULL::MOOD)) ---- @@ -98,3 +113,8 @@ statement error CREATE OR REPLACE TABLE enums(e ENUM('sad', 'happy')); ---- Enums in Postgres must be named + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_interval.test b/test/sql/storage/attach_types_interval.test index e9fd77229..ce37bc6ec 100644 --- a/test/sql/storage/attach_types_interval.test +++ b/test/sql/storage/attach_types_interval.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE attach_intervals(i INTERVAL); @@ -38,3 +43,8 @@ NULL 00:30:15 2 months 2 days 00:15:00 -2 days + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_json.test b/test/sql/storage/attach_types_json.test index 9652ae4cc..384cd9400 100644 --- a/test/sql/storage/attach_types_json.test +++ b/test/sql/storage/attach_types_json.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + query I SELECT * FROM pg_json ---- @@ -27,3 +32,8 @@ query I SELECT * FROM pg_json ---- {"a": 42, "b": "string"} + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_macaddr.test b/test/sql/storage/attach_types_macaddr.test index 9b5d99772..37e20ae4c 100644 --- a/test/sql/storage/attach_types_macaddr.test +++ b/test/sql/storage/attach_types_macaddr.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + # macaddress only statement ok DELETE FROM pg_macaddr @@ -28,6 +33,11 @@ SELECT * FROM pg_macaddr 08:00:2b:01:02:03 NULL +endloop + +statement ok +RESET pg_use_text_protocol + # complex types, including list of macaddr statement ok DELETE FROM pg_complex_types_mix diff --git a/test/sql/storage/attach_types_multidimensional_array.test b/test/sql/storage/attach_types_multidimensional_array.test index 79c4f0c6a..3c42a9c3b 100644 --- a/test/sql/storage/attach_types_multidimensional_array.test +++ b/test/sql/storage/attach_types_multidimensional_array.test @@ -93,7 +93,7 @@ NULL # clear cache, then query again # this forces us to load the metadata from postgres itself statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query I SELECT * FROM two_dimensional_array diff --git a/test/sql/storage/attach_types_table_row.test b/test/sql/storage/attach_types_table_row.test index 0a5cf8766..aa3f958d5 100644 --- a/test/sql/storage/attach_types_table_row.test +++ b/test/sql/storage/attach_types_table_row.test @@ -29,7 +29,7 @@ statement ok CALL postgres_execute('s', 'CREATE VIEW row_types.widget_row AS SELECT w FROM row_types.widget AS w'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types.widget_row @@ -44,7 +44,7 @@ statement ok CALL postgres_execute('s', 'INSERT INTO row_types.wrapper VALUES (1, ROW(2, ''nested''))'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query III SELECT id, payload.id, payload.label FROM row_types.wrapper @@ -56,7 +56,7 @@ statement ok SET pg_use_text_protocol=true; statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types.widget_row @@ -99,7 +99,7 @@ statement ok CALL postgres_execute('s', 'INSERT INTO row_types_b.wrapper VALUES (1, ROW(2, ''nested''))'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types_b.widget_view @@ -115,7 +115,7 @@ statement ok SET pg_use_text_protocol=true; statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types_b.widget_view diff --git a/test/sql/storage/attach_types_time.test b/test/sql/storage/attach_types_time.test index 470d16259..11670954b 100644 --- a/test/sql/storage/attach_types_time.test +++ b/test/sql/storage/attach_types_time.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE times(t TIME); @@ -53,3 +58,8 @@ SELECT * FROM timestz NULL 23:59:59+00 23:59:59+06 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_timestamp.test b/test/sql/storage/attach_types_timestamp.test index a6b2d13ed..01c46daf6 100644 --- a/test/sql/storage/attach_types_timestamp.test +++ b/test/sql/storage/attach_types_timestamp.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE timestamps(t TIMESTAMP); @@ -49,6 +54,11 @@ insert into timestamps values (TIMESTAMP 'infinity'); statement ok insert into timestamps values (TIMESTAMP '-infinity'); +# TODO: BC with text proto + +statement ok +SET pg_use_text_protocol = FALSE + query I SELECT * FROM timestamps ---- @@ -60,6 +70,9 @@ NULL infinity -infinity +statement ok +SET pg_use_text_protocol = {plain} + # timestamptz statement ok CREATE OR REPLACE TABLE timestamptz(t TIMESTAMPTZ); @@ -137,3 +150,8 @@ SELECT * FROM timestamp_tbl 2000-01-01 00:00:00 NULL 1980-01-01 23:59:59.123456 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_uuid.test b/test/sql/storage/attach_types_uuid.test index b92094e9a..a53e887be 100644 --- a/test/sql/storage/attach_types_uuid.test +++ b/test/sql/storage/attach_types_uuid.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE uuids(u UUID); @@ -34,3 +39,8 @@ SELECT * FROM uuids NULL 00000000-0000-0000-0000-000000000001 ffffffff-ffff-ffff-ffff-ffffffffffff + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_varchar.test b/test/sql/storage/attach_types_varchar.test index c6ba6ee5c..d568b88a6 100644 --- a/test/sql/storage/attach_types_varchar.test +++ b/test/sql/storage/attach_types_varchar.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE varchars(v VARCHAR); @@ -34,3 +39,8 @@ SELECT * FROM varchars this is a long string 🦆🦆🦆🦆🦆🦆🦆🦆 NULL + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_varchar_fallback.test b/test/sql/storage/attach_types_varchar_fallback.test new file mode 100644 index 000000000..67329bee1 --- /dev/null +++ b/test/sql/storage/attach_types_varchar_fallback.test @@ -0,0 +1,78 @@ +# name: test/sql/storage/attach_types_varchar_fallback.test +# description: Test for using types from different schema that must fall back to VARCHAR +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# prepare schemas + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS data_s CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS types_s CASCADE') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA types_s') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA data_s') + +statement ok +CALL postgres_execute('s', 'CREATE TYPE types_s.mood AS ENUM (''happy'',''sad'')') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE data_s.users (id int, name text)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE data_s.moods (id int, m types_s.mood)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO data_s.users VALUES (1, ''a'')') + +statement ok +CALL postgres_execute('s', 'INSERT INTO data_s.moods VALUES (1, ''happy'')') + +statement ok +DETACH s + +# attach only the data schema + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA data_s, READ_ONLY) + +statement ok +USE s + +query I +SELECT count(*) FROM s.data_s.users +---- +1 + +# cleanup + +statement ok +USE memory + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA data_s CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA types_s CASCADE') + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_use_ctid_scan.test b/test/sql/storage/attach_use_ctid_scan.test new file mode 100644 index 000000000..86d1c7125 --- /dev/null +++ b/test/sql/storage/attach_use_ctid_scan.test @@ -0,0 +1,73 @@ +# name: test/sql/storage/attach_use_ctid_scan.test +# description: Test the per-attach use_ctid_scan option overriding the global pg_use_ctid_scan setting +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +PRAGMA enable_verification + +# per-attach option disables ctid scan even though the global setting defaults to true +statement ok +SET pg_use_ctid_scan=true; + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN false) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +# per-attach option enables ctid scan even though the global setting is false (override works both ways) +statement ok +SET pg_use_ctid_scan=false; + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN true) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +# when omitted, behavior follows the global setting (here: false) and still produces correct results +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +statement ok +SET pg_use_ctid_scan=true; + +# boolean-typed values are accepted +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN 1) + +statement ok +DETACH s + +# a non-boolean value raises a clear error +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN 'not_a_bool') +---- diff --git a/test/sql/storage/attach_views.test b/test/sql/storage/attach_views.test index c5279c050..2b82431bf 100644 --- a/test/sql/storage/attach_views.test +++ b/test/sql/storage/attach_views.test @@ -28,10 +28,11 @@ CREATE VIEW v1 AS SELECT 84; ---- already exists +# the error is raised by Postgres - "cannot insert into view" when the INSERT is pushed down statement error INSERT INTO v1 VALUES (42); ---- -cannot copy to view +:.*(cannot copy to view|cannot insert into view).* # FIXME - error message here is not very descriptive statement error @@ -45,7 +46,7 @@ DELETE FROM v1 statement error INSERT INTO v1 VALUES (1, 1); ---- -table v1 has 1 columns but 2 values were supplied +has 1 columns but 2 values were supplied statement ok CREATE VIEW IF NOT EXISTS v1 AS SELECT 84; diff --git a/test/sql/storage/catalog_cache_external_ddl_visibility.test b/test/sql/storage/catalog_cache_external_ddl_visibility.test new file mode 100644 index 000000000..cc277fb2b --- /dev/null +++ b/test/sql/storage/catalog_cache_external_ddl_visibility.test @@ -0,0 +1,58 @@ +# name: test/sql/storage/catalog_cache_external_ddl_visibility.test +# description: DDL from a separate ClientContext must become visible without an explicit postgres_clear_cache() call +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# con1: prime an empty-schema cache +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS ext_ddl_visibility_tbl + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'ext_ddl_visibility_tbl' +---- +0 + +# con2: a genuinely separate ClientContext, attaching the same physical database +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +statement ok con2 +CREATE TABLE ext_ddl_visibility_tbl AS SELECT 1 AS x + +# con1: re-query without any postgres_clear_cache() call from either connection +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'ext_ddl_visibility_tbl' +---- +1 + +query I con1 +SELECT x FROM ext_ddl_visibility_tbl +---- +1 + +statement ok con1 +DROP TABLE ext_ddl_visibility_tbl + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_identity_swap_detected.test b/test/sql/storage/catalog_cache_identity_swap_detected.test new file mode 100644 index 000000000..6ad856c84 --- /dev/null +++ b/test/sql/storage/catalog_cache_identity_swap_detected.test @@ -0,0 +1,64 @@ +# name: test/sql/storage/catalog_cache_identity_swap_detected.test +# description: drop+create in the same schema that leaves row count unchanged must still be detected as stale (adversarial case that rules out a pure cardinality or xmin-aggregation staleness signal) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS identity_swap_foo + +statement ok con1 +DROP TABLE IF EXISTS identity_swap_bar + +statement ok con1 +CREATE TABLE identity_swap_foo AS SELECT 1 AS x + +# prime the cache with foo present +query I con1 +SELECT table_name FROM duckdb_tables() WHERE table_name LIKE 'identity_swap_%' ORDER BY table_name +---- +identity_swap_foo + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +# net pg_class row count for this schema is unchanged: one dropped, one created +statement ok con2 +DROP TABLE identity_swap_foo + +statement ok con2 +CREATE TABLE identity_swap_bar AS SELECT 2 AS y + +# con1 must see bar and must NOT still see foo, despite unchanged cardinality +# (filtered to database_name = 's' - con2's own s2 attach of the same physical +# DB would otherwise also surface identity_swap_bar a second time) +query I con1 +SELECT table_name FROM duckdb_tables() WHERE database_name = 's' AND table_name LIKE 'identity_swap_%' ORDER BY table_name +---- +identity_swap_bar + +statement ok con1 +DROP TABLE identity_swap_bar + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_staged_signature_cross_connection.test b/test/sql/storage/catalog_cache_staged_signature_cross_connection.test new file mode 100644 index 000000000..374ecc856 --- /dev/null +++ b/test/sql/storage/catalog_cache_staged_signature_cross_connection.test @@ -0,0 +1,89 @@ +# name: test/sql/storage/catalog_cache_staged_signature_cross_connection.test +# description: a pending staleness signature staged by one connection's open explicit transaction must not be promoted or discarded by a concurrent, unrelated autocommit write from a different connection +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS staged_sig_test + +statement ok con1 +CREATE TABLE staged_sig_test(i INTEGER) + +statement ok con1 +INSERT INTO s.staged_sig_test VALUES (42) + +# prime con1's cache +query I con1 +SELECT column_name FROM duckdb_columns() WHERE table_name = 'staged_sig_test' ORDER BY column_name +---- +i + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +# con1 opens an explicit transaction, alters the table, and forces a reload that stages a +# pending signature (not yet committed to con1's PostgresTableSet::staleness_signature) +statement ok con1 +BEGIN + +statement ok con1 +ALTER TABLE s.staged_sig_test ADD COLUMN j INTEGER + +query I con1 +SELECT j FROM s.staged_sig_test +---- +NULL + +# concurrently, con2 (autocommit, unrelated connection) writes a *different* table in the +# same schema and commits - this must not promote or clobber con1's still-pending signature +statement ok con2 +DROP TABLE IF EXISTS staged_sig_unrelated + +statement ok con2 +CREATE TABLE staged_sig_unrelated(x INTEGER) + +# con1 rolls back its own transaction - its pending signature (with column j) must be discarded, +# not promoted, regardless of what con2 did concurrently +statement ok con1 +ROLLBACK + +statement error con1 +SELECT j FROM s.staged_sig_test +---- +not found + +# con1's cache must still correctly see con2's committed, unrelated change on the next access +query I con1 +SELECT table_name FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'staged_sig_unrelated' +---- +staged_sig_unrelated + +statement ok con1 +DROP TABLE staged_sig_test + +statement ok con1 +DROP TABLE staged_sig_unrelated + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test b/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test new file mode 100644 index 000000000..f1520dd7f --- /dev/null +++ b/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test @@ -0,0 +1,259 @@ +# name: test/sql/storage/catalog_cache_staleness_wire_compat_gating.test +# description: pg_staleness_query_enabled gates automatic staleness detection, default tied to pg_use_information_schema_introspection +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# --------------------------------------------------------------------------- +# Default-on for real Postgres (unchanged behavior) +# --------------------------------------------------------------------------- +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc1 (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +DROP TABLE IF EXISTS wc1.public.staleness_gating_tbl1 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc1' AND table_name = 'staleness_gating_tbl1' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc1b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc1b.public.staleness_gating_tbl1 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc1' AND table_name = 'staleness_gating_tbl1' +---- +1 + +statement ok con1 +DROP TABLE wc1.public.staleness_gating_tbl1 + +statement ok con2 +DETACH wc1b + +statement ok con1 +DETACH wc1 + +# --------------------------------------------------------------------------- +# Default-off when compat mode is signaled +# --------------------------------------------------------------------------- +statement ok +SET GLOBAL pg_staleness_query_enabled = NULL + +statement ok con1 +SET pg_use_information_schema_introspection=true + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc2 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc2.public.staleness_gating_tbl2 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc2' AND table_name = 'staleness_gating_tbl2' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc2b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc2b.public.staleness_gating_tbl2 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc2' AND table_name = 'staleness_gating_tbl2' +---- +0 + +statement ok con2 +DROP TABLE wc2b.public.staleness_gating_tbl2 + +statement ok con2 +DETACH wc2b + +statement ok con1 +RESET pg_use_information_schema_introspection + +statement ok con1 +DETACH wc2 + +# --------------------------------------------------------------------------- +# Explicit override wins, on-direction +# --------------------------------------------------------------------------- +statement ok con1 +SET pg_use_information_schema_introspection=true + +statement ok con1 +SET GLOBAL pg_staleness_query_enabled=true + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc3 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc3.public.staleness_gating_tbl3 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc3' AND table_name = 'staleness_gating_tbl3' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc3b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc3b.public.staleness_gating_tbl3 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc3' AND table_name = 'staleness_gating_tbl3' +---- +1 + +statement ok con1 +DROP TABLE wc3.public.staleness_gating_tbl3 + +statement ok con2 +DETACH wc3b + +statement ok con1 +RESET pg_use_information_schema_introspection + +statement ok con1 +RESET GLOBAL pg_staleness_query_enabled + +statement ok con1 +DETACH wc3 + +# --------------------------------------------------------------------------- +# Explicit override wins, off-direction +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query_enabled=false + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc4 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc4.public.staleness_gating_tbl4 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc4' AND table_name = 'staleness_gating_tbl4' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc4b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc4b.public.staleness_gating_tbl4 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc4' AND table_name = 'staleness_gating_tbl4' +---- +0 + +statement ok con2 +DROP TABLE wc4b.public.staleness_gating_tbl4 + +statement ok con2 +DETACH wc4b + +statement ok con1 +RESET GLOBAL pg_staleness_query_enabled + +statement ok con1 +DETACH wc4 + +# --------------------------------------------------------------------------- +# Custom query used in place of the default +# --------------------------------------------------------------------------- +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid, relname, pg_class.xmin FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid WHERE relkind IN (''r'',''v'',''m'',''f'',''p'') AND pg_namespace.nspname = ${SCHEMA} ORDER BY pg_class.oid' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc5 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc5.public.staleness_gating_tbl5 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc5' AND table_name = 'staleness_gating_tbl5' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc5b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc5b.public.staleness_gating_tbl5 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc5' AND table_name = 'staleness_gating_tbl5' +---- +1 + +statement ok con1 +DROP TABLE wc5.public.staleness_gating_tbl5 + +statement ok con2 +DETACH wc5b + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc5 + +# --------------------------------------------------------------------------- +# Missing ${SCHEMA} placeholder is rejected before running +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid, relname, pg_class.xmin FROM pg_class ORDER BY pg_class.oid' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc6 (TYPE POSTGRES) + +statement error con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc6' +---- +:.*pg_staleness_query must contain a \$\{SCHEMA\} placeholder.* + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc6 + +# --------------------------------------------------------------------------- +# Wrong column count is rejected, malformed query fails loud +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid WHERE pg_namespace.nspname = ${SCHEMA}' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc7 (TYPE POSTGRES) + +statement error con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc7' +---- +:.*pg_staleness_query must return at least 3 columns.* + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc7 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow b/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow new file mode 100644 index 000000000..1c519efea --- /dev/null +++ b/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow @@ -0,0 +1,49 @@ +# name: test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow +# description: Concurrent postgres_clear_cache() vs duckdb_tables()/duckdb_indexes() UAF stress repro (run under ASAN) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +require-env SANITIZER_BUILD + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +DROP TABLE IF EXISTS scan_stress_tbl + +statement ok +CREATE TABLE scan_stress_tbl (i INTEGER) + +statement ok +INSERT INTO scan_stress_tbl VALUES (1) + +concurrentloop i 0 200 + +statement maybe +CALL postgres_clear_cache(); +---- + +statement maybe +SELECT * FROM duckdb_tables(); +---- + +statement maybe +SELECT * FROM duckdb_indexes(); +---- + +statement maybe +SELECT * FROM scan_stress_tbl; +---- + +endloop + +query I +SELECT * FROM scan_stress_tbl +---- +1 diff --git a/test/sql/storage/catalog_scan_entry_survives_clear.test b/test/sql/storage/catalog_scan_entry_survives_clear.test new file mode 100644 index 000000000..12168af7e --- /dev/null +++ b/test/sql/storage/catalog_scan_entry_survives_clear.test @@ -0,0 +1,35 @@ +# name: test/sql/storage/catalog_scan_entry_survives_clear.test +# description: PostgresCatalogSet::Scan entries must survive a concurrent postgres_clear_cache() +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +DROP TABLE IF EXISTS scan_survives_clear_tbl + +statement ok +CREATE TABLE scan_survives_clear_tbl AS SELECT 1 AS x + +query I +SELECT count(*) >= 0 FROM duckdb_tables() +---- +1 + +statement ok +CALL postgres_clear_cache() + +query I +SELECT table_name FROM duckdb_tables() WHERE table_name = 'scan_survives_clear_tbl' +---- +scan_survives_clear_tbl + +statement ok +DROP TABLE scan_survives_clear_tbl diff --git a/test/sql/storage/detach_prepared.test b/test/sql/storage/detach_prepared.test new file mode 100644 index 000000000..0dd4061ae --- /dev/null +++ b/test/sql/storage/detach_prepared.test @@ -0,0 +1,126 @@ +# name: test/sql/storage/detach_prepared.test +# description: Test the behaviour of DETACH with PREPARE/EXECUTE +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.detach_prepare_test AS FROM range(41, 43) r + +statement ok +PREPARE p1 AS SELECT divide(r, 1) FROM s.detach_prepare_test + +query I +EXECUTE p1 +---- +41 +42 + +query I +EXECUTE p1 +---- +41 +42 + +statement ok +DETACH s + +statement error +EXECUTE p1 +---- +Catalog Error: Table with name "s.detach_prepare_test" does not exist because schema "s" does not exist. + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +EXECUTE p1 +---- +41 +42 + +statement ok +DROP TABLE s.detach_prepare_test + +statement error +EXECUTE p1 +---- +Catalog Error: Table with name detach_prepare_test does not exist! + +statement ok +DEALLOCATE p1 + +statement error +EXECUTE p1 +---- +Binder Error: Prepared statement "p1" does not exist + +statement ok +DETACh s + +# prepared postgres_query, no cached tables + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_query('s', 'CREATE TABLE detach_prepare_test(r INT)') + +statement ok +CALL postgres_query('s', 'INSERT INTO detach_prepare_test VALUES (41),(42)') + +statement ok +PREPARE p2 AS FROM postgres_query('s', 'SELECT * FROM detach_prepare_test') + +query I +EXECUTE p2 +---- +41 +42 + +query I +EXECUTE p2 +---- +41 +42 + +statement ok +DETACH s + +statement error +EXECUTE p2 +---- +Invalid Input Error: Attached PostgreSQL database not found in the specified client session, name: "s" + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +EXECUTE p2 +---- +41 +42 + +statement ok +CALL postgres_query('s', 'DROP TABLE detach_prepare_test') + +statement error +EXECUTE p2 +---- +ERROR: relation "detach_prepare_test" does not exist + +statement ok +DEALLOCATE p2 + +statement error +EXECUTE p2 +---- +Binder Error: Prepared statement "p2" does not exist + +statement ok +DETACh s diff --git a/test/sql/storage/filter_pushdown_blob_in.test b/test/sql/storage/filter_pushdown_blob_in.test new file mode 100644 index 000000000..9ba0734bd --- /dev/null +++ b/test/sql/storage/filter_pushdown_blob_in.test @@ -0,0 +1,61 @@ +# name: test/sql/storage/filter_pushdown_blob_in.test +# description: Test pushing down IN filters over blob columns +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET pg_enable_filter_pushdown=true + +statement ok +CREATE OR REPLACE TABLE s.blob_in(id INTEGER, b BLOB); + +statement ok +INSERT INTO s.blob_in VALUES (1, '\x01\x02'), (2, '\x03\x04'), (3, 'dpfkg'), (4, 'other'), (5, NULL); + +# blob constants in an IN list must be written as bytea literals, not bare hex digits +query I +SELECT id FROM s.blob_in WHERE b IN ('dpfkg'::BLOB, 'other'::BLOB) ORDER BY id +---- +3 +4 + +query I +SELECT id FROM s.blob_in WHERE b IN ('\x01\x02'::BLOB, '\x03\x04'::BLOB) ORDER BY id +---- +1 +2 + +# a longer list, which the binder is more likely to keep as a COMPARE_IN expression +query I +SELECT id FROM s.blob_in WHERE b IN ('\x01\x02'::BLOB, '\x03\x04'::BLOB, 'dpfkg'::BLOB, 'other'::BLOB, '\xff'::BLOB, '\xfe'::BLOB) ORDER BY id +---- +1 +2 +3 +4 + +query I +SELECT count(*) FROM s.blob_in WHERE b NOT IN ('dpfkg'::BLOB, 'other'::BLOB) +---- +2 + +# blobs containing quote and backslash bytes must survive the round trip +statement ok +INSERT INTO s.blob_in VALUES (6, '\x27\x5c'); + +query I +SELECT id FROM s.blob_in WHERE b IN ('\x27\x5c'::BLOB, '\x00'::BLOB) ORDER BY id +---- +6 + +statement ok +DROP TABLE s.blob_in + +statement ok +DETACH s diff --git a/test/sql/storage/information_schema_introspection.test b/test/sql/storage/information_schema_introspection.test new file mode 100644 index 000000000..b68df5da7 --- /dev/null +++ b/test/sql/storage/information_schema_introspection.test @@ -0,0 +1,90 @@ +# name: test/sql/storage/information_schema_introspection.test +# description: Test pg_use_information_schema_introspection — schema discovery via SQL-standard information_schema instead of pg_catalog +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# baseline ATTACH (pg_catalog path) — collect the table set for comparison +statement ok +ATTACH 'dbname=postgresscanner' AS pgc (TYPE POSTGRES) + +# default is false; default path is unchanged +query I +SELECT current_setting('pg_use_information_schema_introspection') +---- +false + +# snapshot schema as obtained the pg way +statement ok +CREATE TEMP TABLE pgc_tables AS SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='pgc' AND schema_name NOT IN ('pg_catalog', 'information_schema') + +statement ok +DETACH pgc + +statement ok +SET pg_use_information_schema_introspection=true + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# the two paths discover the same tables +query I +SELECT COUNT(*) FROM ( + SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='s' + EXCEPT + SELECT schema_name, table_name FROM pgc_tables +) +---- +0 + +# query it the other way around to ensure exact match +query I +SELECT COUNT(*) FROM ( + SELECT schema_name, table_name FROM pgc_tables + EXCEPT + SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='s' +) +---- +0 + +# round-trip a table with the data_type forms the new path has to map +statement ok +DROP TABLE IF EXISTS s.public.is_types + +statement ok +CREATE TABLE s.public.is_types(i BIGINT, name VARCHAR, ts TIMESTAMPTZ, flag BOOLEAN) + +statement ok +INSERT INTO s.public.is_types VALUES (42, 'hello', TIMESTAMPTZ '2024-01-01 00:00:00+00', true) + +statement ok +INSERT INTO s.public.is_types VALUES (NULL, NULL, NULL, NULL) + +# force a fresh introspection of the new table +statement ok +CALL postgres_clear_cache() + +query IIII +SELECT i, name, ts, flag FROM s.public.is_types ORDER BY i NULLS LAST +---- +42 hello 2024-01-01 00:00:00+00 true +NULL NULL NULL NULL + +# information_schema reports verbose data_type names; verify the mapping landed +# on the right DuckDB types (bigint→BIGINT, character varying→VARCHAR, +# timestamp with time zone→TIMESTAMPTZ (which duckdb then reports back as timestamp with timezone), boolean→BOOLEAN) +query IIII +SELECT typeof(i), typeof(name), typeof(ts), typeof(flag) FROM s.public.is_types LIMIT 1 +---- +BIGINT VARCHAR TIMESTAMP WITH TIME ZONE BOOLEAN + +statement ok +DROP TABLE s.public.is_types + +statement ok +DETACH s + +statement ok +RESET pg_use_information_schema_introspection diff --git a/test/sql/storage/limit_parallel_bug.test b/test/sql/storage/limit_parallel_bug.test_slow similarity index 96% rename from test/sql/storage/limit_parallel_bug.test rename to test/sql/storage/limit_parallel_bug.test_slow index dc325a9e7..76570f50d 100644 --- a/test/sql/storage/limit_parallel_bug.test +++ b/test/sql/storage/limit_parallel_bug.test_slow @@ -1,4 +1,4 @@ -# name: test/sql/storage/limit_parallel_bug.test +# name: test/sql/storage/limit_parallel_bug.test_slow # description: Test LIMIT bug with parallelism (issues #380, #395) # group: [storage] diff --git a/test/sql/storage/multi_scan_aggregate.test b/test/sql/storage/multi_scan_aggregate.test new file mode 100644 index 000000000..9ae996a94 --- /dev/null +++ b/test/sql/storage/multi_scan_aggregate.test @@ -0,0 +1,55 @@ +# name: test/sql/storage/multi_scan_aggregate.test +# description: Test multiple simultaneous scans that are aggregated on DuckDB side +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_a AS +SELECT '2020-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_b AS +SELECT '2021-01-01' insert_date, unnest(range(1<<14)) amount + +# Note, 'divide()' is necessary to prevent remote-execute pushdown + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_a t +GROUP BY 1 +---- +2020-01-01 134209536 + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_b t +GROUP BY 1 +---- +2021-01-01 134209536 + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_a t +GROUP BY 1 +UNION ALL +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_b t +GROUP BY 1 +---- +2020-01-01 134209536 +2021-01-01 134209536 + +statement ok +DROP TABLE s.aggr_union_test_a + +statement ok +DROP TABLE s.aggr_union_test_b + +statement ok +DETACH s; diff --git a/test/sql/storage/postgres_execute_transaction.test b/test/sql/storage/postgres_execute_transaction.test index 8ce709d7a..ed8b1c7e3 100644 --- a/test/sql/storage/postgres_execute_transaction.test +++ b/test/sql/storage/postgres_execute_transaction.test @@ -39,7 +39,7 @@ CALL postgres_query('s', 'SELECT 42') 42 statement ok -CALL postgres_execute('s', 'INSERT INTO postgres_execute_attempt VALUES (42); INSERT INTO postgres_execute_attempt VALUES (84)') +CALL postgres_execute('s', 'INSERT INTO postgres_execute_attempt VALUES (42), (84)') statement ok COMMIT @@ -57,7 +57,11 @@ statement ok DESCRIBE query I -SELECT query FROM duckdb_logs_parsed('PostgresQueryLog'); +SELECT count(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query LIKE 'BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ%' ---- -BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ -COMMIT +1 + +query I +SELECT count(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'COMMIT' +---- +1 diff --git a/test/sql/storage/postgres_execute_use_transaction.test b/test/sql/storage/postgres_execute_use_transaction.test index 5257dd3a6..2c083720b 100644 --- a/test/sql/storage/postgres_execute_use_transaction.test +++ b/test/sql/storage/postgres_execute_use_transaction.test @@ -12,20 +12,17 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement error CALL postgres_execute('s', 'VACUUM') ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement error CALL postgres_execute('s', 'VACUUM', use_transaction=true) ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement error CALL postgres_execute('s', 'VACUUM', use_transaction=true) ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement ok CALL postgres_execute('s', 'VACUUM', use_transaction=false) diff --git a/vcpkg.json b/vcpkg.json index fb16ba635..d64381d70 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,5 @@ { "dependencies": [ - "openssl", "libpq" ], "vcpkg-configuration": { diff --git a/vcpkg_ports/libpq/Makefile b/vcpkg_ports/libpq/Makefile deleted file mode 100644 index d743b5335..000000000 --- a/vcpkg_ports/libpq/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -subdir = . -top_builddir = . -include src/Makefile.global - -ifeq ($(LIBPQ_LIBRARY_TYPE), static) -LIBPQ_INSTALL_LIBS = install-stlib -endif - -ifeq ($(LIBPQ_LIBRARY_TYPE), shared) -ifeq ($(LIBPQ_USING_MINGW), yes) -# The import library name is the same as the static library name -EXTRA_TARGET = install-lib-static -endif -endif - -.PHONY: all -all: - $(MAKE) -C src/include - $(MAKE) -C src/common - $(MAKE) -C src/port - $(MAKE) -C src/interfaces/libpq all-$(LIBPQ_LIBRARY_TYPE)-lib - $(MAKE) -C src/interfaces/ecpg/include - $(MAKE) -C src/interfaces/ecpg/pgtypeslib - $(MAKE) -C src/interfaces/ecpg/ecpglib - $(MAKE) -C src/interfaces/ecpg/compatlib -ifeq ($(LIBPQ_LIBRARY_TYPE), static) - $(AR) $(AROPT) src/interfaces/libpq/libpq.a src/common/encnames_shlib.o -endif -ifeq ($(LIBPQ_ENABLE_TOOLS), yes) - $(MAKE) -C src/interfaces/ecpg/preproc - $(MAKE) -C src/fe_utils - $(MAKE) -C src/bin -else - $(MAKE) -C src/bin/pg_config -endif - -.PHONY: install-stlib -install-stlib: - $(MAKE) -C src/common install -o all - mv -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' '$(DESTDIR)$(libdir)/libpgcommon.a' - $(MAKE) -C src/port install -o all - rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' - -.PHONY: install -install: $(LIBPQ_INSTALL_LIBS) - $(MAKE) -C src/include install - $(MAKE) -C src/interfaces/libpq $(EXTRA_TARGET) install-lib-$(LIBPQ_LIBRARY_TYPE) install-lib-pc install -o all -o install-lib - $(MAKE) -C src/interfaces/ecpg/include install - $(MAKE) -C src/interfaces/ecpg/pgtypeslib install - $(MAKE) -C src/interfaces/ecpg/ecpglib install - $(MAKE) -C src/interfaces/ecpg/compatlib install -ifeq ($(LIBPQ_ENABLE_TOOLS), yes) - $(MAKE) -C src/interfaces/ecpg/preproc install - $(MAKE) -C src/bin install -else - $(MAKE) -C src/bin/pg_config install -endif diff --git a/vcpkg_ports/libpq/build-msvc.cmake b/vcpkg_ports/libpq/build-msvc.cmake deleted file mode 100644 index 36dd79391..000000000 --- a/vcpkg_ports/libpq/build-msvc.cmake +++ /dev/null @@ -1,221 +0,0 @@ -function(build_msvc source_path) - # Strip meson build to only compile client libraries and tools. - # The full PostgreSQL meson build includes the server backend, timezone data, - # PL languages, contrib modules, and tests - none of which we need. - - # src/meson.build: keep only bin (client tools) and interfaces (ecpg) - file(WRITE "${source_path}/src/meson.build" [=[ -subdir('bin') -subdir('interfaces') -]=]) - - # src/bin/meson.build: only client tools (mirrors unix/no-server-tools.patch) - file(WRITE "${source_path}/src/bin/meson.build" [=[ -subdir('pg_amcheck') -subdir('pg_basebackup') -subdir('pg_config') -subdir('pg_dump') -subdir('pg_verifybackup') -subdir('pgbench') -subdir('pgevent') -subdir('psql') -subdir('scripts') -]=]) - - # Truncate meson.build before the test/pseudo-target sections that reference - # variables from skipped subdirs (pg_regress, regress_module, docs, etc.). - # The subdirs themselves are already commented out by windows/meson-vcpkg.patch, - # but the test infrastructure section still references variables from them. - file(READ "${source_path}/meson.build" meson_content) - string(FIND "${meson_content}" "# all targets that require building code" truncate_pos) - if(NOT truncate_pos EQUAL -1) - string(SUBSTRING "${meson_content}" 0 ${truncate_pos} meson_content) - file(WRITE "${source_path}/meson.build" "${meson_content}") - endif() - - # For static builds, remove __declspec(dllimport) from installed headers - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_replace_string("${source_path}/src/include/port/win32.h" - "__declspec (dllimport)" "") - endif() - - # The meson build compiles the static pgcommon with -DUSE_PRIVATE_ENCODING_FUNCS, - # which renames pg_char_to_encoding -> pg_char_to_encoding_private. But libpq - # (compiled as FRONTEND without that flag) calls the non-private names, causing - # unresolved symbols for consumers. The autoconf Makefile works around this by - # installing pgcommon_shlib (non-private names) as pgcommon. Since we don't build - # the server backend, there's no symbol conflict risk, so just remove the flag. - vcpkg_replace_string("${source_path}/src/common/meson.build" - "'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS']," - "# 'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS'], # vcpkg: use non-private names") - - # Map vcpkg features to meson options - vcpkg_list(SET MESON_OPTIONS) - - # Disable auto-detection so we don't pick up random system libraries - list(APPEND MESON_OPTIONS -Dauto_features=disabled) - - # Features that map directly to meson option names - foreach(option IN ITEMS icu lz4 zlib zstd) - if(option IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -D${option}=enabled) - endif() - endforeach() - - # Features with different meson option names - if("openssl" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dssl=openssl) - else() - list(APPEND MESON_OPTIONS -Dssl=none) - endif() - if("xml" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dlibxml=enabled) - endif() - if("xslt" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dlibxslt=enabled) - endif() - if("nls" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dnls=enabled) - # Static intl depends on iconv, but PostgreSQL's meson build uses - # cc.find_library('intl') which doesn't resolve transitive deps. - # Patch to also link iconv when intl is found. - vcpkg_replace_string("${source_path}/meson.build" - "i18n = import('i18n')" - "iconv_dep = cc.find_library('iconv', required: false, dirs: test_lib_d)\n if iconv_dep.found()\n libintl = declare_dependency(dependencies: [libintl, iconv_dep])\n endif\n i18n = import('i18n')") - endif() - # plpython requires matching debug/release Python libraries. - # vcpkg's Python is release-only, so only enable for release builds. - vcpkg_list(SET MESON_OPTIONS_RELEASE) - vcpkg_list(SET MESON_OPTIONS_DEBUG) - if("python" IN_LIST FEATURES) - if(VCPKG_CROSSCOMPILING) - # plpython can't be configured when cross-compiling because meson - # needs to run the Python interpreter at configure time, but the - # target binary won't execute on the build host. - message(STATUS "Disabling plpython for cross-compilation build") - list(APPEND MESON_OPTIONS -Dplpython=disabled) - else() - # Use the vcpkg python3 interpreter (which has matching dev libraries) - # instead of the standalone tools Python (which has no SDK). - string(REPLACE [[\]] [[/]] VCPKG_PYTHON3_PATH "${CURRENT_INSTALLED_DIR}/tools/python3/python.exe") - list(APPEND MESON_OPTIONS_RELEASE -Dplpython=enabled "-DPYTHON=${VCPKG_PYTHON3_PATH}") - list(APPEND MESON_OPTIONS_DEBUG -Dplpython=disabled) - endif() - endif() - if("tcl" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dpltcl=enabled) - # The vcpkg tcl port doesn't generate pkg-config files on Windows, - # so meson's dependency('tcl90') fails. Its fallback cc.find_library() - # also fails because meson's link-test probe doesn't work reliably - # with vcpkg's directory layout. Generate .pc files so meson can - # find Tcl through its preferred pkg-config path. - # Tcl naming: tcl90 (dynamic), tcl90s (static), +g suffix for debug. - set(_tcl_pc_name "tcl90") - # Determine the release library name - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - set(_tcl_rel_libname "tcl90s") - else() - set(_tcl_rel_libname "tcl90") - endif() - # Generate release .pc file - set(_tcl_pc_dir "${CURRENT_INSTALLED_DIR}/lib/pkgconfig") - file(MAKE_DIRECTORY "${_tcl_pc_dir}") - file(WRITE "${_tcl_pc_dir}/${_tcl_pc_name}.pc" -"prefix=${CURRENT_INSTALLED_DIR}\nlibdir=\${prefix}/lib\nincludedir=\${prefix}/include\n\nName: ${_tcl_pc_name}\nDescription: Tcl scripting language\nVersion: 9.0\nLibs: -L\${libdir} -l${_tcl_rel_libname}\nCflags: -I\${includedir}\n") - # Determine the debug library name - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - set(_tcl_dbg_libname "tcl90sg") - else() - set(_tcl_dbg_libname "tcl90g") - endif() - # Generate debug .pc file - if(NOT VCPKG_BUILD_TYPE) - set(_tcl_pc_dbg_dir "${CURRENT_INSTALLED_DIR}/debug/lib/pkgconfig") - file(MAKE_DIRECTORY "${_tcl_pc_dbg_dir}") - file(WRITE "${_tcl_pc_dbg_dir}/${_tcl_pc_name}.pc" -"prefix=${CURRENT_INSTALLED_DIR}/debug\nlibdir=\${prefix}/lib\nincludedir=${CURRENT_INSTALLED_DIR}/include\n\nName: ${_tcl_pc_name}\nDescription: Tcl scripting language\nVersion: 9.0\nLibs: -L\${libdir} -l${_tcl_dbg_libname}\nCflags: -I\${includedir}\n") - endif() - list(APPEND MESON_OPTIONS "-Dtcl_version=${_tcl_pc_name}") - endif() - - # Provide paths to required tools - vcpkg_list(SET ADDITIONAL_BINARIES) - string(REPLACE [[\]] [[/]] BISON_PATH "${BISON}") - string(REPLACE [[\]] [[/]] FLEX_PATH "${FLEX}") - string(REPLACE [[\]] [[/]] PERL_PATH "${PERL}") - list(APPEND ADDITIONAL_BINARIES - "bison = ['${BISON_PATH}']" - "flex = ['${FLEX_PATH}']" - "perl = ['${PERL_PATH}']" - ) - - # Extra include/lib dirs for vcpkg dependencies (only if they exist; - # with no optional deps installed, CURRENT_INSTALLED_DIR may lack these). - # Debug and release use different lib dirs (debug libs have different - # suffixes, e.g. tcl90g.lib vs tcl90.lib). - if(EXISTS "${CURRENT_INSTALLED_DIR}/include") - list(APPEND MESON_OPTIONS "-Dextra_include_dirs=['${CURRENT_INSTALLED_DIR}/include']") - endif() - if(EXISTS "${CURRENT_INSTALLED_DIR}/lib") - list(APPEND MESON_OPTIONS_RELEASE "-Dextra_lib_dirs=['${CURRENT_INSTALLED_DIR}/lib']") - endif() - if(EXISTS "${CURRENT_INSTALLED_DIR}/debug/lib") - list(APPEND MESON_OPTIONS_DEBUG "-Dextra_lib_dirs=['${CURRENT_INSTALLED_DIR}/debug/lib']") - endif() - - vcpkg_configure_meson( - SOURCE_PATH "${source_path}" - OPTIONS - ${MESON_OPTIONS} - OPTIONS_RELEASE - ${MESON_OPTIONS_RELEASE} - OPTIONS_DEBUG - ${MESON_OPTIONS_DEBUG} - LANGUAGES C - ADDITIONAL_BINARIES - ${ADDITIONAL_BINARIES} - ) - vcpkg_install_meson() - - # Remove _shlib variants of pgcommon/pgport that consumers don't need - file(GLOB _shlib_libs - "${CURRENT_PACKAGES_DIR}/lib/*pgcommon_shlib*" - "${CURRENT_PACKAGES_DIR}/lib/*pgport_shlib*" - "${CURRENT_PACKAGES_DIR}/debug/lib/*pgcommon_shlib*" - "${CURRENT_PACKAGES_DIR}/debug/lib/*pgport_shlib*" - ) - if(_shlib_libs) - file(REMOVE ${_shlib_libs}) - endif() - - # Remove server-related installed files we don't need - file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/lib/postgresql" - "${CURRENT_PACKAGES_DIR}/debug/lib/postgresql" - "${CURRENT_PACKAGES_DIR}/share/postgresql" - ) - - if(HAS_TOOLS) - set(TOOL_NAMES - clusterdb createdb createuser dropdb dropuser ecpg pgbench - pg_amcheck pg_basebackup pg_config pg_createsubscriber - pg_dump pg_dumpall pg_isready pg_receivewal pg_recvlogical - pg_restore pg_verifybackup psql reindexdb vacuumdb - ) - vcpkg_copy_tools(TOOL_NAMES ${TOOL_NAMES} AUTO_CLEAN) - else() - # Remove all executables (keep DLLs in bin/ for dynamic linkage) - file(GLOB exe_files - "${CURRENT_PACKAGES_DIR}/bin/*.exe" - "${CURRENT_PACKAGES_DIR}/debug/bin/*.exe" - ) - if(exe_files) - file(REMOVE ${exe_files}) - endif() - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools") - endif() - - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") - endif() -endfunction() diff --git a/vcpkg_ports/libpq/libintl.diff b/vcpkg_ports/libpq/libintl.diff new file mode 100644 index 000000000..a5d116731 --- /dev/null +++ b/vcpkg_ports/libpq/libintl.diff @@ -0,0 +1,17 @@ +diff --git a/meson.build b/meson.build +index dd7ca47..8e0964a 100644 +--- a/meson.build ++++ b/meson.build +@@ -3023,6 +3023,12 @@ if not nlsopt.disabled() + endif + + if libintl.found() ++ if build_system == 'windows' ++ iconv_dep = cc.find_library('iconv', required: false, dirs: test_lib_d) ++ if iconv_dep.found() ++ libintl = declare_dependency(dependencies: [libintl, iconv_dep]) ++ endif ++ endif + i18n = import('i18n') + cdata.set('ENABLE_NLS', 1) + endif diff --git a/vcpkg_ports/libpq/libpq-and-client-tools.diff b/vcpkg_ports/libpq/libpq-and-client-tools.diff new file mode 100644 index 000000000..2f0cddb90 --- /dev/null +++ b/vcpkg_ports/libpq/libpq-and-client-tools.diff @@ -0,0 +1,127 @@ +diff --git a/config/meson.build b/config/meson.build +index ab19c38..b6203b5 100644 +--- a/config/meson.build ++++ b/config/meson.build +@@ -1,3 +1,4 @@ ++subdir_done() + install_data( + 'install-sh', 'missing', + install_dir: dir_pgxs / 'config' +diff --git a/meson.build b/meson.build +index 4016033..dd7ca47 100644 +--- a/meson.build ++++ b/meson.build +@@ -3381,6 +3381,10 @@ test_install_libs = [] + + subdir('src') + ++contrib_data_dir = dir_data_extension ++ ++if false # skip for vcpkg port ++ + subdir('contrib') + + subdir('src/test') +@@ -3924,6 +3928,7 @@ if not meson.is_subproject() + endif + endif + ++endif # skip for vcpkg port libpq + + + ############################################################### +diff --git a/meson_options.txt b/meson_options.txt +index 06bf562..5505bfd 100644 +--- a/meson_options.txt ++++ b/meson_options.txt +@@ -1,3 +1,6 @@ ++option('tools', type: 'feature', value: 'disabled', ++ description: 'Enable tool executables') ++ + # Copyright (c) 2022-2025, PostgreSQL Global Development Group + + # Data layout influencing options +diff --git a/src/backend/meson.build b/src/backend/meson.build +index 2b0db21..75d8f68 100644 +--- a/src/backend/meson.build ++++ b/src/backend/meson.build +@@ -40,6 +40,8 @@ subdir('po', if_found: libintl) + backend_link_args = [] + backend_link_depends = [] + ++subdir_done() ++if false # skip for vcpkg port + + # On windows when compiling with msvc we need to make postgres export all its + # symbols so that extension libraries can use them. For that we need to scan +@@ -140,6 +142,8 @@ postgres = executable('postgres', + + backend_targets += postgres + ++endif # skip for vcpkg port ++ + pg_mod_c_args = cflags_mod + pg_mod_cpp_args = cxxflags_mod + pg_mod_link_args = ldflags_sl + ldflags_mod +@@ -187,6 +191,7 @@ pg_test_mod_args = pg_mod_args + { + } + + ++subdir_done() + + # Shared modules that, on some system, link against the server binary. Only + # enter these after we defined the server build. +diff --git a/src/bin/meson.build b/src/bin/meson.build +index b33cb6c..b08941f 100644 +--- a/src/bin/meson.build ++++ b/src/bin/meson.build +@@ -1,24 +1,13 @@ + # Copyright (c) 2022-2025, PostgreSQL Global Development Group + +-subdir('initdb') ++# vcpkg: Reduced to documented "Client applications" (and scripts) ++# Cf. https://www.postgresql.org/docs/current/reference-client.html + subdir('pg_amcheck') +-subdir('pg_archivecleanup') + subdir('pg_basebackup') +-subdir('pg_checksums') + subdir('pg_combinebackup') + subdir('pg_config') +-subdir('pg_controldata') +-subdir('pg_ctl') + subdir('pg_dump') +-subdir('pg_resetwal') +-subdir('pg_rewind') +-subdir('pg_test_fsync') +-subdir('pg_test_timing') +-subdir('pg_upgrade') + subdir('pg_verifybackup') +-subdir('pg_waldump') +-subdir('pg_walsummary') + subdir('pgbench') +-subdir('pgevent') + subdir('psql') + subdir('scripts') +diff --git a/src/meson.build b/src/meson.build +index f098f9f..28eb09c 100644 +--- a/src/meson.build ++++ b/src/meson.build +@@ -3,12 +3,18 @@ + # libraries that other subsystems might depend upon first, in their respective + # dependency order + ++subdir('interfaces') ++if not get_option('tools').disabled() ++ + subdir('timezone') + + subdir('backend') + + subdir('bin') + ++endif ++subdir_done() ++ + subdir('pl') + + subdir('interfaces') diff --git a/vcpkg_ports/libpq/libpq.props.in b/vcpkg_ports/libpq/libpq.props.in deleted file mode 100644 index 223e528fe..000000000 --- a/vcpkg_ports/libpq/libpq.props.in +++ /dev/null @@ -1,25 +0,0 @@ - - - - - %(AdditionalOptions) @VCPKG_COMBINED_C_FLAGS_DEBUG@ - - - %(AdditionalOptions) @VCPKG_COMBINED_SHARED_LINKER_FLAGS_DEBUG@ - - - %(AdditionalOptions) @VCPKG_COMBINED_STATIC_LINKER_FLAGS_DEBUG@ - - - - - %(AdditionalOptions) @VCPKG_COMBINED_C_FLAGS_RELEASE@ - - - %(AdditionalOptions) @VCPKG_COMBINED_SHARED_LINKER_FLAGS_RELEASE@ - - - %(AdditionalOptions) @VCPKG_COMBINED_STATIC_LINKER_FLAGS_RELEASE@ - - - diff --git a/vcpkg_ports/libpq/library-linkage.diff b/vcpkg_ports/libpq/library-linkage.diff new file mode 100644 index 000000000..6c3c260d3 --- /dev/null +++ b/vcpkg_ports/libpq/library-linkage.diff @@ -0,0 +1,241 @@ +diff --git a/meson.build b/meson.build +index 6e1c867..4016033 100644 +--- a/meson.build ++++ b/meson.build +@@ -3293,6 +3293,12 @@ frontend_shlib_code = declare_dependency( + dependencies: [shlib_code, os_deps, libintl], + ) + ++if get_option('default_library') == 'shared' ++ frontend_lib_code = frontend_shlib_code ++else ++ frontend_lib_code = frontend_stlib_code ++endif ++ + # For frontend code that doesn't use fe_utils - this mainly exists for libpq's + # tests, which are defined before fe_utils is defined, as fe_utils depends on + # libpq. +@@ -3326,7 +3332,7 @@ subdir('src/interfaces/libpq-oauth') + # for frontend binaries + frontend_code = declare_dependency( + include_directories: [postgres_inc], +- link_with: [fe_utils, common_static, pgport_static], ++ link_with: [fe_utils, common_default, pgport_default], + sources: generated_headers, + dependencies: [os_deps, libintl], + ) +diff --git a/src/bin/initdb/meson.build b/src/bin/initdb/meson.build +index 06958e3..a25abd1 100644 +--- a/src/bin/initdb/meson.build ++++ b/src/bin/initdb/meson.build +@@ -20,7 +20,7 @@ initdb = executable('initdb', + # from libpq, else we have risks of version skew if we run with a libpq + # shared library from a different PG version. Define + # USE_PRIVATE_ENCODING_FUNCS to ensure that that happens. +- c_args: ['-DUSE_PRIVATE_ENCODING_FUNCS'], ++ # But vcpkg cannot build the libs with this definition. + dependencies: [frontend_code, libpq, icu, icu_i18n], + kwargs: default_bin_args, + ) +diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build +index bbc4c57..9ea8597 100644 +--- a/src/bin/pg_combinebackup/meson.build ++++ b/src/bin/pg_combinebackup/meson.build +@@ -17,7 +17,7 @@ endif + + pg_combinebackup = executable('pg_combinebackup', + pg_combinebackup_sources, +- dependencies: [frontend_code], ++ dependencies: [frontend_code, libpq], + kwargs: default_bin_args, + ) + bin_targets += pg_combinebackup +diff --git a/src/common/meson.build b/src/common/meson.build +index 1540ba6..e29d594 100644 +--- a/src/common/meson.build ++++ b/src/common/meson.build +@@ -114,6 +114,8 @@ common_sources_frontend_shlib += files( + ) + + common_sources_excluded_shlib = files( ++) ++common_sources_frontend_shlib += files( + 'fe_memutils.c', + 'logging.c', + ) +@@ -144,7 +146,7 @@ pgcommon_variants = { + 'dependencies': [frontend_common_code], + # Files in libpgcommon.a should use/export the "xxx_private" versions + # of pg_char_to_encoding() and friends. +- 'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS'], ++ # But vcpkg's static library linkage must not add the private suffix + }, + '_shlib': default_lib_args + { + 'pic': true, +@@ -159,6 +161,19 @@ pgcommon_variants = { + + foreach name, opts : pgcommon_variants + ++ if name == '_srv' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ if name == '' and get_option('default_library') == 'shared' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ if name == '_shlib' and get_option('default_library') == 'static' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ + # Build internal static libraries for sets of files that need to be built + # with different cflags + cflag_libs = [] +@@ -195,6 +210,7 @@ foreach name, opts : pgcommon_variants + } + ) + pgcommon += {name: lib} ++ common_default = lib + endforeach + + common_srv = pgcommon['_srv'] +diff --git a/src/interfaces/ecpg/compatlib/meson.build b/src/interfaces/ecpg/compatlib/meson.build +index 56e0a21..e3c8675 100644 +--- a/src/interfaces/ecpg/compatlib/meson.build ++++ b/src/interfaces/ecpg/compatlib/meson.build +@@ -41,11 +41,17 @@ ecpg_compat_so = shared_library('libecpg_compat', + ) + ecpg_targets += ecpg_compat_so + ++if get_option('default_library') == 'shared' ++ ecpg_compat_libraries = ecpg_compat_so ++else ++ ecpg_compat_libraries = ecpg_compat_st ++endif ++ + pkgconfig.generate( + name: 'libecpg_compat', + description: 'PostgreSQL libecpg_compat library', + url: pg_url, +- libraries: ecpg_compat_so, +- libraries_private: [frontend_stlib_code, thread_dep], ++ libraries: ecpg_compat_libraries, ++ libraries_private: [frontend_lib_code, thread_dep], + requires_private: ['libecpg', 'libpgtypes'], + ) +diff --git a/src/interfaces/ecpg/ecpglib/meson.build b/src/interfaces/ecpg/ecpglib/meson.build +index 8f478c6..fbae057 100644 +--- a/src/interfaces/ecpg/ecpglib/meson.build ++++ b/src/interfaces/ecpg/ecpglib/meson.build +@@ -52,12 +52,18 @@ ecpglib_so = shared_library('libecpg', + ) + ecpg_targets += ecpglib_so + ++if get_option('default_library') == 'shared' ++ ecpglib_libraries = ecpglib_so ++else ++ ecpglib_libraries = ecpglib_st ++endif ++ + pkgconfig.generate( + name: 'libecpg', + description: 'PostgreSQL libecpg library', + url: pg_url, +- libraries: ecpglib_so, +- libraries_private: [frontend_stlib_code, thread_dep], ++ libraries: ecpglib_libraries, ++ libraries_private: [frontend_lib_code, thread_dep], + requires_private: ['libpgtypes', 'libpq'], + ) + +diff --git a/src/interfaces/ecpg/pgtypeslib/meson.build b/src/interfaces/ecpg/pgtypeslib/meson.build +index 02301ec..e695b98 100644 +--- a/src/interfaces/ecpg/pgtypeslib/meson.build ++++ b/src/interfaces/ecpg/pgtypeslib/meson.build +@@ -46,10 +46,16 @@ ecpg_pgtypes_so = shared_library('libpgtypes', + ) + ecpg_targets += ecpg_pgtypes_so + ++if get_option('default_library') == 'shared' ++ ecpg_pgtypes_libraries = ecpg_pgtypes_so ++else ++ ecpg_pgtypes_libraries = ecpg_pgtypes_st ++endif ++ + pkgconfig.generate( + name: 'libpgtypes', + description: 'PostgreSQL libpgtypes library', + url: pg_url, +- libraries: ecpg_pgtypes_so, +- libraries_private: [frontend_stlib_code], ++ libraries: ecpg_pgtypes_libraries, ++ libraries_private: [frontend_lib_code], + ) +diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build +index a74e885..7354286 100644 +--- a/src/interfaces/libpq/meson.build ++++ b/src/interfaces/libpq/meson.build +@@ -60,7 +60,7 @@ libpq_so_c_args = ['-DUSE_DYNAMIC_OAUTH'] + libpq_st = static_library('libpq', + libpq_sources, + include_directories: [libpq_inc], +- c_args: libpq_c_args, ++ c_args: libpq_c_args + ['-DFRONTEND'], + c_pch: pch_postgres_fe_h, + dependencies: [frontend_stlib_code, libpq_deps], + kwargs: default_lib_args, +@@ -80,13 +80,19 @@ libpq_so = shared_library('libpq', + kwargs: default_lib_args, + ) + ++if get_option('default_library') == 'shared' ++ libpq_link_with = [libpq_so] ++else ++ libpq_link_with = [libpq_st] ++endif ++ + libpq = declare_dependency( +- link_with: [libpq_so], ++ link_with: libpq_link_with, + include_directories: [include_directories('.')] + ) + + private_deps = [ +- frontend_stlib_code, ++ frontend_lib_code, + libpq_deps, + ] + +diff --git a/src/port/meson.build b/src/port/meson.build +index fc7b059..3b8cf40 100644 +--- a/src/port/meson.build ++++ b/src/port/meson.build +@@ -167,6 +167,19 @@ pgport_variants = { + + foreach name, opts : pgport_variants + ++ if name == '_srv' ++ pgport += {name: disabler()} ++ continue ++ endif ++ if name == '' and get_option('default_library') == 'shared' ++ pgport += {name: disabler()} ++ continue ++ endif ++ if name == '_shlib' and get_option('default_library') == 'static' ++ pgport += {name: disabler()} ++ continue ++ endif ++ + # Build internal static libraries for sets of files that need to be built + # with different cflags + cflag_libs = [] +@@ -196,6 +209,7 @@ foreach name, opts : pgport_variants + } + ) + pgport += {name: lib} ++ pgport_default = lib + endforeach + + pgport_srv = pgport['_srv'] diff --git a/vcpkg_ports/libpq/portfile.cmake b/vcpkg_ports/libpq/portfile.cmake index b40d25866..97758ed1f 100644 --- a/vcpkg_ports/libpq/portfile.cmake +++ b/vcpkg_ports/libpq/portfile.cmake @@ -2,148 +2,111 @@ vcpkg_download_distfile(ARCHIVE URLS "https://ftp.postgresql.org/pub/source/v${VERSION}/postgresql-${VERSION}.tar.bz2" "https://www.mirrorservice.org/sites/ftp.postgresql.org/source/v${VERSION}/postgresql-${VERSION}.tar.bz2" FILENAME "postgresql-${VERSION}.tar.bz2" - SHA512 fdbe6d726f46738cf14acab96e5c05f7d65aefe78563281b416bb14a27c7c42e4df921e26b32816a5030ddbe506b95767e2c74a35afc589916504df38d1cb11c + SHA512 1a7606b5b2460a4fc817d491e1e03262df17ece6869fa3d2b80647baffcfb790fbcfa019c3b10fab9f9a14218651b6a358864e40a5d8c5ddc796db2d6e5b469c ) vcpkg_extract_source_archive( SOURCE_PATH ARCHIVE "${ARCHIVE}" PATCHES - unix/installdirs.patch - unix/fix-configure.patch - unix/single-linkage.patch - unix/no-server-tools.patch - unix/mingw-install.patch - unix/mingw-static-importlib-fix.patch - unix/python.patch + library-linkage.diff + libpq-and-client-tools.diff + libintl.diff + zic.diff windows/macro-def.patch windows/spin_delay.patch - windows/tcl-9.0-alpha.patch - windows/meson-vcpkg.patch windows/getopt.patch + windows/clang-cl-c99-test.patch + windows/clang-cl-msvc-abi.patch ) file(GLOB _py3_include_path "${CURRENT_HOST_INSTALLED_DIR}/include/python3*") string(REGEX MATCH "python3\\.([0-9]+)" _python_version_tmp "${_py3_include_path}") set(PYTHON_VERSION_MINOR "${CMAKE_MATCH_1}") -if("client" IN_LIST FEATURES) - set(HAS_TOOLS TRUE) -else() - set(HAS_TOOLS FALSE) -endif() - vcpkg_cmake_get_vars(cmake_vars_file) include("${cmake_vars_file}") -set(required_programs BISON FLEX) -if(VCPKG_DETECTED_MSVC OR NOT VCPKG_HOST_IS_WINDOWS) - list(APPEND required_programs PERL) -endif() -foreach(program_name IN LISTS required_programs) - vcpkg_find_acquire_program(${program_name}) - get_filename_component(program_dir ${${program_name}} DIRECTORY) - vcpkg_add_to_path(PREPEND "${program_dir}") +vcpkg_find_acquire_program(BISON) +vcpkg_find_acquire_program(FLEX) +vcpkg_find_acquire_program(PERL) + +vcpkg_list(SET MESON_OPTIONS_RELEASE) +vcpkg_list(SET MESON_OPTIONS) +foreach(option IN ITEMS icu lz4 zlib zstd) + if(option IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -D${option}=enabled) + endif() endforeach() -if(VCPKG_DETECTED_MSVC) - include("${CMAKE_CURRENT_LIST_DIR}/build-msvc.cmake") - build_msvc("${SOURCE_PATH}") +if("openssl" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -Dssl=openssl) else() - file(COPY "${CMAKE_CURRENT_LIST_DIR}/Makefile" DESTINATION "${SOURCE_PATH}") + list(APPEND MESON_OPTIONS -Dssl=none) +endif() - vcpkg_list(SET BUILD_OPTS) - foreach(option IN ITEMS bonjour icu lz4 readline zlib zstd) - if(option IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-${option}) - else() - list(APPEND BUILD_OPTS --without-${option}) - endif() - endforeach() - if("openssl" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-openssl) - else() - list(APPEND BUILD_OPTS --without-openssl) - endif() - if("xml" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-libxml) - else() - list(APPEND BUILD_OPTS --without-libxml) - endif() - if("xslt" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-libxslt) - else() - list(APPEND BUILD_OPTS --without-libxslt) - endif() - if("nls" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --enable-nls) - else() - list(APPEND BUILD_OPTS --disable-nls) - endif() - if("nls" IN_LIST FEATURES) - set(ENV{MSGFMT} "${CURRENT_HOST_INSTALLED_DIR}/tools/gettext/bin/msgfmt${VCPKG_HOST_EXECUTABLE_SUFFIX}") - endif() - if("python" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-python=3.${PYTHON_VERSION_MINOR}) - vcpkg_find_acquire_program(PYTHON3) - list(APPEND BUILD_OPTS "PYTHON=${PYTHON3}") - endif() - if(VCPKG_TARGET_IS_ANDROID AND (NOT VCPKG_CMAKE_SYSTEM_VERSION OR VCPKG_CMAKE_SYSTEM_VERSION LESS "26")) - list(APPEND BUILD_OPTS ac_cv_header_langinfo_h=no) - endif() - if(VCPKG_DETECTED_CMAKE_OSX_SYSROOT) - list(APPEND BUILD_OPTS "PG_SYSROOT=${VCPKG_DETECTED_CMAKE_OSX_SYSROOT}") - endif() - if(VCPKG_TARGET_IS_OSX) - set(ENV{LDFLAGS} "$ENV{LDFLAGS} -headerpad_max_install_names") - endif() - vcpkg_configure_make( - SOURCE_PATH "${SOURCE_PATH}" - COPY_SOURCE - AUTOCONFIG - ADDITIONAL_MSYS_PACKAGES autoconf-archive - DIRECT_PACKAGES - "https://mirror.msys2.org/msys/x86_64/tzcode-2025b-1-x86_64.pkg.tar.zst" - 824779e3ac4857bb21cbdc92fa881fa24bf89dfa8bc2f9ca816e9a9837a6d963805e8e0991499c43337a134552215fdee50010e643ddc8bd699170433a4c83de - OPTIONS - ${BUILD_OPTS} - OPTIONS_DEBUG - --enable-debug - ) +if("nls" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -Dnls=enabled) +endif() - if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") - set(ENV{LIBPQ_LIBRARY_TYPE} shared) - else() - set(ENV{LIBPQ_LIBRARY_TYPE} static) - endif() - if(VCPKG_TARGET_IS_MINGW) - set(ENV{LIBPQ_USING_MINGW} yes) - endif() - if(HAS_TOOLS) - set(ENV{LIBPQ_ENABLE_TOOLS} yes) +if("client" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS_RELEASE -Dtools=enabled) + if(VCPKG_CROSSCOMPILING) + list(APPEND MESON_OPTIONS "-DZIC=${CURRENT_HOST_INSTALLED_DIR}/manual-tools/${PORT}/zic${VCPKG_HOST_EXECUTABLE_SUFFIX}") endif() +endif() - # https://stackoverflow.com/a/76688525 - set(ENV{MAKELEVEL} 0) +vcpkg_configure_meson( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -Dauto_features=disabled + ${MESON_OPTIONS} + # cannot use ADDITIONAL_BINARIES for "native" programs + "-DBISON=['${BISON}']" + "-DFLEX=['${FLEX}']" + "-DPERL=${PERL}" + OPTIONS_RELEASE + ${MESON_OPTIONS_RELEASE} +) +vcpkg_install_meson() +vcpkg_fixup_pkgconfig() - vcpkg_install_make() +if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW AND VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + file(GLOB pc_files "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/*.pc" "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/*.pc") + foreach(file IN LISTS pc_files) + vcpkg_replace_string("${file}" " -l(pq|pg|ecpg)" " -llib\\1" REGEX) + endforeach() +endif() - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/postgresql/server/pg_config.h" "#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/pg_config.h" "#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS") +vcpkg_copy_tools(TOOL_NAMES ecpg AUTO_CLEAN) +if("client" IN_LIST FEATURES) + if(NOT VCPKG_CROSSCOMPILING) + vcpkg_copy_tools(TOOL_NAMES zic AUTO_CLEAN DESTINATION "${CURRENT_PACKAGES_DIR}/manual-tools/${PORT}") + endif() + vcpkg_copy_tools( + TOOL_NAMES + clusterdb createdb createuser + dropdb dropuser + pg_amcheck + pg_basebackup pgbench + pg_combinebackup pg_config pg_createsubscriber + pg_dump pg_dumpall + pg_isready + pg_receivewal pg_recvlogical pg_restore + pg_verifybackup + psql + reindexdb + vacuumdb + AUTO_CLEAN + ) endif() -vcpkg_fixup_pkgconfig() -configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" "${CURRENT_PACKAGES_DIR}/share/postgresql/vcpkg-cmake-wrapper.cmake" @ONLY) file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/doc" "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/debug/symbols" - "${CURRENT_PACKAGES_DIR}/debug/tools" - "${CURRENT_PACKAGES_DIR}/symbols" - "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug" ) +configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" "${CURRENT_PACKAGES_DIR}/share/postgresql/vcpkg-cmake-wrapper.cmake" @ONLY) file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYRIGHT") diff --git a/vcpkg_ports/libpq/unix/fix-configure.patch b/vcpkg_ports/libpq/unix/fix-configure.patch deleted file mode 100644 index c7cc663a2..000000000 --- a/vcpkg_ports/libpq/unix/fix-configure.patch +++ /dev/null @@ -1,60 +0,0 @@ -diff --git a/configure.ac b/configure.ac ---- a/configure.ac -+++ b/configure.ac -@@ -20,6 +20,7 @@ AC_INIT([PostgreSQL], [18.3], [pgsql-bugs@lists.postgresql.org], [], [https://ww --m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. --Untested combinations of 'autoconf' and PostgreSQL versions are not --recommended. You can remove the check from 'configure.ac' but it is then --your responsibility whether the result works or not.])]) -+cross_compiling=yes # Avoid conftest loading shared objects -+m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_warn([unsupported],[Autoconf version 2.69 is required. -+Untested combinations of 'autoconf' and PostgreSQL versions are not -+recommended. You can remove the check from 'configure.ac' but it is then -+your responsibility whether the result works or not.])]) - AC_COPYRIGHT([Copyright (c) 1996-2025, PostgreSQL Global Development Group]) - AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) -@@ -1347,3 +1348,4 @@ AC_SEARCH_LIBS(pthread_barrier_wait, pthread) - if test "$with_readline" = yes; then -- PGAC_CHECK_READLINE -+ PKG_CHECK_MODULES([READLINE], [readline], [HAVE_LIBREADLINE=1], [pgac_cv_check_readline=no]) -+ LIBS="$READLINE_LIBS $LIBS" - if test x"$pgac_cv_check_readline" = x"no"; then -@@ -1357,3 +1359,3 @@ fi - if test "$with_zlib" = yes; then -- AC_CHECK_LIB(z, inflate, [], -+ PKG_CHECK_MODULES([ZLIB], [zlib], [LIBS="$ZLIB_LIBS $LIBS"], - [AC_MSG_ERROR([zlib library not found -@@ -1394,5 +1396,9 @@ fi - AC_DEFINE(OPENSSL_API_COMPAT, [0x10101000L], - [Define to the OpenSSL API version in use. This avoids deprecation warnings from newer OpenSSL versions.]) -+ PKG_CHECK_MODULES([CRYPTO_new_ex_data], [libcrypto], [LIBS="$CRYPTO_new_ex_data_LIBS $LIBS"], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])]) -+ PKG_CHECK_MODULES([SSL_new], [libssl], [LIBS="$SSL_new_LIBS $LIBS"], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) -+ if false ; then - AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, [], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])]) - AC_CHECK_LIB(ssl, SSL_new, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) -+ fi - # Functions introduced in OpenSSL 1.1.1. -@@ -1415,3 +1421,4 @@ fi - if test "$with_libxml" = yes ; then -- AC_CHECK_LIB(xml2, xmlSaveToBuffer, [], [AC_MSG_ERROR([library 'xml2' (version >= 2.6.23) is required for XML support])]) -+ PKG_CHECK_MODULES([LIBXML2], [libxml-2.0 >= 2.6.23], [AC_DEFINE(HAVE_LIBXML2,1,[Define to 1 if with xml2])], [AC_MSG_ERROR([library 'xml2' (version >= 2.6.23) is required for XML support])]) -+ LIBS="$LIBXML2_LIBS $LIBS" - fi -@@ -1419,3 +1426,4 @@ fi - if test "$with_libxslt" = yes ; then -- AC_CHECK_LIB(xslt, xsltCleanupGlobals, [], [AC_MSG_ERROR([library 'xslt' is required for XSLT support])]) -+ PKG_CHECK_MODULES([LIBXSLT], [libxslt], [AC_DEFINE(HAVE_LIBXSLT,1,[Define to 1 if with xslt])], [AC_MSG_ERROR([library 'xslt' is required for XSLT support])]) -+ LIBS="$LIBXSLT_LIBS $LIBS" - fi -@@ -1430,3 +1438,4 @@ fi - if test "$with_lz4" = yes ; then -- AC_CHECK_LIB(lz4, LZ4_compress_default, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) -+ PKG_CHECK_MODULES([LZ4], [liblz4], [AC_DEFINE(HAVE_LIBLZ4,1,[Define to 1 if with lz4])], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) -+ LIBS="$LZ4_LIBS $LIBS" - fi -@@ -1434,3 +1443,4 @@ fi - if test "$with_zstd" = yes ; then -- AC_CHECK_LIB(zstd, ZSTD_compress, [], [AC_MSG_ERROR([library 'zstd' is required for ZSTD support])]) -+ PKG_CHECK_MODULES([ZSTD], [libzstd], [AC_DEFINE(HAVE_LIBZSTD,1,[Define to 1 if with zstd])], [AC_MSG_ERROR([library 'zstd' is required for ZSTD support])]) -+ LIBS="$ZSTD_LIBS $LIBS" - fi diff --git a/vcpkg_ports/libpq/unix/installdirs.patch b/vcpkg_ports/libpq/unix/installdirs.patch deleted file mode 100644 index bbba0fe07..000000000 --- a/vcpkg_ports/libpq/unix/installdirs.patch +++ /dev/null @@ -1,52 +0,0 @@ -diff --git a/src/Makefile.global.in b/src/Makefile.global.in ---- a/src/Makefile.global.in -+++ b/src/Makefile.global.in -@@ -113,42 +113,42 @@ bindir := @bindir@ - --datadir := @datadir@ -+datadir := @datadir@/postgresql - ifeq "$(findstring pgsql, $(datadir))" "" - ifeq "$(findstring postgres, $(datadir))" "" - override datadir := $(datadir)/postgresql - endif - endif - --sysconfdir := @sysconfdir@ -+sysconfdir := @sysconfdir@/postgresql - ifeq "$(findstring pgsql, $(sysconfdir))" "" - ifeq "$(findstring postgres, $(sysconfdir))" "" - override sysconfdir := $(sysconfdir)/postgresql - endif - endif - - libdir := @libdir@ - --pkglibdir = $(libdir) -+pkglibdir = $(libdir)/postgresql - ifeq "$(findstring pgsql, $(pkglibdir))" "" - ifeq "$(findstring postgres, $(pkglibdir))" "" - override pkglibdir := $(pkglibdir)/postgresql - endif - endif - - includedir := @includedir@ - --pkgincludedir = $(includedir) -+pkgincludedir = $(includedir)/postgresql - ifeq "$(findstring pgsql, $(pkgincludedir))" "" - ifeq "$(findstring postgres, $(pkgincludedir))" "" - override pkgincludedir := $(pkgincludedir)/postgresql - endif - endif - - mandir := @mandir@ - --docdir := @docdir@ -+docdir := @docdir@/postgresql - ifeq "$(findstring pgsql, $(docdir))" "" - ifeq "$(findstring postgres, $(docdir))" "" - override docdir := $(docdir)/postgresql - endif - endif - - htmldir := @htmldir@ diff --git a/vcpkg_ports/libpq/unix/mingw-install.patch b/vcpkg_ports/libpq/unix/mingw-install.patch deleted file mode 100644 index ffeea023c..000000000 --- a/vcpkg_ports/libpq/unix/mingw-install.patch +++ /dev/null @@ -1,36 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index 16255d7..51e9ed2 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -85,7 +85,11 @@ else - # Naming convention for dynamically loadable modules - shlib = $(NAME)$(DLSUFFIX) - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+stlib = lib$(NAME).dll.a -+else - stlib = lib$(NAME).a -+endif - - ifndef soname - # additional flags for backend modules -@@ -468,6 +472,9 @@ endif - else # no soname - $(INSTALL_SHLIB) $< '$(DESTDIR)$(pkglibdir)/$(shlib)' - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+ mv '$(DESTDIR)$(libdir)/$(shlib)' '$(DESTDIR)$(libdir)/../bin/$(shlib)' -+endif - - - installdirs-lib: -@@ -476,6 +483,9 @@ ifdef soname - else - $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+ $(MKDIR_P) '$(DESTDIR)$(libdir)/../bin' -+endif - - - ## diff --git a/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch b/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch deleted file mode 100644 index 44d465710..000000000 --- a/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index f94d59d..77aa6bf 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -216,8 +216,10 @@ ifeq ($(PORTNAME), win32) - ifdef SO_MAJOR_VERSION - shlib = lib$(NAME)$(DLSUFFIX) - endif -+ifneq ($(LIBPQ_LIBRARY_TYPE), static) - haslibarule = yes - endif -+endif - - - # If the shared library doesn't have an export file, mark all symbols not -@@ -323,8 +325,10 @@ else - # Win32 case - - # See notes in src/backend/parser/Makefile about the following two rules -+ifneq ($(LIBPQ_LIBRARY_TYPE), static) - $(stlib): $(shlib) - touch $@ -+endif - - # XXX A backend that loads a module linked with libgcc_s_dw2-1.dll will exit - # uncleanly, hence -static-libgcc. (Last verified with MinGW-w64 compilers - -diff --git a/src/makefiles/Makefile.win32 b/src/makefiles/Makefile.win32: -index dc1aafa..1ab97a8 100644 ---- a/src/makefiles/Makefile.win32 -+++ b/src/makefiles/Makefile.win32 -@@ -22,21 +22,32 @@ endif - endif - endif - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring src/common,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring src/port,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring timezone,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring ecpg/ecpglib,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif - - # required by Python headers - ifneq (,$(findstring src/pl/plpython,$(subdir))) diff --git a/vcpkg_ports/libpq/unix/no-server-tools.patch b/vcpkg_ports/libpq/unix/no-server-tools.patch deleted file mode 100644 index 1677edb75..000000000 --- a/vcpkg_ports/libpq/unix/no-server-tools.patch +++ /dev/null @@ -1,39 +0,0 @@ -diff --git a/src/bin/Makefile b/src/bin/Makefile -index 7f9dde9..bc6d835 100644 ---- a/src/bin/Makefile -+++ b/src/bin/Makefile -@@ -13,31 +13,20 @@ subdir = src/bin - top_builddir = ../.. - include $(top_builddir)/src/Makefile.global - -+.NOTPARALLEL: -+# incl. https://www.postgresql.org/docs/current/reference-client.html -+# excl. https://www.postgresql.org/docs/current/reference-server.html - SUBDIRS = \ -- initdb \ - pg_amcheck \ -- pg_archivecleanup \ - pg_basebackup \ -- pg_checksums \ -- pg_combinebackup \ - pg_config \ -- pg_controldata \ -- pg_ctl \ - pg_dump \ -- pg_resetwal \ -- pg_rewind \ -- pg_test_fsync \ -- pg_test_timing \ -- pg_upgrade \ - pg_verifybackup \ -- pg_waldump \ -- pg_walsummary \ - pgbench \ - psql \ - scripts - - ifeq ($(PORTNAME), win32) --SUBDIRS += pgevent - else - ALWAYS_SUBDIRS += pgevent - endif diff --git a/vcpkg_ports/libpq/unix/python.patch b/vcpkg_ports/libpq/unix/python.patch deleted file mode 100644 index 7d8f58d4a..000000000 --- a/vcpkg_ports/libpq/unix/python.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/configure.ac b/configure.ac -index 7f97248992..33b6c84fc4 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -891,7 +891,9 @@ AC_SUBST(with_perl) - # Optionally build Python modules (PL/Python) - # - AC_MSG_CHECKING([whether to build Python modules]) --PGAC_ARG_BOOL(with, python, no, [build Python modules (PL/Python)]) -+PGAC_ARG_OPTARG(with, python, [PYTHON_VERSION], [build Python modules (PL/Python)], -+ [], -+ [python_version=$withval]) - AC_MSG_RESULT([$with_python]) - AC_SUBST(with_python) - -@@ -1214,7 +1216,18 @@ fi - - if test "$with_python" = yes; then - PGAC_PATH_PYTHON -- PGAC_CHECK_PYTHON_EMBED_SETUP -+ python_majorversion=3 -+ PKG_CHECK_MODULES(PYTHON_EMBED, python-${python_version}-embed) -+ python_includespec="${PYTHON_EMBED_CFLAGS}" -+ python_libdir=[$(echo " ${PYTHON_EMBED_LIBS}" | sed -e 's/\( -L[^ ]*\).*/\1/' -e 's/^.* -L//')] -+ python_libspec="${PYTHON_EMBED_LIBS}" -+ python_additional_libs="" -+ AC_SUBST(python_majorversion) -+ AC_SUBST(python_version) -+ AC_SUBST(python_includespec) -+ AC_SUBST(python_libdir) -+ AC_SUBST(python_libspec) -+ AC_SUBST(python_additional_libs) - fi - - if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then diff --git a/vcpkg_ports/libpq/unix/single-linkage.patch b/vcpkg_ports/libpq/unix/single-linkage.patch deleted file mode 100644 index 1c334a58a..000000000 --- a/vcpkg_ports/libpq/unix/single-linkage.patch +++ /dev/null @@ -1,56 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index 551023c..16255d7 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -260,10 +260,14 @@ endif - - .PHONY: all-lib all-static-lib all-shared-lib - -+ifndef LIBPQ_LIBRARY_TYPE - all-lib: all-shared-lib - ifdef soname - # no static library when building a dynamically loadable module - all-lib: all-static-lib -+endif -+else -+all-lib: all-$(LIBPQ_LIBRARY_TYPE)-lib - all-lib: lib$(NAME).pc - endif - -@@ -417,9 +421,13 @@ endif # PORTNAME == cygwin || PORTNAME == win32 - ## - - .PHONY: install-lib install-lib-static install-lib-shared installdirs-lib -+ifndef LIBPQ_LIBRARY_TYPE - install-lib: install-lib-shared - ifdef soname - install-lib: install-lib-static -+endif -+else -+install-lib: install-lib-$(LIBPQ_LIBRARY_TYPE) - install-lib: install-lib-pc - endif - -diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile -index 8abdb09..185461e 100644 ---- a/src/interfaces/libpq/Makefile -+++ b/src/interfaces/libpq/Makefile -@@ -115,6 +115,7 @@ backend_src = $(top_srcdir)/src/backend - # Also skip the test on platforms where libpq infrastructure may be provided - # by statically-linked libraries, as we can't expect them to honor this - # coding rule. -+ifeq ($(LIBPQ_LIBRARY_TYPE), shared) - libpq-refs-stamp: $(shlib) - ifneq ($(enable_coverage), yes) - ifeq (,$(filter solaris,$(PORTNAME))) -@@ -124,6 +125,10 @@ ifeq (,$(filter solaris,$(PORTNAME))) - endif - endif - touch $@ -+else -+.PHONY: libpq-refs-stamp -+libpq-refs-stamp: -+endif - - # Make dependencies on pg_config_paths.h visible in all builds. - fe-connect.o: fe-connect.c $(top_builddir)/src/port/pg_config_paths.h diff --git a/vcpkg_ports/libpq/usage b/vcpkg_ports/libpq/usage index 8dfcd2754..1f36ee18b 100644 --- a/vcpkg_ports/libpq/usage +++ b/vcpkg_ports/libpq/usage @@ -1,4 +1,4 @@ -The package libpq provides CMake integration: +libpq provides CMake integration: - find_package(PostgreSQL REQUIRED) - target_link_libraries(main PRIVATE PostgreSQL::PostgreSQL) + find_package(PostgreSQL REQUIRED) + target_link_libraries(main PRIVATE PostgreSQL::PostgreSQL) diff --git a/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake b/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake index fa7d3b56e..fdb714590 100644 --- a/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake +++ b/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake @@ -44,9 +44,9 @@ if(PostgreSQL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") endforeach() if(WIN32) if(TARGET PostgreSQL::PostgreSQL) - set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "Secur32.lib") + set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "secur32.lib") endif() - list(APPEND PostgreSQL_LIBRARIES Secur32.lib) + list(APPEND PostgreSQL_LIBRARIES secur32.lib) endif() cmake_policy(PUSH) cmake_policy(SET CMP0057 NEW) @@ -56,13 +56,13 @@ if(PostgreSQL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") if(TARGET PostgreSQL::PostgreSQL) set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "OpenSSL::SSL") endif() - list(APPEND PostgreSQL_LIBRARIES OpenSSL::SSL) + list(APPEND PostgreSQL_LIBRARIES ${OPENSSL_LIBRARIES}) endif() if(WIN32) if(TARGET PostgreSQL::PostgreSQL) - set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "Wldap32.lib") + set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "wldap32.lib") endif() - list(APPEND PostgreSQL_LIBRARIES Wldap32.lib) + list(APPEND PostgreSQL_LIBRARIES wldap32.lib) endif() unset(Z_VCPKG_PORT_FEATURES) cmake_policy(POP) diff --git a/vcpkg_ports/libpq/vcpkg-libs.props.in b/vcpkg_ports/libpq/vcpkg-libs.props.in deleted file mode 100644 index cb7e5f92c..000000000 --- a/vcpkg_ports/libpq/vcpkg-libs.props.in +++ /dev/null @@ -1,31 +0,0 @@ - - - - @CURRENT_INSTALLED_DIR@/debug/lib/icuind.lib;@CURRENT_INSTALLED_DIR@/debug/lib/icuucd.lib;@CURRENT_INSTALLED_DIR@/debug/lib/icudtd.lib; - @CURRENT_INSTALLED_DIR@/debug/lib/lz4d.lib - @CURRENT_INSTALLED_DIR@/debug/lib/intl.lib;@CURRENT_INSTALLED_DIR@/debug/lib/iconv.lib;@CURRENT_INSTALLED_DIR@/debug/lib/charset.lib - @CURRENT_INSTALLED_DIR@/debug/lib/libssl.lib;@CURRENT_INSTALLED_DIR@/debug/lib/libcrypto.lib;crypt32.lib;ws2_32.lib;secur32.lib - @CURRENT_INSTALLED_DIR@/debug/lib/python3@PYTHON_VERSION_MINOR@_d.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90g.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90sg.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90sgx.lib - @LIBXML2_LIBS_DEBUG@ - @LIBXSLT_LIBS_DEBUG@ - @CURRENT_INSTALLED_DIR@/debug/lib/zlibd.lib - @CURRENT_INSTALLED_DIR@/debug/lib/zstd.lib - - - @CURRENT_INSTALLED_DIR@/lib/icuin.lib;@CURRENT_INSTALLED_DIR@/lib/icuuc.lib;@CURRENT_INSTALLED_DIR@/lib/icudt.lib; - @CURRENT_INSTALLED_DIR@/lib/lz4.lib - @CURRENT_INSTALLED_DIR@/lib/intl.lib;@CURRENT_INSTALLED_DIR@/lib/iconv.lib;@CURRENT_INSTALLED_DIR@/lib/charset.lib - @CURRENT_INSTALLED_DIR@/lib/libssl.lib;@CURRENT_INSTALLED_DIR@/lib/libcrypto.lib;crypt32.lib;ws2_32.lib;secur32.lib - @CURRENT_INSTALLED_DIR@/lib/python3@PYTHON_VERSION_MINOR@.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90s.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90sx.lib - @LIBXML2_LIBS_RELEASE@ - @LIBXSLT_LIBS_RELEASE@ - @CURRENT_INSTALLED_DIR@/lib/zlib.lib - @CURRENT_INSTALLED_DIR@/lib/zstd.lib - - diff --git a/vcpkg_ports/libpq/vcpkg.json b/vcpkg_ports/libpq/vcpkg.json index 782747bda..2dd780c6a 100644 --- a/vcpkg_ports/libpq/vcpkg.json +++ b/vcpkg_ports/libpq/vcpkg.json @@ -1,27 +1,18 @@ { "name": "libpq", - "version": "18.3", + "version": "18.4", "description": "The official database access API of postgresql", "homepage": "https://www.postgresql.org/", "license": "PostgreSQL", "supports": "!uwp & !emscripten", "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "bonjour" - ], - "platform": "osx" - }, { "name": "vcpkg-cmake-get-vars", "host": true }, { "name": "vcpkg-tool-meson", - "host": true, - "platform": "windows & !mingw" + "host": true } ], "default-features": [ @@ -38,55 +29,35 @@ "features": [ "client", "icu", - "xml", - "xslt", + "lz4", + "nls", + "openssl", + "zlib", "zstd" ] }, - { - "name": "libpq", - "features": [ - "bonjour" - ], - "platform": "osx" - }, - { - "name": "libpq", - "features": [ - "nls" - ], - "platform": "!osx" - }, { "name": "libpq", "features": [ "readline" ], "platform": "!windows" - }, - { - "name": "libpq", - "features": [ - "python" - ], - "platform": "!android & !mingw" - }, + } + ] + }, + "client": { + "description": "Build all client tools and libraries", + "dependencies": [ { "name": "libpq", + "host": true, + "default-features": false, "features": [ - "tcl" - ], - "platform": "windows & !mingw & !arm" + "client" + ] } ] }, - "bonjour": { - "description": "Build with Bonjour support", - "supports": "osx" - }, - "client": { - "description": "Build all client tools and libraries" - }, "icu": { "description": "Build with support for the ICU library", "dependencies": [ @@ -119,20 +90,6 @@ "openssl" ] }, - "python": { - "$supports": "!(windows & (static | mingw))", - "description": "build the PL/Python server programming language", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "client" - ] - }, - "python3" - ] - }, "readline": { "description": "Use readline", "supports": "!windows, mingw", @@ -140,43 +97,6 @@ "readline" ] }, - "tcl": { - "$supports": "!(windows & (static | mingw))", - "description": "build the PL/Tcl procedural language", - "supports": "windows & !mingw", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "client" - ] - }, - "tcl" - ] - }, - "xml": { - "description": "Build with libxml", - "dependencies": [ - { - "name": "libxml2", - "default-features": false - } - ] - }, - "xslt": { - "description": "Build with libxslt", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "xml" - ] - }, - "libxslt" - ] - }, "zlib": { "description": "Use zlib", "dependencies": [ diff --git a/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch b/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch new file mode 100644 index 000000000..1a200cbde --- /dev/null +++ b/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch @@ -0,0 +1,15 @@ +diff --git a/meson.build b/meson.build +index 5b9ea5f..a552243 100644 +--- a/meson.build ++++ b/meson.build +@@ -1711,8 +1711,10 @@ + # unnecessarily, because we optionally rely on newer features. + c99_test = ''' + #include ++#ifndef _MSC_VER + #include + #include ++#endif + #include + + struct named_init_test { diff --git a/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch b/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch new file mode 100644 index 000000000..db890396d --- /dev/null +++ b/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch @@ -0,0 +1,316 @@ +diff --git a/meson.build b/meson.build +index 2c94236..e42075d 100644 +--- a/meson.build ++++ b/meson.build +@@ -40,6 +40,7 @@ build_system = build_machine.system() + host_cpu = host_machine.cpu_family() + + cc = meson.get_compiler('c') ++msvc_abi = cc.get_argument_syntax() == 'msvc' + + not_found_dep = dependency('', required: false) + thread_dep = dependency('threads') +@@ -275,14 +276,14 @@ elif host_system == 'windows' + exesuffix = '.exe' + dlsuffix = '.dll' + library_path_var = '' +- if cc.get_id() != 'msvc' ++ if not msvc_abi + # define before including for getting localtime_r() etc. on MinGW + cppflags += '-D_POSIX_C_SOURCE' + endif + + export_file_format = 'win' + export_file_suffix = 'def' +- if cc.get_id() == 'msvc' ++ if msvc_abi + export_fmt = '/DEF:@0@' + mod_link_with_name = '@0@.lib' + else +@@ -296,7 +297,7 @@ elif host_system == 'windows' + sema_kind = 'win32' + + cdata.set('WIN32_STACK_RLIMIT', 4194304) +- if cc.get_id() == 'msvc' ++ if msvc_abi + ldflags += '/INCREMENTAL:NO' + ldflags += '/STACK:@0@'.format(cdata.get('WIN32_STACK_RLIMIT')) + # ldflags += '/nxcompat' # generated by msbuild, should have it for ninja? +@@ -314,7 +315,7 @@ elif host_system == 'windows' + libpq_deps += secur32_dep + + postgres_inc_d += 'src/include/port/win32' +- if cc.get_id() == 'msvc' ++ if msvc_abi + postgres_inc_d += 'src/include/port/win32_msvc' + endif + +@@ -1216,7 +1217,7 @@ if not perlopt.disabled() + if host_system == 'windows' + perl_ccflags += ['-DPLPERL_HAVE_UID_GID'] + +- if cc.get_id() == 'msvc' ++ if msvc_abi + # prevent binary mismatch between MSVC built plperl and Strawberry or + # msys ucrt perl libraries + perl_v = run_command(perl, '-V', check: false).stdout() +@@ -1429,7 +1430,7 @@ Use -Dreadline=disabled to disable readline support.'''.format(readline_dep)) + + # On windows with mingw readline requires auto-import to successfully + # link, as the headers don't use declspec(dllimport) +- if host_system == 'windows' and cc.get_id() != 'msvc' ++ if host_system == 'windows' and not msvc_abi + readline = declare_dependency(dependencies: readline, + link_args: '-Wl,--enable-auto-import') + endif +@@ -2018,50 +2019,78 @@ if cc.links(''' + endif + + # Check for __get_cpuid() and __cpuid(). +-if cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); +- } +- ''', name: '__get_cpuid', +- args: test_c_args) +- cdata.set('HAVE__GET_CPUID', 1) +-elif cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __cpuid(exx, 1); +- } +- ''', name: '__cpuid', +- args: test_c_args) +- cdata.set('HAVE__CPUID', 1) ++if msvc_abi ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuid(exx, 1); ++ } ++ ''', name: '__cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__CPUID', 1) ++ endif ++else ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ unsigned int exx[4] = {0, 0, 0, 0}; ++ __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); ++ } ++ ''', name: '__get_cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__GET_CPUID', 1) ++ elif cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuid(exx, 1); ++ } ++ ''', name: '__cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__CPUID', 1) ++ endif + endif + + + # Check for __get_cpuid_count() and __cpuidex() in a similar fashion. +-if cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); +- } +- ''', name: '__get_cpuid_count', +- args: test_c_args) +- cdata.set('HAVE__GET_CPUID_COUNT', 1) +-elif cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __cpuidex(exx, 7, 0); +- } +- ''', name: '__cpuidex', +- args: test_c_args) +- cdata.set('HAVE__CPUIDEX', 1) ++if msvc_abi ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuidex(exx, 7, 0); ++ } ++ ''', name: '__cpuidex', ++ args: test_c_args) ++ cdata.set('HAVE__CPUIDEX', 1) ++ endif ++else ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ unsigned int exx[4] = {0, 0, 0, 0}; ++ __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); ++ } ++ ''', name: '__get_cpuid_count', ++ args: test_c_args) ++ cdata.set('HAVE__GET_CPUID_COUNT', 1) ++ elif cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuidex(exx, 7, 0); ++ } ++ ''', name: '__cpuidex', ++ args: test_c_args) ++ cdata.set('HAVE__CPUIDEX', 1) ++ endif + endif + + +@@ -2183,7 +2212,7 @@ foreach w : negative_warning_flags + endforeach + + +-if cc.get_id() == 'msvc' ++if msvc_abi + cflags_warn += [ + '/wd4018', # signed/unsigned mismatch + '/wd4244', # conversion from 'type1' to 'type2', possible loss of data +@@ -2442,7 +2471,7 @@ have_optimized_crc = false + cflags_crc = [] + if host_cpu == 'x86' or host_cpu == 'x86_64' + +- if cc.get_id() == 'msvc' ++ if msvc_abi + cdata.set('USE_SSE42_CRC32C', false) + cdata.set('USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 1) + have_optimized_crc = true +@@ -2591,7 +2620,7 @@ endif + # Other CPU specific stuff + ############################################################### + +-if host_cpu == 'x86_64' ++if host_cpu == 'x86_64' and not msvc_abi + + if cc.compiles(''' + void main(void) +diff --git a/src/backend/meson.build b/src/backend/meson.build +index 2b0db21..aad9a3b 100644 +--- a/src/backend/meson.build ++++ b/src/backend/meson.build +@@ -70,7 +70,7 @@ postgres_lib = static_library('postgres_lib', + kwargs: internal_lib_args, + ) + +-if cc.get_id() == 'msvc' ++if msvc_abi + postgres_def = custom_target('postgres.def', + command: [perl, files('../tools/msvc_gendef.pl'), + '--arch', host_cpu, +diff --git a/src/bin/pgevent/meson.build b/src/bin/pgevent/meson.build +index 6ec04e9..24b2bd7 100644 +--- a/src/bin/pgevent/meson.build ++++ b/src/bin/pgevent/meson.build +@@ -17,7 +17,7 @@ pgevent_sources += windows.compile_resources('pgmsgevent.rc') + # XXX: We may want to revisit the addition of this flag and improve the + # way symbols are exported. + pgevent_link_args = [] +-if cc.get_id() == 'msvc' ++if msvc_abi + pgevent_link_args += '/ignore:4104' + endif + +diff --git a/src/port/meson.build b/src/port/meson.build +index fc7b059..57a6a06 100644 +--- a/src/port/meson.build ++++ b/src/port/meson.build +@@ -54,7 +54,7 @@ elif host_system == 'cygwin' + ) + endif + +-if cc.get_id() == 'msvc' ++if msvc_abi + pgport_sources += files( + 'dirent.c', + 'win32gettimeofday.c', +@@ -136,7 +136,7 @@ endforeach + + + if (host_system == 'windows' or host_system == 'cygwin') and \ +- (cc.get_id() != 'msvc' or cc.version().version_compare('<14.0')) ++ (not msvc_abi or cc.version().version_compare('<14.0')) + + # Cygwin and (apparently, based on test results) Mingw both + # have a broken strtof(), so substitute its implementation. +diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c +index 61c7388..941e1b5 100644 +--- a/src/port/pg_bitutils.c ++++ b/src/port/pg_bitutils.c +@@ -143,7 +143,7 @@ pg_popcount_available(void) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +diff --git a/src/port/pg_crc32c_sse42_choose.c b/src/port/pg_crc32c_sse42_choose.c +index 74d2421..f18ea81 100644 +--- a/src/port/pg_crc32c_sse42_choose.c ++++ b/src/port/pg_crc32c_sse42_choose.c +@@ -73,7 +73,7 @@ pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +@@ -92,7 +92,7 @@ pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) + #if defined(HAVE__GET_CPUID_COUNT) + __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUIDEX) +- __cpuidex(exx, 7, 0); ++ __cpuidex((int *) exx, 7, 0); + #endif + + #ifdef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK +diff --git a/src/port/pg_popcount_avx512.c b/src/port/pg_popcount_avx512.c +index 896bc74..9a0aa75 100644 +--- a/src/port/pg_popcount_avx512.c ++++ b/src/port/pg_popcount_avx512.c +@@ -44,7 +44,7 @@ xsave_available(void) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +@@ -82,7 +82,7 @@ avx512_popcnt_available(void) + #if defined(HAVE__GET_CPUID_COUNT) + __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUIDEX) +- __cpuidex(exx, 7, 0); ++ __cpuidex((int *) exx, 7, 0); + #else + #error cpuid instruction not available + #endif diff --git a/vcpkg_ports/libpq/windows/getopt.patch b/vcpkg_ports/libpq/windows/getopt.patch index 073b32adb..c6e0126a3 100644 --- a/vcpkg_ports/libpq/windows/getopt.patch +++ b/vcpkg_ports/libpq/windows/getopt.patch @@ -1,21 +1,14 @@ diff --git a/meson.build b/meson.build +index 34456b3..052e8c8 100644 --- a/meson.build +++ b/meson.build -@@ -2624,7 +2624,7 @@ header_checks = [ - 'copyfile.h', - 'crtdefs.h', - 'execinfo.h', -- 'getopt.h', -+ # 'getopt.h', # vcpkg: skip to avoid getopt-win32 header pollution - 'ifaddrs.h', - 'mbarrier.h', - 'strings.h', -@@ -2714,7 +2714,7 @@ endforeach +@@ -2650,6 +2650,9 @@ foreach header : header_checks + description: 'Define to 1 if you have the <@0@> header file.'.format(header)) + endforeach - if cc.has_type('struct option', - args: test_c_args, include_directories: postgres_inc, -- prefix: '@0@'.format(cdata.get('HAVE_GETOPT_H')) == '1' ? '#include ' : '') -+ prefix: '@0@'.format(cdata.get('HAVE_GETOPT_H', 0)) == '1' ? '#include ' : '') - cdata.set('HAVE_STRUCT_OPTION', 1) - endif ++if cc.get_id() == 'msvc' ++ cdata.set('HAVE_GETOPT_H', false) ++endif + decl_checks = [ + ['F_FULLFSYNC', 'fcntl.h'], diff --git a/vcpkg_ports/libpq/windows/meson-vcpkg.patch b/vcpkg_ports/libpq/windows/meson-vcpkg.patch deleted file mode 100644 index 84543bcd0..000000000 --- a/vcpkg_ports/libpq/windows/meson-vcpkg.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/meson.build b/meson.build ---- a/meson.build -+++ b/meson.build -@@ -83,7 +83,7 @@ endif - ############################################################### - - postgres_inc_d = ['src/include'] --postgres_inc_d += get_option('extra_include_dirs') -+# postgres_inc_d += get_option('extra_include_dirs') # vcpkg: moved below - - postgres_lib_d = get_option('extra_lib_dirs') - -@@ -316,6 +316,7 @@ elif host_system == 'windows' - postgres_inc_d += 'src/include/port/win32' - if cc.get_id() == 'msvc' - postgres_inc_d += 'src/include/port/win32_msvc' -+ postgres_inc_d += get_option('extra_include_dirs') # vcpkg: moved here for correct priority - endif - - windows = import('windows') -@@ -3218,6 +3219,7 @@ endif - generated_headers = [] - # headers that the backend build depends on - generated_backend_headers = [] -+generated_backend_sources = [] - # configure_files() output, needs a way of converting to file names - configure_files = [] - -@@ -3372,12 +3374,12 @@ test_install_libs = [] - - subdir('src') - --subdir('contrib') -+# subdir('contrib') # vcpkg: skip - --subdir('src/test') --subdir('src/interfaces/ecpg/test') -+# subdir('src/test') # vcpkg: skip -+# subdir('src/interfaces/ecpg/test') # vcpkg: skip - --subdir('doc/src/sgml') -+# subdir('doc/src/sgml') # vcpkg: skip - - generated_sources_ac += {'': ['GNUmakefile']} - diff --git a/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch b/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch deleted file mode 100644 index 154efcb8f..000000000 --- a/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -Naur postgresql-16.9/src/pl/tcl/pltcl.c postgresql-16.9-vcpkg/src/pl/tcl/pltcl.c ---- postgresql-16.9/src/pl/tcl/pltcl.c 2025-05-05 22:30:08.000000000 +0200 -+++ postgresql-16.9-vcpkg/src/pl/tcl/pltcl.c 2025-05-23 22:53:43.485828671 +0200 -@@ -56,7 +56,7 @@ - #define CONST86 - #endif - --#if !HAVE_TCL_VERSION(8,7) -+#if !HAVE_TCL_VERSION(8,7) || (TCL_MAJOR_VERSION == 9 && TCL_MINOR_VERSION == 0 && TCL_RELEASE_LEVEL == TCL_ALPHA_RELEASE) - typedef int Tcl_Size; - #endif - diff --git a/vcpkg_ports/libpq/zic.diff b/vcpkg_ports/libpq/zic.diff new file mode 100644 index 000000000..737182f2d --- /dev/null +++ b/vcpkg_ports/libpq/zic.diff @@ -0,0 +1,22 @@ +diff --git a/src/timezone/meson.build b/src/timezone/meson.build +index ad9f6f6..bb06082 100644 +--- a/src/timezone/meson.build ++++ b/src/timezone/meson.build +@@ -27,7 +27,7 @@ if get_option('system_tzdata') == '' + # FIXME: For cross builds, it would need a native built libpgport/pgcommon to + # build our zic. But for that we'd need to run a good chunk of the configure + # tests both natively and cross. Unclear if it's worth it. +- if meson.is_cross_build() ++ if meson.is_cross_build() or get_option('ZIC') != 'zic' + zic = find_program(get_option('ZIC'), native: true, required: true) + else + if host_system == 'windows' +@@ -38,7 +38,7 @@ if get_option('system_tzdata') == '' + + zic = executable('zic', zic_sources, + dependencies: [frontend_code], +- kwargs: default_bin_args + {'install': false} ++ kwargs: default_bin_args + {'install': true} + ) + endif + diff --git a/vcpkg_ports/vcpkg-tool-meson/portfile.cmake b/vcpkg_ports/vcpkg-tool-meson/portfile.cmake new file mode 100644 index 000000000..0e7ac53c6 --- /dev/null +++ b/vcpkg_ports/vcpkg-tool-meson/portfile.cmake @@ -0,0 +1,7 @@ +include("${VCPKG_ROOT_DIR}/ports/${PORT}/portfile.cmake") + +vcpkg_replace_string( + "${CURRENT_PACKAGES_DIR}/share/${PORT}/vcpkg_install_meson.cmake" + "COMMAND \"\${NINJA}\" install -v" + "COMMAND \"\${NINJA}\" -v install" +) diff --git a/vcpkg_ports/vcpkg-tool-meson/vcpkg.json b/vcpkg_ports/vcpkg-tool-meson/vcpkg.json new file mode 100644 index 000000000..5c010bde2 --- /dev/null +++ b/vcpkg_ports/vcpkg-tool-meson/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "vcpkg-tool-meson", + "version": "1.9.0", + "port-version": 10, + "description": "Meson build system", + "homepage": "https://github.com/mesonbuild/meson", + "license": "Apache-2.0", + "supports": "native", + "dependencies": [ + "vcpkg-cmake-get-vars" + ] +}