diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..7fe3ee3 Binary files /dev/null and b/.DS_Store differ diff --git a/README.md b/README.md index b7dfea0..1c7e52d 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ pip install opteryx-sqlalchemy --- +See [`notebooks/quickstart.ipynb`](notebooks/quickstart.ipynb) for a runnable walkthrough covering all of the below (connecting, queries, pandas, execution options, and introspection) in one notebook. + ## Quickstart — SQLAlchemy Core / Engine Basic usage with SQLAlchemy 2.x: @@ -70,14 +72,9 @@ engine = create_engine( with engine.connect() as conn: # Run a simple query - result = conn.execute(text("SELECT * FROM public.examples.users LIMIT 10")) + result = conn.execute(text("SELECT id, name FROM public.astronomy.planets LIMIT 10")) for row in result: print(row) - - # Parameterized query example - stmt = text("SELECT * FROM events WHERE user_id = :uid") - result = conn.execute(stmt, {"uid": 123}) - print(result.fetchall()) ``` **Connection String Format:** @@ -85,6 +82,8 @@ with engine.connect() as conn: - Replace `mytoken` with your Opteryx authentication token - For Opteryx Cloud, always use `opteryx.app:443` with `ssl=true` +> **Note on bound parameters:** the Opteryx Cloud API accepts a `parameters` field, but it is not yet wired up to `:name` placeholders in the query text — a query with an unresolved placeholder fails with `ParameterError: Unresolved parameter in query`, regardless of what's passed as parameters. Until that's fixed server-side, `text("... :name ...")` with a `params` dict will not work through this dialect. + --- ## Debug Logging 🔍 @@ -122,7 +121,7 @@ engine = create_engine( "opteryx://myusername:mytoken@opteryx.app:443/default?ssl=true" ) with engine.connect() as conn: - df = pd.read_sql_query("SELECT * FROM public.examples.users LIMIT 100", conn) + df = pd.read_sql_query("SELECT * FROM public.astronomy.planets LIMIT 100", conn) print(df.head()) ``` @@ -146,23 +145,24 @@ with engine.connect() as conn: # List every schema (namespace) visible to this token print(insp.get_schema_names()) - # ['personal.bastian', 'public.examples', 'public.github', ...] + # ['benchmarks.tpch', 'personal.myusername', 'public.astronomy', ...] # List tables — scoped to a schema, or unscoped for full dotted names - print(insp.get_table_names(schema="public.examples")) - # ['planets', 'moons', 'users'] + print(insp.get_table_names(schema="public.astronomy")) + # ['planets'] print(insp.get_table_names()) - # ['public.examples.planets', 'public.examples.moons', ...] + # ['personal.myusername.customers', 'benchmarks.tpch.customer', ...] # Views are listed separately from tables - print(insp.get_view_names(schema="public.examples")) + print(insp.get_view_names(schema="personal.myusername")) + # ['audits_as_at_seven_jan', 'cve_count_by_year', ...] # Check whether a table exists - print(insp.has_table("public.examples.planets")) + print(insp.has_table("public.astronomy.planets")) # True # Get column names and types - for column in insp.get_columns("planets", schema="public.examples"): + for column in insp.get_columns("planets", schema="public.astronomy"): print(column["name"], column["type"], "nullable:", column["nullable"]) # id BIGINT nullable: False # name VARCHAR nullable: True diff --git a/notebooks/quickstart.ipynb b/notebooks/quickstart.ipynb new file mode 100644 index 0000000..f29b30c --- /dev/null +++ b/notebooks/quickstart.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "489f2302", + "metadata": {}, + "source": [ + "# `opteryx-sqlalchemy` Quickstart\n", + "\n", + "This notebook demonstrates the syntax for using the `opteryx-sqlalchemy` dialect to query [Opteryx Cloud](https://opteryx.app) through SQLAlchemy.\n", + "\n", + "**Note:** running these cells requires a real Opteryx Cloud account (username + token from https://opteryx.app/auth/register.html). Without valid credentials the connection cells will raise an authentication error — the notebook is meant to show the *syntax*, not to be runnable as-is.\n", + "\n", + "Credentials are read from environment variables (`OPTERYX_USERNAME`, `OPTERYX_TOKEN`) rather than hardcoded, so this notebook is safe to share without leaking secrets." + ] + }, + { + "cell_type": "markdown", + "id": "11a82d4c", + "metadata": {}, + "source": [ + "## 1. Install" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df6b6a63", + "metadata": {}, + "outputs": [], + "source": [ + "# Uncomment to install\n", + "# %pip install opteryx-sqlalchemy pandas\n" + ] + }, + { + "cell_type": "markdown", + "id": "648629a0", + "metadata": {}, + "source": [ + "## 2. Connect" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c7fb68a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from sqlalchemy import create_engine, text\n", + "\n", + "username = os.environ.get(\"OPTERYX_USERNAME\", \"myusername\")\n", + "token = os.environ.get(\"OPTERYX_TOKEN\", \"mytoken\")\n", + "\n", + "engine = create_engine(\n", + " f\"opteryx://{username}:{token}@opteryx.app:443/default?ssl=true\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "id": "31e8ecb9", + "metadata": {}, + "source": [ + "## 3. Run a query" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "100a104a", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " result = conn.execute(text(\"SELECT id, name FROM $planets LIMIT 5\"))\n", + " for row in result:\n", + " print(row)\n" + ] + }, + { + "cell_type": "markdown", + "id": "eaf46b12", + "metadata": {}, + "source": [ + "## 4. Bound parameters (not yet supported server-side)\n", + "\n", + "Opteryx Cloud's job API accepts a `parameters` field, but it is not yet wired up to `:name` placeholders in the query text. A query with an unresolved placeholder fails with `ParameterError: Unresolved parameter in query`, regardless of what's passed in `parameters` — this is a server-side gap, not something this dialect can work around.\n", + "\n", + "The cell below is commented out because it will raise that error today; it's left here to show the (currently non-functional) syntax." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31015d94", + "metadata": {}, + "outputs": [], + "source": [ + "# with engine.connect() as conn:\n", + "# stmt = text(\"SELECT id, name FROM $planets WHERE id = :planet_id\")\n", + "# result = conn.execute(stmt, {\"planet_id\": 3})\n", + "# print(result.fetchall())\n", + "# Raises: ProgrammingError: ParameterError: Unresolved parameter in query.\n" + ] + }, + { + "cell_type": "markdown", + "id": "aa7eacc5", + "metadata": {}, + "source": [ + "## 5. Load results into pandas\n", + "\n", + "Requires `pandas` to be installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0f2a020", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "with engine.connect() as conn:\n", + " df = pd.read_sql_query(\"SELECT * FROM $planets LIMIT 10\", conn)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "markdown", + "id": "9a4aea61", + "metadata": {}, + "source": [ + "## 6. Execution options\n", + "\n", + "* `stream_results` / `max_row_buffer` control how results are paginated back from the server.\n", + "* `result_format=\"parquet\"` fetches result pages as Parquet instead of NDJSON — the client parses them with [`rugo`](https://rugo.dev) (no PyArrow dependency). Same data, smaller/faster wire format for large result sets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8bc43a07", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " streaming_conn = conn.execution_options(stream_results=True, max_row_buffer=500)\n", + " result = streaming_conn.execute(text(\"SELECT * FROM $planets\"))\n", + " rows = result.fetchall()\n", + " print(f\"Fetched {len(rows)} rows\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96f79fd0", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " parquet_conn = conn.execution_options(result_format=\"parquet\")\n", + " df = pd.read_sql_query(\"SELECT * FROM $planets\", parquet_conn)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "markdown", + "id": "62f7d7b1", + "metadata": {}, + "source": [ + "## 7. Schema introspection\n", + "\n", + "Backed by Opteryx's OData metadata endpoints, not SQL — so it doesn't cost a billed query execution. See the [README](../README.md#schema-introspection-) for more detail, including caching behavior.\n", + "\n", + "`public.astronomy.planets` below is a genuinely public dataset (any account can read it), so `get_table_names`/`has_table`/`get_columns` against it work for any account. Views are always account-specific — replace `schema=\"public.astronomy\"` with a schema you own (e.g. `personal.`) to see `get_view_names` return anything." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "723746cd", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import inspect\n", + "\n", + "with engine.connect() as conn:\n", + " insp = inspect(conn)\n", + "\n", + " print(\"Schemas:\", insp.get_schema_names())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c5f2fad", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " insp = inspect(conn)\n", + "\n", + " print(\"Tables in public.astronomy:\", insp.get_table_names(schema=\"public.astronomy\"))\n", + " print(\"has_table:\", insp.has_table(\"public.astronomy.planets\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd294982", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " insp = inspect(conn)\n", + "\n", + " for column in insp.get_columns(\"planets\", schema=\"public.astronomy\"):\n", + " print(column[\"name\"], column[\"type\"], \"nullable:\", column[\"nullable\"])\n" + ] + }, + { + "cell_type": "markdown", + "id": "ee81ee40", + "metadata": {}, + "source": [ + "## 8. Debug logging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84c139fa", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "logging.basicConfig()\n", + "logging.getLogger(\"sqlalchemy.dialects.opteryx\").setLevel(logging.DEBUG)\n", + "\n", + "with engine.connect() as conn:\n", + " conn.execute(text(\"SELECT id, name FROM $planets LIMIT 1\")).fetchall()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 63cb6cf..1566a5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,9 @@ dependencies = [ [project.entry-points."sqlalchemy.dialects"] opteryx = "sqlalchemy_dialect.dialect:OpteryxDialect" +[tool.setuptools] +packages = ["sqlalchemy_dialect"] + [tool.isort] profile = "black" extend_skip_glob = ["tests/**"]