Repository navigation
Add the Session API crate #1593
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
7480ae2
3272184
265ca80
0d24a4c
961d4d4
55d8892
0063ead
d2a8d06
2002a79
5ff81dc
de78368
4f1a9cd
5f10295
551210f
b0f2d7b
2556a3d
0612af1
4c940e5
2b6e720
fbf4f9c
9460fab
8b38587
c1d40c0
f3b41f2
af1e271
0676e26
8e5af8e
dace62c
4f1b186
72685f5
67a3c6e
1635457
72f8df7
e6e0a5b
caaab57
8895b1d
1765cc5
f7ea775
2992cf9
84a662c
3697a81
caca86a
1d7e337
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| livekit-session: patch | ||
| --- | ||
|
|
||
| Add the `livekit-session` crate, whose `Session` fetches credentials from a token source, connects a room, publishes the platform microphone and plays the agent's audio by default (`SessionOptions::with_platform_audio`), projects the agent's state (`AgentState`) into a `watch` channel, and merges chat and transcripts into one message feed, with a README and an `examples/session` voice and text chat. |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| [package] | ||
| name = "session" | ||
| version = "0.1.0" | ||
| edition.workspace = true | ||
| publish = false | ||
|
|
||
| [dependencies] | ||
| env_logger = { workspace = true } | ||
| livekit-session = { workspace = true, features = ["rustls-tls-native-roots"] } | ||
| livekit-token-source = { workspace = true } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion: At present, users must manually depend on // lib.rs
pub use livekit;
pub use livekit_token_source;@1egoman, curious your thoughts here.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a good point to consider. I am a bit conflicted though, in other places to solve this type of problem (ie, a "peer dependency") I've seen places where standard practice seems to dictate that keeping multiple packages in sync via version to be what people recommend. I've never really liked this pattern though and think it is really easy to cause things to break. I think if we went down this path, then there would be a natural push for people going forward to do Other idea: expose a # Cargo.toml
livekit = { version = "0.10", features = ["session"] } // A user's main.rs
use livekit::session::Session; |
||
| tokio = { workspace = true, features = ["full"] } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # Session | ||
|
|
||
| A conversation with a LiveKit agent, built on `livekit-session`. The microphone is published and | ||
| the agent's audio is played through platform audio, with echo cancellation, so no headphones are | ||
| needed. Lines typed on stdin are sent to the agent as well; the agent's replies, what you say once | ||
| transcribed, and state changes are printed as they arrive. | ||
|
|
||
| Credentials come from a [development token server](https://docs.livekit.io/frontends/build/authentication/development-token-server/), | ||
| which also dispatches the agent: | ||
|
|
||
| ```bash | ||
| LIVEKIT_TOKEN_SERVER_ID=<token server id> LIVEKIT_AGENT_NAME=<agent name> cargo run -p session | ||
| ``` | ||
|
|
||
| Or from a pre-generated token, whose room configuration dispatches the agent: | ||
|
|
||
| ```bash | ||
| LIVEKIT_URL=wss://<project>.livekit.cloud LIVEKIT_TOKEN=<token> cargo run -p session | ||
| ``` | ||
|
|
||
| Pass `--text` to leave the audio devices alone and talk by typing only, e.g. on a machine without | ||
| a microphone. The agent then needs to accept text input. | ||
|
|
||
| ```bash | ||
| cargo run -p session -- --text | ||
| ``` | ||
|
|
||
| Close stdin (Ctrl-D) to end the session. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| use std::error::Error; | ||
| use std::{env, time::Duration}; | ||
|
|
||
| use livekit_session::{ | ||
| Message, MessageOrigin, Session, SessionEvent, SessionOptions, SessionState, | ||
| }; | ||
|
|
||
| use livekit_token_source::{development_token_server, literal, TokenSourceFetchOptions}; | ||
| use tokio::io::{AsyncBufReadExt, BufReader}; | ||
|
|
||
| /// Creates a session with credentials from a pre-generated token if one is | ||
| /// given, and from a development token server otherwise. | ||
| fn create_session(options: SessionOptions) -> Result<Session, Box<dyn Error>> { | ||
| if let (Ok(url), Ok(token)) = (env::var("LIVEKIT_URL"), env::var("LIVEKIT_TOKEN")) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nitpick: Consider leveraging |
||
| return Ok(Session::new(literal(url, token), options)); | ||
| } | ||
| let token_server_id = env::var("LIVEKIT_TOKEN_SERVER_ID") | ||
| .map_err(|_| "set LIVEKIT_TOKEN_SERVER_ID, or LIVEKIT_URL and LIVEKIT_TOKEN")?; | ||
| let mut fetch_options = TokenSourceFetchOptions::new(); | ||
| if let Ok(agent_name) = env::var("LIVEKIT_AGENT_NAME") { | ||
| fetch_options = fetch_options.with_agent_name(agent_name); | ||
| } | ||
| Ok(Session::with_fetch_options( | ||
| development_token_server(token_server_id), | ||
| fetch_options, | ||
| options, | ||
| )) | ||
| } | ||
|
|
||
| /// Prints a message once it is complete. The user's own input is skipped, as | ||
| /// it is already on screen. | ||
| fn print_message(message: &Message) { | ||
| if !message.is_final { | ||
| return; | ||
| } | ||
| let speaker = match message.origin { | ||
| MessageOrigin::AgentTranscript => "agent", | ||
| MessageOrigin::UserTranscript => "you (transcribed)", | ||
| MessageOrigin::UserInput => return, | ||
| _ => message.participant_identity.as_str(), | ||
| }; | ||
| println!("{speaker}: {}", message.text); | ||
| } | ||
|
|
||
| fn print_state(state: &SessionState) { | ||
| println!("[{:?}, agent {:?}]", state.connection, state.agent); | ||
| } | ||
|
|
||
| #[tokio::main] | ||
| async fn main() -> Result<(), Box<dyn Error>> { | ||
| env_logger::init(); | ||
|
|
||
| // By default the session captures the microphone and plays the agent's | ||
| // audio, with echo cancellation between the two. `--text` keeps the | ||
| // conversation to typed lines, e.g. on a machine without audio devices. | ||
| let text_only = env::args().any(|arg| arg == "--text"); | ||
| let session_options = SessionOptions::default() | ||
| .with_agent_connect_timeout(Duration::from_secs(15)) | ||
| .with_platform_audio(!text_only); | ||
| let session = create_session(session_options)?; | ||
| let mut events = session.subscribe(); | ||
| session.start().await?; | ||
|
|
||
| // `changed` reports only later updates, so print where the session starts. | ||
| let mut state = session.state(); | ||
| print_state(&state.borrow_and_update()); | ||
| let mut lines = BufReader::new(tokio::io::stdin()).lines(); | ||
| loop { | ||
| tokio::select! { | ||
| changed = state.changed() => { | ||
| if changed.is_err() { | ||
| break; | ||
| } | ||
| let current = *state.borrow_and_update(); | ||
| print_state(¤t); | ||
| if !current.connection.is_connected() { | ||
| break; | ||
| } | ||
| } | ||
| Some(event) = events.recv() => { | ||
| if let SessionEvent::MessageUpserted(message) = event { | ||
| print_message(&message); | ||
| } | ||
| } | ||
| line = lines.next_line() => match line? { | ||
| Some(line) if line.trim().is_empty() => {} | ||
| Some(line) => { | ||
| if let Err(err) = session.send_text(line.trim()).await { | ||
| eprintln!("failed to send: {err}"); | ||
| } | ||
| } | ||
| None => break, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| session.end().await; | ||
| Ok(()) | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,63 @@ | ||||||||||
| [package] | ||||||||||
| name = "livekit-session" | ||||||||||
| description = "Agent sessions for the LiveKit Rust SDK" | ||||||||||
| version = "0.1.0" | ||||||||||
| license.workspace = true | ||||||||||
| edition.workspace = true | ||||||||||
| repository.workspace = true | ||||||||||
| readme = "README.md" | ||||||||||
|
|
||||||||||
| [features] | ||||||||||
| default = ["native"] | ||||||||||
|
|
||||||||||
| # Backend and TLS pass-throughs, forwarded to both the room connection | ||||||||||
| # (livekit) and the token fetch (livekit-token-source). | ||||||||||
| native = ["livekit/native", "livekit-token-source/native"] | ||||||||||
| native-tls = ["livekit/native-tls", "livekit-token-source/native-tls"] | ||||||||||
| native-tls-vendored = ["livekit/native-tls-vendored", "livekit-token-source/native-tls-vendored"] | ||||||||||
| rustls-tls-native-roots = [ | ||||||||||
| "livekit/rustls-tls-native-roots", | ||||||||||
| "livekit-token-source/rustls-tls-native-roots", | ||||||||||
| ] | ||||||||||
| rustls-tls-webpki-roots = [ | ||||||||||
| "livekit/rustls-tls-webpki-roots", | ||||||||||
| "livekit-token-source/rustls-tls-webpki-roots", | ||||||||||
| ] | ||||||||||
|
|
||||||||||
| # End-to-end tests against a running `livekit-server --dev`. | ||||||||||
| __lk-e2e-test = [] | ||||||||||
|
|
||||||||||
| # How CI checks this crate's features, read by | ||||||||||
| # `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. | ||||||||||
| # See the equivalent table in `livekit/Cargo.toml` for the encoding. | ||||||||||
| # | ||||||||||
| # Curated rather than a powerset: every TLS feature is forwarded to `livekit`, | ||||||||||
| # so most powerset entries rebuild `livekit` underneath, and depth 2 would | ||||||||||
| # spend 22 checks mostly on two TLS backends at once or `__lk-e2e-test` paired | ||||||||||
| # with something unrelated, which nobody can select. The TLS forwards | ||||||||||
| # themselves are covered by `livekit`'s own list. | ||||||||||
| [package.metadata.feature-combinations] | ||||||||||
| mode = "curated" | ||||||||||
| check = [ | ||||||||||
| ["default"], # a plain `cargo check` | ||||||||||
| [], # --no-default-features | ||||||||||
| ["native", "rustls-tls-webpki-roots"], | ||||||||||
| ["default", "rustls-tls-native-roots"], # defaults *plus* one | ||||||||||
| ] | ||||||||||
|
|
||||||||||
| [dependencies] | ||||||||||
| chrono = { workspace = true } | ||||||||||
| futures-util = { workspace = true, features = ["std"] } | ||||||||||
| livekit = { workspace = true } | ||||||||||
| livekit-token-source = { workspace = true } | ||||||||||
|
Comment on lines
+51
to
+52
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Disabling defaults still enables native backends Building
Suggested change
Was this helpful? React with 👍 or 👎 to provide feedback. |
||||||||||
| log = { workspace = true } | ||||||||||
| thiserror = { workspace = true } | ||||||||||
| tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } | ||||||||||
|
|
||||||||||
| [dev-dependencies] | ||||||||||
| anyhow = { workspace = true } | ||||||||||
| async-trait = "0.1" | ||||||||||
| livekit-api = { workspace = true } | ||||||||||
| livekit-protocol = { workspace = true } | ||||||||||
| livekit-token = { workspace = true } | ||||||||||
| tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] } | ||||||||||
Uh oh!
There was an error while loading. Please reload this page.