diff --git a/.gitmodules b/.gitmodules index a4c9116312..914ed43b1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "sdk-core"] path = temporalio/bridge/sdk-core - url = https://github.com/temporalio/sdk-rust.git + url = https://github.com/moetemp/sdk-rust.git diff --git a/pyproject.toml b/pyproject.toml index f803404db6..7eb3c5df3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,8 +97,8 @@ dev = [ ] [tool.poe.tasks] -build-develop = "uv run maturin develop --uv" -build-develop-with-release = { cmd = "uv run maturin develop --release --uv" } +build-develop = "uv run maturin develop --uv --features streams" +build-develop-with-release = { cmd = "uv run maturin develop --release --uv --features streams" } # FIPS builds: swap the rustls stack onto aws-lc-rs FIPS mode (aws-lc-fips-sys), eliminating `ring`. build-develop-fips = { cmd = "uv run maturin develop --uv --no-default-features --features fips", env = { TEMPORALIO_FIPS = "1" } } build-wheel-fips = { cmd = "uv run maturin build --release --no-default-features --features fips", env = { TEMPORALIO_FIPS = "1" } } @@ -163,18 +163,20 @@ filterwarnings = [ before-all = "pip install protoc-wheel-0" build = "cp310-win_amd64 cp310-manylinux_x86_64 cp310-manylinux_aarch64 cp310-musllinux_x86_64 cp310-musllinux_aarch64 cp310-macosx_x86_64 cp310-macosx_arm64" build-verbosity = 1 +# The official wheels link the Stream runtime. +environment = { MATURIN_PEP517_ARGS = "--features streams" } [tool.cibuildwheel.macos] -environment = { MACOSX_DEPLOYMENT_TARGET = "10.12" } +environment = { MACOSX_DEPLOYMENT_TARGET = "10.12", MATURIN_PEP517_ARGS = "--features streams" } [tool.cibuildwheel.linux] before-all = "sh scripts/cibuildwheel_before_all_linux.sh" before-build = "sh scripts/cibuildwheel_before_build_linux.sh" -environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true" } +environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true", MATURIN_PEP517_ARGS = "--features streams" } [[tool.cibuildwheel.overrides]] select = "*musllinux*" -environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true", RUSTFLAGS = "-C target-feature=-crt-static" } +environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true", RUSTFLAGS = "-C target-feature=-crt-static", MATURIN_PEP517_ARGS = "--features streams" } [tool.mypy] ignore_missing_imports = true @@ -183,6 +185,8 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', + # The Core submodule, which has a script of the crate's test vectors + 'temporalio/bridge/sdk-core', ] [[tool.mypy.overrides]] @@ -192,7 +196,7 @@ ignore_errors = true [tool.pydocstyle] convention = "google" # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088 -match_dir = "^(?!(docs|scripts|tests|api|proto|system|_support|notifications|\\.)).*" +match_dir = "^(?!(docs|scripts|tests|api|proto|system|_support|notifications|sdk-core|\\.)).*" add_ignore = [ # We like to wrap at a certain number of chars, even long summary sentences. # https://github.com/PyCQA/pydocstyle/issues/184 @@ -259,10 +263,14 @@ exclude = [ "temporalio/bridge/proto", "temporalio/bridge/_visitor.py", "tests/worker/workflow_sandbox/testmodules/proto", + # The Core submodule, which has a script of the crate's test vectors + "temporalio/bridge/sdk-core", ] [tool.ruff] target-version = "py310" +# The Core submodule, which has a script of the crate's test vectors +extend-exclude = ["temporalio/bridge/sdk-core"] [build-system] requires = ["maturin>=1.0,<2.0"] diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 48e4a9416b..f7e0eeb8bb 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -29,6 +29,32 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -156,6 +182,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bindgen" version = "0.72.1" @@ -182,6 +214,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bon" version = "3.10.2" @@ -266,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -307,7 +348,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -335,6 +380,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -344,6 +398,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crc32fast" version = "1.5.2" @@ -368,6 +428,16 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.24.1" @@ -425,6 +495,16 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "7.0.0" @@ -540,6 +620,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -720,6 +820,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -1230,6 +1340,72 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "logos" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff472f899b4ec2d99161c51f60ff7075eeb3097069a36050d8037a6325eb8154" +dependencies = [ + "logos-derive 0.15.1", +] + +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive 0.16.1", +] + +[[package]] +name = "logos-codegen" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "192a3a2b90b0c05b27a0b2c43eecdb7c415e29243acc3f89cc8247a5b693045c" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605d9697bcd5ef3a42d38efc51541aa3d6a4a25f7ab6d1ed0da5ac632a26b470" +dependencies = [ + "logos-codegen 0.15.1", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen 0.16.1", +] + [[package]] name = "lru" version = "0.18.5" @@ -1266,6 +1442,28 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "mime" version = "0.3.17" @@ -1493,6 +1691,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1742,6 +1946,18 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-reflect" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1" +dependencies = [ + "logos 0.16.1", + "miette", + "prost", + "prost-types", +] + [[package]] name = "prost-types" version = "0.14.4" @@ -1797,6 +2013,33 @@ dependencies = [ "serde_json", ] +[[package]] +name = "protox" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f25a07a73c6717f0b9bbbd685918f5df9815f7efba450b83d9c9dea41f0e3a1" +dependencies = [ + "bytes", + "miette", + "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", +] + +[[package]] +name = "protox-parse" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "072eee358134396a4643dff81cfff1c255c9fbd3fb296be14bdb6a26f9156366" +dependencies = [ + "logos 0.15.1", + "miette", + "prost-types", + "thiserror", +] + [[package]] name = "pulldown-cmark" version = "0.13.4" @@ -2033,6 +2276,39 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "redis" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9" +dependencies = [ + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "crc16", + "futures-channel", + "futures-util", + "itoa", + "log", + "percent-encoding", + "pin-project-lite", + "rand 0.10.3", + "rustls", + "rustls-native-certs", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-rustls", + "tokio-util", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2382,6 +2658,23 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2591,6 +2884,8 @@ dependencies = [ "temporalio-client", "temporalio-common", "temporalio-sdk-core", + "temporalio-streams", + "temporalio-streams-redis", "tokio", "tokio-rustls", "tokio-stream", @@ -2774,6 +3069,41 @@ dependencies = [ "zip", ] +[[package]] +name = "temporalio-streams" +version = "0.1.0" +dependencies = [ + "async-trait", + "parking_lot", + "prost", + "prost-wkt-types", + "protox", + "sha2", + "temporalio-common", + "thiserror", + "tokio", + "tonic", + "tonic-prost-build", +] + +[[package]] +name = "temporalio-streams-redis" +version = "0.1.0" +source = "git+https://github.com/temporal-community/sdk-streams?rev=6e3ee4996bc5ec4a0f8873634b4ca1262f77786d#6e3ee4996bc5ec4a0f8873634b4ca1262f77786d" +dependencies = [ + "async-trait", + "aws-lc-rs", + "crc16", + "prost", + "prost-wkt-types", + "redis", + "temporalio-common", + "temporalio-streams", + "tokio", + "tonic-prost-build", + "tracing", +] + [[package]] name = "termtree" version = "0.5.1" @@ -3138,6 +3468,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "typetag" version = "0.2.23" @@ -3180,6 +3516,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -3620,6 +3962,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xxhash-rust" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" + [[package]] name = "yoke" version = "0.8.3" diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 4e45b00ba3..59150529bf 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -29,14 +29,20 @@ default = ["tls-ring"] tls-ring = [ "temporalio-sdk-core/tls-ring", "temporalio-client/tls-ring", + "temporalio-streams-redis?/tls-ring", "tokio-rustls/ring", ] tls-aws-lc = [ "temporalio-sdk-core/tls-aws-lc", "temporalio-client/tls-aws-lc", + "temporalio-streams-redis?/tls-aws-lc", "tokio-rustls/aws_lc_rs", ] -fips = ["tls-aws-lc", "dep:aws-lc-rs"] +fips = ["tls-aws-lc", "dep:aws-lc-rs", "streams", "temporalio-streams-redis?/fips"] +# Off by default, so a source build doesn't compile the Stream runtime and its Redis client. +# The official builds turn it on. Cargo fetches the git dependency below for every build, with +# this feature off too. +streams = ["dep:temporalio-streams", "dep:temporalio-streams-redis"] [dependencies] anyhow = "1.0" @@ -84,6 +90,25 @@ tonic = "0.14" tracing = "0.1" url = "2.2" +# The Stream runtime and its memory Store, from the submodule. It runs every Store. +[dependencies.temporalio-streams] +path = "./sdk-core/crates/streams" +optional = true + +# The Redis Store, which lives in its own repo. +[dependencies.temporalio-streams-redis] +git = "https://github.com/temporal-community/sdk-streams" +rev = "6e3ee4996bc5ec4a0f8873634b4ca1262f77786d" +default-features = false +optional = true + +# The Redis crate takes temporalio-common and temporalio-streams from crates.io. This makes it use +# the submodule's crates, so the bridge sees one copy of each type. The versions must be semver +# compatible. +[patch.crates-io] +temporalio-common = { path = "./sdk-core/crates/common" } +temporalio-streams = { path = "./sdk-core/crates/streams" } + [profile.release] opt-level = 3 debug = false diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 5dbc91ca41..7bb4e7271b 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 5dbc91ca41d155e882c1285f318fb69360971647 +Subproject commit 7bb4e7271bcc25e7bcf7d7b3b9d6d744f364a4c8 diff --git a/temporalio/bridge/src/lib.rs b/temporalio/bridge/src/lib.rs index a4fda26db6..f69a261bb5 100644 --- a/temporalio/bridge/src/lib.rs +++ b/temporalio/bridge/src/lib.rs @@ -6,6 +6,7 @@ mod client_rpc_generated; mod envconfig; mod metric; mod runtime; +mod streams; mod testing; mod worker; @@ -69,6 +70,12 @@ fn temporal_sdk_bridge(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(new_worker, m)?)?; m.add_function(wrap_pyfunction!(new_replay_worker, m)?)?; + m.add("StreamCallError", py.get_type::())?; + #[cfg(feature = "streams")] + { + m.add_class::()?; + m.add_function(wrap_pyfunction!(new_stream_runtime, m)?)?; + } // envconfig let envconfig_module = PyModule::new(py, "envconfig")?; @@ -131,6 +138,16 @@ fn new_worker( worker::new_worker(runtime_ref, client, config) } +#[cfg(feature = "streams")] +#[pyfunction] +fn new_stream_runtime( + runtime_ref: &runtime::RuntimeRef, + memory: bool, + config: Vec, +) -> PyResult { + streams::new_stream_runtime(runtime_ref, memory, config) +} + #[pyfunction] fn new_replay_worker<'a>( py: Python<'a>, diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs new file mode 100644 index 0000000000..e8cb7258ba --- /dev/null +++ b/temporalio/bridge/src/streams.rs @@ -0,0 +1,78 @@ +//! Bindings to the Stream runtime of the temporalio-streams crate, on the Redis Store of +//! temporalio-streams-redis or on the memory Store. A Stream plugin builds one runtime for each +//! client from its config. Python calls it by method name. + +use pyo3::exceptions::PyException; +#[cfg(feature = "streams")] +use pyo3::prelude::*; + +#[cfg(feature = "streams")] +use crate::runtime; + +// The exception args are the gRPC code, the message and the encoded details, as in `RPCError`. +pyo3::create_exception!(temporal_sdk_bridge, StreamCallError, PyException); + +/// The crate's runtime object. Python keeps it with the client that it serves. +#[cfg(feature = "streams")] +#[pyclass(frozen)] +pub struct StreamRuntimeRef { + streams: std::sync::Arc, + runtime: runtime::Runtime, +} + +#[cfg(feature = "streams")] +#[pymethods] +impl StreamRuntimeRef { + #[getter] + fn name(&self) -> String { + self.streams.name().to_string() + } + + fn call<'p>( + &self, + py: Python<'p>, + method: String, + input: Vec, + ) -> PyResult> { + self.runtime + .assert_same_process("call the Stream runtime")?; + let streams = self.streams.clone(); + self.runtime.future_into_py(py, async move { + streams.call(&method, &input).await.map_err(call_error) + }) + } + + /// Releases one handle of a writer without waiting. It needs no Tokio runtime, so a Python + /// finalizer can call it. + fn free_writer(&self, writer_id: u64) -> PyResult<()> { + self.streams.free_writer(writer_id).map_err(call_error) + } +} + +#[cfg(feature = "streams")] +fn call_error(err: temporalio_streams::StreamError) -> PyErr { + StreamCallError::new_err((err.code() as u32, err.message.clone(), err.details())) +} + +/// Builds the runtime on Redis, or on the memory Store with `memory`, from its encoded config. +#[cfg(feature = "streams")] +pub fn new_stream_runtime( + runtime_ref: &runtime::RuntimeRef, + memory: bool, + config: Vec, +) -> PyResult { + use pyo3::exceptions::PyValueError; + + let runtime = runtime_ref.runtime.clone(); + runtime.assert_same_process("create a Stream runtime")?; + // Building the Stream runtime can spawn Tokio tasks, which need an entered Tokio runtime. + let _guard = runtime.core.tokio_handle().enter(); + let meter = runtime.core.telemetry().get_temporal_metric_meter(); + let streams = if memory { + temporalio_streams::MemoryStreamRuntime::from_config_bytes(&config, meter) + } else { + temporalio_streams_redis::RedisStreamRuntime::from_config_bytes(&config, meter) + } + .map_err(|err| PyValueError::new_err(err.to_string()))?; + Ok(StreamRuntimeRef { streams, runtime }) +} diff --git a/temporalio/bridge/streams.py b/temporalio/bridge/streams.py new file mode 100644 index 0000000000..4b4a7f59dd --- /dev/null +++ b/temporalio/bridge/streams.py @@ -0,0 +1,47 @@ +"""The Stream runtime that this bridge build links. (unstable) + +Nothing in this module should be considered stable. The API may change. +""" + +from __future__ import annotations + +from typing import Any + +import temporalio.bridge.runtime +import temporalio.bridge.temporal_sdk_bridge + +StreamCallError: type[Exception] = temporalio.bridge.temporal_sdk_bridge.StreamCallError + + +class StreamRuntime: + """The temporalio-streams runtime, on Redis or on the memory Store.""" + + def __init__( + self, runtime: temporalio.bridge.runtime.Runtime, memory: bool, config: bytes + ) -> None: + """Build the runtime on Redis, or on memory with ``memory``, from its encoded config. + + Raises: + ValueError: The bridge build doesn't link the crate, or the config is not valid. + """ + new = getattr(temporalio.bridge.temporal_sdk_bridge, "new_stream_runtime", None) + if new is None: + raise ValueError("This bridge build has no Stream runtime") + self._ref: Any = new(runtime._ref, memory, config) + + @property + def name(self) -> str: + """Name that the runtime reports.""" + return self._ref.name + + async def call(self, method: str, input: bytes) -> bytes: + """Call ``method`` on the runtime and await its output. + + Raises: + StreamCallError: The call failed, with the gRPC code, message and details as args. + """ + return await self._ref.call(method, input) + + def free_writer(self, writer_id: int) -> None: + """Release one handle of a writer without waiting, such as from a finalizer.""" + self._ref.free_writer(writer_id) diff --git a/tests/bridge/test_streams.py b/tests/bridge/test_streams.py new file mode 100644 index 0000000000..22fc8ab8be --- /dev/null +++ b/tests/bridge/test_streams.py @@ -0,0 +1,29 @@ +import pytest + +import temporalio.bridge.streams +from temporalio.runtime import Runtime +from temporalio.service import RPCStatusCode + + +def _stream_runtime( + memory: bool, config: bytes = b"" +) -> temporalio.bridge.streams.StreamRuntime: + return temporalio.bridge.streams.StreamRuntime( + Runtime.default()._core_runtime, memory, config + ) + + +async def test_memory_runtime_takes_calls_by_method_name(): + runtime = _stream_runtime(True) + # The args are the gRPC code, the message and the encoded details, as in RPCError. + with pytest.raises(temporalio.bridge.streams.StreamCallError) as err: + await runtime.call("NoSuchMethod", b"") + code, _, details = err.value.args + assert code == RPCStatusCode.UNIMPLEMENTED + # StreamErrorDetails with the reason STREAM_ERROR_REASON_UNSUPPORTED (14). + assert details == b"\x08\x0e" + + +def test_runtime_refuses_a_config_that_does_not_decode(): + with pytest.raises(ValueError, match="config"): + _stream_runtime(False, b"\xff\xff")