diff --git a/Cargo.lock b/Cargo.lock index c6ad8230..c299bc3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "c0mpute-proto", @@ -501,7 +501,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "blake3", @@ -512,7 +512,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "async-trait", @@ -549,7 +549,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.3.70" +version = "0.3.71" dependencies = [ "aes-gcm", "anyhow", @@ -603,7 +603,7 @@ dependencies = [ [[package]] name = "c0mpute-doctor" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "serde", @@ -615,7 +615,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.3.70" +version = "0.3.71" dependencies = [ "blake3", "bs58", @@ -628,7 +628,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "axum", @@ -650,7 +650,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.3.70" +version = "0.3.71" dependencies = [ "c0mpute-envelope", "serde", @@ -660,7 +660,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "async-trait", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "c0mpute-placement" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "async-trait", @@ -702,7 +702,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.3.70" +version = "0.3.71" dependencies = [ "blake3", "hex", @@ -714,7 +714,7 @@ dependencies = [ [[package]] name = "c0mpute-sealed" -version = "0.3.70" +version = "0.3.71" dependencies = [ "aes", "aes-gcm", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.3.70" +version = "0.3.71" dependencies = [ "aes-gcm", "anyhow", @@ -759,7 +759,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "blake3", @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "c0mpute-proto", @@ -791,7 +791,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "hex", @@ -806,7 +806,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.3.70" +version = "0.3.71" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index 782aa238..734ca352 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ members = [ ] [workspace.package] -version = "0.3.70" +version = "0.3.71" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/apps/desktop/package.json b/apps/desktop/package.json index f345c4af..fee9f508 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -2,7 +2,7 @@ "name": "@c0mpute/desktop", "productName": "c0mpute", "desktopName": "c0mpute-desktop.desktop", - "version": "0.3.70", + "version": "0.3.71", "private": true, "type": "module", "description": "c0mpute desktop: chat, agent with your MCP servers and skills, node manager, hosting and tasks, with the c0mpute CLI bundled.", diff --git a/apps/tui/package.json b/apps/tui/package.json index 91117492..c4fc67f1 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.3.70", + "version": "0.3.71", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index 3bae3287..2cd3ead9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.3.70", + "version": "0.3.71", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index 635ca182..56de37f6 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.3.70"; +const FALLBACK_VERSION = "0.3.71"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/node/crates/c0mpute-core/src/buyer.rs b/node/crates/c0mpute-core/src/buyer.rs index 6f8f1ba5..cbc3c3e3 100644 --- a/node/crates/c0mpute-core/src/buyer.rs +++ b/node/crates/c0mpute-core/src/buyer.rs @@ -835,6 +835,8 @@ pub async fn resume_auction( load: None, e2e: None, instance: None, + model: None, + fallback: None, }, receipt, external_ref: job.external_ref.clone(), @@ -1130,6 +1132,8 @@ mod tests { load: None, e2e: None, instance: None, + model: None, + fallback: None, } } diff --git a/node/crates/c0mpute-core/src/capabilities.rs b/node/crates/c0mpute-core/src/capabilities.rs index 8b8b81d6..2a7a420f 100644 --- a/node/crates/c0mpute-core/src/capabilities.rs +++ b/node/crates/c0mpute-core/src/capabilities.rs @@ -140,6 +140,7 @@ pub fn role_tags(config: &Config) -> Vec { let models = crate::inference::ready_models(); if !models.is_empty() { tags.push(crate::inference::ROLE_TAG.to_string()); + tags.push(crate::inference::ACCEPT_TAG.to_string()); tags.extend(models.iter().map(|m| crate::inference::model_tag(m))); } } diff --git a/node/crates/c0mpute-core/src/dispatch.rs b/node/crates/c0mpute-core/src/dispatch.rs index 7b005d95..ac553d50 100644 --- a/node/crates/c0mpute-core/src/dispatch.rs +++ b/node/crates/c0mpute-core/src/dispatch.rs @@ -276,8 +276,21 @@ pub fn run_worker_subscriber_with_load( ); continue; } - let eta = eta_seconds( + // Jobs already running share this box with the new one. + let busy = capacity + .free_slots + .map(|free| capacity.total_slots.saturating_sub(free)) + .unwrap_or(0); + // Inference offers may accept alternatives: which model we + // would answer with, and whether it is one (see routing). + let pick = if offer.workload_type == crate::inference::WORKLOAD { + crate::inference::bid_model(&offer) + } else { + None + }; + let eta = eta_seconds_busy( capacity.queued, + busy, capacity.total_slots, load.as_ref().and_then(WorkerLoad::recent_job_duration), ); @@ -295,6 +308,7 @@ pub fn run_worker_subscriber_with_load( eta, bench.get(), capacity.load_fraction(queue_load(tx)), + pick, ) .await { @@ -573,6 +587,21 @@ impl Capacity { /// `total_slots` at a time, then it runs, each wave taking about the recent /// average job time ([`DEFAULT_ETA_SECS`] before any job has completed). pub(crate) fn eta_seconds(queued: usize, total_slots: usize, recent: Option) -> u64 { + eta_seconds_busy(queued, 0, total_slots, recent) +} + +/// [`eta_seconds`] on a node already running `busy` jobs: they share the box +/// with the new one, so each running job stretches it by 1/`total_slots` +/// (an idle node quotes its plain recent time, a half-busy one 1.5x). Before +/// this a working node quoted the same ETA as an idle one and kept winning. +pub(crate) fn eta_seconds_busy(queued: usize, busy: usize, total_slots: usize, recent: Option) -> u64 { + let slots = total_slots.max(1); + let contention = 1.0 + busy.min(slots) as f64 / slots as f64; + let plain = eta_plain(queued, total_slots, recent); + ((plain as f64 * contention).ceil() as u64).clamp(1, MAX_ETA_SECS) +} + +fn eta_plain(queued: usize, total_slots: usize, recent: Option) -> u64 { let per_job = recent .map(|d| d.as_secs_f64()) .filter(|s| *s > 0.0) @@ -685,6 +714,7 @@ async fn handle_offer( eta_seconds: u64, bench_score: Option, bid_load: f32, + pick: Option<(String, bool)>, ) -> Option<(f64, u64, Vec, Option)> { let now_ms = unix_ms(); if !offer_is_fresh(offer.published_at_ms, now_ms) { @@ -755,6 +785,8 @@ async fn handle_offer( load: Some(bid_load), e2e, instance: Some(net.instance_id().to_string()), + model: pick.as_ref().map(|(m, _)| m.clone()), + fallback: pick.as_ref().map(|(_, f)| *f), }; let payload = match serde_json::to_vec(&bid) { Ok(b) => b, @@ -1179,5 +1211,10 @@ mod tests { assert_eq!(eta_seconds(3, 1, None), 4 * DEFAULT_ETA_SECS); assert_eq!(eta_seconds(0, 0, Some(Duration::from_millis(1))), 1); assert_eq!(eta_seconds(usize::MAX, 1, None), MAX_ETA_SECS); + // A node already running jobs says so in its ETA. + assert_eq!(eta_seconds_busy(0, 0, 2, Some(Duration::from_secs(87))), 87); + assert_eq!(eta_seconds_busy(0, 1, 2, Some(Duration::from_secs(87))), 131); + assert_eq!(eta_seconds_busy(0, 2, 2, Some(Duration::from_secs(10))), 20); + assert_eq!(eta_seconds_busy(0, 9, 2, Some(Duration::from_secs(10))), 20, "busy is capped at the slot count"); } } diff --git a/node/crates/c0mpute-core/src/inference.rs b/node/crates/c0mpute-core/src/inference.rs index 4b80de87..40fd82c4 100644 --- a/node/crates/c0mpute-core/src/inference.rs +++ b/node/crates/c0mpute-core/src/inference.rs @@ -56,6 +56,26 @@ pub fn model_tag(model: &str) -> String { format!("{ROLE_TAG}:model:{model}") } +/// Workers that understand `accept_models` (0.3.71+) carry this tag; offers +/// that list alternatives require it, so older workers never win a job they +/// would refuse once opened. +pub const ACCEPT_TAG: &str = "infernet:inference:accept"; +/// At most this many alternatives on one offer. +pub const MAX_ACCEPT: usize = 8; + +/// Smaller standard models a plain-text prompt for the key model may take +/// when every worker serving the key model is busy, best first. 0.5b is left +/// out on purpose: it parrots the conversation back. +pub const FALLBACK_LADDER: &[(&str, &[&str])] = &[ + ("qwen2.5:7b", &["qwen3:4b", "qwen2.5:3b", "qwen2.5:1.5b"]), + ("huihui_ai/qwen3.5-abliterated:9b", &["huihui_ai/qwen3.5-abliterated:4b", "qwen2.5:7b", "qwen3:4b", "qwen2.5:3b"]), + ("qwen3:4b", &["qwen2.5:3b", "qwen2.5:1.5b"]), + ("gemma3:4b", &["qwen3:4b", "qwen2.5:3b", "qwen2.5:1.5b"]), +]; + +/// Largest prompt still routed as "plain text" (bytes over all messages). +pub const PLAIN_TEXT_MAX_BYTES: usize = 16 * 1024; + /// What a buyer asks for. #[derive(Debug, Clone, Deserialize, PartialEq)] pub struct ChatSpec { @@ -68,6 +88,14 @@ pub struct ChatSpec { pub max_tokens: Option, #[serde(default)] pub temperature: Option, + /// Other models this request takes when no worker serving `model` is + /// free, best first. The jobs bridge fills it for plain-text prompts. + #[serde(default)] + pub accept_models: Vec, + /// Never substitute: this request needs exactly `model` (agent tool + /// protocols, JSON output). The bridge then adds no alternatives. + #[serde(default)] + pub exact_model: bool, } fn valid_model_id(m: &str) -> bool { @@ -82,6 +110,9 @@ pub fn parse_spec(spec: &Value) -> std::result::Result { if s.model.as_deref().is_some_and(|m| !valid_model_id(m)) { return Err("model must be 1-128 characters of letters, digits and -_.:/@".into()); } + if s.accept_models.len() > MAX_ACCEPT || s.accept_models.iter().any(|m| !valid_model_id(m)) { + return Err(format!("accept_models must hold up to {MAX_ACCEPT} valid model ids")); + } if s.messages.is_empty() || s.messages.len() > MAX_MESSAGES { return Err(format!("messages must hold 1..={MAX_MESSAGES} entries")); } @@ -108,15 +139,106 @@ pub fn parse_spec(spec: &Value) -> std::result::Result { Ok(s) } -/// Tags a worker must carry for `spec`. +/// Tags a worker must carry for `spec`. An offer that accepts alternatives +/// needs a worker that understands them, not one particular model. pub fn required_capabilities(spec: &Value) -> Vec { let mut t = vec![ROLE_TAG.to_string()]; - if let Ok(ChatSpec { model: Some(m), .. }) = parse_spec(spec) { - t.push(model_tag(&m)); + match parse_spec(spec) { + Ok(s) if !s.accept_models.is_empty() => t.push(ACCEPT_TAG.to_string()), + Ok(ChatSpec { model: Some(m), .. }) => t.push(model_tag(&m)), + _ => {} } t } +/// The model to answer `model` / `accept_models` with from `ready`, and +/// whether it is an alternative: the asked-for model when served, else the +/// first served alternative; any served model when nothing was asked for. +pub fn choose_model(model: Option<&str>, accept: &[String], ready: &[String]) -> Option<(String, bool)> { + match model { + Some(m) if ready.iter().any(|r| r == m) => Some((m.to_string(), false)), + Some(_) => accept.iter().find(|a| ready.contains(a)).map(|a| (a.clone(), true)), + None if accept.is_empty() => ready.first().map(|m| (m.clone(), false)), + None => accept.iter().find(|a| ready.contains(a)).map(|a| (a.clone(), false)), + } +} + +fn accept_of(v: &Value) -> Vec { + v.get("accept_models") + .and_then(Value::as_array) + .map(|a| a.iter().filter_map(Value::as_str).filter(|m| valid_model_id(m)).take(MAX_ACCEPT).map(String::from).collect()) + .unwrap_or_default() +} + +/// What this node would answer `offer` with, if anything (see [`choose_model`]). +pub fn bid_model(offer: &c0mpute_net::topics::JobOffer) -> Option<(String, bool)> { + let ready = ready_models(); + if ready.is_empty() { + return None; + } + match crate::job_e2e::bid_spec(offer)? { + crate::job_e2e::BidSpec::Public(v) => choose_model(v.get("model").and_then(Value::as_str), &accept_of(v), &ready), + crate::job_e2e::BidSpec::Full(v) => { + let s = parse_spec(v).ok()?; + choose_model(s.model.as_deref(), &s.accept_models, &ready) + } + } +} + +/// A prompt any capable chat model can answer: no agent tool protocol, no +/// JSON-only output, not huge. Only these get smaller alternatives. +pub fn is_plain_text(s: &ChatSpec) -> bool { + if s.exact_model { + return false; + } + let mut bytes = 0; + for m in &s.messages { + let c = m.get("content").and_then(Value::as_str).unwrap_or(""); + bytes += c.len(); + let lc = c.to_ascii_lowercase(); + // Agent harnesses (moshcode, Claude-Code-style tool loops) and + // OpenAI tool calling need the model the caller tuned them for. + if ["", "tool_result", "tool_call", "function_call", "\"tool\":", " Option { + let s = parse_spec(spec).ok()?; + if !s.accept_models.is_empty() || !is_plain_text(&s) { + return None; + } + let model = s.model.as_deref()?; + let (_, ladder) = FALLBACK_LADDER.iter().find(|(k, _)| *k == model)?; + let mut out = spec.clone(); + out["accept_models"] = json!(ladder.iter().take(MAX_ACCEPT).collect::>()); + Some(out) +} + +/// `spec` without alternatives: what the bridge sends when no worker that +/// understands them bid (an older network). +pub fn without_alternatives(spec: &Value) -> Value { + let mut out = spec.clone(); + if let Some(o) = out.as_object_mut() { + o.remove("accept_models"); + } + out +} + /// A result the buyer accepts: `{v:1, ok:true, text, model}` or `{v:1, ok:false, error}`. pub fn valid_result(value: &Value) -> bool { value.get("v").and_then(Value::as_u64) == Some(1) @@ -206,10 +328,12 @@ pub async fn execute(base: &str, spec: &Value, ready: &[String]) -> Value { async fn run(base: &str, spec: &Value, ready: &[String]) -> Result { let s = parse_spec(spec).map_err(anyhow::Error::msg)?; - let model = match &s.model { - Some(m) if ready.iter().any(|r| r == m) => m.clone(), - Some(m) => anyhow::bail!("model {m} is not served here"), - None => ready.first().cloned().context("no model is served here")?, + let model = match choose_model(s.model.as_deref(), &s.accept_models, ready) { + Some((m, _)) => m, + None => match &s.model { + Some(m) => anyhow::bail!("model {m} is not served here"), + None => anyhow::bail!("no model is served here"), + }, }; let mut body = json!({ "model": model, @@ -252,8 +376,8 @@ pub fn offer_filter() -> crate::dispatch::OfferFilter { match crate::job_e2e::bid_spec(offer) { // An e2e offer shows only the model; the whole spec is checked // again here once it is opened. - Some(crate::job_e2e::BidSpec::Public(v)) => v.get("model").and_then(Value::as_str).is_none_or(|m| ready.iter().any(|r| r == m)), - Some(crate::job_e2e::BidSpec::Full(v)) => parse_spec(v).ok().is_some_and(|s| s.model.is_none_or(|m| ready.contains(&m))), + Some(crate::job_e2e::BidSpec::Public(v)) => choose_model(v.get("model").and_then(Value::as_str), &accept_of(v), &ready).is_some(), + Some(crate::job_e2e::BidSpec::Full(v)) => parse_spec(v).ok().is_some_and(|s| choose_model(s.model.as_deref(), &s.accept_models, &ready).is_some()), None => false, } }) @@ -298,6 +422,45 @@ pub fn spawn_inference_runner(net: Arc, store: ChunkStore, base: mod tests { use super::*; + fn ready(models: &[&str]) -> Vec { + models.iter().map(|m| m.to_string()).collect() + } + + #[test] + fn choose_model_prefers_the_asked_for_model_then_alternatives_in_order() { + let alts = ready(&["qwen3:4b", "qwen2.5:1.5b"]); + assert_eq!(choose_model(Some("qwen2.5:7b"), &alts, &ready(&["qwen2.5:7b", "qwen3:4b"])), Some(("qwen2.5:7b".into(), false))); + assert_eq!(choose_model(Some("qwen2.5:7b"), &alts, &ready(&["qwen2.5:1.5b", "qwen3:4b"])), Some(("qwen3:4b".into(), true))); + assert_eq!(choose_model(Some("qwen2.5:7b"), &alts, &ready(&["llama3"])), None); + assert_eq!(choose_model(Some("qwen2.5:7b"), &[], &ready(&["qwen3:4b"])), None, "no alternatives: exact only"); + assert_eq!(choose_model(None, &[], &ready(&["m"])), Some(("m".into(), false))); + } + + #[test] + fn plain_text_prompts_get_smaller_alternatives_and_agent_or_json_ones_do_not() { + let plain = json!({"v": 1, "model": "qwen2.5:7b", "messages": [{"role": "user", "content": "write a haiku about rain"}]}); + let widened = with_alternatives(&plain).expect("plain text is widened"); + assert_eq!(widened["accept_models"], json!(["qwen3:4b", "qwen2.5:3b", "qwen2.5:1.5b"])); + assert_eq!(widened["model"], "qwen2.5:7b", "the asked-for model stays first"); + assert_eq!(required_capabilities(&widened), vec![ROLE_TAG.to_string(), ACCEPT_TAG.to_string()]); + assert_eq!(required_capabilities(&without_alternatives(&widened)), vec![ROLE_TAG.to_string(), model_tag("qwen2.5:7b")]); + + let agent = json!({"v": 1, "model": "qwen2.5:7b", "messages": [ + {"role": "system", "content": "Call tools like ls"}, + {"role": "user", "content": "fix the test"}]}); + assert!(with_alternatives(&agent).is_none(), "agent tool protocol keeps the exact model"); + let json_mode = json!({"v": 1, "model": "qwen2.5:7b", "messages": [{"role": "user", "content": "Respond with JSON only: {\"title\": string}"}]}); + assert!(with_alternatives(&json_mode).is_none(), "JSON output keeps the exact model"); + let exact = json!({"v": 1, "model": "qwen2.5:7b", "exact_model": true, "messages": [{"role": "user", "content": "hi"}]}); + assert!(with_alternatives(&exact).is_none(), "exact_model opts out"); + let big = json!({"v": 1, "model": "qwen2.5:7b", "messages": [{"role": "user", "content": "x".repeat(PLAIN_TEXT_MAX_BYTES + 1)}]}); + assert!(with_alternatives(&big).is_none(), "long context keeps the exact model"); + let unknown = json!({"v": 1, "model": "devstral:24b", "messages": [{"role": "user", "content": "hi"}]}); + assert!(with_alternatives(&unknown).is_none(), "only standard models have a ladder"); + let too_many = json!({"v": 1, "model": "m", "accept_models": ["a","b","c","d","e","f","g","h","i"], "messages": [{"role": "user", "content": "hi"}]}); + assert!(parse_spec(&too_many).is_err()); + } + fn spec(v: Value) -> std::result::Result { parse_spec(&v) } diff --git a/node/crates/c0mpute-core/src/job_e2e.rs b/node/crates/c0mpute-core/src/job_e2e.rs index 74ff6c2b..0815ceb2 100644 --- a/node/crates/c0mpute-core/src/job_e2e.rs +++ b/node/crates/c0mpute-core/src/job_e2e.rs @@ -546,7 +546,7 @@ fn public_keys(workload: &str) -> &'static [&'static str] { "frame_timeout_secs", ], "workflow.step" => &["v", "kind"], - "infernet.inference" => &["v", "model", "max_tokens", "temperature"], + "infernet.inference" => &["v", "model", "accept_models", "max_tokens", "temperature"], "infernet.text.infer" => &["base_model", "max_new_tokens"], "infernet.temporal.infer" => &["model_hash", "ttl_s"], "webhook.deliver" => &["v", "max_attempts", "timeout_ms", "backoff_ms"], diff --git a/node/crates/c0mpute-core/src/jobs_gateway.rs b/node/crates/c0mpute-core/src/jobs_gateway.rs index 221bf140..cf505c99 100644 --- a/node/crates/c0mpute-core/src/jobs_gateway.rs +++ b/node/crates/c0mpute-core/src/jobs_gateway.rs @@ -479,6 +479,17 @@ async fn run_job( ) })?; + // Routing: a plain-text prompt for a standard model also accepts smaller + // standard models, so a free worker answers it instead of the job + // queuing on a working one (inference::with_alternatives). Agent tool + // loops, JSON output and big prompts keep exactly the model they asked for. + let widened = if matches!(workload, Workload::Inference) { + crate::inference::with_alternatives(&spec) + } else { + None + }; + let exact_spec = widened.as_ref().map(|_| spec.clone()); + let spec = widened.unwrap_or(spec); let required = workload.required_capabilities(&spec); // Kept for checking the result (embeddings: one vector per text). let spec_for_check = spec.clone(); @@ -491,14 +502,32 @@ async fn run_job( let remaining = timeout.saturating_sub(started.elapsed()); let auction = JobAuction::new(workload.name(), spec_inline) .with_required_capabilities(required) - .with_allowed_bidders(allowed) + .with_allowed_bidders(allowed.clone()) .with_max_price_usd(max_price) .with_bid_window(state.bid_window) .with_deadline(remaining) .with_receipt_timeout(remaining); - let outcome = run_auction(state.net.clone(), auction) - .await - .map_err(|error| match error.downcast_ref::() { + let outcome = match (run_auction(state.net.clone(), auction).await, exact_spec) { + // Nobody that understands alternatives bid (workers older than + // 0.3.71): ask again for exactly the model the caller named, so the + // widening never costs a request its answer. + (Err(error), Some(exact)) + if !matches!(error.downcast_ref::(), Some(AuctionFailure::NoReceipt(_))) => + { + tracing::info!(err = %error, "jobs bridge: no bids on the widened inference offer, retrying with the exact model"); + let remaining = timeout.saturating_sub(started.elapsed()); + let auction = JobAuction::new(workload.name(), exact.clone()) + .with_required_capabilities(workload.required_capabilities(&exact)) + .with_allowed_bidders(allowed.clone()) + .with_max_price_usd(max_price) + .with_bid_window(state.bid_window) + .with_deadline(remaining) + .with_receipt_timeout(remaining); + run_auction(state.net.clone(), auction).await + } + (result, _) => result, + } + .map_err(|error| match error.downcast_ref::() { Some(AuctionFailure::NoReceipt(_)) => { ApiError(StatusCode::GATEWAY_TIMEOUT, json!({ "error": "timeout" })) } diff --git a/node/crates/c0mpute-core/src/routing.rs b/node/crates/c0mpute-core/src/routing.rs index 2b52bf01..3bfd1ea4 100644 --- a/node/crates/c0mpute-core/src/routing.rs +++ b/node/crates/c0mpute-core/src/routing.rs @@ -103,6 +103,18 @@ fn price_cmp(a: &JobBid, b: &JobBid) -> Ordering { a.price_usd.partial_cmp(&b.price_usd).unwrap_or(Ordering::Equal) } +/// True when some bid for the asked-for model (not an alternative) comes +/// from an idle worker: alternatives then wait their turn. +pub fn preferred_idle(bids: &[JobBid], deadline_secs: u64) -> bool { + bids.iter() + .any(|b| b.fallback != Some(true) && load_of(b) == 0.0 && !is_overloaded(b, deadline_secs)) +} + +/// An alternative-model bid while an idle worker offers the asked-for model. +fn yields(bid: &JobBid, preferred_idle: bool) -> bool { + preferred_idle && bid.fallback == Some(true) +} + /// Order two bids under `policy`, ignoring publish time (the tie-breaker). pub fn compare( a: &JobBid, @@ -111,6 +123,21 @@ pub fn compare( deadline_secs: u64, best: u32, fastest: u64, +) -> Ordering { + compare_with(a, b, policy, deadline_secs, best, fastest, false) +} + +/// [`compare`], knowing whether an idle worker offers the asked-for model +/// ([`preferred_idle`]): smaller alternative models only win once every +/// worker serving the asked-for one is working. +pub fn compare_with( + a: &JobBid, + b: &JobBid, + policy: SelectionPolicy, + deadline_secs: u64, + best: u32, + fastest: u64, + preferred_idle: bool, ) -> Ordering { match policy { SelectionPolicy::Cheapest => price_cmp(a, b), @@ -124,6 +151,7 @@ pub fn compare( Ordering::Equal } }) + .then_with(|| yields(a, preferred_idle).cmp(&yields(b, preferred_idle))) .then_with(|| eta_bucket(a, fastest).cmp(&eta_bucket(b, fastest))) .then_with(|| bench_bucket(a, best).cmp(&bench_bucket(b, best))) .then_with(|| a.eta_seconds.cmp(&b.eta_seconds)) @@ -137,15 +165,16 @@ pub fn compare( pub fn rank(bids: &mut [JobBid], policy: SelectionPolicy, deadline_secs: u64) -> usize { let best = best_bench(bids); let fastest = fastest_eta(bids); + let idle = preferred_idle(bids, deadline_secs); bids.sort_by(|a, b| { - compare(a, b, policy, deadline_secs, best, fastest) + compare_with(a, b, policy, deadline_secs, best, fastest, idle) .then(a.published_at_ms.cmp(&b.published_at_ms)) }); match bids.first() { None => 0, Some(top) => bids .iter() - .take_while(|b| compare(b, top, policy, deadline_secs, best, fastest) == Ordering::Equal) + .take_while(|b| compare_with(b, top, policy, deadline_secs, best, fastest, idle) == Ordering::Equal) .count(), } } @@ -154,6 +183,40 @@ pub fn rank(bids: &mut [JobBid], policy: SelectionPolicy, deadline_secs: u64) -> mod tests { use super::*; + fn alt(mut b: JobBid, fallback: bool) -> JobBid { + b.model = Some(if fallback { "qwen3:4b" } else { "qwen2.5:7b" }.into()); + b.fallback = Some(fallback); + b + } + + #[test] + fn a_smaller_model_waits_while_an_idle_worker_serves_the_asked_for_one() { + // Idle node with the asked-for model vs a much faster idle node with + // an alternative: the asked-for model wins. + let mut bids = vec![ + alt(bid("fast-small", 0.001, 10, Some(9000), Some(0.0)), true), + alt(bid("idle-7b", 0.001, 90, Some(5000), Some(0.0)), false), + ]; + rank(&mut bids, SelectionPolicy::Smart, 600); + assert_eq!(bids[0].bidder_peer_id, "idle-7b"); + } + + #[test] + fn a_free_smaller_model_beats_a_working_node_with_the_asked_for_one() { + // Every 7b worker is busy (load 0.5, ETA stretched): the free node + // with an alternative takes the plain-text prompt. + let mut bids = vec![ + alt(bid("busy-7b", 0.001, 131, Some(12898), Some(0.5)), false), + alt(bid("free-small", 0.001, 40, Some(4000), Some(0.0)), true), + ]; + rank(&mut bids, SelectionPolicy::Smart, 600); + assert_eq!(bids[0].bidder_peer_id, "free-small"); + // Old workers (no model fields) count as the asked-for model. + let mut old = vec![bid("old-idle", 0.001, 90, Some(5000), Some(0.0)), alt(bid("free-small", 0.001, 10, Some(9000), Some(0.0)), true)]; + rank(&mut old, SelectionPolicy::Smart, 600); + assert_eq!(old[0].bidder_peer_id, "old-idle"); + } + fn bid(peer: &str, price: f64, eta: u64, bench: Option, load: Option) -> JobBid { JobBid { job_id: "j".into(), @@ -167,6 +230,8 @@ mod tests { load, e2e: None, instance: None, + model: None, + fallback: None, } } diff --git a/node/crates/c0mpute-net/src/topics.rs b/node/crates/c0mpute-net/src/topics.rs index 499132e7..52592e5a 100644 --- a/node/crates/c0mpute-net/src/topics.rs +++ b/node/crates/c0mpute-net/src/topics.rs @@ -271,6 +271,16 @@ pub struct JobBid { /// instance on the accept. `None` from workers older than 0.3.20. #[serde(default, skip_serializing_if = "Option::is_none")] pub instance: Option, + /// The model the worker would answer with, on inference offers. `None` + /// from other workloads and from workers older than 0.3.71. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + /// True when `model` is one of the offer's accepted alternatives (a + /// smaller model a plain-text prompt can take) rather than the model it + /// asked for. Buyers keep the asked-for model while a worker serving it + /// is idle; once those are all working, a free alternative wins. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback: Option, } impl JobBid { @@ -399,6 +409,8 @@ mod tests { load: None, e2e: None, instance: None, + model: None, + fallback: None, }; let json = serde_json::to_value(&bid).unwrap(); assert_eq!(json["expires_at_ms"], 1000); @@ -447,6 +459,8 @@ mod tests { load: None, e2e: None, instance: None, + model: None, + fallback: None, }; assert!(bid.is_live(u64::MAX), "old bids never expire"); bid.expires_at_ms = Some(100); diff --git a/package.json b/package.json index 10d9567c..69a9b4c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.3.70", + "version": "0.3.71", "private": true, "workspaces": [ "apps/*", diff --git a/packages/captcha/package.json b/packages/captcha/package.json index 5792c8a0..0bc89619 100644 --- a/packages/captcha/package.json +++ b/packages/captcha/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/c0mpute-captcha", - "version": "0.3.70", + "version": "0.3.71", "description": "Solve image, slider, gap, click-text, audio and math captchas on the c0mpute network: library (fetch().then(solveCaptcha)), CLI and stdio MCP server.", "type": "module", "license": "MIT", diff --git a/packages/shared/package.json b/packages/shared/package.json index a89fdf8f..eab990c3 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.3.70", + "version": "0.3.71", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 5f721854..847cd0d6 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index b2c1cc93..d55a289e 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -38,7 +38,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "partial" diff --git a/plugins/apps/module.toml b/plugins/apps/module.toml index b593576c..c69e6ff5 100644 --- a/plugins/apps/module.toml +++ b/plugins/apps/module.toml @@ -18,7 +18,7 @@ [module] id = "apps" name = "Apps — backends with 3x redundancy" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/captcha/module.toml b/plugins/captcha/module.toml index 14131fb4..6094a462 100644 --- a/plugins/captcha/module.toml +++ b/plugins/captcha/module.toml @@ -22,7 +22,7 @@ [module] id = "captcha" name = "Captcha — image, slider, audio and math solving" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index f436d5a0..3198c46f 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "available" diff --git a/plugins/containers/module.toml b/plugins/containers/module.toml index 27199b38..52de2d7d 100644 --- a/plugins/containers/module.toml +++ b/plugins/containers/module.toml @@ -15,7 +15,7 @@ [module] id = "containers" name = "Containers — remote docker builds and jobs" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "developer" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index b8e40495..71601193 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "data" status = "available" diff --git a/plugins/crowdgpt/module.toml b/plugins/crowdgpt/module.toml index e09ce3d8..cfaa4c79 100644 --- a/plugins/crowdgpt/module.toml +++ b/plugins/crowdgpt/module.toml @@ -43,7 +43,7 @@ [module] id = "crowdgpt" name = "CrowdGPT: donate idle compute" -version = "0.3.70" +version = "0.3.71" kind = "donation" category = "ai" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index 5f529b53..3f01bdc1 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -28,7 +28,7 @@ [module] id = "ctv-ads" name = "CTV Ads — VAST and VMAP from every gateway" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "partial" diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index e3213d9a..8faaacf5 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index 9296694a..79191da6 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -19,7 +19,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index d2f1b387..7952537d 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -55,7 +55,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/git/module.toml b/plugins/git/module.toml index 7d830cf8..adf6f526 100644 --- a/plugins/git/module.toml +++ b/plugins/git/module.toml @@ -32,7 +32,7 @@ [module] id = "git" name = "Git — p2p repo hosting" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "data" status = "partial" diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index ad739172..7d257c84 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -30,7 +30,7 @@ [module] id = "hosting" name = "Hosting — p2p static sites" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "data" status = "available" diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index 99cff6e6..abf47fcf 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -22,7 +22,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index a2a24cde..6fcef075 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -34,7 +34,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "media" status = "available" diff --git a/plugins/music/module.toml b/plugins/music/module.toml index 7e4e8004..f4e9ab87 100644 --- a/plugins/music/module.toml +++ b/plugins/music/module.toml @@ -15,7 +15,7 @@ [module] id = "music" name = "Music — song generation (ACE-Step 1.5)" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index 36b423e1..4b739c1d 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -19,7 +19,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index df644fc3..cec97bf6 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -31,7 +31,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — dynamic ad insertion" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "partial" diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index dcdc47dd..fd4d6f88 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "media" status = "available" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index 1799dc95..80574f4e 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "media" status = "available" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index 6da1b141..7b6c34fa 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -33,7 +33,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "media" status = "available" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index a1ecf75e..9fe8d44e 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -21,7 +21,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "available" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index dcc58063..5b18d783 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "data" status = "available" diff --git a/plugins/threatcrush/module.toml b/plugins/threatcrush/module.toml index 9b717132..71b1fe89 100644 --- a/plugins/threatcrush/module.toml +++ b/plugins/threatcrush/module.toml @@ -19,7 +19,7 @@ [module] id = "threatcrush" name = "ThreatCrush — protect this node" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "infrastructure" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 99dd325b..dee59212 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -55,7 +55,7 @@ [module] id = "train" name = "Fine-tune — LoRA training" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index 9c11d730..15d26a4e 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index 7d584870..e36f04f0 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index b6b7b189..d7edca2e 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -65,7 +65,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, Wan 2.1 1.3B on 8 GiB or CPU, LTX-2.3 with audio)" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 2321417c..aed2315b 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index bcf7a403..64da6391 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index 200a5b3c..fb959afc 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.3.70" +version = "0.3.71" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/x402/module.toml b/plugins/x402/module.toml index 93d8eedf..66fd2895 100644 --- a/plugins/x402/module.toml +++ b/plugins/x402/module.toml @@ -21,7 +21,7 @@ [module] id = "x402" name = "x402 — charge bots for hosted files" -version = "0.3.70" +version = "0.3.71" kind = "service" category = "applications" status = "available"