69 lines
2.1 KiB
Rust
69 lines
2.1 KiB
Rust
|
use std::sync::LazyLock;
|
||
|
|
use std::time::Duration;
|
||
|
|
use serde::Deserialize;
|
||
|
|
use tokio::process::Command;
|
||
|
|
use tokio::time::sleep;
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct ServerConfig {
|
||
|
|
llama_server: LlamaServerConfig,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct LlamaServerConfig {
|
||
|
|
binary: String,
|
||
|
|
model_path: String,
|
||
|
|
host: String,
|
||
|
|
port: u16,
|
||
|
|
context_size: u32,
|
||
|
|
}
|
||
|
|
|
||
|
|
static SERVER_CONFIG: LazyLock<ServerConfig> = LazyLock::new(|| {
|
||
|
|
toml::from_str(include_str!("server.toml")).expect("Could not parse server.toml")
|
||
|
|
});
|
||
|
|
|
||
|
|
static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new);
|
||
|
|
|
||
|
|
pub(crate) fn url() -> String {
|
||
|
|
format!("http://{}:{}", SERVER_CONFIG.llama_server.host, SERVER_CONFIG.llama_server.port)
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn is_healthy(health_url: &str) -> bool {
|
||
|
|
HTTP_CLIENT.get(health_url).send().await.is_ok_and(|r| r.status().is_success())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Checks whether llama-server is already serving on the configured host/port,
|
||
|
|
/// and if not, spawns it from the configured binary/model path and waits for
|
||
|
|
/// it to report healthy before returning.
|
||
|
|
pub(crate) async fn ensure_running() -> anyhow::Result<()> {
|
||
|
|
let base_url = url();
|
||
|
|
let health_url = format!("{base_url}/health");
|
||
|
|
|
||
|
|
if is_healthy(&health_url).await {
|
||
|
|
return Ok(());
|
||
|
|
}
|
||
|
|
|
||
|
|
println!("llama-server not running at {base_url}, starting it...");
|
||
|
|
|
||
|
|
Command::new(&SERVER_CONFIG.llama_server.binary)
|
||
|
|
.args([
|
||
|
|
"-m", &SERVER_CONFIG.llama_server.model_path,
|
||
|
|
"--jinja",
|
||
|
|
"-c", &SERVER_CONFIG.llama_server.context_size.to_string(),
|
||
|
|
"--host", &SERVER_CONFIG.llama_server.host,
|
||
|
|
"--port", &SERVER_CONFIG.llama_server.port.to_string(),
|
||
|
|
])
|
||
|
|
.spawn()
|
||
|
|
.map_err(|e| anyhow::anyhow!("failed to spawn llama-server at {}: {e}", SERVER_CONFIG.llama_server.binary))?;
|
||
|
|
|
||
|
|
for _ in 0..60 {
|
||
|
|
if is_healthy(&health_url).await {
|
||
|
|
println!("llama-server is up.");
|
||
|
|
return Ok(());
|
||
|
|
}
|
||
|
|
sleep(Duration::from_secs(1)).await;
|
||
|
|
}
|
||
|
|
|
||
|
|
anyhow::bail!("llama-server did not become healthy within 60s")
|
||
|
|
}
|