Harden server with panic reduction and resource limits

Cargo.toml

@@ -43,6 +43,7 @@ tempfile = "3"
 thiserror = "2"
 tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] }
 tracing = "0.1"
+tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
 unicode-segmentation = "1.12"
 unicode-width = "0.2"

README.md

@@ -87,6 +87,44 @@ Useful subcommand groups include:
 - `node`: `node zoom`, `node swap`, `node break`, `node join-buffer`, `node move-before`, `node move-after`
 - `popup`: `display-popup`, `kill-popup`
 
+## Logging
+
+The tracing filter is resolved from the first of these that is set, highest
+precedence first: `--log <FILTER>` (alias `--log-level`), `-v`/`-vv`,
+`EMBERS_LOG`, `RUST_LOG`, then the default of `info`. `<FILTER>` accepts the full
+`tracing` env-filter syntax (a bare level like `debug`, or per-target directives
+like `embers_server=trace,info`).
+
+```sh
+embers --log-level debug list-sessions
+EMBERS_LOG=embers_server=trace,info embers
+```
+
+Foreground commands log to stderr. The background server writes to a
+daily-rotating file in the socket's directory, named
+`embers-server.<date>.log`, retaining the most recent 7 days. The launching
+command's filter is propagated to the server, and server panics are recorded in
+the same log.
+
+## Resource Limits
+
+The server enforces operator-tunable ceilings so a runaway client cannot exhaust
+host resources. Each is overridable via an environment variable:
+
+- `EMBERS_MAX_SESSIONS` (default `256`)
+- `EMBERS_MAX_BUFFERS` (default `2048`) — each buffer owns a PTY-backed process plus scrollback, so this is the dominant resource bound
+- `EMBERS_MAX_SCROLLBACK_LINES` (default `10000`)
+
+Requests that would exceed a limit are rejected with an error naming the limit;
+existing sessions are unaffected.
+
+```sh
+EMBERS_MAX_BUFFERS=256 EMBERS_MAX_SCROLLBACK_LINES=2000 embers
+```
+
+See [`docs/configuration.md`](docs/configuration.md) for the full reference of
+operational environment variables and flags.
+
 ## Configuration
 
 Embers loads configuration in this order:
@@ -108,6 +146,10 @@ embers --config ./config.rhai
 
 The generated config API reference lives in [`docs/config-api`](docs/config-api/index.md), with a rendered mdBook copy in [`docs/config-api-book`](docs/config-api-book/index.html).
 
+Operational configuration (socket path, logging, and resource limits via
+environment variables and flags) is documented in
+[`docs/configuration.md`](docs/configuration.md).
+
 ## Development
 
 Run the test suite:

crates/embers-cli/src/bin/embers-cli.rs

@@ -1,11 +1,15 @@
 use clap::Parser;
-use embers_cli::{Cli, run};
+use embers_cli::{Cli, Command, run};
 use embers_core::init_tracing;
 
 #[tokio::main]
 async fn main() {
     let cli = Cli::parse();
-    init_tracing(&cli.log_filter());
+    // The detached server sets up its own rotating file logger in `run_server`;
+    // every other invocation logs to stderr here.
+    if !matches!(cli.command, Some(Command::Serve)) {
+        init_tracing(&cli.log_filter());
+    }
 
     if let Err(error) = run(cli).await {
         eprintln!("{}", format_error_chain(&error));

crates/embers-cli/src/lib.rs

@@ -32,14 +32,22 @@ use embers_server::{SOCKET_ENV_VAR, Server, ServerConfig};
 use tokio::time::{Duration, sleep};
 use tracing::warn;
 
+/// Environment variable selecting the tracing filter (e.g. `info`, `embers=debug`).
+pub const EMBERS_LOG_ENV_VAR: &str = "EMBERS_LOG";
+
 #[derive(Debug, Parser)]
 #[command(name = "embers", about = "headless terminal multiplexer for embers")]
 pub struct Cli {
     #[arg(long, global = true)]
     pub socket: Option<PathBuf>,
     #[arg(long, global = true)]
     pub config: Option<PathBuf>,
-    #[arg(long, global = true, value_name = "FILTER")]
+    #[arg(
+        long,
+        visible_alias = "log-level",
+        global = true,
+        value_name = "FILTER"
+    )]
     pub log: Option<String>,
     #[arg(short = 'v', long = "verbose", global = true, action = clap::ArgAction::Count)]
     pub verbose: u8,
@@ -57,7 +65,7 @@ impl Cli {
             1 => return "debug".to_owned(),
             _ => return "trace".to_owned(),
         }
-        if let Some(filter) = std::env::var("EMBERS_LOG")
+        if let Some(filter) = std::env::var(EMBERS_LOG_ENV_VAR)
             .ok()
             .filter(|value| !value.trim().is_empty())
         {
@@ -1038,6 +1046,7 @@ async fn execute_command(connection: &mut CliConnection, command: Command) -> Re
 }
 
 pub async fn run(cli: Cli) -> Result<()> {
+    let log_filter = cli.log_filter();
     let Cli {
         socket,
         config,
@@ -1066,7 +1075,7 @@ pub async fn run(cli: Cli) -> Result<()> {
 
             match command {
                 None => {
-                    ensure_server_process(&socket).await?;
+                    ensure_server_process(&socket, &log_filter).await?;
                     interactive::run(socket, None, config).await
                 }
                 Some(Command::Attach { target }) => {
@@ -1082,12 +1091,12 @@ pub async fn run(cli: Cli) -> Result<()> {
                     target,
                     all_sessions,
                 }) => {
-                    ensure_server_process(&socket).await?;
+                    ensure_server_process(&socket, &log_filter).await?;
                     automation::run(socket, target, all_sessions).await
                 }
-                Some(Command::Serve) => run_server(socket).await,
+                Some(Command::Serve) => run_server(socket, &log_filter).await,
                 Some(command) => {
-                    ensure_server_process(&socket).await?;
+                    ensure_server_process(&socket, &log_filter).await?;
                     let output = execute(&socket, command).await?;
                     if !output.is_empty() {
                         println!("{output}");
@@ -1196,7 +1205,7 @@ async fn server_is_available(socket_path: &Path) -> bool {
     CliConnection::connect(socket_path).await.is_ok()
 }
 
-async fn ensure_server_process(socket_path: &Path) -> Result<()> {
+async fn ensure_server_process(socket_path: &Path, log_filter: &str) -> Result<()> {
     if server_is_available(socket_path).await {
         return Ok(());
     }
@@ -1208,6 +1217,10 @@ async fn ensure_server_process(socket_path: &Path) -> Result<()> {
         .arg("__serve")
         .arg("--socket")
         .arg(socket_path)
+        // Propagate the resolved filter so a `--log`/`-v` flag reaches the
+        // detached server, not just the inherited EMBERS_LOG/RUST_LOG env. The
+        // server writes its own rotating log file, so its stdio is discarded.
+        .env(EMBERS_LOG_ENV_VAR, log_filter)
         .stdin(Stdio::null())
         .stdout(Stdio::null())
         .stderr(Stdio::null())
@@ -1242,8 +1255,15 @@ async fn ensure_server_process(socket_path: &Path) -> Result<()> {
     }
 }
 
-async fn run_server(socket_path: PathBuf) -> Result<()> {
+async fn run_server(socket_path: PathBuf, log_filter: &str) -> Result<()> {
     ensure_socket_parent(&socket_path)?;
+    // The detached server logs to a daily-rotating file next to the socket. Set
+    // this up before anything else so startup is captured, then route panics
+    // through tracing so a crash lands in the same log instead of a dead stderr.
+    if let Some(dir) = socket_path.parent() {
+        embers_core::init_server_tracing(log_filter, dir)?;
+        install_server_panic_hook();
+    }
     let secure_parent = socket_path
         .parent()
         .is_some_and(|parent| parent == default_runtime_dir().as_path());
@@ -1253,6 +1273,17 @@ async fn run_server(socket_path: PathBuf) -> Result<()> {
     handle.shutdown().await
 }
 
+/// Routes panics in the detached server through `tracing` (chained after the
+/// default hook) so a crash is recorded in the rotating log file, which is the
+/// server's only output sink once its stdio is discarded.
+fn install_server_panic_hook() {
+    let previous = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |info| {
+        tracing::error!(target: "panic", "server panicked: {info}");
+        previous(info);
+    }));
+}
+
 fn ensure_socket_parent(socket_path: &Path) -> Result<()> {
     let Some(parent) = socket_path.parent() else {
         return Ok(());