polymech/zeroclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(channels): add automatic media understanding pipeline for inbound messages

Browse Source 
Add MediaPipeline that pre-processes inbound channel message attachments
before the agent sees them:

- Audio: transcribed via existing transcription infrastructure, annotated
  as [Audio transcription: ...]
- Images: annotated with [Image: <file> attached] (vision-aware)
- Video: annotated with [Video: <file> attached] (placeholder for future API)

The pipeline is opt-in via [media_pipeline] config section (default: disabled).
Individual media types can be toggled independently.

Changes:
- New src/channels/media_pipeline.rs with MediaPipeline struct and tests
- New MediaPipelineConfig in config/schema.rs
- Added attachments field to ChannelMessage for media pass-through
- Wired pipeline into process_channel_message after hooks, before agent

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Giulio V 2026-03-21 15:23:57 +01:00
parent 87b5bca449
commit 6b33e50bfd
32 changed files with 636 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/channels/bluesky.rs
View File

@ -252,6 +252,7 @@ impl BlueskyChannel {
timestamp,
thread_ts: Some(notif.uri.clone()),
interruption_scope_id: None,
attachments: vec![],
})
}

3
src/channels/cli.rs
View File

@ -49,6 +49,7 @@ impl Channel for CliChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(msg).await.is_err() {
@ -113,6 +114,7 @@ mod tests {
timestamp: 1_234_567_890,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
assert_eq!(msg.id, "test-id");
assert_eq!(msg.sender, "user");
@ -133,6 +135,7 @@ mod tests {
timestamp: 0,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
let cloned = msg.clone();
assert_eq!(cloned.id, msg.id);

1
src/channels/dingtalk.rs
View File

@ -285,6 +285,7 @@ impl Channel for DingTalkChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

1
src/channels/discord.rs
View File

@ -799,6 +799,7 @@ impl Channel for DiscordChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

1
src/channels/email_channel.rs
View File

@ -468,6 +468,7 @@ impl EmailChannel {
timestamp: email.timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(msg).await.is_err() {

1
src/channels/imessage.rs
View File

@ -295,6 +295,7 @@ end tell"#
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(msg).await.is_err() {

1
src/channels/irc.rs
View File

@ -581,6 +581,7 @@ impl Channel for IrcChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

2
src/channels/lark.rs
View File

@ -909,6 +909,7 @@ impl LarkChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
tracing::debug!("Lark WS: message in {}", lark_msg.chat_id);
@ -1207,6 +1208,7 @@ impl LarkChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
messages

1
src/channels/linq.rs
View File

@ -268,6 +268,7 @@ impl LinqChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
messages

1
src/channels/matrix.rs
View File

@ -901,6 +901,7 @@ impl Channel for MatrixChannel {
.as_secs(),
thread_ts: thread_ts.clone(),
interruption_scope_id: thread_ts,
attachments: vec![],
};
let _ = tx.send(msg).await;

1
src/channels/mattermost.rs
View File

@ -332,6 +332,7 @@ impl MattermostChannel {
timestamp: (create_at / 1000) as u64,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
}
}

409
src/channels/media_pipeline.rs Normal file
View File

@ -0,0 +1,409 @@
//! Automatic media understanding pipeline for inbound channel messages.
//!
//! Pre-processes media attachments (audio, images, video) before the agent sees
//! the message, enriching the text with human-readable annotations:
//!
//! - **Audio**: transcribed via the existing [`super::transcription`] infrastructure,
//! prepended as `[Audio transcription: ...]`.
//! - **Images**: when a vision-capable provider is active, described as `[Image: <description>]`.
//! Falls back to `[Image: attached]` when vision is unavailable.
//! - **Video**: summarised as `[Video summary: ...]` when an API is available,
//! otherwise `[Video: attached]`.
//!
//! The pipeline is **opt-in** via `[media_pipeline] enabled = true` in config.
use crate::config::{MediaPipelineConfig, TranscriptionConfig};
/// Classifies an attachment by MIME type or file extension.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MediaKind {
Audio,
Image,
Video,
Unknown,
}
/// A single media attachment on an inbound message.
#[derive(Debug, Clone)]
pub struct MediaAttachment {
/// Original file name (e.g. `voice.ogg`, `photo.jpg`).
pub file_name: String,
/// Raw bytes of the attachment.
pub data: Vec<u8>,
/// MIME type if known (e.g. `audio/ogg`, `image/jpeg`).
pub mime_type: Option<String>,
}
impl MediaAttachment {
/// Classify this attachment into a [`MediaKind`].
pub fn kind(&self) -> MediaKind {
// Try MIME type first.
if let Some(ref mime) = self.mime_type {
let lower = mime.to_ascii_lowercase();
if lower.starts_with("audio/") {
return MediaKind::Audio;
}
if lower.starts_with("image/") {
return MediaKind::Image;
}
if lower.starts_with("video/") {
return MediaKind::Video;
}
}
// Fall back to file extension.
let ext = self
.file_name
.rsplit_once('.')
.map(|(_, e)| e.to_ascii_lowercase())
.unwrap_or_default();
match ext.as_str() {
"flac" | "mp3" | "mpeg" | "mpga" | "m4a" | "ogg" | "oga" | "opus" | "wav" | "webm" => {
MediaKind::Audio
}
"png" | "jpg" | "jpeg" | "gif" | "bmp" | "webp" | "heic" | "tiff" | "svg" => {
MediaKind::Image
}
"mp4" | "mkv" | "avi" | "mov" | "wmv" | "flv" => MediaKind::Video,
_ => MediaKind::Unknown,
}
}
}
/// The media understanding pipeline.
///
/// Consumes a message's text and attachments, returning enriched text with
/// media annotations prepended.
pub struct MediaPipeline<'a> {
config: &'a MediaPipelineConfig,
transcription_config: &'a TranscriptionConfig,
vision_available: bool,
}
impl<'a> MediaPipeline<'a> {
/// Create a new pipeline. `vision_available` indicates whether the current
/// provider supports vision (image description).
pub fn new(
config: &'a MediaPipelineConfig,
transcription_config: &'a TranscriptionConfig,
vision_available: bool,
) -> Self {
Self {
config,
transcription_config,
vision_available,
}
}
/// Process a message's attachments and return enriched text.
///
/// If the pipeline is disabled via config, returns `original_text` unchanged.
pub async fn process(&self, original_text: &str, attachments: &[MediaAttachment]) -> String {
if !self.config.enabled || attachments.is_empty() {
return original_text.to_string();
}
let mut annotations = Vec::new();
for attachment in attachments {
match attachment.kind() {
MediaKind::Audio if self.config.transcribe_audio => {
let annotation = self.process_audio(attachment).await;
annotations.push(annotation);
}
MediaKind::Image if self.config.describe_images => {
let annotation = self.process_image(attachment);
annotations.push(annotation);
}
MediaKind::Video if self.config.summarize_video => {
let annotation = self.process_video(attachment);
annotations.push(annotation);
}
_ => {}
}
}
if annotations.is_empty() {
return original_text.to_string();
}
let mut enriched = String::with_capacity(
annotations.iter().map(|a| a.len() + 1).sum::<usize>() + original_text.len() + 2,
);
for annotation in &annotations {
enriched.push_str(annotation);
enriched.push('\n');
}
if !original_text.is_empty() {
enriched.push('\n');
enriched.push_str(original_text);
}
enriched.trim().to_string()
}
/// Transcribe an audio attachment using the existing transcription infra.
async fn process_audio(&self, attachment: &MediaAttachment) -> String {
if !self.transcription_config.enabled {
return "[Audio: attached]".to_string();
}
match super::transcription::transcribe_audio(
attachment.data.clone(),
&attachment.file_name,
self.transcription_config,
)
.await
{
Ok(text) => {
let trimmed = text.trim();
if trimmed.is_empty() {
"[Audio transcription: (empty)]".to_string()
} else {
format!("[Audio transcription: {trimmed}]")
}
}
Err(err) => {
tracing::warn!(
file = %attachment.file_name,
error = %err,
"Media pipeline: audio transcription failed"
);
"[Audio: transcription failed]".to_string()
}
}
}
/// Describe an image attachment.
///
/// When vision is available, the image will be passed through to the
/// provider as an `[IMAGE:]` marker and described by the model in the
/// normal flow. Here we only add a placeholder annotation so the agent
/// knows an image is present.
fn process_image(&self, attachment: &MediaAttachment) -> String {
if self.vision_available {
format!(
"[Image: {} attached, will be processed by vision model]",
attachment.file_name
)
} else {
format!("[Image: {} attached]", attachment.file_name)
}
}
/// Summarize a video attachment.
///
/// Video analysis requires external APIs not currently integrated.
/// For now we add a placeholder annotation.
fn process_video(&self, attachment: &MediaAttachment) -> String {
format!("[Video: {} attached]", attachment.file_name)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn default_pipeline_config(enabled: bool) -> MediaPipelineConfig {
MediaPipelineConfig {
enabled,
transcribe_audio: true,
describe_images: true,
summarize_video: true,
}
}
fn sample_audio() -> MediaAttachment {
MediaAttachment {
file_name: "voice.ogg".to_string(),
data: vec![0u8; 100],
mime_type: Some("audio/ogg".to_string()),
}
}
fn sample_image() -> MediaAttachment {
MediaAttachment {
file_name: "photo.jpg".to_string(),
data: vec![0u8; 50],
mime_type: Some("image/jpeg".to_string()),
}
}
fn sample_video() -> MediaAttachment {
MediaAttachment {
file_name: "clip.mp4".to_string(),
data: vec![0u8; 200],
mime_type: Some("video/mp4".to_string()),
}
}
#[test]
fn media_kind_from_mime() {
let audio = MediaAttachment {
file_name: "file".to_string(),
data: vec![],
mime_type: Some("audio/ogg".to_string()),
};
assert_eq!(audio.kind(), MediaKind::Audio);
let image = MediaAttachment {
file_name: "file".to_string(),
data: vec![],
mime_type: Some("image/png".to_string()),
};
assert_eq!(image.kind(), MediaKind::Image);
let video = MediaAttachment {
file_name: "file".to_string(),
data: vec![],
mime_type: Some("video/mp4".to_string()),
};
assert_eq!(video.kind(), MediaKind::Video);
}
#[test]
fn media_kind_from_extension() {
let audio = MediaAttachment {
file_name: "voice.ogg".to_string(),
data: vec![],
mime_type: None,
};
assert_eq!(audio.kind(), MediaKind::Audio);
let image = MediaAttachment {
file_name: "photo.png".to_string(),
data: vec![],
mime_type: None,
};
assert_eq!(image.kind(), MediaKind::Image);
let video = MediaAttachment {
file_name: "clip.mp4".to_string(),
data: vec![],
mime_type: None,
};
assert_eq!(video.kind(), MediaKind::Video);
let unknown = MediaAttachment {
file_name: "data.bin".to_string(),
data: vec![],
mime_type: None,
};
assert_eq!(unknown.kind(), MediaKind::Unknown);
}
#[tokio::test]
async fn disabled_pipeline_returns_original_text() {
let config = default_pipeline_config(false);
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, false);
let result = pipeline.process("hello", &[sample_audio()]).await;
assert_eq!(result, "hello");
}
#[tokio::test]
async fn empty_attachments_returns_original_text() {
let config = default_pipeline_config(true);
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, false);
let result = pipeline.process("hello", &[]).await;
assert_eq!(result, "hello");
}
#[tokio::test]
async fn image_annotation_with_vision() {
let config = default_pipeline_config(true);
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, true);
let result = pipeline.process("check this", &[sample_image()]).await;
assert!(
result.contains("[Image: photo.jpg attached, will be processed by vision model]"),
"expected vision annotation, got: {result}"
);
assert!(result.contains("check this"));
}
#[tokio::test]
async fn image_annotation_without_vision() {
let config = default_pipeline_config(true);
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, false);
let result = pipeline.process("check this", &[sample_image()]).await;
assert!(
result.contains("[Image: photo.jpg attached]"),
"expected basic image annotation, got: {result}"
);
}
#[tokio::test]
async fn video_annotation() {
let config = default_pipeline_config(true);
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, false);
let result = pipeline.process("watch", &[sample_video()]).await;
assert!(
result.contains("[Video: clip.mp4 attached]"),
"expected video annotation, got: {result}"
);
}
#[tokio::test]
async fn audio_without_transcription_enabled() {
let config = default_pipeline_config(true);
let mut tc = TranscriptionConfig::default();
tc.enabled = false;
let pipeline = MediaPipeline::new(&config, &tc, false);
let result = pipeline.process("", &[sample_audio()]).await;
assert_eq!(result, "[Audio: attached]");
}
#[tokio::test]
async fn multiple_attachments_produce_multiple_annotations() {
let config = default_pipeline_config(true);
let mut tc = TranscriptionConfig::default();
tc.enabled = false;
let pipeline = MediaPipeline::new(&config, &tc, false);
let attachments = vec![sample_audio(), sample_image(), sample_video()];
let result = pipeline.process("context", &attachments).await;
assert!(
result.contains("[Audio: attached]"),
"missing audio annotation"
);
assert!(
result.contains("[Image: photo.jpg attached]"),
"missing image annotation"
);
assert!(
result.contains("[Video: clip.mp4 attached]"),
"missing video annotation"
);
assert!(result.contains("context"), "missing original text");
}
#[tokio::test]
async fn disabled_sub_features_skip_processing() {
let config = MediaPipelineConfig {
enabled: true,
transcribe_audio: false,
describe_images: false,
summarize_video: false,
};
let tc = TranscriptionConfig::default();
let pipeline = MediaPipeline::new(&config, &tc, false);
let attachments = vec![sample_audio(), sample_image(), sample_video()];
let result = pipeline.process("hello", &attachments).await;
assert_eq!(result, "hello");
}
}

1
src/channels/mochat.rs
View File

@ -199,6 +199,7 @@ impl Channel for MochatChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

124
src/channels/mod.rs
View File

@ -28,6 +28,7 @@ pub mod linq;
#[cfg(feature = "channel-matrix")]
pub mod matrix;
pub mod mattermost;
pub mod media_pipeline;
pub mod mochat;
pub mod nextcloud_talk;
#[cfg(feature = "channel-nostr")]
@ -358,6 +359,8 @@ struct ChannelRuntimeContext {
message_timeout_secs: u64,
interrupt_on_new_message: InterruptOnNewMessageConfig,
multimodal: crate::config::MultimodalConfig,
media_pipeline: crate::config::MediaPipelineConfig,
transcription_config: crate::config::TranscriptionConfig,
hooks: Option<Arc<crate::hooks::HookRunner>>,
non_cli_excluded_tools: Arc<Vec<String>>,
autonomy_level: AutonomyLevel,
@ -2058,6 +2061,17 @@ async fn process_channel_message(
msg
};
// ── Media pipeline: enrich inbound message with media annotations ──
if ctx.media_pipeline.enabled && !msg.attachments.is_empty() {
let vision = ctx.provider.supports_vision();
let pipeline = media_pipeline::MediaPipeline::new(
&ctx.media_pipeline,
&ctx.transcription_config,
vision,
);
msg.content = Box::pin(pipeline.process(&msg.content, &msg.attachments)).await;
}
let target_channel = ctx
.channels_by_name
.get(&msg.channel)
@ -4617,6 +4631,8 @@ pub async fn start_channels(config: Config) -> Result<()> {
matrix: interrupt_on_new_message_matrix,
},
multimodal: config.multimodal.clone(),
media_pipeline: config.media_pipeline.clone(),
transcription_config: config.transcription.clone(),
hooks: if config.hooks.enabled {
let mut runner = crate::hooks::HookRunner::new();
if config.hooks.builtin.command_logger {
@ -4988,6 +5004,8 @@ mod tests {
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
provider_runtime_options: providers::ProviderRuntimeOptions::default(),
workspace_dir: Arc::new(std::env::temp_dir()),
@ -5105,6 +5123,8 @@ mod tests {
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
provider_runtime_options: providers::ProviderRuntimeOptions::default(),
workspace_dir: Arc::new(std::env::temp_dir()),
@ -5178,6 +5198,8 @@ mod tests {
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
provider_runtime_options: providers::ProviderRuntimeOptions::default(),
workspace_dir: Arc::new(std::env::temp_dir()),
@ -5270,6 +5292,8 @@ mod tests {
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
provider_runtime_options: providers::ProviderRuntimeOptions::default(),
workspace_dir: Arc::new(std::env::temp_dir()),
@ -5819,6 +5843,8 @@ BTC is currently around $65,000 based on latest tool output."#
autonomy_level: AutonomyLevel::default(),
tool_call_dedup_exempt: Arc::new(Vec::new()),
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
model_routes: Arc::new(Vec::new()),
query_classification: crate::config::QueryClassificationConfig::default(),
@ -5844,6 +5870,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -5901,6 +5928,8 @@ BTC is currently around $65,000 based on latest tool output."#
autonomy_level: AutonomyLevel::default(),
tool_call_dedup_exempt: Arc::new(Vec::new()),
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
model_routes: Arc::new(Vec::new()),
query_classification: crate::config::QueryClassificationConfig::default(),
@ -5926,6 +5955,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -5994,6 +6024,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6022,6 +6054,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 3,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6075,6 +6108,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6103,6 +6138,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6166,6 +6202,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6194,6 +6232,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6278,6 +6317,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6306,6 +6347,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6371,6 +6413,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6399,6 +6443,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 3,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6479,6 +6524,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6507,6 +6554,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 4,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6572,6 +6620,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6600,6 +6650,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6655,6 +6706,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6683,6 +6736,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -6853,6 +6907,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6880,6 +6936,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -6892,6 +6949,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -6956,6 +7014,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -6984,6 +7044,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -6997,6 +7058,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -7077,6 +7139,8 @@ BTC is currently around $65,000 based on latest tool output."#
show_tool_calls: true,
session_store: None,
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -7102,6 +7166,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: Some("1741234567.100001".to_string()),
interruption_scope_id: Some("1741234567.100001".to_string()),
attachments: vec![],
})
.await
.unwrap();
@ -7115,6 +7180,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: Some("1741234567.100001".to_string()),
interruption_scope_id: Some("1741234567.100001".to_string()),
attachments: vec![],
})
.await
.unwrap();
@ -7189,6 +7255,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -7217,6 +7285,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -7230,6 +7299,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.unwrap();
@ -7286,6 +7356,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -7314,6 +7386,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -7367,6 +7440,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -7395,6 +7470,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -7920,6 +7996,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
assert_eq!(conversation_memory_key(&msg), "slack_U123_msg_abc123");
@ -7936,6 +8013,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: Some("1741234567.123456".into()),
interruption_scope_id: None,
attachments: vec![],
};
assert_eq!(
@ -7955,6 +8033,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
assert_eq!(followup_thread_id(&msg).as_deref(), Some("msg_abc123"));
@ -7971,6 +8050,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
let msg2 = traits::ChannelMessage {
id: "msg_2".into(),
@ -7981,6 +8061,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
assert_ne!(
@ -8003,6 +8084,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
let msg2 = traits::ChannelMessage {
id: "msg_2".into(),
@ -8013,6 +8095,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
mem.store(
@ -8134,6 +8217,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -8162,6 +8247,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8178,6 +8264,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8266,6 +8353,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -8294,6 +8383,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8326,6 +8416,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8364,6 +8455,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 3,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8438,6 +8530,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -8466,6 +8560,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -8547,6 +8642,8 @@ BTC is currently around $65,000 based on latest tool output."#
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -8575,6 +8672,7 @@ BTC is currently around $65,000 based on latest tool output."#
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9120,6 +9218,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9149,6 +9249,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9208,6 +9309,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9236,6 +9339,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9252,6 +9356,7 @@ This is an example JSON object for profile settings."#;
timestamp: 2,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9371,6 +9476,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9399,6 +9506,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9483,6 +9591,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9511,6 +9621,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9587,6 +9698,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9615,6 +9728,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9711,6 +9825,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -9739,6 +9855,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
},
CancellationToken::new(),
)
@ -9896,6 +10013,7 @@ This is an example JSON object for profile settings."#;
timestamp: 0,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
assert_eq!(interruption_scope_key(&msg), "matrix_room_alice");
}
@ -9911,6 +10029,7 @@ This is an example JSON object for profile settings."#;
timestamp: 0,
thread_ts: Some("$thread1".into()),
interruption_scope_id: Some("$thread1".into()),
attachments: vec![],
};
assert_eq!(interruption_scope_key(&msg), "matrix_room_alice_$thread1");
}
@ -9927,6 +10046,7 @@ This is an example JSON object for profile settings."#;
timestamp: 0,
thread_ts: Some("1234567890.000100".into()), // Slack top-level fallback
interruption_scope_id: None, // but NOT a thread reply
attachments: vec![],
};
assert_eq!(interruption_scope_key(&msg), "slack_C123_alice");
}
@ -9973,6 +10093,8 @@ This is an example JSON object for profile settings."#;
matrix: false,
},
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
transcription_config: crate::config::TranscriptionConfig::default(),
hooks: None,
non_cli_excluded_tools: Arc::new(Vec::new()),
autonomy_level: AutonomyLevel::default(),
@ -10003,6 +10125,7 @@ This is an example JSON object for profile settings."#;
timestamp: 1,
thread_ts: Some("1741234567.100001".to_string()),
interruption_scope_id: Some("1741234567.100001".to_string()),
attachments: vec![],
})
.await
.unwrap();
@ -10016,6 +10139,7 @@ This is an example JSON object for profile settings."#;
timestamp: 2,
thread_ts: Some("1741234567.200002".to_string()),
interruption_scope_id: Some("1741234567.200002".to_string()),
attachments: vec![],
})
.await
.unwrap();

2
src/channels/nextcloud_talk.rs
View File

@ -206,6 +206,7 @@ impl NextcloudTalkChannel {
timestamp: Self::now_unix_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
messages
@ -308,6 +309,7 @@ impl NextcloudTalkChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
messages

1
src/channels/nostr.rs
View File

@ -254,6 +254,7 @@ impl Channel for NostrChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(msg).await.is_err() {
tracing::info!("Nostr listener: message bus closed, stopping");

1
src/channels/notion.rs
View File

@ -361,6 +361,7 @@ impl Channel for NotionChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.is_err()

2
src/channels/qq.rs
View File

@ -1139,6 +1139,7 @@ impl Channel for QQChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {
@ -1178,6 +1179,7 @@ impl Channel for QQChannel {
.as_secs(),
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

1
src/channels/reddit.rs
View File

@ -226,6 +226,7 @@ impl RedditChannel {
timestamp,
thread_ts: item.parent_id.clone(),
interruption_scope_id: None,
attachments: vec![],
})
}
}

1
src/channels/signal.rs
View File

@ -280,6 +280,7 @@ impl SignalChannel {
timestamp: timestamp / 1000, // millis → secs
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
}
}

5
src/channels/slack.rs
View File

@ -1908,6 +1908,7 @@ impl SlackChannel {
Self::inbound_thread_ts_genuine_only(event)
},
interruption_scope_id: Self::inbound_interruption_scope_id(event, ts),
attachments: vec![],
};
// Track thread context so start_typing can set assistant status.
@ -2582,6 +2583,7 @@ impl Channel for SlackChannel {
Self::inbound_thread_ts_genuine_only(msg)
},
interruption_scope_id: Self::inbound_interruption_scope_id(msg, ts),
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {
@ -2667,6 +2669,7 @@ impl Channel for SlackChannel {
.as_secs(),
thread_ts: Some(thread_ts.clone()),
interruption_scope_id: Some(thread_ts.clone()),
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {
@ -3619,6 +3622,7 @@ mod tests {
timestamp: 0,
thread_ts: None, // thread_replies=false → no fallback to ts
interruption_scope_id: None,
attachments: vec![],
};
let msg1 = make_msg("100.000");
@ -3644,6 +3648,7 @@ mod tests {
timestamp: 0,
thread_ts: Some(ts.to_string()), // thread_replies=true → ts as thread_ts
interruption_scope_id: None,
attachments: vec![],
};
let msg1 = make_msg("100.000");

3
src/channels/telegram.rs
View File

@ -1152,6 +1152,7 @@ Allowlist Telegram username (without '@') or numeric user ID.",
.as_secs(),
thread_ts: thread_id,
interruption_scope_id: None,
attachments: vec![],
})
}
@ -1275,6 +1276,7 @@ Allowlist Telegram username (without '@') or numeric user ID.",
.as_secs(),
thread_ts: thread_id,
interruption_scope_id: None,
attachments: vec![],
})
}
@ -1437,6 +1439,7 @@ Allowlist Telegram username (without '@') or numeric user ID.",
.as_secs(),
thread_ts: thread_id,
interruption_scope_id: None,
attachments: vec![],
})
}

6
src/channels/traits.rs
View File

@ -17,6 +17,10 @@ pub struct ChannelMessage {
/// is genuinely inside a reply thread and should be isolated from other threads.
/// `None` means top-level — scope is sender+channel only.
pub interruption_scope_id: Option<String>,
/// Media attachments (audio, images, video) for the media pipeline.
/// Channels populate this when they receive media alongside a text message.
/// Defaults to empty — existing channels are unaffected.
pub attachments: Vec<super::media_pipeline::MediaAttachment>,
}
/// Message to send through a channel
@ -188,6 +192,7 @@ mod tests {
timestamp: 123,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
.map_err(|e| anyhow::anyhow!(e.to_string()))
@ -205,6 +210,7 @@ mod tests {
timestamp: 999,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
let cloned = message.clone();

1
src/channels/twitter.rs
View File

@ -289,6 +289,7 @@ impl Channel for TwitterChannel {
.and_then(|c| c.as_str())
.map(|s| s.to_string()),
interruption_scope_id: None,
attachments: vec![],
};
if tx.send(channel_msg).await.is_err() {

1
src/channels/wati.rs
View File

@ -174,6 +174,7 @@ impl WatiChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
messages

1
src/channels/webhook.rs
View File

@ -238,6 +238,7 @@ impl Channel for WebhookChannel {
timestamp,
thread_ts: payload.thread_id,
interruption_scope_id: None,
attachments: vec![],
};
if state.tx.send(msg).await.is_err() {

1
src/channels/whatsapp.rs
View File

@ -152,6 +152,7 @@ impl WhatsAppChannel {
timestamp,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
});
}
}

1
src/channels/whatsapp_web.rs
View File

@ -822,6 +822,7 @@ impl Channel for WhatsAppWebChannel {
timestamp: chrono::Utc::now().timestamp() as u64,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
})
.await
{

25
src/config/mod.rs
View File

@ -18,18 +18,19 @@ pub use schema::{
IMessageConfig, IdentityConfig, ImageProviderDalleConfig, ImageProviderFluxConfig,
ImageProviderImagenConfig, ImageProviderStabilityConfig, JiraConfig, KnowledgeConfig,
LarkConfig, LinkedInConfig, LinkedInContentConfig, LinkedInImageConfig, LocalWhisperConfig,
MatrixConfig, McpConfig, McpServerConfig, McpTransport, MemoryConfig, Microsoft365Config,
ModelRouteConfig, MultimodalConfig, NextcloudTalkConfig, NodeTransportConfig, NodesConfig,
NotionConfig, ObservabilityConfig, OpenAiSttConfig, OpenAiTtsConfig, OpenVpnTunnelConfig,
OtpConfig, OtpMethod, PacingConfig, PeripheralBoardConfig, PeripheralsConfig, PluginsConfig,
ProjectIntelConfig, ProxyConfig, ProxyScope, QdrantConfig, QueryClassificationConfig,
ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig,
SchedulerConfig, SecretsConfig, SecurityConfig, SecurityOpsConfig, SkillCreationConfig,
SkillsConfig, SkillsPromptInjectionMode, SlackConfig, StorageConfig, StorageProviderConfig,
StorageProviderSection, StreamMode, SwarmConfig, SwarmStrategy, TelegramConfig,
TextBrowserConfig, ToolFilterGroup, ToolFilterGroupMode, TranscriptionConfig, TtsConfig,
TunnelConfig, VerifiableIntentConfig, WebFetchConfig, WebSearchConfig, WebhookConfig,
WhatsAppChatPolicy, WhatsAppWebMode, WorkspaceConfig, DEFAULT_GWS_SERVICES,
MatrixConfig, McpConfig, McpServerConfig, McpTransport, MediaPipelineConfig, MemoryConfig,
Microsoft365Config, ModelRouteConfig, MultimodalConfig, NextcloudTalkConfig,
NodeTransportConfig, NodesConfig, NotionConfig, ObservabilityConfig, OpenAiSttConfig,
OpenAiTtsConfig, OpenVpnTunnelConfig, OtpConfig, OtpMethod, PacingConfig,
PeripheralBoardConfig, PeripheralsConfig, PluginsConfig, ProjectIntelConfig, ProxyConfig,
ProxyScope, QdrantConfig, QueryClassificationConfig, ReliabilityConfig, ResourceLimitsConfig,
RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, SecretsConfig, SecurityConfig,
SecurityOpsConfig, SkillCreationConfig, SkillsConfig, SkillsPromptInjectionMode, SlackConfig,
StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, SwarmConfig,
SwarmStrategy, TelegramConfig, TextBrowserConfig, ToolFilterGroup, ToolFilterGroupMode,
TranscriptionConfig, TtsConfig, TunnelConfig, VerifiableIntentConfig, WebFetchConfig,
WebSearchConfig, WebhookConfig, WhatsAppChatPolicy, WhatsAppWebMode, WorkspaceConfig,
DEFAULT_GWS_SERVICES,
};
pub fn name_and_presence<T: traits::ChannelConfig>(channel: Option<&T>) -> (&'static str, bool) {

45
src/config/schema.rs
View File

@ -261,6 +261,10 @@ pub struct Config {
#[serde(default)]
pub multimodal: MultimodalConfig,
/// Automatic media understanding pipeline (`[media_pipeline]`).
#[serde(default)]
pub media_pipeline: MediaPipelineConfig,
/// Web fetch tool configuration (`[web_fetch]`).
#[serde(default)]
pub web_fetch: WebFetchConfig,
@ -1427,6 +1431,44 @@ impl Default for MultimodalConfig {
}
}
// ── Media Pipeline ──────────────────────────────────────────────
/// Automatic media understanding pipeline configuration (`[media_pipeline]`).
///
/// When enabled, inbound channel messages with media attachments are
/// pre-processed before reaching the agent: audio is transcribed, images are
/// annotated, and videos are summarised.
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct MediaPipelineConfig {
/// Master toggle for the media pipeline (default: false).
#[serde(default)]
pub enabled: bool,
/// Transcribe audio attachments using the configured transcription provider.
#[serde(default = "default_true")]
pub transcribe_audio: bool,
/// Add image descriptions when a vision-capable model is active.
#[serde(default = "default_true")]
pub describe_images: bool,
/// Summarize video attachments (placeholder — requires external API).
#[serde(default = "default_true")]
pub summarize_video: bool,
}
impl Default for MediaPipelineConfig {
fn default() -> Self {
Self {
enabled: false,
transcribe_audio: true,
describe_images: true,
summarize_video: true,
}
}
}
// ── Identity (AIEOS / OpenClaw format) ──────────────────────────
/// Identity format configuration (`[identity]` section).
@ -6786,6 +6828,7 @@ impl Default for Config {
browser_delegate: crate::tools::browser_delegate::BrowserDelegateConfig::default(),
http_request: HttpRequestConfig::default(),
multimodal: MultimodalConfig::default(),
media_pipeline: MediaPipelineConfig::default(),
web_fetch: WebFetchConfig::default(),
text_browser: TextBrowserConfig::default(),
web_search: WebSearchConfig::default(),
@ -9708,6 +9751,7 @@ default_temperature = 0.7
browser_delegate: crate::tools::browser_delegate::BrowserDelegateConfig::default(),
http_request: HttpRequestConfig::default(),
multimodal: MultimodalConfig::default(),
media_pipeline: MediaPipelineConfig::default(),
web_fetch: WebFetchConfig::default(),
text_browser: TextBrowserConfig::default(),
web_search: WebSearchConfig::default(),
@ -10089,6 +10133,7 @@ default_temperature = 0.7
browser_delegate: crate::tools::browser_delegate::BrowserDelegateConfig::default(),
http_request: HttpRequestConfig::default(),
multimodal: MultimodalConfig::default(),
media_pipeline: MediaPipelineConfig::default(),
web_fetch: WebFetchConfig::default(),
text_browser: TextBrowserConfig::default(),
web_search: WebSearchConfig::default(),

1
src/gateway/mod.rs
View File

@ -2273,6 +2273,7 @@ mod tests {
timestamp: 1,
thread_ts: None,
interruption_scope_id: None,
attachments: vec![],
};
let key = whatsapp_memory_key(&msg);

2
src/onboard/wizard.rs
View File

@ -172,6 +172,7 @@ pub async fn run_wizard(force: bool) -> Result<Config> {
browser_delegate: crate::tools::browser_delegate::BrowserDelegateConfig::default(),
http_request: crate::config::HttpRequestConfig::default(),
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
web_fetch: crate::config::WebFetchConfig::default(),
text_browser: crate::config::TextBrowserConfig::default(),
web_search: crate::config::WebSearchConfig::default(),
@ -595,6 +596,7 @@ async fn run_quick_setup_with_home(
browser_delegate: crate::tools::browser_delegate::BrowserDelegateConfig::default(),
http_request: crate::config::HttpRequestConfig::default(),
multimodal: crate::config::MultimodalConfig::default(),
media_pipeline: crate::config::MediaPipelineConfig::default(),
web_fetch: crate::config::WebFetchConfig::default(),
text_browser: crate::config::TextBrowserConfig::default(),
web_search: crate::config::WebSearchConfig::default(),