diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index d88f156..56990ff 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -33,6 +33,7 @@ from raven.agent.tools.filesystem import EditFileTool, ListDirTool, ReadFileTool, WriteFileTool from raven.agent.tools.media_gen import ( ImageGenerateTool, + MusicGenerateTool, SpeechGenerateTool, VideoGenerateTool, ) @@ -647,6 +648,7 @@ def _register_default_tools(self) -> None: (ImageGenerateTool, media.image), (SpeechGenerateTool, media.speech), (VideoGenerateTool, media.video), + (MusicGenerateTool, media.music), ) for cls, tool_cfg in media_tools: if tool_cfg.api_key or tool_cfg.model: diff --git a/raven/agent/tools/media_gen.py b/raven/agent/tools/media_gen.py index 6f64bae..a217d6e 100644 --- a/raven/agent/tools/media_gen.py +++ b/raven/agent/tools/media_gen.py @@ -25,6 +25,16 @@ ``kwaivgi/kling-v3.0-std`` (Kling v3 Standard). Requires postpaid billing / credits enabled on the OpenRouter account. +Music uses the MiniMax REST API (also not chat-completions): + +- ``music_generate`` → ``POST {base}/music_generation`` with + ``{model, prompt, lyrics, output_format, ...}`` → ``base_resp.status_code`` + ``0`` on success, ``data.status`` ``2`` when the track is ready, and + ``data.audio`` as either a hex-encoded payload (``output_format:"hex"``) or + a list of downloadable URLs (``output_format:"url"``). Default model + ``music-3.0``; cover models ``music-cover`` / ``music-cover-free`` take a + reference track via ``audio_url`` / ``audio_base64``. + Generated files are written under ``/`` and the path is returned so the agent can forward it with the ``message`` tool's ``media`` field. A denied request (HTTP 403) hints at setting ``tools.media.proxy``. @@ -584,3 +594,299 @@ async def _poll(self, client: httpx.AsyncClient, poll_url: str, headers: dict[st await asyncio.sleep(self._POLL_INTERVAL_S) waited += self._POLL_INTERVAL_S return {"status": "timeout"} + + +_MUSIC_DEFAULT_BASE = "https://api.minimax.io/v1" +_MUSIC_AUDIO_FORMATS = ("mp3", "wav", "pcm") + + +class MusicGenerateTool(Tool): + """Generate music or a cover via MiniMax's ``/music_generation`` endpoint. + + MiniMax is a dedicated REST API, not chat-completions: one synchronous POST + returns the finished track. Success is ``base_resp.status_code == 0``; + ``data.status`` is ``2`` when the track is ready. ``data.audio`` carries the + result as a hex-encoded payload (``output_format:"hex"``, the only form + ``stream`` accepts) or a list of downloadable URLs (``output_format:"url"``, + which expire after 24h so the tool downloads them immediately). Cover models + (``music-cover`` / ``music-cover-free``) take a reference track through + ``audio_url`` / ``audio_base64``. + """ + + name = "music_generate" + default_model = "music-3.0" + description = ( + "Generate music (or a cover of a reference track) from a text prompt. " + "Saves the audio under the workspace and returns its path; forward it " + "to the user with the `message` tool's `media` field." + ) + timeout_seconds = 600.0 + + def __init__( + self, + config: "MediaToolConfig | None" = None, + *, + workspace: Path | None = None, + proxy: str | None = None, + output_subdir: str = "generated", + ): + self._config = config + self._workspace = Path(workspace) if workspace else Path.cwd() + self._proxy = proxy + self._output_subdir = output_subdir + + @property + def api_key(self) -> str: + cfg_key = getattr(self._config, "api_key", "") if self._config else "" + return cfg_key or os.environ.get("MINIMAX_API_KEY", "") + + @property + def api_base(self) -> str: + cfg_base = getattr(self._config, "api_base", "") if self._config else "" + return (cfg_base or _MUSIC_DEFAULT_BASE).rstrip("/") + + def _model(self, override: str | None) -> str: + cfg_model = getattr(self._config, "model", "") if self._config else "" + return override or cfg_model or self.default_model + + def _output_path(self, ext: str) -> Path: + out_dir = self._workspace / self._output_subdir + out_dir.mkdir(parents=True, exist_ok=True) + return out_dir / f"{self.name}-{uuid.uuid4().hex[:12]}.{ext}" + + def _no_key_error(self) -> str: + return json.dumps( + { + "error": ( + "music_generate: no API key configured. Set it in " + "~/.raven/config.json under tools.media.music.apiKey or " + "providers.minimax.apiKey, or export MINIMAX_API_KEY, then " + "restart the gateway." + ) + }, + ensure_ascii=False, + ) + + def _format_http_error(self, e: httpx.HTTPStatusError) -> str: + body = e.response.text[:400] + logger.error("{} HTTP {}: {}", self.name, e.response.status_code, body) + return json.dumps({"error": f"HTTP {e.response.status_code}: {body}"}, ensure_ascii=False) + + parameters = { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "Style/mood description of the music to create, or the target cover style", + }, + "lyrics": { + "type": "string", + "description": "Song lyrics, one line per verse, with optional section tags like [Verse]/[Chorus]", + }, + "model": { + "type": "string", + "description": ( + "Optional MiniMax music model override. Generation: music-3.0 " + "(default), music-2.6, music-3.0-free, music-2.6-free. Cover: " + "music-cover, music-cover-free." + ), + }, + "output_format": { + "type": "string", + "enum": ["url", "hex"], + "default": "url", + "description": "'url' returns downloadable links (expire in 24h, downloaded here); 'hex' returns the raw audio bytes", + }, + "audio_setting": { + "type": "object", + "description": 'Audio container settings, e.g. {"format": "mp3", "sample_rate": 44100, "bitrate": 256000}', + }, + "is_instrumental": {"type": "boolean", "description": "Generate instrumental music with no vocals"}, + "lyrics_optimizer": { + "type": "boolean", + "description": "Auto-generate lyrics from the prompt when lyrics is empty", + }, + "stream": { + "type": "boolean", + "default": False, + "description": "Stream the audio; only hex output is supported", + }, + "audio_url": { + "type": "string", + "description": "Reference audio URL (cover models only); local paths are read and base64-encoded", + }, + "audio_base64": {"type": "string", "description": "Base64-encoded reference audio (cover models only)"}, + "cover_feature_id": { + "type": "string", + "description": "Feature id from the music cover preprocess API (cover models only)", + }, + "aigc_watermark": { + "type": "integer", + "description": "Required on the China (api.minimaxi.com) endpoint: 1 embeds an AIGC watermark, 0 does not", + }, + }, + "required": [], + } + + async def execute( + self, + prompt: str | None = None, + lyrics: str | None = None, + model: str | None = None, + output_format: str = "url", + audio_setting: dict[str, Any] | None = None, + is_instrumental: bool | None = None, + lyrics_optimizer: bool | None = None, + stream: bool = False, + audio_url: str | None = None, + audio_base64: str | None = None, + cover_feature_id: str | None = None, + aigc_watermark: int | None = None, + **kwargs: Any, + ) -> str: + if not self.api_key: + return self._no_key_error() + if not (prompt or lyrics or audio_url or audio_base64 or cover_feature_id): + return json.dumps( + {"error": "music_generate: provide a prompt, lyrics, or a reference audio"}, + ensure_ascii=False, + ) + + model_id = self._model(model) + payload: dict[str, Any] = {"model": model_id} + if prompt: + payload["prompt"] = prompt + if lyrics: + payload["lyrics"] = lyrics + fmt = "hex" if stream else (output_format or "url") + payload["output_format"] = fmt + if stream: + payload["stream"] = True + if audio_setting: + payload["audio_setting"] = audio_setting + if is_instrumental is not None: + payload["is_instrumental"] = bool(is_instrumental) + if lyrics_optimizer is not None: + payload["lyrics_optimizer"] = bool(lyrics_optimizer) + if cover_feature_id: + payload["cover_feature_id"] = cover_feature_id + if aigc_watermark is not None: + payload["aigc_watermark"] = int(aigc_watermark) + if audio_url: + if audio_url.startswith(("http://", "https://", "data:")): + payload["audio_url"] = audio_url + else: + try: + payload["audio_base64"] = base64.b64encode(Path(audio_url).expanduser().read_bytes()).decode( + "ascii" + ) + except OSError as e: + return json.dumps({"error": f"could not read reference audio: {e}"}, ensure_ascii=False) + elif audio_base64: + payload["audio_base64"] = audio_base64 + + headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"} + try: + async with httpx.AsyncClient(proxy=self._proxy, timeout=300.0) as client: + if stream: + audio_hex = await self._stream_hex(client, headers, payload) + resp = {"data": {"audio": audio_hex, "status": 2}, "base_resp": {"status_code": 0}} + else: + r = await client.post(f"{self.api_base}/music_generation", headers=headers, json=payload) + r.raise_for_status() + resp = r.json() + base_resp = resp.get("base_resp") or {} + if base_resp.get("status_code", 0) != 0: + return json.dumps( + { + "error": base_resp.get("status_msg") or f"status_code={base_resp.get('status_code')}", + "model": model_id, + }, + ensure_ascii=False, + ) + data = resp.get("data") or {} + audio = data.get("audio") + if not audio: + status = data.get("status") + return json.dumps( + { + "error": f"music generation status={status}" if status else "no audio returned", + "model": model_id, + }, + ensure_ascii=False, + ) + if fmt == "url": + paths = await self._download_urls(client, audio, audio_setting) + else: + paths = [str(self._save_hex(audio, audio_setting))] + except httpx.HTTPStatusError as e: + return self._format_http_error(e) + except Exception as e: + logger.error("music_generate error: {}", e) + return json.dumps({"error": str(e)}, ensure_ascii=False) + + logger.info("music_generate: {} file(s) via {} -> {}", len(paths), model_id, paths) + return json.dumps({"success": True, "model": model_id, "paths": paths}, ensure_ascii=False) + + async def _stream_hex(self, client: httpx.AsyncClient, headers: dict[str, str], payload: dict[str, Any]) -> str: + """Concatenate the hex chunks of a streamed music response. + + Each SSE ``data:`` event carries either a JSON object whose ``data.audio`` + is a hex chunk or a bare hex string; ``[DONE]`` ends the stream. + """ + hex_parts: list[str] = [] + async with client.stream("POST", f"{self.api_base}/music_generation", headers=headers, json=payload) as r: + if r.status_code >= 400: + await r.aread() + r.raise_for_status() + async for line in r.aiter_lines(): + if not line.startswith("data:"): + continue + data = line[len("data:") :].strip() + if not data or data == "[DONE]": + continue + try: + obj = json.loads(data) + except json.JSONDecodeError: + hex_parts.append(data) + continue + if isinstance(obj, dict): + audio = (obj.get("data") or {}).get("audio") + if audio: + hex_parts.append(audio) + return "".join(hex_parts) + + async def _download_urls( + self, + client: httpx.AsyncClient, + audio: Any, + audio_setting: dict[str, Any] | None, + ) -> list[str]: + """Download each audio URL (pre-signed, no auth) and save it locally.""" + urls = audio if isinstance(audio, list) else [audio] + paths: list[str] = [] + for url in urls: + if not isinstance(url, str) or not url.startswith(("http://", "https://")): + continue + r = await client.get(url, timeout=180.0) + r.raise_for_status() + path = self._output_path(self._ext_for_url(url, audio_setting)) + path.write_bytes(r.content) + paths.append(str(path)) + if not paths: + raise ValueError("no downloadable audio URLs in response") + return paths + + def _ext_for_url(self, url: str, audio_setting: dict[str, Any] | None) -> str: + suffix = Path(url.split("?", 1)[0]).suffix.lstrip(".").lower() + if suffix in _MUSIC_AUDIO_FORMATS: + return suffix + fmt = (audio_setting or {}).get("format", "mp3") + return fmt if fmt in _MUSIC_AUDIO_FORMATS else "mp3" + + def _save_hex(self, audio_hex: str, audio_setting: dict[str, Any] | None) -> Path: + fmt = (audio_setting or {}).get("format", "mp3") + fmt = fmt if fmt in _MUSIC_AUDIO_FORMATS else "mp3" + path = self._output_path(fmt) + path.write_bytes(bytes.fromhex(audio_hex)) + return path diff --git a/raven/config/schema.py b/raven/config/schema.py index b4cefb3..3c5cbeb 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -630,26 +630,29 @@ class ExecToolConfig(Base): class MediaToolConfig(Base): """Config for a media-generation tool (key + base + model). - Empty fields fall back at call time: ``api_key`` → ``providers.openrouter`` - / ``OPENROUTER_API_KEY``; ``api_base`` → OpenRouter; ``model`` → the tool's - default (Nano Banana for images). + Empty fields fall back at call time: ``api_key`` → the backend's provider + config / env var (OpenRouter for image/speech/video, MiniMax for music); + ``api_base`` → the backend default; ``model`` → the tool's default. """ api_key: str = "" - api_base: str = "" # defaults to https://openrouter.ai/api/v1 + api_base: str = "" # backend base URL, e.g. https://openrouter.ai/api/v1 model: str = "" class MediaGenConfig(Base): """Multimodal generation tools configuration. - OpenRouter is the only backend: image + speech via chat-completions output - modalities, and video via the async ``/videos`` endpoint (Kling). + Image + speech go through OpenRouter chat-completions output modalities, + video through the async OpenRouter ``/videos`` endpoint (Kling), and music + through the MiniMax ``/music_generation`` endpoint (``api.minimax.io`` + global, ``api.minimaxi.com`` for mainland China). """ image: MediaToolConfig = Field(default_factory=MediaToolConfig) speech: MediaToolConfig = Field(default_factory=MediaToolConfig) video: MediaToolConfig = Field(default_factory=MediaToolConfig) + music: MediaToolConfig = Field(default_factory=MediaToolConfig) proxy: str | None = None # HTTP/SOCKS proxy for media API calls output_subdir: str = "generated" # where generated files are written under workspace @@ -755,10 +758,14 @@ def effective_media_config(self) -> MediaGenConfig: media = self.tools.media.model_copy(deep=True) openrouter = self.providers.get("openrouter") or_key = openrouter.api_key if openrouter else "" + minimax = self.providers.get("minimax") + mm_key = minimax.api_key if minimax else "" for tool in (media.image, media.speech, media.video): configured = bool(tool.api_key or tool.model) if configured and or_key and not tool.api_key: tool.api_key = or_key + if (media.music.api_key or media.music.model) and mm_key and not media.music.api_key: + media.music.api_key = mm_key return media def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | None", str | None]: diff --git a/tests/test_media_gen_music.py b/tests/test_media_gen_music.py new file mode 100644 index 0000000..761e280 --- /dev/null +++ b/tests/test_media_gen_music.py @@ -0,0 +1,183 @@ +"""Unit tests for the MiniMax music_generate tool. + +Hermetic: an httpx.MockTransport feeds canned responses for the +``/music_generation`` POST and the audio downloads, so no real network or key +is touched. Covers the url/hex output formats, cover reference audio, streamed +hex, and error response parsing (``base_resp.status_code`` / ``data.status``). +""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import httpx +import pytest + +from raven.agent.tools import media_gen as mg +from raven.agent.tools.media_gen import MusicGenerateTool +from raven.config.schema import MediaToolConfig + + +def _patch(monkeypatch, handler) -> None: + real_client = httpx.AsyncClient + + def factory(*args, **kwargs): + return real_client(transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(mg.httpx, "AsyncClient", factory) + + +@pytest.fixture +def tool(tmp_path: Path) -> MusicGenerateTool: + return MusicGenerateTool(MediaToolConfig(api_key="mk-test"), workspace=tmp_path) + + +async def test_url_output_downloads_audio(tool: MusicGenerateTool, monkeypatch) -> None: + mp3_bytes = b"ID3\x00\x00fake mp3" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/music_generation"): + assert request.url.host == "api.minimax.io" + body = json.loads(request.content) + assert body["model"] == "music-3.0" + assert body["output_format"] == "url" + return httpx.Response( + 200, + json={ + "base_resp": {"status_code": 0}, + "data": {"status": 2, "audio": ["https://cdn.example.com/track.mp3"]}, + }, + ) + if "cdn.example.com" in str(request.url): + return httpx.Response(200, content=mp3_bytes) + return httpx.Response(404) + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="indie folk")) + + assert result["success"] is True + assert result["model"] == "music-3.0" + path = Path(result["paths"][0]) + assert path.read_bytes() == mp3_bytes + assert path.suffix == ".mp3" + + +async def test_hex_output_decodes_audio(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/music_generation") + body = json.loads(request.content) + assert body["output_format"] == "hex" + assert body["audio_setting"] == {"format": "mp3"} + return httpx.Response(200, json={"base_resp": {"status_code": 0}, "data": {"status": 2, "audio": "ffd8ff"}}) + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="lofi", output_format="hex", audio_setting={"format": "mp3"})) + + assert result["success"] is True + assert Path(result["paths"][0]).read_bytes() == bytes.fromhex("ffd8ff") + + +async def test_default_region_endpoint_is_global(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.host == "api.minimax.io" + return httpx.Response(200, json={"base_resp": {"status_code": 0}, "data": {"status": 2, "audio": "abcd"}}) + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="ambient", output_format="hex")) + assert result["success"] is True + + +async def test_cn_region_endpoint_from_api_base(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.host == "api.minimaxi.com" + body = json.loads(request.content) + assert body["aigc_watermark"] == 1 + return httpx.Response(200, json={"base_resp": {"status_code": 0}, "data": {"status": 2, "audio": "abcd"}}) + + _patch(monkeypatch, handler) + tool._config = MediaToolConfig(api_key="mk-test", api_base="https://api.minimaxi.com/v1") + result = json.loads(await tool.execute(prompt="ambient", output_format="hex", aigc_watermark=1)) + assert result["success"] is True + + +async def test_cover_model_accepts_local_reference_audio(tool: MusicGenerateTool, monkeypatch, tmp_path: Path) -> None: + ref = tmp_path / "ref.mp3" + ref.write_bytes(b"\x00\x01\x02") + seen: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json={"base_resp": {"status_code": 0}, "data": {"status": 2, "audio": "abcd"}}) + + _patch(monkeypatch, handler) + result = json.loads( + await tool.execute(model="music-cover", prompt="rock cover", audio_url=str(ref), output_format="hex") + ) + + assert seen["model"] == "music-cover" + assert seen.get("audio_url") is None + assert base64.b64decode(seen["audio_base64"]) == b"\x00\x01\x02" + assert result["success"] is True + + +async def test_stream_hex_concatenates_chunks(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["stream"] is True + assert body["output_format"] == "hex" + events = ( + 'data: {"data": {"status": 1, "audio": "ff"}}\n\n' + 'data: {"data": {"status": 1, "audio": "d8"}}\n\n' + "data: [DONE]\n\n" + ) + return httpx.Response(200, content=events.encode()) + + _patch(monkeypatch, handler) + # output_format is forced to hex when stream is set. + result = json.loads(await tool.execute(prompt="synthwave", stream=True, output_format="url")) + + assert result["success"] is True + assert Path(result["paths"][0]).read_bytes() == bytes.fromhex("ffd8") + + +async def test_api_error_status_code(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, json={"base_resp": {"status_code": 1004, "status_msg": "Authentication failed"}, "data": {}} + ) + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="x")) + assert result["error"] == "Authentication failed" + + +async def test_in_progress_status_reports_no_audio(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"base_resp": {"status_code": 0}, "data": {"status": 1}}) + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="x")) + assert result["error"] == "music generation status=1" + + +async def test_http_error_is_formatted(tool: MusicGenerateTool, monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="upstream boom") + + _patch(monkeypatch, handler) + result = json.loads(await tool.execute(prompt="x")) + assert "HTTP 500" in result["error"] + + +async def test_no_key_error(tool: MusicGenerateTool, monkeypatch) -> None: + tool._config = MediaToolConfig() + monkeypatch.delenv("MINIMAX_API_KEY", raising=False) + result = json.loads(await tool.execute(prompt="x")) + assert "no API key" in result["error"] + + +async def test_requires_some_input(tool: MusicGenerateTool) -> None: + result = json.loads(await tool.execute()) + assert "provide a prompt, lyrics, or a reference audio" in result["error"] diff --git a/tests/test_tool_registry_timeout.py b/tests/test_tool_registry_timeout.py index 4db47b0..15ecd45 100644 --- a/tests/test_tool_registry_timeout.py +++ b/tests/test_tool_registry_timeout.py @@ -95,12 +95,13 @@ async def execute(self, **kwargs) -> str: @pytest.mark.asyncio async def test_long_running_tools_keep_generous_ceilings(): # Guard against regressing the overrides on the genuinely-slow tools. - from raven.agent.tools.media_gen import VideoGenerateTool + from raven.agent.tools.media_gen import MusicGenerateTool, VideoGenerateTool from raven.agent.tools.shell import ExecTool from raven.agent.tools.spawn import SpawnTool assert ExecTool.timeout_seconds >= 600 assert VideoGenerateTool.timeout_seconds >= 600 + assert MusicGenerateTool.timeout_seconds >= 600 assert SpawnTool.timeout_seconds >= 600 # Default-class tools inherit None -> registry default applies. assert Tool.timeout_seconds is None