diff --git a/tinyflow-core/src/main/java/dev/tinyflow/core/llm/Llm.java b/tinyflow-core/src/main/java/dev/tinyflow/core/llm/Llm.java index 2bb22e6f9e71a95cfdc6d47fd992f39ef8d44bd1..fcce091ae14477755fb24caab6eee1168ea4a32b 100644 --- a/tinyflow-core/src/main/java/dev/tinyflow/core/llm/Llm.java +++ b/tinyflow-core/src/main/java/dev/tinyflow/core/llm/Llm.java @@ -50,6 +50,10 @@ public interface Llm { private String message; private String systemMessage; private List images; + /** + * 视频 URL 列表，对应多模态消息中的 video_url 类型 + */ + private List videos; public String getMessage() { return message; @@ -74,6 +78,14 @@ public interface Llm { public void setImages(List images) { this.images = images; } + + public List getVideos() { + return videos; + } + + public void setVideos(List videos) { + this.videos = videos; + } } /** diff --git a/tinyflow-core/src/main/java/dev/tinyflow/core/node/LlmNode.java b/tinyflow-core/src/main/java/dev/tinyflow/core/node/LlmNode.java index 2093e1ef6de664938401b9c0fbc7f53ba94dd1e5..477657cfa2993935208d9c50a9997421171db6a0 100644 --- a/tinyflow-core/src/main/java/dev/tinyflow/core/node/LlmNode.java +++ b/tinyflow-core/src/main/java/dev/tinyflow/core/node/LlmNode.java @@ -34,6 +34,10 @@ public class LlmNode extends BaseNode { protected String jsonSchema; protected String outType = "text"; //text markdown json protected List images; + /** + * 视频识别输入参数，运行时解析为 video_url 多模态内容 + */ + protected List videos; public LlmNode() { } @@ -94,6 +98,14 @@ public class LlmNode extends BaseNode { this.images = images; } + public List getVideos() { + return videos; + } + + public void setVideos(List videos) { + this.videos = videos; + } + @Override public Map execute(Chain chain) { Map formatParameters = getFormatParameters(chain); @@ -116,21 +128,15 @@ public class LlmNode extends BaseNode { messageInfo.setMessage(userPromptString); messageInfo.setSystemMessage(systemPromptString); - if (images != null && !images.isEmpty()) { - Map filesMap = chain.getState().resolveParameters(this, images); - List imagesUrls = new ArrayList<>(); - filesMap.forEach((s, o) -> { - if (o instanceof String) { - imagesUrls.add((String) o); - } else if (o instanceof File) { - byte[] bytes = IOUtil.readBytes((File) o); - String base64 = Base64.getEncoder().encodeToString(bytes); - imagesUrls.add(base64); - } - }); - messageInfo.setImages(imagesUrls); + List imageUrls = resolveMediaUrls(chain, images); + if (imageUrls != null && !imageUrls.isEmpty()) { + messageInfo.setImages(imageUrls); } + List videoUrls = resolveMediaUrls(chain, videos); + if (videoUrls != null && !videoUrls.isEmpty()) { + messageInfo.setVideos(videoUrls); + } String responseContent = llm.chat(messageInfo, chatOptions, this, chain); @@ -166,6 +172,27 @@ public class LlmNode extends BaseNode { } + /** + * 将图片/视频参数解析为 URL 或 Base64 字符串列表 + */ + private List resolveMediaUrls(Chain chain, List mediaParameters) { + if (mediaParameters == null || mediaParameters.isEmpty()) { + return null; + } + Map filesMap = chain.getState().resolveParameters(this, mediaParameters); + List mediaUrls = new ArrayList<>(); + filesMap.forEach((s, o) -> { + if (o instanceof String) { + mediaUrls.add((String) o); + } else if (o instanceof File) { + byte[] bytes = IOUtil.readBytes((File) o); + String base64 = Base64.getEncoder().encodeToString(bytes); + mediaUrls.add(base64); + } + }); + return mediaUrls; + } + /** * 移除 ``` 或者 ```json 等 * @@ -201,6 +228,7 @@ public class LlmNode extends BaseNode { ", systemPrompt='" + systemPrompt + '\'' + ", outType='" + outType + '\'' + ", images=" + images + + ", videos=" + videos + ", parameters=" + parameters + ", outputDefs=" + outputDefs + ", id='" + id + '\'' + diff --git a/tinyflow-core/src/main/java/dev/tinyflow/core/parser/impl/LlmNodeParser.java b/tinyflow-core/src/main/java/dev/tinyflow/core/parser/impl/LlmNodeParser.java index 831760bf47c9920a6c94a1f50a2082f600d10e00..89dc2353d67cd272729efabfcc24a0ae85422505 100644 --- a/tinyflow-core/src/main/java/dev/tinyflow/core/parser/impl/LlmNodeParser.java +++ b/tinyflow-core/src/main/java/dev/tinyflow/core/parser/impl/LlmNodeParser.java @@ -36,6 +36,10 @@ public class LlmNodeParser extends BaseNodeParser { List images = getParameters(data, "images"); llmNode.setImages(images); + // 支持视频识别输入（video_url） + List videos = getParameters(data, "videos"); + llmNode.setVideos(videos); + return llmNode; } } diff --git a/tinyflow-core/src/test/java/dev/tinyflow/core/test/LlmNodeVideoTest.java b/tinyflow-core/src/test/java/dev/tinyflow/core/test/LlmNodeVideoTest.java new file mode 100644 index 0000000000000000000000000000000000000000..e302b9a175451edcf1802def1c02d6ba2f93aa5a --- /dev/null +++ b/tinyflow-core/src/test/java/dev/tinyflow/core/test/LlmNodeVideoTest.java @@ -0,0 +1,302 @@ +package dev.tinyflow.core.test; + +import dev.tinyflow.core.chain.Chain; +import dev.tinyflow.core.chain.ChainDefinition; +import dev.tinyflow.core.chain.Parameter; +import dev.tinyflow.core.chain.repository.ChainDefinitionRepository; +import dev.tinyflow.core.chain.repository.InMemoryChainStateRepository; +import dev.tinyflow.core.chain.repository.InMemoryNodeStateRepository; +import dev.tinyflow.core.chain.runtime.ChainExecutor; +import dev.tinyflow.core.llm.Llm; +import dev.tinyflow.core.llm.LlmManager; +import dev.tinyflow.core.llm.LlmProvider; +import dev.tinyflow.core.node.LlmNode; +import dev.tinyflow.core.parser.ChainParser; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 大模型节点视频识别（video_url）测试。 + *

+ * 工作流 JSON 来自前端 demo 导出的 data，llmNode.data.videos 为 List<Parameter>。 + */ +public class LlmNodeVideoTest { + + private static final String LLM_NODE_ID = "node_x4SxWFcJ6Z9AbdPm"; + private static final String START_NODE_ID = "node_xmHgVB6Bm9QMzSAK"; + private static final String END_NODE_ID = "node_rzhnGQRbH3TTZGwL"; + private static final String TEST_LLM_ID = "test-llm"; + + /** + * 示例视频地址（Qwen 官方文档常用样例） + */ + private static final String TEST_VIDEO_URL = + "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-VL/space_woaudio.mp4"; + + private static final CapturingLlm CAPTURING_LLM = new CapturingLlm(); + + /** + * 前端 demo 导出的工作流 JSON。 + * start(传入 videoUrl) -> llmNode(引用 videoUrl) -> end + *

+ * 补充 llmId、userPrompt 仅为满足 LlmNode 执行所需，不影响 videos 参数结构。 + */ + private static final String WORKFLOW_JSON = + "{\n" + + " \"nodes\": [\n" + + " {\n" + + " \"id\": \"node_x4SxWFcJ6Z9AbdPm\",\n" + + " \"position\": {\n" + + " \"x\": 338,\n" + + " \"y\": -10\n" + + " },\n" + + " \"data\": {\n" + + " \"title\": \"大模型\",\n" + + " \"description\": \"使用大模型处理问题\",\n" + + " \"llmId\": \"test-llm\",\n" + + " \"userPrompt\": \"请描述这段视频的内容\",\n" + + " \"outType\": \"text\",\n" + + " \"outputDefs\": [\n" + + " {\n" + + " \"name\": \"output\",\n" + + " \"dataType\": \"String\",\n" + + " \"dataTypeDisabled\": true,\n" + + " \"deleteDisabled\": true\n" + + " }\n" + + " ],\n" + + " \"expand\": true,\n" + + " \"videos\": [\n" + + " {\n" + + " \"name\": \"videoUrl\",\n" + + " \"refType\": \"ref\",\n" + + " \"id\": \"7sEVjzeLDjAgZaUf\",\n" + + " \"ref\": \"node_xmHgVB6Bm9QMzSAK.videoUrl\"\n" + + " }\n" + + " ],\n" + + " \"images\": []\n" + + " },\n" + + " \"type\": \"llmNode\",\n" + + " \"selected\": false,\n" + + " \"measured\": {\n" + + " \"width\": 385,\n" + + " \"height\": 815\n" + + " },\n" + + " \"dragging\": false\n" + + " },\n" + + " {\n" + + " \"id\": \"node_xmHgVB6Bm9QMzSAK\",\n" + + " \"position\": {\n" + + " \"x\": -130,\n" + + " \"y\": -5\n" + + " },\n" + + " \"data\": {\n" + + " \"title\": \"开始节点\",\n" + + " \"description\": \"开始定义输入参数\",\n" + + " \"expand\": true,\n" + + " \"parameters\": [\n" + + " {\n" + + " \"name\": \"videoUrl\",\n" + + " \"refType\": \"input\",\n" + + " \"dataType\": \"String\",\n" + + " \"id\": \"BWBkbGxHEAAoTYsu\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"type\": \"startNode\",\n" + + " \"selected\": false,\n" + + " \"measured\": {\n" + + " \"width\": 305,\n" + + " \"height\": 211\n" + + " },\n" + + " \"dragging\": false\n" + + " },\n" + + " {\n" + + " \"id\": \"node_rzhnGQRbH3TTZGwL\",\n" + + " \"position\": {\n" + + " \"x\": 904,\n" + + " \"y\": 14\n" + + " },\n" + + " \"data\": {\n" + + " \"title\": \"结束节点\",\n" + + " \"description\": \"结束定义输出参数\",\n" + + " \"expand\": true,\n" + + " \"outputDefs\": [\n" + + " {\n" + + " \"name\": \"output\",\n" + + " \"refType\": \"ref\",\n" + + " \"id\": \"2UddkIAxGZ1eGKfN\",\n" + + " \"ref\": \"node_x4SxWFcJ6Z9AbdPm.output\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"type\": \"endNode\",\n" + + " \"selected\": false,\n" + + " \"measured\": {\n" + + " \"width\": 388,\n" + + " \"height\": 211\n" + + " },\n" + + " \"dragging\": false\n" + + " }\n" + + " ],\n" + + " \"edges\": [\n" + + " {\n" + + " \"source\": \"node_xmHgVB6Bm9QMzSAK\",\n" + + " \"target\": \"node_x4SxWFcJ6Z9AbdPm\",\n" + + " \"id\": \"iOuqIfd4Lmw6oUhw\"\n" + + " },\n" + + " {\n" + + " \"source\": \"node_x4SxWFcJ6Z9AbdPm\",\n" + + " \"target\": \"node_rzhnGQRbH3TTZGwL\",\n" + + " \"id\": \"2EKpG86r8dJPVnlW\"\n" + + " }\n" + + " ],\n" + + " \"viewport\": {\n" + + " \"x\": 429.70804962858574,\n" + + " \"y\": 155.26548373932576,\n" + + " \"zoom\": 0.8705505632961246\n" + + " }\n" + + "}"; + + public static void main(String[] args) { + registerMockLlmOnce(); + testParseVideosFromJson(); + testExecuteWithRefVideo(); + testExecuteFullWorkflow(); + System.out.println("LlmNodeVideoTest: all passed."); + } + + /** + * 验证 LlmNodeParser 能正确解析前端导出的 videos 参数 + */ + private static void testParseVideosFromJson() { + LlmNode llmNode = parseLlmNode(WORKFLOW_JSON); + + List videos = llmNode.getVideos(); + assertNotNull(videos, "videos should not be null"); + assertEquals(1, videos.size(), "videos size"); + assertEquals("videoUrl", videos.get(0).getName(), "video parameter name"); + assertEquals(START_NODE_ID + ".videoUrl", videos.get(0).getRef(), "video parameter ref"); + assertEquals("test-llm", llmNode.getLlmId(), "llmId"); + assertEquals("请描述这段视频的内容", llmNode.getUserPrompt(), "userPrompt"); + + System.out.println("[OK] testParseVideosFromJson"); + } + + /** + * 验证通过 start 节点引用传入的视频 URL 能传递到 LLM MessageInfo.videos + */ + private static void testExecuteWithRefVideo() { + CAPTURING_LLM.reset(); + + Map variables = new HashMap<>(); + variables.put("videoUrl", TEST_VIDEO_URL); + + Map result = executeWorkflow(WORKFLOW_JSON, variables); + + assertNotNull(CAPTURING_LLM.lastMessageInfo, "messageInfo"); + List videos = CAPTURING_LLM.lastMessageInfo.getVideos(); + assertNotNull(videos, "messageInfo.videos"); + assertEquals(1, videos.size(), "messageInfo.videos size"); + assertEquals(TEST_VIDEO_URL, videos.get(0), "messageInfo video url"); + assertEquals("请描述这段视频的内容", CAPTURING_LLM.lastMessageInfo.getMessage(), "user prompt"); + assertEquals("video analyzed", result.get("output"), "end node output"); + + System.out.println("[OK] testExecuteWithRefVideo, videos=" + videos); + } + + /** + * 验证完整工作流节点 ID 与连线解析正确 + */ + private static void testExecuteFullWorkflow() { + ChainDefinition definition = parseWorkflow(WORKFLOW_JSON); + + assertNotNull(definition.getNodeById(START_NODE_ID), "start node"); + assertNotNull(definition.getNodeById(LLM_NODE_ID), "llm node"); + assertNotNull(definition.getNodeById(END_NODE_ID), "end node"); + assertEquals(2, definition.getEdges().size(), "edge count"); + + System.out.println("[OK] testExecuteFullWorkflow"); + } + + private static LlmNode parseLlmNode(String workflowJson) { + return (LlmNode) parseWorkflow(workflowJson).getNodeById(LLM_NODE_ID); + } + + private static ChainDefinition parseWorkflow(String workflowJson) { + ChainParser chainParser = ChainParser.builder() + .withDefaultParsers(true) + .build(); + return chainParser.parse(workflowJson); + } + + private static Map executeWorkflow(String workflowJson, Map variables) { + ChainParser chainParser = ChainParser.builder() + .withDefaultParsers(true) + .build(); + + ChainExecutor executor = new ChainExecutor( + new ChainDefinitionRepository() { + @Override + public ChainDefinition getChainDefinitionById(String id) { + ChainDefinition definition = chainParser.parse(workflowJson); + definition.setId(id); + return definition; + } + }, + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository() + ); + + return executor.execute("llm-video-test", variables); + } + + private static void registerMockLlmOnce() { + LlmProvider provider = new LlmProvider() { + @Override + public Llm getChatModel(Object modelId) { + if (TEST_LLM_ID.equals(String.valueOf(modelId))) { + return CAPTURING_LLM; + } + return null; + } + }; + LlmManager.getInstance().registerProvider(provider); + } + + /** + * 用于捕获 LlmNode 执行时传入的 MessageInfo，验证 videos 是否生效 + */ + private static class CapturingLlm implements Llm { + private Llm.MessageInfo lastMessageInfo; + + void reset() { + this.lastMessageInfo = null; + } + + @Override + public String chat(MessageInfo messageInfo, ChatOptions options, dev.tinyflow.core.node.LlmNode llmNode, Chain chain) { + this.lastMessageInfo = messageInfo; + return "video analyzed"; + } + } + + private static void assertNotNull(Object value, String name) { + if (value == null) { + throw new AssertionError(name + " should not be null"); + } + } + + private static void assertEquals(Object expected, Object actual, String name) { + if (expected == null ? actual != null : !expected.equals(actual)) { + throw new AssertionError(name + " expected: " + expected + ", actual: " + actual); + } + } + + private static void assertEquals(int expected, int actual, String name) { + if (expected != actual) { + throw new AssertionError(name + " expected: " + expected + ", actual: " + actual); + } + } +} diff --git a/tinyflow-support-agentsflex/src/main/java/dev/tinyflow/agentsflex/provider/AgentsFlexLlm.java b/tinyflow-support-agentsflex/src/main/java/dev/tinyflow/agentsflex/provider/AgentsFlexLlm.java index 51fe7802e9ce3754ff8e236f637dae4670dd6ce0..e7498647a29bbc8c01cbb105ca6504542aaf548f 100644 --- a/tinyflow-support-agentsflex/src/main/java/dev/tinyflow/agentsflex/provider/AgentsFlexLlm.java +++ b/tinyflow-support-agentsflex/src/main/java/dev/tinyflow/agentsflex/provider/AgentsFlexLlm.java @@ -53,6 +53,13 @@ public class AgentsFlexLlm implements Llm { } } + // 视频（video_url） + List videos = messageInfo.getVideos(); + if (videos != null && !videos.isEmpty()) { + for (String video : videos) { + prompt.addVideoUrl(video); + } + } com.agentsflex.core.model.chat.ChatOptions chatOptions = new com.agentsflex.core.model.chat.ChatOptions(); chatOptions.setSeed(options.getSeed()); diff --git a/tinyflow-support-langchain4j/src/main/java/dev/tinyflow/langchain4j/provider/Langchain4jLlm.java b/tinyflow-support-langchain4j/src/main/java/dev/tinyflow/langchain4j/provider/Langchain4jLlm.java index 305fef941e31684b9552bee1a1f6c8b859b2f94d..21e673861d0a02af2fcc8f1f3b837b6ab5cc6627 100644 --- a/tinyflow-support-langchain4j/src/main/java/dev/tinyflow/langchain4j/provider/Langchain4jLlm.java +++ b/tinyflow-support-langchain4j/src/main/java/dev/tinyflow/langchain4j/provider/Langchain4jLlm.java @@ -43,6 +43,13 @@ public class Langchain4jLlm implements Llm { userContents.add(ImageContent.from(imageUrl)); } } + // 添加视频内容（video_url） + List videos = messageInfo.getVideos(); + if (videos != null && !videos.isEmpty()) { + for (String videoUrl : videos) { + userContents.add(VideoContent.from(videoUrl)); + } + } // 添加用户消息 chatMessages.add(UserMessage.from(userContents)); diff --git a/tinyflow-support-solonai/src/main/java/dev/tinyflow/solon/provider/SolonAiLlm.java b/tinyflow-support-solonai/src/main/java/dev/tinyflow/solon/provider/SolonAiLlm.java index c39ffa3849401bbaa0e1215fb04ecf7e6fc3d274..049e3d77eba2234c407d53c6360c1165cc696eaf 100644 --- a/tinyflow-support-solonai/src/main/java/dev/tinyflow/solon/provider/SolonAiLlm.java +++ b/tinyflow-support-solonai/src/main/java/dev/tinyflow/solon/provider/SolonAiLlm.java @@ -9,6 +9,7 @@ import org.noear.solon.ai.chat.ChatResponse; import org.noear.solon.ai.chat.message.ChatMessage; import org.noear.solon.ai.chat.prompt.Prompt; import org.noear.solon.ai.media.Image; +import org.noear.solon.ai.media.Video; import org.noear.solon.core.util.Assert; import java.util.List; @@ -50,6 +51,14 @@ public class SolonAiLlm implements Llm { } } + // 用户视频（video_url） + List videoUrls = messageInfo.getVideos(); + if (Assert.isNotEmpty(videoUrls)) { + for (String url : videoUrls) { + prompt.addMessage(ChatMessage.ofUser(Video.ofUrl(url))); + } + } + ChatResponse response = null; try { diff --git a/tinyflow-support-springai/src/main/java/dev/tinyflow/springai/provider/SpringAiLlm.java b/tinyflow-support-springai/src/main/java/dev/tinyflow/springai/provider/SpringAiLlm.java index 61c2c0d09557809d19c076a8c0909336f6fcce1c..22f6a9b9f1b1b9cf39ec2847869acc12b971ad5c 100644 --- a/tinyflow-support-springai/src/main/java/dev/tinyflow/springai/provider/SpringAiLlm.java +++ b/tinyflow-support-springai/src/main/java/dev/tinyflow/springai/provider/SpringAiLlm.java @@ -38,15 +38,30 @@ public class SpringAiLlm implements Llm { } List images = messageInfo.getImages(); - if (images != null && !images.isEmpty()) { + List videos = messageInfo.getVideos(); + boolean hasImages = images != null && !images.isEmpty(); + boolean hasVideos = videos != null && !videos.isEmpty(); + + if (hasImages || hasVideos) { StringBuilder contentBuilder = new StringBuilder(); if (StringUtils.hasText(messageInfo.getMessage())) { contentBuilder.append(messageInfo.getMessage()).append("\n"); } - contentBuilder.append("包含以下图片："); - for (String imageUrl : images) { - contentBuilder.append("\n").append(imageUrl); + if (hasImages) { + contentBuilder.append("包含以下图片："); + for (String imageUrl : images) { + contentBuilder.append("\n").append(imageUrl); + } + } + if (hasVideos) { + if (hasImages) { + contentBuilder.append("\n"); + } + contentBuilder.append("包含以下视频："); + for (String videoUrl : videos) { + contentBuilder.append("\n").append(videoUrl); + } } messages.add(new UserMessage(contentBuilder.toString())); } else {