|
|
@@ -13,20 +13,35 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
|
|
import java.io.IOException;
|
|
|
import java.nio.file.Path;
|
|
|
import java.util.ArrayList;
|
|
|
-import java.util.Collections;
|
|
|
import java.util.HashSet;
|
|
|
import java.util.LinkedHashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
import java.util.Set;
|
|
|
import java.util.concurrent.CompletableFuture;
|
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|
|
+import java.util.concurrent.ExecutionException;
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
import java.util.concurrent.Executors;
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.concurrent.TimeoutException;
|
|
|
+import java.util.concurrent.atomic.AtomicBoolean;
|
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|
|
+import java.util.concurrent.atomic.AtomicReference;
|
|
|
|
|
|
/**
|
|
|
- * 按 DAG 拓扑层级逐层执行节点。
|
|
|
- * 从 WorkflowEngine.execute() 抽取,承担节点遍历、状态推进与失败策略处理。
|
|
|
+ * 按 DAG 拓扑执行节点(Dataflow 模型)。
|
|
|
+ *
|
|
|
+ * <p>调度规则:节点所有前驱完成后立即被调度,不等同层兄弟节点。
|
|
|
+ * 通过 {@link CompletableFuture#whenComplete} 回调链驱动后继节点;
|
|
|
+ * 不再有"层"barrier,慢节点不再阻塞快节点的下游链路。</p>
|
|
|
+ *
|
|
|
+ * <p>线程模型:</p>
|
|
|
+ * <ul>
|
|
|
+ * <li>{@link #nodeExecutor}:节点 worker 线程池,{@link #executeOneNode} 在此执行</li>
|
|
|
+ * <li>回调链(onNodeCompleted / tryScheduleSuccessors)也在 nodeExecutor 中执行</li>
|
|
|
+ * <li>主线程仅在 {@code ctx.done.get(...)} 上等待</li>
|
|
|
+ * </ul>
|
|
|
*/
|
|
|
@Slf4j
|
|
|
@Component
|
|
|
@@ -37,12 +52,15 @@ public class WorkflowLevelExecutor {
|
|
|
private static final ObjectMapper MAPPER = new ObjectMapper()
|
|
|
.registerModule(new JavaTimeModule());
|
|
|
|
|
|
+ /** dataflow 调度主线程等待超时(兜底,避免死锁时永久阻塞) */
|
|
|
+ private static final long EXECUTION_TIMEOUT_MINUTES = 30L;
|
|
|
+
|
|
|
private final Map<String, NodeExecutor> executorMap;
|
|
|
private final SseEventBus eventBus;
|
|
|
private final NodeWorkspaceBuilder workspaceBuilder;
|
|
|
/**
|
|
|
- * 同层节点并行执行的线程池。独立于 WorkflowEngine.executor(工作流主线程池),
|
|
|
- * 避免层内并行与层间串行争用同一池导致死锁。
|
|
|
+ * 节点执行的线程池。独立于 WorkflowEngine.executor(工作流主线程池),
|
|
|
+ * 避免节点任务与外层 wrapper 争用同一池导致死锁。
|
|
|
*/
|
|
|
private final ExecutorService nodeExecutor = Executors.newFixedThreadPool(
|
|
|
Math.max(8, Runtime.getRuntime().availableProcessors() * 2),
|
|
|
@@ -100,7 +118,68 @@ public class WorkflowLevelExecutor {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 按层执行 DAG。
|
|
|
+ * Dataflow 调度的共享上下文。
|
|
|
+ *
|
|
|
+ * <p>所有并发访问的可变状态集中在此对象,便于审计与生命周期管理。
|
|
|
+ * dag / context / runId / runRecordId / emitter 为不可变引用,回调链中通过 ctx.dag / ctx.context 直接访问。</p>
|
|
|
+ */
|
|
|
+ private static final class DataflowCtx {
|
|
|
+ final DagResolver.ResolvedDag dag;
|
|
|
+ final WorkflowContext context;
|
|
|
+ final String runId;
|
|
|
+ final Long runRecordId;
|
|
|
+ final SseEmitter emitter;
|
|
|
+
|
|
|
+ /** 活跃节点集合(待执行或执行中);非活跃节点入度归零后会被标记 SKIPPED */
|
|
|
+ final Set<String> activeNodes = ConcurrentHashMap.newKeySet();
|
|
|
+ /** 活跃边集合(条件分支按选中 sourceHandle 过滤后实际生效的边) */
|
|
|
+ final Set<String> activeEdges = ConcurrentHashMap.newKeySet();
|
|
|
+ /** 节点的剩余入度(按 source 去重,与 DagResolver 入度规则一致);归零则可调度 */
|
|
|
+ final Map<String, AtomicInteger> pendingInputs = new ConcurrentHashMap<>();
|
|
|
+ /** nodeId → 在飞 future;abort 时用于 cancel */
|
|
|
+ final Map<String, CompletableFuture<NodeExecutionResult>> inFlightFutures = new ConcurrentHashMap<>();
|
|
|
+ /** 已完成(含 SKIPPED)的节点集合;用于去重 onNodeCompleted 与 triggerAbort 之间的双重处理 */
|
|
|
+ final Set<String> completedNodes = ConcurrentHashMap.newKeySet();
|
|
|
+
|
|
|
+ /** 剩余未完成节点数;归零时唤醒主线程(complete done) */
|
|
|
+ final AtomicInteger remainingNodes;
|
|
|
+ /** abort 标志(CAS 去重,保证 workflow_error 只推一次) */
|
|
|
+ final AtomicBoolean abortFlag = new AtomicBoolean(false);
|
|
|
+ /** abort 错误信息(首个失败的 abort 节点写入) */
|
|
|
+ final AtomicReference<String> abortMsgRef = new AtomicReference<>(null);
|
|
|
+ /** 全局递增的 sortOrder(按完成顺序赋值) */
|
|
|
+ final AtomicInteger sortOrderSeq = new AtomicInteger(0);
|
|
|
+
|
|
|
+ /** 最终输出(output 节点完成时写入);持锁访问 */
|
|
|
+ final Map<String, Object> finalOutputs = new LinkedHashMap<>();
|
|
|
+ /** 节点执行记录;持锁访问 */
|
|
|
+ final List<WorkflowRunNode> nodeRecords = new ArrayList<>();
|
|
|
+ /** 保护 finalOutputs 与 nodeRecords 的锁 */
|
|
|
+ final Object recordsLock = new Object();
|
|
|
+
|
|
|
+ /** 主线程等待句柄;remainingNodes 归零时 complete */
|
|
|
+ final CompletableFuture<Void> done = new CompletableFuture<>();
|
|
|
+
|
|
|
+ DataflowCtx(DagResolver.ResolvedDag dag, WorkflowContext context,
|
|
|
+ String runId, Long runRecordId, SseEmitter emitter) {
|
|
|
+ this.dag = dag;
|
|
|
+ this.context = context;
|
|
|
+ this.runId = runId;
|
|
|
+ this.runRecordId = runRecordId;
|
|
|
+ this.emitter = emitter;
|
|
|
+ this.remainingNodes = new AtomicInteger(dag.getNodeDataMap().size());
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 标记一个节点完成(含 skipped);remainingNodes 归零时唤醒主线程 */
|
|
|
+ void markCompleted() {
|
|
|
+ if (remainingNodes.decrementAndGet() == 0) {
|
|
|
+ done.complete(null);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 按 DAG 数据流驱动执行。
|
|
|
*
|
|
|
* @param dag 已解析的 DAG
|
|
|
* @param context 工作流上下文
|
|
|
@@ -111,105 +190,287 @@ public class WorkflowLevelExecutor {
|
|
|
*/
|
|
|
public ExecutionOutcome executeByLevel(DagResolver.ResolvedDag dag, WorkflowContext context,
|
|
|
String runId, Long runRecordId, SseEmitter emitter) {
|
|
|
- Set<String> activeNodes = new HashSet<>();
|
|
|
- if (!dag.getLevels().isEmpty()) {
|
|
|
- activeNodes.addAll(dag.getLevels().get(0));
|
|
|
- }
|
|
|
-
|
|
|
- // 运行时活跃边集合,用于条件分支的动态可达集计算
|
|
|
- Set<String> activeEdges = new HashSet<>();
|
|
|
+ DataflowCtx ctx = new DataflowCtx(dag, context, runId, runRecordId, emitter);
|
|
|
|
|
|
// 注入流式回调:executor 调用 sink.emit(...) 时实时包装为 node_stream 事件推给前端
|
|
|
context.setStreamSink((nodeId, kind, content, toolName) ->
|
|
|
safeSend(emitter, WorkflowRunEvent.nodeStream(runId, nodeId, kind, content, toolName)));
|
|
|
|
|
|
- Map<String, Object> finalOutputs = new LinkedHashMap<>();
|
|
|
- List<WorkflowRunNode> nodeRecords = new ArrayList<>();
|
|
|
- int sortOrder = 0;
|
|
|
+ // === 初始化入度(按 source 去重,与 DagResolver 入度计算规则保持一致) ===
|
|
|
+ for (Map.Entry<String, List<DagResolver.EdgeInfo>> e : dag.getIncomingEdges().entrySet()) {
|
|
|
+ Set<String> sources = new HashSet<>();
|
|
|
+ for (DagResolver.EdgeInfo edge : e.getValue()) {
|
|
|
+ sources.add(edge.getSource());
|
|
|
+ }
|
|
|
+ ctx.pendingInputs.put(e.getKey(), new AtomicInteger(sources.size()));
|
|
|
+ }
|
|
|
+ for (String nodeId : dag.getNodeDataMap().keySet()) {
|
|
|
+ ctx.pendingInputs.computeIfAbsent(nodeId, k -> new AtomicInteger(0));
|
|
|
+ }
|
|
|
+
|
|
|
+ // === 所有节点初始都视为活跃;条件分支未选 target 由 tryScheduleSuccessors 标记移除,
|
|
|
+ // === SKIPPED 节点的下游由 handleSkippedNode 传播时移除 ===
|
|
|
+ for (String nodeId : dag.getNodeDataMap().keySet()) {
|
|
|
+ ctx.activeNodes.add(nodeId);
|
|
|
+ }
|
|
|
+ // 入度为 0 的节点(DAG 起点)提交调度
|
|
|
+ for (Map.Entry<String, AtomicInteger> e : ctx.pendingInputs.entrySet()) {
|
|
|
+ if (e.getValue().get() == 0) {
|
|
|
+ scheduleNode(e.getKey(), ctx);
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- for (int levelIdx = 0; levelIdx < dag.getLevels().size(); levelIdx++) {
|
|
|
- List<String> level = dag.getLevels().get(levelIdx);
|
|
|
- log.debug("[WorkflowEngine] 执行第 {} 层, {} 个节点, 活跃: {}", levelIdx, level.size(),
|
|
|
- level.stream().filter(activeNodes::contains).count());
|
|
|
+ // === 主线程等待所有节点完成(含 abort 后的清理) ===
|
|
|
+ try {
|
|
|
+ ctx.done.get(EXECUTION_TIMEOUT_MINUTES, TimeUnit.MINUTES);
|
|
|
+ } catch (TimeoutException e) {
|
|
|
+ log.error("[WorkflowEngine] 工作流执行超时({} 分钟),强制结束: runId={}",
|
|
|
+ EXECUTION_TIMEOUT_MINUTES, runId);
|
|
|
+ triggerAbort(ctx, "工作流执行超时");
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ Thread.currentThread().interrupt();
|
|
|
+ log.error("[WorkflowEngine] 工作流执行被中断: runId={}", runId);
|
|
|
+ } catch (ExecutionException e) {
|
|
|
+ log.error("[WorkflowEngine] 工作流执行异常: runId={}", runId, e);
|
|
|
+ }
|
|
|
+
|
|
|
+ String errorMsg = ctx.abortMsgRef.get();
|
|
|
+ return new ExecutionOutcome(ctx.finalOutputs, ctx.nodeRecords, errorMsg != null, errorMsg);
|
|
|
+ }
|
|
|
|
|
|
- // 1. 分离活跃与非活跃节点;非活跃节点直接标记为 SKIPPED(保持原 sortOrder 顺序)
|
|
|
- List<String> activeInLevel = new ArrayList<>();
|
|
|
- for (String nodeId : level) {
|
|
|
- if (activeNodes.contains(nodeId)) {
|
|
|
- activeInLevel.add(nodeId);
|
|
|
+ /**
|
|
|
+ * 调度单个节点:提交到线程池执行,挂载 whenComplete 回调。
|
|
|
+ *
|
|
|
+ * <p>并发安全:</p>
|
|
|
+ * <ul>
|
|
|
+ * <li>abort 时直接 return,不再提交新任务,未启动节点转交 handleSkippedNode 收尾</li>
|
|
|
+ * <li>重复调度同一节点(理论不应发生)由 inFlightFutures 的 putIfAbsent 检测</li>
|
|
|
+ * </ul>
|
|
|
+ */
|
|
|
+ private void scheduleNode(String nodeId, DataflowCtx ctx) {
|
|
|
+ if (ctx.abortFlag.get()) {
|
|
|
+ // 已 abort:未启动节点不再调度,直接按 skipped 处理(仅递减 remaining,不写 record/SSE)
|
|
|
+ handleSkippedNode(nodeId, ctx);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ CompletableFuture<NodeExecutionResult> existing = ctx.inFlightFutures.get(nodeId);
|
|
|
+ if (existing != null) {
|
|
|
+ // 已在飞 / 已处理,避免重复调度
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ CompletableFuture<NodeExecutionResult> future = CompletableFuture.supplyAsync(
|
|
|
+ () -> executeOneNode(nodeId, ctx.dag, ctx.context, ctx.runId, ctx.emitter, ctx.activeEdges),
|
|
|
+ nodeExecutor);
|
|
|
+ // putIfAbsent 防止并发场景下重复提交(如多前驱同时归零)
|
|
|
+ CompletableFuture<NodeExecutionResult> race = ctx.inFlightFutures.putIfAbsent(nodeId, future);
|
|
|
+ if (race != null) {
|
|
|
+ // 极端竞态:另一线程已先提交,取消当前 future 并复用既有
|
|
|
+ future.cancel(false);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ future.whenComplete((result, ex) -> onNodeCompleted(nodeId, ctx, result, ex));
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 节点完成回调:处理结果(成功/失败/异常)、写 records、推 SSE、激活下游。
|
|
|
+ *
|
|
|
+ * <p>无论 abort 与否,都会执行 markCompleted;abort 时仅清理 inFlightFutures,
|
|
|
+ * 不写 records / 不推 SSE / 不激活下游(避免与已发的 workflow_error 乱序)。</p>
|
|
|
+ */
|
|
|
+ private void onNodeCompleted(String nodeId, DataflowCtx ctx,
|
|
|
+ NodeExecutionResult result, Throwable ex) {
|
|
|
+ ctx.inFlightFutures.remove(nodeId);
|
|
|
+ // 去重:同一节点可能因 cancel + 正常完成进入两次回调
|
|
|
+ if (!ctx.completedNodes.add(nodeId)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 异常降级为 failed 结果
|
|
|
+ if (ex != null) {
|
|
|
+ result = NodeExecutionResult.failed(nodeId, "节点并行执行异常: " + ex.getMessage());
|
|
|
+ } else if (result == null) {
|
|
|
+ result = NodeExecutionResult.failed(nodeId, "节点执行返回 null");
|
|
|
+ }
|
|
|
+
|
|
|
+ // abort 后只清理计数,不处理副作用
|
|
|
+ if (ctx.abortFlag.get()) {
|
|
|
+ ctx.markCompleted();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ String nodeType = ctx.dag.getNodeTypeMap().get(nodeId);
|
|
|
+ JsonNode nodeData = ctx.dag.getNodeDataMap().get(nodeId);
|
|
|
+ String label = nodeData.path("label").asText(nodeId);
|
|
|
+
|
|
|
+ // 持锁处理 records / finalOutputs / sortOrder(dataflow 下多 worker 并发进入)
|
|
|
+ boolean shouldAbort = false;
|
|
|
+ String abortMsg = null;
|
|
|
+ synchronized (ctx.recordsLock) {
|
|
|
+ int sortOrder = ctx.sortOrderSeq.getAndIncrement();
|
|
|
+ ctx.nodeRecords.add(buildNodeRecord(ctx.runRecordId, nodeId, nodeType, label,
|
|
|
+ result.getStatus().name(), result.getOutput(), result.getError(),
|
|
|
+ result.getSelectedBranch(), result.getLogs(), result.getContextSnapshot(), sortOrder));
|
|
|
+
|
|
|
+ if (result.getStatus() == NodeExecutionResult.Status.FAILED) {
|
|
|
+ String failStrategy = nodeData.path("failStrategy").asText("abort");
|
|
|
+ if ("skip".equals(failStrategy)) {
|
|
|
+ log.info("[WorkflowEngine] 节点 {} 执行失败,策略为跳过,继续工作流", nodeId);
|
|
|
} else {
|
|
|
- String nodeType = dag.getNodeTypeMap().get(nodeId);
|
|
|
- JsonNode nodeData = dag.getNodeDataMap().get(nodeId);
|
|
|
- String label = nodeData.path("label").asText(nodeId);
|
|
|
- safeSend(emitter, WorkflowRunEvent.nodeResult(runId, NodeExecutionResult.skipped(nodeId)));
|
|
|
- nodeRecords.add(buildNodeRecord(runRecordId, nodeId, nodeType, label,
|
|
|
- "SKIPPED", null, null, null, null, null, sortOrder++));
|
|
|
+ shouldAbort = true;
|
|
|
+ abortMsg = "节点 " + nodeId + " 执行失败: " + result.getError();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- if (activeInLevel.isEmpty()) continue;
|
|
|
+ if ("output".equals(nodeType) && result.getOutput() != null) {
|
|
|
+ // result.getOutput() 已是 envelope.toMap(),整体作为工作流最终输出
|
|
|
+ ctx.finalOutputs.clear();
|
|
|
+ ctx.finalOutputs.putAll(result.getOutput());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 推送 nodeResult(在 recordsLock 之外,避免与 safeSend 形成嵌套锁)
|
|
|
+ safeSend(ctx.emitter, WorkflowRunEvent.nodeResult(ctx.runId, result));
|
|
|
+
|
|
|
+ // 失败 + abort:跳过下游调度,直接 triggerAbort(未启动节点由 triggerAbort 内 handleSkippedNode 收尾)
|
|
|
+ if (shouldAbort) {
|
|
|
+ triggerAbort(ctx, abortMsg);
|
|
|
+ ctx.markCompleted();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 激活下游(含条件分支入度补偿)
|
|
|
+ tryScheduleSuccessors(nodeId, nodeType, result, ctx);
|
|
|
|
|
|
- // 2. 同层活跃节点并行执行:每个节点提交到 nodeExecutor 线程池
|
|
|
- List<CompletableFuture<NodeExecutionResult>> futures = new ArrayList<>(activeInLevel.size());
|
|
|
- Set<String> capturedActiveEdges = activeEdges;
|
|
|
- for (String nodeId : activeInLevel) {
|
|
|
- final String finalNodeId = nodeId;
|
|
|
- futures.add(CompletableFuture.supplyAsync(
|
|
|
- () -> executeOneNode(finalNodeId, dag, context, runId, emitter, capturedActiveEdges),
|
|
|
- nodeExecutor));
|
|
|
+ ctx.markCompleted();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 激活后继节点:递减 target 入度,归零则调度。
|
|
|
+ *
|
|
|
+ * <p>关键规则(与 DagResolver 入度去重保持一致):</p>
|
|
|
+ * <ul>
|
|
|
+ * <li>每个 source 完成时,对其所有 target 各递减 1 次(即使同 source 多条边到同 target)</li>
|
|
|
+ * <li>条件分支节点:未选中分支的 target 仍要递减入度,但 activeEdges 不加</li>
|
|
|
+ * <li>顺序:先 add activeEdges 再 decrement 入度,保证新 worker 启动时能看到活跃边(happens-before)</li>
|
|
|
+ * </ul>
|
|
|
+ */
|
|
|
+ private void tryScheduleSuccessors(String nodeId, String nodeType, NodeExecutionResult result,
|
|
|
+ DataflowCtx ctx) {
|
|
|
+ List<DagResolver.EdgeInfo> outEdges = ctx.dag.getOutgoingEdges().get(nodeId);
|
|
|
+ if (outEdges == null || outEdges.isEmpty()) return;
|
|
|
+
|
|
|
+ boolean isConditionBranch = "condition".equals(nodeType) && result.getSelectedBranch() != null;
|
|
|
+ // 同一 target 在本节点完成时只递减一次(无论几条边、几个分支)
|
|
|
+ Set<String> decreased = new HashSet<>();
|
|
|
+
|
|
|
+ // 条件分支:先把未选 target 从 activeNodes 移除,使其入度归零时走 SKIPPED 路径而非调度
|
|
|
+ if (isConditionBranch) {
|
|
|
+ for (DagResolver.EdgeInfo edge : outEdges) {
|
|
|
+ if (!result.getSelectedBranch().equals(edge.getSourceHandle())) {
|
|
|
+ ctx.activeNodes.remove(edge.getTarget());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ for (DagResolver.EdgeInfo edge : outEdges) {
|
|
|
+ boolean edgeActive = !isConditionBranch
|
|
|
+ || result.getSelectedBranch().equals(edge.getSourceHandle());
|
|
|
+ if (edgeActive) {
|
|
|
+ // 先加活跃边,确保下游 worker 构建工作区时能看到(happens-before)
|
|
|
+ ctx.activeEdges.add(WorkflowScopeResolver.activeEdgeKey(
|
|
|
+ edge.getSource(), edge.getSourceHandle(), edge.getTarget()));
|
|
|
}
|
|
|
- // 等待当前层所有并行节点完成(任一节点抛出的异常都被封装为 failed 结果,不会从 join 传播)
|
|
|
- CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
|
|
|
-
|
|
|
- // 3. 串行处理结果:按提交顺序写 nodeRecords / sortOrder,按 failStrategy 决定是否 abort
|
|
|
- // 结果处理阶段是单线程,无需同步 nodeRecords / finalOutputs / sortOrder
|
|
|
- String abortMsg = null;
|
|
|
- for (int i = 0; i < activeInLevel.size(); i++) {
|
|
|
- String nodeId = activeInLevel.get(i);
|
|
|
- NodeExecutionResult result = joinResult(futures.get(i), nodeId);
|
|
|
- String nodeType = dag.getNodeTypeMap().get(nodeId);
|
|
|
- JsonNode nodeData = dag.getNodeDataMap().get(nodeId);
|
|
|
- String label = nodeData.path("label").asText(nodeId);
|
|
|
-
|
|
|
- safeSend(emitter, WorkflowRunEvent.nodeResult(runId, result));
|
|
|
- nodeRecords.add(buildNodeRecord(runRecordId, nodeId, nodeType, label,
|
|
|
- result.getStatus().name(), result.getOutput(), result.getError(),
|
|
|
- result.getSelectedBranch(), result.getLogs(), result.getContextSnapshot(), sortOrder++));
|
|
|
-
|
|
|
- if (result.getStatus() == NodeExecutionResult.Status.FAILED) {
|
|
|
- String failStrategy = nodeData.path("failStrategy").asText("abort");
|
|
|
- if ("skip".equals(failStrategy)) {
|
|
|
- log.info("[WorkflowEngine] 节点 {} 执行失败,策略为跳过,继续工作流", nodeId);
|
|
|
- } else if (abortMsg == null) {
|
|
|
- // 同层多个失败时,仅记录第一个 abort 原因;继续处理剩余结果以保证 nodeRecords 完整
|
|
|
- abortMsg = "节点 " + nodeId + " 执行失败: " + result.getError();
|
|
|
+ if (decreased.add(edge.getTarget())) {
|
|
|
+ AtomicInteger pending = ctx.pendingInputs.get(edge.getTarget());
|
|
|
+ if (pending != null && pending.decrementAndGet() == 0) {
|
|
|
+ // 入度归零:决定调度或 SKIPPED
|
|
|
+ if (ctx.activeNodes.contains(edge.getTarget())) {
|
|
|
+ scheduleNode(edge.getTarget(), ctx);
|
|
|
+ } else {
|
|
|
+ handleSkippedNode(edge.getTarget(), ctx);
|
|
|
}
|
|
|
}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- // 激活下游节点并记录活跃边
|
|
|
- activateDownstream(nodeId, nodeType, result, dag, activeNodes, activeEdges);
|
|
|
+ /**
|
|
|
+ * 处理 SKIPPED 节点:推送 skipped SSE + 写 record + 继续传播下游入度。
|
|
|
+ *
|
|
|
+ * <p>触发场景:</p>
|
|
|
+ * <ol>
|
|
|
+ * <li>条件分支未选中路径上的节点(前驱完成时发现 target 不在 activeNodes 中)</li>
|
|
|
+ * <li>abort 后未启动的节点(scheduleNode 入口检测 abortFlag 后直接走此分支)</li>
|
|
|
+ * </ol>
|
|
|
+ */
|
|
|
+ private void handleSkippedNode(String nodeId, DataflowCtx ctx) {
|
|
|
+ if (!ctx.completedNodes.add(nodeId)) {
|
|
|
+ return; // 已处理(如 abort 时已遍历过)
|
|
|
+ }
|
|
|
+ // 标记本节点为非活跃,确保其下游入度归零时也走 SKIPPED 而不是误调度
|
|
|
+ ctx.activeNodes.remove(nodeId);
|
|
|
+ String nodeType = ctx.dag.getNodeTypeMap().get(nodeId);
|
|
|
+ JsonNode nodeData = ctx.dag.getNodeDataMap().get(nodeId);
|
|
|
+ String label = nodeData.path("label").asText(nodeId);
|
|
|
+ NodeExecutionResult skipped = NodeExecutionResult.skipped(nodeId);
|
|
|
+ // 始终写 record(abort 时也写,让前端能看到所有节点的最终状态);
|
|
|
+ // 但 abort 时不推 SSE(避免与已发的 workflow_error 乱序)
|
|
|
+ synchronized (ctx.recordsLock) {
|
|
|
+ int sortOrder = ctx.sortOrderSeq.getAndIncrement();
|
|
|
+ ctx.nodeRecords.add(buildNodeRecord(ctx.runRecordId, nodeId, nodeType, label,
|
|
|
+ "SKIPPED", null, null, null, null, null, sortOrder));
|
|
|
+ }
|
|
|
+ if (!ctx.abortFlag.get()) {
|
|
|
+ safeSend(ctx.emitter, WorkflowRunEvent.nodeResult(ctx.runId, skipped));
|
|
|
+ }
|
|
|
|
|
|
- if ("output".equals(nodeType) && result.getOutput() != null) {
|
|
|
- // result.getOutput() 已是 envelope.toMap(),整体作为工作流最终输出
|
|
|
- // (含 status / message / data 三个壳字段)
|
|
|
- finalOutputs.clear();
|
|
|
- finalOutputs.putAll(result.getOutput());
|
|
|
+ // 传播下游:本节点 SKIPPED 等价于"前驱已处理",对下游入度递减;
|
|
|
+ // 同时把下游 target 也标记为非活跃(即使其他前驱正常完成,target 也应 SKIPPED)
|
|
|
+ List<DagResolver.EdgeInfo> outEdges = ctx.dag.getOutgoingEdges().get(nodeId);
|
|
|
+ if (outEdges != null) {
|
|
|
+ Set<String> decreased = new HashSet<>();
|
|
|
+ for (DagResolver.EdgeInfo edge : outEdges) {
|
|
|
+ ctx.activeNodes.remove(edge.getTarget());
|
|
|
+ if (decreased.add(edge.getTarget())) {
|
|
|
+ AtomicInteger pending = ctx.pendingInputs.get(edge.getTarget());
|
|
|
+ if (pending != null && pending.decrementAndGet() == 0) {
|
|
|
+ handleSkippedNode(edge.getTarget(), ctx);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
+ }
|
|
|
+ ctx.markCompleted();
|
|
|
+ }
|
|
|
|
|
|
- if (abortMsg != null) {
|
|
|
- safeSend(emitter, WorkflowRunEvent.workflowError(runId, abortMsg));
|
|
|
- return new ExecutionOutcome(finalOutputs, nodeRecords, true, abortMsg);
|
|
|
- }
|
|
|
+ /**
|
|
|
+ * 触发 abort:CAS 去重 + 推送 workflow_error(仅一次)+ 取消在飞 future + 收尾未启动节点。
|
|
|
+ */
|
|
|
+ private void triggerAbort(DataflowCtx ctx, String msg) {
|
|
|
+ if (!ctx.abortFlag.compareAndSet(false, true)) {
|
|
|
+ return; // 已 abort,仅首个失败节点推 workflow_error
|
|
|
}
|
|
|
+ ctx.abortMsgRef.set(msg);
|
|
|
+ safeSend(ctx.emitter, WorkflowRunEvent.workflowError(ctx.runId, msg));
|
|
|
|
|
|
- return new ExecutionOutcome(finalOutputs, nodeRecords, false, null);
|
|
|
+ // 取消所有在飞 future(cancel 只能中断响应中断的任务,对已返回的不影响)
|
|
|
+ for (Map.Entry<String, CompletableFuture<NodeExecutionResult>> e : ctx.inFlightFutures.entrySet()) {
|
|
|
+ e.getValue().cancel(true);
|
|
|
+ }
|
|
|
+
|
|
|
+ // 未启动的节点(pendingInputs 未归零、不在 inFlightFutures、不在 completedNodes)按 SKIPPED 收尾
|
|
|
+ // 保证 remainingNodes 最终归零,主线程能退出
|
|
|
+ for (String nodeId : ctx.pendingInputs.keySet()) {
|
|
|
+ if (!ctx.completedNodes.contains(nodeId) && !ctx.inFlightFutures.containsKey(nodeId)) {
|
|
|
+ handleSkippedNode(nodeId, ctx);
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 执行单个节点(线程池 worker 中调用)。
|
|
|
* 把节点类型查找、前置条件校验、executor.execute、上下文写入等放在同一个并行任务里。
|
|
|
- * 异常一律封装为 failed 结果返回,不向上抛(避免中断 CompletableFuture.allOf)。
|
|
|
+ * 异常一律封装为 failed 结果返回,不向上抛(避免中断 CompletableFuture 链)。
|
|
|
*/
|
|
|
private NodeExecutionResult executeOneNode(String nodeId, DagResolver.ResolvedDag dag,
|
|
|
WorkflowContext context, String runId,
|
|
|
@@ -274,38 +535,6 @@ public class WorkflowLevelExecutor {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * 从 CompletableFuture 取出结果;future 内部异常一律降级为 failed。
|
|
|
- */
|
|
|
- private static NodeExecutionResult joinResult(CompletableFuture<NodeExecutionResult> future, String nodeId) {
|
|
|
- try {
|
|
|
- return future.join();
|
|
|
- } catch (Exception e) {
|
|
|
- return NodeExecutionResult.failed(nodeId, "节点并行执行异常: " + e.getMessage());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 根据当前节点执行结果激活下游节点;条件分支按选中 sourceHandle 过滤,
|
|
|
- * 同时把实际激活的边加入 activeEdges 用于下游节点的动态可达集计算。
|
|
|
- */
|
|
|
- private void activateDownstream(String nodeId, String nodeType, NodeExecutionResult result,
|
|
|
- DagResolver.ResolvedDag dag, Set<String> activeNodes,
|
|
|
- Set<String> activeEdges) {
|
|
|
- List<DagResolver.EdgeInfo> outEdges = dag.getOutgoingEdges().get(nodeId);
|
|
|
- if (outEdges == null) return;
|
|
|
-
|
|
|
- boolean isConditionBranch = "condition".equals(nodeType) && result.getSelectedBranch() != null;
|
|
|
- for (DagResolver.EdgeInfo edge : outEdges) {
|
|
|
- if (isConditionBranch && !result.getSelectedBranch().equals(edge.getSourceHandle())) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- activeNodes.add(edge.getTarget());
|
|
|
- activeEdges.add(WorkflowScopeResolver.activeEdgeKey(
|
|
|
- edge.getSource(), edge.getSourceHandle(), edge.getTarget()));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
private WorkflowRunNode buildNodeRecord(Long runId, String nodeId, String nodeType, String label,
|
|
|
String status, Map<String, Object> output, String error,
|
|
|
String selectedBranch, List<ExecutionLog> logs,
|
|
|
@@ -432,7 +661,7 @@ public class WorkflowLevelExecutor {
|
|
|
|
|
|
/**
|
|
|
* 安全发送 SSE 事件
|
|
|
- * <p>加 synchronized:同层多节点并行执行时,多个 worker 线程会并发调用 safeSend
|
|
|
+ * <p>加 synchronized:dataflow 下多 worker 并发调用 safeSend
|
|
|
* (nodeRunning / nodeStream / nodeResult),而 SseEmitter.send 非线程安全。</p>
|
|
|
*/
|
|
|
private synchronized void safeSend(SseEmitter emitter, WorkflowRunEvent event) {
|