2 * Copyright (C) 2014 SDN Hub, LLC.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 * Authors : Srini Seetharaman, Madhu Venugopal
10 package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.services;
12 import java.math.BigInteger;
13 import java.util.List;
16 import org.opendaylight.controller.sal.utils.Status;
17 import org.opendaylight.controller.sal.utils.StatusCode;
18 import org.opendaylight.controller.sal.core.Node;
19 import org.opendaylight.controller.sal.core.Node.NodeIDType;
20 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration;
21 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration.LoadBalancerPoolMember;
22 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerProvider;
23 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.AbstractServiceInstance;
24 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.Service;
25 import org.opendaylight.ovsdb.utils.mdsal.openflow.ActionUtils;
26 import org.opendaylight.ovsdb.utils.mdsal.openflow.MatchUtils;
27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.PortNumber;
29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
30 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.address.address.Ipv4Builder;
31 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionBuilder;
33 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionKey;
34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.dst.choice.grouping.dst.choice.DstNxRegCaseBuilder;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.ovs.nx.action.rev140421.OfjNxHashFields;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.ovs.nx.action.rev140421.OfjNxMpAlgorithm;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg0;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
53 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
55 import com.google.common.collect.Lists;
56 import org.slf4j.Logger;
57 import org.slf4j.LoggerFactory;
59 public class LoadBalancerService extends AbstractServiceInstance implements LoadBalancerProvider {
61 private static final Logger logger = LoggerFactory.getLogger(LoadBalancerProvider.class);
62 private static final int DEFAULT_FLOW_PRIORITY = 32768;
63 private static final Long FIRST_PASS_REG0_MATCH_VALUE = 0L;
64 private static final Long SECOND_PASS_REG0_MATCH_VALUE = 1L;
66 private static final Class<? extends NxmNxReg> REG_FIELD_A = NxmNxReg0.class;
67 private static final Class<? extends NxmNxReg> REG_FIELD_B = NxmNxReg1.class;
69 public LoadBalancerService() {
70 super(Service.LOAD_BALANCER);
73 public LoadBalancerService(Service service) {
78 public boolean isBridgeInPipeline (String nodeId) {
83 * When this method is called, we do the following for minimizing flow updates:
84 * 1. Overwrite the solo multipath rule that applies to all members
85 * 2. Append second pass rule for the header rewriting specific to this member
86 * 3. Append reverse rules specific to this member
89 public Status programLoadBalancerPoolMemberRules(Node node,
90 LoadBalancerConfiguration lbConfig, LoadBalancerPoolMember member, org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
91 if (!node.getType().equals(NodeIDType.OPENFLOW)) {
92 logger.trace("Ignoring non-OpenFlow node {} from flow programming", node);
93 return new Status(StatusCode.BADREQUEST);
95 logger.debug("Performing {} rules for member {} with index {} on LB with VIP {} and total members {}",
96 action, member.getIP(), member.getIndex(), lbConfig.getVip(), lbConfig.getMembers().size());
98 NodeBuilder nodeBuilder = new NodeBuilder();
99 nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + String.valueOf(node.getID())));
100 nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
102 //Update the multipath rule
103 insertLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig);
105 if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
106 insertLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig.getVip(), member);
107 insertLoadBalancerMemberReverseRules(nodeBuilder, lbConfig.getVip(), member);
108 return new Status(StatusCode.SUCCESS);
110 /* TODO: Delete single member.
111 * For now, removing a member requires deleting the full LB instance and re-adding
113 return new Status(StatusCode.NOTIMPLEMENTED);
117 * When this method is called, we perform the following:
118 * 1. Write the solo multipath rule that applies to all members
119 * 2. Append second pass rules for the header rewriting for all members
120 * 3. Append reverse rules for all the members, specific to the protocol/port
123 public Status programLoadBalancerRules(Node node, LoadBalancerConfiguration lbConfig, org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
124 if (!node.getType().equals(NodeIDType.OPENFLOW)) {
125 logger.trace("Ignoring non-OpenFlow node {} from flow programming", node);
126 return new Status(StatusCode.BADREQUEST);
128 logger.debug("Performing {} rules for VIP {} and {} members", action, lbConfig.getVip(), lbConfig.getMembers().size());
130 NodeBuilder nodeBuilder = new NodeBuilder();
131 nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + String.valueOf(node.getID())));
132 nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
134 if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
135 insertLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig);
136 insertLoadBalancerVIPRulesSecondPass(nodeBuilder, lbConfig);
137 insertLoadBalancerReverseRules(nodeBuilder, lbConfig);
138 return new Status(StatusCode.SUCCESS);
140 else if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.DELETE)) {
141 removeLoadBalancerVIPRules(nodeBuilder, lbConfig);
142 removeLoadBalancerReverseRules(nodeBuilder, lbConfig);
143 return new Status(StatusCode.SUCCESS);
146 return new Status(StatusCode.NOTIMPLEMENTED);
149 private void insertLoadBalancerVIPRulesFirstPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
150 MatchBuilder matchBuilder = new MatchBuilder();
151 FlowBuilder flowBuilder = new FlowBuilder();
153 // Match VIP, and Reg0==0
154 MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
155 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, FIRST_PASS_REG0_MATCH_VALUE));
157 // Create the OF Actions and Instructions
158 InstructionsBuilder isb = new InstructionsBuilder();
160 // Instructions List Stores Individual Instructions
161 List<Instruction> instructions = Lists.newArrayList();
163 List<Action> actionList = Lists.newArrayList();
165 ActionBuilder ab = new ActionBuilder();
166 ab.setAction(ActionUtils.nxLoadRegAction(new DstNxRegCaseBuilder().setNxReg(REG_FIELD_A).build(),
167 BigInteger.valueOf(SECOND_PASS_REG0_MATCH_VALUE)));
169 ab.setKey(new ActionKey(0));
170 actionList.add(ab.build());
172 ab = new ActionBuilder();
173 ab.setAction(ActionUtils.nxMultipathAction(OfjNxHashFields.NXHASHFIELDSSYMMETRICL4,
174 (Integer)0, OfjNxMpAlgorithm.NXMPALGMODULON,
175 (Integer)lbConfig.getMembers().size()-1, //By Nicira-Ext spec, this field is max_link minus 1
176 (Long)0L, new DstNxRegCaseBuilder().setNxReg(REG_FIELD_B).build(),
177 (Integer)0, (Integer)31));
179 ab.setKey(new ActionKey(1));
180 actionList.add(ab.build());
182 ab = new ActionBuilder();
183 ab.setAction(ActionUtils.nxResubmitAction(null, this.getTable()));
185 ab.setKey(new ActionKey(2));
186 actionList.add(ab.build());
188 // Create an Apply Action
189 ApplyActionsBuilder aab = new ApplyActionsBuilder();
190 aab.setAction(actionList);
191 InstructionBuilder ib = new InstructionBuilder();
192 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
194 // Call the InstructionBuilder Methods Containing Actions
196 ib.setKey(new InstructionKey(0));
197 instructions.add(ib.build());
199 // Add InstructionBuilder to the Instruction(s)Builder List
200 isb.setInstruction(instructions);
202 // Add InstructionsBuilder to FlowBuilder
203 flowBuilder.setInstructions(isb.build());
205 String flowId = "LOADBALANCER_FORWARD_FLOW1_" + lbConfig.getVip();
206 flowBuilder.setId(new FlowId(flowId));
207 FlowKey key = new FlowKey(new FlowId(flowId));
208 flowBuilder.setMatch(matchBuilder.build());
209 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
210 flowBuilder.setBarrier(true);
211 flowBuilder.setTableId((short) this.getTable());
212 flowBuilder.setKey(key);
213 flowBuilder.setFlowName(flowId);
214 flowBuilder.setHardTimeout(0);
215 flowBuilder.setIdleTimeout(0);
216 writeFlow(flowBuilder, nodeBuilder);
220 * Method to program each rule that matches on Reg0 and Reg1 to insert appropriate header rewriting
221 * rules for all members. This function calls insertLoadBalancerMemberVIPRulesSecondPass in turn.
222 * @param nodeBuilder Node to insert rule to
223 * @param lbConfig Configuration for this LoadBalancer instance
225 private void insertLoadBalancerVIPRulesSecondPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
226 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
227 insertLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig.getVip(), (LoadBalancerPoolMember)entry.getValue());
231 private void insertLoadBalancerMemberVIPRulesSecondPass(NodeBuilder nodeBuilder, String vip, LoadBalancerPoolMember member) {
232 MatchBuilder matchBuilder = new MatchBuilder();
233 FlowBuilder flowBuilder = new FlowBuilder();
235 // Match VIP, Reg0==1 and Reg1==Index of member
236 MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(vip));
237 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, SECOND_PASS_REG0_MATCH_VALUE),
238 new MatchUtils.RegMatch(REG_FIELD_B, (long)member.getIndex()));
240 // Create the OF Actions and Instructions
241 InstructionsBuilder isb = new InstructionsBuilder();
243 // Instructions List Stores Individual Instructions
244 List<Instruction> instructions = Lists.newArrayList();
246 List<Action> actionList = Lists.newArrayList();
247 ActionBuilder ab = new ActionBuilder();
248 ab.setAction(ActionUtils.setDlDstAction(new MacAddress(member.getMAC())));
250 ab.setKey(new ActionKey(0));
251 actionList.add(ab.build());
253 ab = new ActionBuilder();
254 Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(new Ipv4Prefix(member.getIP()));
255 ab.setAction(ActionUtils.setNwDstAction(ipb.build()));
257 ab.setKey(new ActionKey(1));
258 actionList.add(ab.build());
260 // Create an Apply Action
261 ApplyActionsBuilder aab = new ApplyActionsBuilder();
262 aab.setAction(actionList);
264 // Call the InstructionBuilder Methods Containing Actions
265 InstructionBuilder ib = new InstructionBuilder();
266 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
268 ib.setKey(new InstructionKey(0));
269 instructions.add(ib.build());
271 // Call the InstructionBuilder Methods Containing Actions
272 ib = this.getMutablePipelineInstructionBuilder();
274 ib.setKey(new InstructionKey(1));
275 instructions.add(ib.build());
277 // Add InstructionBuilder to the Instruction(s)Builder List
278 isb.setInstruction(instructions);
280 // Add InstructionsBuilder to FlowBuilder
281 flowBuilder.setInstructions(isb.build());
283 String flowId = "LOADBALANCER_FORWARD_FLOW2_" + vip + "_" + member.getIP();
284 flowBuilder.setId(new FlowId(flowId));
285 FlowKey key = new FlowKey(new FlowId(flowId));
286 flowBuilder.setMatch(matchBuilder.build());
287 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY+1);
288 flowBuilder.setBarrier(true);
289 flowBuilder.setTableId((short) this.getTable());
290 flowBuilder.setKey(key);
291 flowBuilder.setFlowName(flowId);
292 flowBuilder.setHardTimeout(0);
293 flowBuilder.setIdleTimeout(0);
294 writeFlow(flowBuilder, nodeBuilder);
298 * Method to program all reverse rules that matches member {IP, Protocol, Port} for all members.
299 * This function calls insertLoadBalancerMemberReverseRules in turn.
300 * @param nodeBuilder Node to insert rule to
301 * @param lbConfig Configuration for this LoadBalancer instance
303 private void insertLoadBalancerReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
304 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
305 insertLoadBalancerMemberReverseRules(nodeBuilder, lbConfig.getVip(), (LoadBalancerPoolMember)entry.getValue());
309 private void insertLoadBalancerMemberReverseRules(NodeBuilder nodeBuilder, String vip, LoadBalancerPoolMember member) {
310 MatchBuilder matchBuilder = new MatchBuilder();
311 FlowBuilder flowBuilder = new FlowBuilder();
313 // Match MemberIP, and Protocol/Port
314 MatchUtils.createSrcL3IPv4Match(matchBuilder, new Ipv4Prefix(member.getIP()));
315 if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTP))
316 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTP_PORT));
317 else if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTPS))
318 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTPS_PORT));
322 // Create the OF Actions and Instructions
323 InstructionsBuilder isb = new InstructionsBuilder();
325 // Instructions List Stores Individual Instructions
326 List<Instruction> instructions = Lists.newArrayList();
328 List<Action> actionList = Lists.newArrayList();
329 ActionBuilder ab = new ActionBuilder();
330 Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(new Ipv4Prefix(vip));
331 ab.setAction(ActionUtils.setNwSrcAction(ipb.build()));
333 ab.setKey(new ActionKey(0));
334 actionList.add(ab.build());
336 // Create an Apply Action
337 ApplyActionsBuilder aab = new ApplyActionsBuilder();
338 aab.setAction(actionList);
340 // Call the InstructionBuilder Methods Containing Actions
341 InstructionBuilder ib = new InstructionBuilder();
342 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
344 ib.setKey(new InstructionKey(0));
345 instructions.add(ib.build());
347 // Call the InstructionBuilder Methods Containing Actions
348 ib = this.getMutablePipelineInstructionBuilder();
350 ib.setKey(new InstructionKey(1));
351 instructions.add(ib.build());
353 // Add InstructionBuilder to the Instruction(s)Builder List
354 isb.setInstruction(instructions);
356 // Add InstructionsBuilder to FlowBuilder
357 flowBuilder.setInstructions(isb.build());
359 String flowId = "LOADBALANCER_REVERSE_FLOW_" + vip + "_" + member.getIP();
360 flowBuilder.setId(new FlowId(flowId));
361 FlowKey key = new FlowKey(new FlowId(flowId));
362 flowBuilder.setMatch(matchBuilder.build());
363 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
364 flowBuilder.setBarrier(true);
365 flowBuilder.setTableId((short) this.getTable());
366 flowBuilder.setKey(key);
367 flowBuilder.setFlowName(flowId);
368 flowBuilder.setHardTimeout(0);
369 flowBuilder.setIdleTimeout(0);
370 writeFlow(flowBuilder, nodeBuilder);
374 * Method to remove all rules that are regarding traffic destined to the VIP
375 * (both first and second pass rules)
376 * @param nodeBuilder NodeBuilder
377 * @param lbConfig LoadBalancerConfiguration
379 private void removeLoadBalancerVIPRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
380 MatchBuilder matchBuilder = new MatchBuilder();
381 FlowBuilder flowBuilder = new FlowBuilder();
383 // Match all first pass rules
384 MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
385 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, FIRST_PASS_REG0_MATCH_VALUE));
387 flowBuilder.setMatch(matchBuilder.build());
388 String flowId = "LOADBALANCER_FORWARD_FLOW1_" + lbConfig.getVip();
389 flowBuilder.setId(new FlowId(flowId));
390 FlowKey key = new FlowKey(new FlowId(flowId));
391 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
392 flowBuilder.setBarrier(true);
393 flowBuilder.setTableId((short) this.getTable());
394 flowBuilder.setKey(key);
395 removeFlow(flowBuilder, nodeBuilder);
397 // Match all second pass rules
398 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
399 LoadBalancerPoolMember member = (LoadBalancerPoolMember) entry.getValue();
400 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, SECOND_PASS_REG0_MATCH_VALUE),
401 new MatchUtils.RegMatch(REG_FIELD_B, (long)member.getIndex()));
402 MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
404 flowBuilder.setMatch(matchBuilder.build());
405 flowId = "LOADBALANCER_FORWARD_FLOW2_" + lbConfig.getVip() + "_" + member.getIP();
406 flowBuilder.setId(new FlowId(flowId));
407 key = new FlowKey(new FlowId(flowId));
408 flowBuilder.setKey(key);
409 removeFlow(flowBuilder, nodeBuilder);
414 * Method to remove all reverse traffic from LB member VMs
415 * @param nodeBuilder NodeBuilder
416 * @param lbConfig LoadBalancerConfiguration
418 private void removeLoadBalancerReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
419 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
420 LoadBalancerPoolMember member = (LoadBalancerPoolMember) entry.getValue();
422 MatchBuilder matchBuilder = new MatchBuilder();
423 FlowBuilder flowBuilder = new FlowBuilder();
425 // Match MemberIP, and Protocol/Port
426 MatchUtils.createSrcL3IPv4Match(matchBuilder, new Ipv4Prefix(member.getIP()));
427 if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTP))
428 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTP_PORT));
429 else if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTPS))
430 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTPS_PORT));
434 String flowId = "LOADBALANCER_REVERSE_FLOW_" + lbConfig.getVip() + "_" + member.getIP();
435 flowBuilder.setId(new FlowId(flowId));
436 FlowKey key = new FlowKey(new FlowId(flowId));
437 flowBuilder.setMatch(matchBuilder.build());
438 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
439 flowBuilder.setBarrier(true);
440 flowBuilder.setTableId((short) this.getTable());
441 flowBuilder.setKey(key);
442 removeFlow(flowBuilder, nodeBuilder);