openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/services/LoadBalancerService.java

   1 /*
   2  * Copyright (C) 2014 SDN Hub, LLC.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Srini Seetharaman, Madhu Venugopal
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.services;
  11
  12 import java.math.BigInteger;
  13 import java.util.List;
  14 import java.util.Map;
  15
  16 import org.opendaylight.controller.sal.utils.Status;
  17 import org.opendaylight.controller.sal.utils.StatusCode;
  18 import org.opendaylight.controller.sal.core.Node;
  19 import org.opendaylight.controller.sal.core.Node.NodeIDType;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration.LoadBalancerPoolMember;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerProvider;
  23 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.AbstractServiceInstance;
  24 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.Service;
  25 import org.opendaylight.ovsdb.utils.mdsal.openflow.ActionUtils;
  26 import org.opendaylight.ovsdb.utils.mdsal.openflow.MatchUtils;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.PortNumber;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.address.address.Ipv4Builder;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionBuilder;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionKey;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.dst.choice.grouping.dst.choice.DstNxRegCaseBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.ovs.nx.action.rev140421.OfjNxHashFields;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.ovs.nx.action.rev140421.OfjNxMpAlgorithm;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  53 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  54
  55 import com.google.common.collect.Lists;
  56 import org.slf4j.Logger;
  57 import org.slf4j.LoggerFactory;
  58
  59 public class LoadBalancerService extends AbstractServiceInstance implements LoadBalancerProvider {
  60
  61     private static final Logger logger = LoggerFactory.getLogger(LoadBalancerProvider.class);
  62     private static final int DEFAULT_FLOW_PRIORITY = 32768;
  63     private static final Long FIRST_PASS_REGA_MATCH_VALUE = 0L;
  64     private static final Long SECOND_PASS_REGA_MATCH_VALUE = 1L;
  65
  66     private static final Class<? extends NxmNxReg> REG_FIELD_A = NxmNxReg1.class;
  67     private static final Class<? extends NxmNxReg> REG_FIELD_B = NxmNxReg2.class;
  68
  69     public LoadBalancerService() {
  70         super(Service.LOAD_BALANCER);
  71     }
  72
  73     public LoadBalancerService(Service service) {
  74         super(service);
  75     }
  76
  77     @Override
  78     public boolean isBridgeInPipeline (String nodeId) {
  79         return true;
  80     }
  81
  82     /**
  83      * When this method is called, we do the following for minimizing flow updates:
  84      * 1. Overwrite the solo multipath rule that applies to all members
  85      * 2. Append second pass rule for the header rewriting specific to this member
  86      * 3. Append reverse rules specific to this member
  87      */
  88     @Override
  89     public Status programLoadBalancerPoolMemberRules(Node node,
  90             LoadBalancerConfiguration lbConfig, LoadBalancerPoolMember member, org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
  91         if (!node.getType().equals(NodeIDType.OPENFLOW)) {
  92             logger.trace("Ignoring non-OpenFlow node {} from flow programming", node);
  93             return new Status(StatusCode.BADREQUEST);
  94         }
  95         logger.debug("Performing {} rules for member {} with index {} on LB with VIP {} and total members {}",
  96                 action, member.getIP(), member.getIndex(), lbConfig.getVip(), lbConfig.getMembers().size());
  97
  98         NodeBuilder nodeBuilder = new NodeBuilder();
  99         nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + String.valueOf(node.getID())));
 100         nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
 101
 102         //Update the multipath rule
 103         insertLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig);
 104
 105         if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
 106             insertLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig.getVip(), member);
 107             insertLoadBalancerMemberReverseRules(nodeBuilder, lbConfig.getVip(), member);
 108             return new Status(StatusCode.SUCCESS);
 109         }
 110         /* TODO: Delete single member.
 111          * For now, removing a member requires deleting the full LB instance and re-adding
 112          */
 113         return new Status(StatusCode.NOTIMPLEMENTED);
 114     }
 115
 116     /**
 117      * When this method is called, we perform the following:
 118      * 1. Write the solo multipath rule that applies to all members
 119      * 2. Append second pass rules for the header rewriting for all members
 120      * 3. Append reverse rules for all the members, specific to the protocol/port
 121      */
 122     @Override
 123     public Status programLoadBalancerRules(Node node, LoadBalancerConfiguration lbConfig, org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
 124         if (!node.getType().equals(NodeIDType.OPENFLOW)) {
 125             logger.trace("Ignoring non-OpenFlow node {} from flow programming", node);
 126             return new Status(StatusCode.BADREQUEST);
 127         }
 128         logger.debug("Performing {} rules for VIP {} and {} members", action, lbConfig.getVip(), lbConfig.getMembers().size());
 129
 130         NodeBuilder nodeBuilder = new NodeBuilder();
 131         nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + String.valueOf(node.getID())));
 132         nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
 133
 134         if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
 135             insertLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig);
 136             insertLoadBalancerVIPRulesSecondPass(nodeBuilder, lbConfig);
 137             insertLoadBalancerReverseRules(nodeBuilder, lbConfig);
 138             return new Status(StatusCode.SUCCESS);
 139         }
 140         else if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.DELETE)) {
 141             removeLoadBalancerVIPRules(nodeBuilder, lbConfig);
 142             removeLoadBalancerReverseRules(nodeBuilder, lbConfig);
 143             return new Status(StatusCode.SUCCESS);
 144         }
 145
 146         return new Status(StatusCode.NOTIMPLEMENTED);
 147     }
 148
 149     private void insertLoadBalancerVIPRulesFirstPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
 150         MatchBuilder matchBuilder = new MatchBuilder();
 151         FlowBuilder flowBuilder = new FlowBuilder();
 152
 153         // Match VIP, and Reg0==0
 154         MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
 155         MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, FIRST_PASS_REGA_MATCH_VALUE));
 156
 157         // Create the OF Actions and Instructions
 158         InstructionsBuilder isb = new InstructionsBuilder();
 159
 160         // Instructions List Stores Individual Instructions
 161         List<Instruction> instructions = Lists.newArrayList();
 162
 163         List<Action> actionList = Lists.newArrayList();
 164
 165         ActionBuilder ab = new ActionBuilder();
 166         ab.setAction(ActionUtils.nxLoadRegAction(new DstNxRegCaseBuilder().setNxReg(REG_FIELD_A).build(),
 167                 BigInteger.valueOf(SECOND_PASS_REGA_MATCH_VALUE)));
 168         ab.setOrder(0);
 169         ab.setKey(new ActionKey(0));
 170         actionList.add(ab.build());
 171
 172         ab = new ActionBuilder();
 173         ab.setAction(ActionUtils.nxMultipathAction(OfjNxHashFields.NXHASHFIELDSSYMMETRICL4,
 174                 (Integer)0, OfjNxMpAlgorithm.NXMPALGMODULON,
 175                 (Integer)lbConfig.getMembers().size()-1, //By Nicira-Ext spec, this field is max_link minus 1
 176                 (Long)0L, new DstNxRegCaseBuilder().setNxReg(REG_FIELD_B).build(),
 177                 (Integer)0, (Integer)31));
 178         ab.setOrder(1);
 179         ab.setKey(new ActionKey(1));
 180         actionList.add(ab.build());
 181
 182         ab = new ActionBuilder();
 183         ab.setAction(ActionUtils.nxResubmitAction(null, this.getTable()));
 184         ab.setOrder(2);
 185         ab.setKey(new ActionKey(2));
 186         actionList.add(ab.build());
 187
 188         // Create an Apply Action
 189         ApplyActionsBuilder aab = new ApplyActionsBuilder();
 190         aab.setAction(actionList);
 191         InstructionBuilder ib = new InstructionBuilder();
 192         ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
 193
 194         // Call the InstructionBuilder Methods Containing Actions
 195         ib.setOrder(0);
 196         ib.setKey(new InstructionKey(0));
 197         instructions.add(ib.build());
 198
 199         // Add InstructionBuilder to the Instruction(s)Builder List
 200         isb.setInstruction(instructions);
 201
 202         // Add InstructionsBuilder to FlowBuilder
 203         flowBuilder.setInstructions(isb.build());
 204
 205         String flowId = "LOADBALANCER_FORWARD_FLOW1_" + lbConfig.getVip();
 206         flowBuilder.setId(new FlowId(flowId));
 207         FlowKey key = new FlowKey(new FlowId(flowId));
 208         flowBuilder.setMatch(matchBuilder.build());
 209         flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
 210         flowBuilder.setBarrier(true);
 211         flowBuilder.setTableId((short) this.getTable());
 212         flowBuilder.setKey(key);
 213         flowBuilder.setFlowName(flowId);
 214         flowBuilder.setHardTimeout(0);
 215         flowBuilder.setIdleTimeout(0);
 216         writeFlow(flowBuilder, nodeBuilder);
 217     }
 218
 219     /*
 220      * Method to program each rule that matches on Reg0 and Reg1 to insert appropriate header rewriting
 221      * rules for all members. This function calls insertLoadBalancerMemberVIPRulesSecondPass in turn.
 222      * @param nodeBuilder Node to insert rule to
 223      * @param lbConfig Configuration for this LoadBalancer instance
 224      */
 225     private void insertLoadBalancerVIPRulesSecondPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
 226         for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
 227             insertLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig.getVip(), (LoadBalancerPoolMember)entry.getValue());
 228         }
 229     }
 230
 231     private void insertLoadBalancerMemberVIPRulesSecondPass(NodeBuilder nodeBuilder, String vip, LoadBalancerPoolMember member) {
 232         MatchBuilder matchBuilder = new MatchBuilder();
 233         FlowBuilder flowBuilder = new FlowBuilder();
 234
 235         // Match VIP, Reg0==1 and Reg1==Index of member
 236         MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(vip));
 237         MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, SECOND_PASS_REGA_MATCH_VALUE),
 238                                                new MatchUtils.RegMatch(REG_FIELD_B, (long)member.getIndex()));
 239
 240         // Create the OF Actions and Instructions
 241         InstructionsBuilder isb = new InstructionsBuilder();
 242
 243         // Instructions List Stores Individual Instructions
 244         List<Instruction> instructions = Lists.newArrayList();
 245
 246         List<Action> actionList = Lists.newArrayList();
 247         ActionBuilder ab = new ActionBuilder();
 248         ab.setAction(ActionUtils.setDlDstAction(new MacAddress(member.getMAC())));
 249         ab.setOrder(0);
 250         ab.setKey(new ActionKey(0));
 251         actionList.add(ab.build());
 252
 253         ab = new ActionBuilder();
 254         Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(new Ipv4Prefix(member.getIP()));
 255         ab.setAction(ActionUtils.setNwDstAction(ipb.build()));
 256         ab.setOrder(1);
 257         ab.setKey(new ActionKey(1));
 258         actionList.add(ab.build());
 259
 260         // Create an Apply Action
 261         ApplyActionsBuilder aab = new ApplyActionsBuilder();
 262         aab.setAction(actionList);
 263
 264         // Call the InstructionBuilder Methods Containing Actions
 265         InstructionBuilder ib = new InstructionBuilder();
 266         ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
 267         ib.setOrder(0);
 268         ib.setKey(new InstructionKey(0));
 269         instructions.add(ib.build());
 270
 271         // Call the InstructionBuilder Methods Containing Actions
 272         ib = this.getMutablePipelineInstructionBuilder();
 273         ib.setOrder(1);
 274         ib.setKey(new InstructionKey(1));
 275         instructions.add(ib.build());
 276
 277         // Add InstructionBuilder to the Instruction(s)Builder List
 278         isb.setInstruction(instructions);
 279
 280         // Add InstructionsBuilder to FlowBuilder
 281         flowBuilder.setInstructions(isb.build());
 282
 283         String flowId = "LOADBALANCER_FORWARD_FLOW2_" + vip + "_" + member.getIP();
 284         flowBuilder.setId(new FlowId(flowId));
 285         FlowKey key = new FlowKey(new FlowId(flowId));
 286         flowBuilder.setMatch(matchBuilder.build());
 287         flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY+1);
 288         flowBuilder.setBarrier(true);
 289         flowBuilder.setTableId((short) this.getTable());
 290         flowBuilder.setKey(key);
 291         flowBuilder.setFlowName(flowId);
 292         flowBuilder.setHardTimeout(0);
 293         flowBuilder.setIdleTimeout(0);
 294         writeFlow(flowBuilder, nodeBuilder);
 295     }
 296
 297     /**
 298      * Method to program all reverse rules that matches member {IP, Protocol, Port} for all members.
 299      * This function calls insertLoadBalancerMemberReverseRules in turn.
 300      * @param nodeBuilder Node to insert rule to
 301      * @param lbConfig Configuration for this LoadBalancer instance
 302      */
 303     private void insertLoadBalancerReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
 304         for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
 305             insertLoadBalancerMemberReverseRules(nodeBuilder, lbConfig.getVip(), (LoadBalancerPoolMember)entry.getValue());
 306         }
 307     }
 308
 309     private void insertLoadBalancerMemberReverseRules(NodeBuilder nodeBuilder, String vip, LoadBalancerPoolMember member) {
 310         MatchBuilder matchBuilder = new MatchBuilder();
 311         FlowBuilder flowBuilder = new FlowBuilder();
 312
 313         // Match MemberIP, and Protocol/Port
 314         MatchUtils.createSrcL3IPv4Match(matchBuilder, new Ipv4Prefix(member.getIP()));
 315         if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTP))
 316             MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTP_PORT));
 317         else if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTPS))
 318             MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTPS_PORT));
 319         else //Not possible
 320             return;
 321
 322         // Create the OF Actions and Instructions
 323         InstructionsBuilder isb = new InstructionsBuilder();
 324
 325         // Instructions List Stores Individual Instructions
 326         List<Instruction> instructions = Lists.newArrayList();
 327
 328         List<Action> actionList = Lists.newArrayList();
 329         ActionBuilder ab = new ActionBuilder();
 330         Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(new Ipv4Prefix(vip));
 331         ab.setAction(ActionUtils.setNwSrcAction(ipb.build()));
 332         ab.setOrder(0);
 333         ab.setKey(new ActionKey(0));
 334         actionList.add(ab.build());
 335
 336         // Create an Apply Action
 337         ApplyActionsBuilder aab = new ApplyActionsBuilder();
 338         aab.setAction(actionList);
 339
 340         // Call the InstructionBuilder Methods Containing Actions
 341         InstructionBuilder ib = new InstructionBuilder();
 342         ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
 343         ib.setOrder(0);
 344         ib.setKey(new InstructionKey(0));
 345         instructions.add(ib.build());
 346
 347         // Call the InstructionBuilder Methods Containing Actions
 348         ib = this.getMutablePipelineInstructionBuilder();
 349         ib.setOrder(1);
 350         ib.setKey(new InstructionKey(1));
 351         instructions.add(ib.build());
 352
 353         // Add InstructionBuilder to the Instruction(s)Builder List
 354         isb.setInstruction(instructions);
 355
 356         // Add InstructionsBuilder to FlowBuilder
 357         flowBuilder.setInstructions(isb.build());
 358
 359         String flowId = "LOADBALANCER_REVERSE_FLOW_" + vip + "_" + member.getIP();
 360         flowBuilder.setId(new FlowId(flowId));
 361         FlowKey key = new FlowKey(new FlowId(flowId));
 362         flowBuilder.setMatch(matchBuilder.build());
 363         flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
 364         flowBuilder.setBarrier(true);
 365         flowBuilder.setTableId((short) this.getTable());
 366         flowBuilder.setKey(key);
 367         flowBuilder.setFlowName(flowId);
 368         flowBuilder.setHardTimeout(0);
 369         flowBuilder.setIdleTimeout(0);
 370         writeFlow(flowBuilder, nodeBuilder);
 371     }
 372
 373     /**
 374      * Method to remove all rules that are regarding traffic destined to the VIP
 375      * (both first and second pass rules)
 376      * @param nodeBuilder NodeBuilder
 377      * @param lbConfig LoadBalancerConfiguration
 378      */
 379     private void removeLoadBalancerVIPRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
 380         MatchBuilder matchBuilder = new MatchBuilder();
 381         FlowBuilder flowBuilder = new FlowBuilder();
 382
 383         // Match all first pass rules
 384         MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
 385         MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, FIRST_PASS_REGA_MATCH_VALUE));
 386
 387         flowBuilder.setMatch(matchBuilder.build());
 388         String flowId = "LOADBALANCER_FORWARD_FLOW1_" + lbConfig.getVip();
 389         flowBuilder.setId(new FlowId(flowId));
 390         FlowKey key = new FlowKey(new FlowId(flowId));
 391         flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
 392         flowBuilder.setBarrier(true);
 393         flowBuilder.setTableId((short) this.getTable());
 394         flowBuilder.setKey(key);
 395         removeFlow(flowBuilder, nodeBuilder);
 396
 397         // Match all second pass rules
 398         for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
 399             LoadBalancerPoolMember member = (LoadBalancerPoolMember) entry.getValue();
 400             MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, SECOND_PASS_REGA_MATCH_VALUE),
 401                                                    new MatchUtils.RegMatch(REG_FIELD_B, (long)member.getIndex()));
 402             MatchUtils.createDstL3IPv4Match(matchBuilder, new Ipv4Prefix(lbConfig.getVip()));
 403
 404             flowBuilder.setMatch(matchBuilder.build());
 405             flowId = "LOADBALANCER_FORWARD_FLOW2_" + lbConfig.getVip() + "_" + member.getIP();
 406             flowBuilder.setId(new FlowId(flowId));
 407             key = new FlowKey(new FlowId(flowId));
 408             flowBuilder.setKey(key);
 409             removeFlow(flowBuilder, nodeBuilder);
 410         }
 411     }
 412
 413     /**
 414      * Method to remove all reverse traffic from LB member VMs
 415      * @param nodeBuilder NodeBuilder
 416      * @param lbConfig LoadBalancerConfiguration
 417      */
 418     private void removeLoadBalancerReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig) {
 419         for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
 420             LoadBalancerPoolMember member = (LoadBalancerPoolMember) entry.getValue();
 421
 422             MatchBuilder matchBuilder = new MatchBuilder();
 423             FlowBuilder flowBuilder = new FlowBuilder();
 424
 425             // Match MemberIP, and Protocol/Port
 426             MatchUtils.createSrcL3IPv4Match(matchBuilder, new Ipv4Prefix(member.getIP()));
 427             if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTP))
 428                 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTP_PORT));
 429             else if (member.getProtocol().equalsIgnoreCase(LoadBalancerConfiguration.PROTOCOL_HTTPS))
 430                 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(LoadBalancerConfiguration.PROTOCOL_HTTPS_PORT));
 431             else //Not possible
 432                 return;
 433
 434             String flowId = "LOADBALANCER_REVERSE_FLOW_" + lbConfig.getVip() + "_" + member.getIP();
 435             flowBuilder.setId(new FlowId(flowId));
 436             FlowKey key = new FlowKey(new FlowId(flowId));
 437             flowBuilder.setMatch(matchBuilder.build());
 438             flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
 439             flowBuilder.setBarrier(true);
 440             flowBuilder.setTableId((short) this.getTable());
 441             flowBuilder.setKey(key);
 442             removeFlow(flowBuilder, nodeBuilder);
 443         }
 444     }
 445 }