vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.util.concurrent.AsyncFunction;
  11 import com.google.common.util.concurrent.FutureCallback;
  12 import com.google.common.util.concurrent.Futures;
  13 import com.google.common.util.concurrent.JdkFutureAdapters;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import java.math.BigInteger;
  16 import java.util.ArrayList;
  17 import java.util.List;
  18 import java.util.concurrent.Future;
  19 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  20 import org.opendaylight.genius.mdsalutil.ActionInfo;
  21 import org.opendaylight.genius.mdsalutil.ActionType;
  22 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  23 import org.opendaylight.genius.mdsalutil.InstructionType;
  24 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  25 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  26 import org.opendaylight.genius.mdsalutil.MatchInfo;
  27 import org.opendaylight.genius.mdsalutil.NwConstants;
  28 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  29 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  30 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  31 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  32 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronConstants;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  35 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  56 import org.opendaylight.yangtools.yang.common.RpcResult;
  57 import org.slf4j.Logger;
  58 import org.slf4j.LoggerFactory;
  59
  60 public class VpnFloatingIpHandler implements FloatingIPHandler {
  61     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  62     private final DataBroker dataBroker;
  63     private final IMdsalApiManager mdsalManager;
  64     private final VpnRpcService vpnService;
  65     private final IBgpManager bgpManager;
  66     private final FibRpcService fibService;
  67     private final FloatingIPListener floatingIPListener;
  68     private final IFibManager fibManager;
  69     private final OdlArputilService arpUtilService;
  70     static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  71     static final String FLOWID_PREFIX = "NAT.";
  72
  73     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  74                                 final VpnRpcService vpnService,
  75                                 final IBgpManager bgpManager,
  76                                 final FibRpcService fibService,
  77                                 final FloatingIPListener floatingIPListener,
  78                                 final IFibManager fibManager,
  79                                 final OdlArputilService arputilService) {
  80         this.dataBroker = dataBroker;
  81         this.mdsalManager = mdsalManager;
  82         this.vpnService = vpnService;
  83         this.bgpManager = bgpManager;
  84         this.fibService = fibService;
  85         this.floatingIPListener = floatingIPListener;
  86         this.fibManager = fibManager;
  87         this.arpUtilService = arputilService;
  88     }
  89
  90     @Override
  91     public void onAddFloatingIp(final BigInteger dpnId, final String routerId,
  92                                 Uuid networkId, final String interfaceName, final String externalIp,
  93                                 final String internalIp) {
  94         final String vpnName = getAssociatedVPN(networkId, routerId);
  95         if (vpnName == null) {
  96             LOG.info("No VPN associated with ext nw {} to handle add floating ip configuration {} in router {}",
  97                     networkId, externalIp, routerId);
  98             return;
  99         }
 100
 101         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 102                 .setIpPrefix(externalIp).build();
 103         Future<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 104
 105         ListenableFuture<RpcResult<Void>> future = Futures.transform(JdkFutureAdapters.listenInPoolThread(labelFuture),
 106                 new AsyncFunction<RpcResult<GenerateVpnLabelOutput>, RpcResult<Void>>() {
 107
 108             @Override
 109             public ListenableFuture<RpcResult<Void>> apply(RpcResult<GenerateVpnLabelOutput> result) throws Exception {
 110                 if(result.isSuccessful()) {
 111                     GenerateVpnLabelOutput output = result.getResult();
 112                     long label = output.getLabel();
 113                     LOG.debug("Generated label {} for prefix {}", label, externalIp);
 114                     floatingIPListener.updateOperationalDS(routerId, interfaceName, label, internalIp, externalIp);
 115
 116                     //Inform BGP
 117                     String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 118                     String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 119                     LOG.debug("Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 120                     NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, rd, externalIp + "/32", nextHopIp,
 121                             label, LOG, RouteOrigin.STATIC);
 122
 123                     List<Instruction> instructions = new ArrayList<>();
 124                     List<ActionInfo> actionsInfos = new ArrayList<>();
 125                     actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[] { Integer.toString(NwConstants.PDNAT_TABLE) }));
 126                     instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos).buildInstruction(0));
 127                     makeTunnelTableEntry(dpnId, label, instructions);
 128
 129                     //Install custom FIB routes
 130                     List<Instruction> customInstructions = new ArrayList<>();
 131                     customInstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.PDNAT_TABLE }).buildInstruction(0));
 132                     makeLFibTableEntry(dpnId, label, NwConstants.PDNAT_TABLE);
 133                     CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName).setSourceDpid(dpnId).setInstruction(customInstructions)
 134                             .setIpAddress(externalIp + "/32").setServiceId(label).setInstruction(customInstructions).build();
 135                     //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 136                     Future<RpcResult<Void>> future = fibService.createFibEntry(input);
 137                     return JdkFutureAdapters.listenInPoolThread(future);
 138                 } else {
 139                     String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s", externalIp, vpnName, result.getErrors());
 140                     LOG.error(errMsg);
 141                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 142                 }
 143             }
 144         });
 145
 146         Futures.addCallback(future, new FutureCallback<RpcResult<Void>>() {
 147
 148             @Override
 149             public void onFailure(Throwable error) {
 150                 LOG.error("Error in generate label or fib install process", error);
 151             }
 152
 153             @Override
 154             public void onSuccess(RpcResult<Void> result) {
 155                 if(result.isSuccessful()) {
 156                     LOG.info("Successfully installed custom FIB routes for prefix {}", externalIp);
 157                 } else {
 158                     LOG.error("Error in rpc call to create custom Fib entries for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
 159                 }
 160             }
 161         });
 162
 163         // Handle GARP transmission
 164         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 165         final IpAddress internalAddress = IpAddressBuilder.getDefaultInstance(internalIp);
 166         Port neutronPortForIp = NatUtil.getNeutronPortForIp(dataBroker,internalAddress,
 167                 NeutronConstants.DEVICE_OWNER_NEUTRON_PORT);
 168         if (neutronPortForIp == null) {
 169             LOG.warn("No neutron port was found for external ip {} in router {}", internalIp, routerId);
 170             NatServiceCounters.port_not_found_for_floating.inc();
 171             return;
 172         }
 173         sendGarpOnInterface(neutronPortForIp.getUuid().getValue(), extrenalAddress, routerId);
 174
 175     }
 176
 177     @Override
 178     public void onRemoveFloatingIp(final BigInteger dpnId, String routerId, Uuid networkId, final String externalIp,
 179                                    String internalIp, final long label) {
 180         final String vpnName = getAssociatedVPN(networkId, routerId);
 181         if (vpnName == null) {
 182             LOG.info("No VPN associated with ext nw {} to handle remove floating ip configuration {} in router {}",
 183                     networkId, externalIp, routerId);
 184             return;
 185         }
 186         //Remove Prefix from BGP
 187         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 188         NatUtil.removePrefixFromBGP(dataBroker, bgpManager, fibManager, rd, externalIp + "/32", LOG);
 189
 190         //Remove custom FIB routes
 191         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 192         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp + "/32").setServiceId(label).build();
 193         Future<RpcResult<Void>> future = fibService.removeFibEntry(input);
 194
 195         ListenableFuture<RpcResult<Void>> labelFuture = Futures.transform(JdkFutureAdapters.listenInPoolThread(future), new AsyncFunction<RpcResult<Void>, RpcResult<Void>>() {
 196
 197             @Override
 198             public ListenableFuture<RpcResult<Void>> apply(RpcResult<Void> result) throws Exception {
 199                 //Release label
 200                 if(result.isSuccessful()) {
 201                     removeTunnelTableEntry(dpnId, label);
 202                     removeLFibTableEntry(dpnId, label);
 203                     RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder().setVpnName(vpnName).setIpPrefix(externalIp).build();
 204                     Future<RpcResult<Void>> labelFuture = vpnService.removeVpnLabel(labelInput);
 205                     return JdkFutureAdapters.listenInPoolThread(labelFuture);
 206                 } else {
 207                     String errMsg = String.format("RPC call to remove custom FIB entries on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 208                     LOG.error(errMsg);
 209                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 210                 }
 211             }
 212         });
 213
 214         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<Void>>() {
 215
 216             @Override
 217             public void onFailure(Throwable error) {
 218                 LOG.error("Error in removing the label or custom fib entries", error);
 219             }
 220
 221             @Override
 222             public void onSuccess(RpcResult<Void> result) {
 223                 if(result.isSuccessful()) {
 224                     LOG.debug("Successfully removed the label for the prefix {} from VPN {}", externalIp, vpnName);
 225                 } else {
 226                     LOG.error("Error in removing the label for prefix {} from VPN {}, {}", externalIp, vpnName, result.getErrors());
 227                 }
 228             }
 229         });
 230     }
 231
 232     void cleanupFibEntries(final BigInteger dpnId, final String vpnName, final String externalIp, final long label ) {
 233         //Remove Prefix from BGP
 234         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 235         NatUtil.removePrefixFromBGP(dataBroker, bgpManager, fibManager, rd, externalIp + "/32", LOG);
 236
 237         //Remove custom FIB routes
 238
 239         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 240         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp + "/32").setServiceId(label).build();
 241         Future<RpcResult<Void>> future = fibService.removeFibEntry(input);
 242
 243         ListenableFuture<RpcResult<Void>> labelFuture = Futures.transform(JdkFutureAdapters.listenInPoolThread(future),
 244             new AsyncFunction<RpcResult<Void>, RpcResult<Void>>() {
 245
 246             @Override
 247             public ListenableFuture<RpcResult<Void>> apply(RpcResult<Void> result) throws Exception {
 248                 //Release label
 249                 if(result.isSuccessful()) {
 250                     removeTunnelTableEntry(dpnId, label);
 251                     removeLFibTableEntry(dpnId, label);
 252                     RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder().setVpnName(vpnName).setIpPrefix(externalIp).build();
 253                     Future<RpcResult<Void>> labelFuture = vpnService.removeVpnLabel(labelInput);
 254                     return JdkFutureAdapters.listenInPoolThread(labelFuture);
 255                 } else {
 256                     String errMsg = String.format("RPC call to remove custom FIB entries on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 257                     LOG.error(errMsg);
 258                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 259                 }
 260             }
 261         });
 262
 263         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<Void>>() {
 264
 265             @Override
 266             public void onFailure(Throwable error) {
 267                 LOG.error("Error in removing the label or custom fib entries", error);
 268             }
 269
 270             @Override
 271             public void onSuccess(RpcResult<Void> result) {
 272                 if(result.isSuccessful()) {
 273                     LOG.debug("Successfully removed the label for the prefix {} from VPN {}", externalIp, vpnName);
 274                 } else {
 275                     LOG.error("Error in removing the label for prefix {} from VPN {}, {}", externalIp, vpnName, result.getErrors());
 276                 }
 277             }
 278         });
 279     }
 280
 281     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
 282         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
 283                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
 284                 .append(id).append(NwConstants.FLOWID_SEPARATOR).append(ipAddress).toString();
 285     }
 286
 287     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId) {
 288         LOG.info("remove terminatingServiceActions called with DpnId = {} and label = {}", dpnId , serviceId);
 289         List<MatchInfo> mkMatches = new ArrayList<>();
 290         // Matching metadata
 291         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(serviceId)}));
 292         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 293                 getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
 294                 5, String.format("%s:%d","TST Flow Entry ",serviceId), 0, 0,
 295                 COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
 296         mdsalManager.removeFlow(dpnId, flowEntity);
 297         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully {}",dpnId, serviceId);
 298     }
 299
 300     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, List<Instruction> customInstructions) {
 301         List<MatchInfo> mkMatches = new ArrayList<>();
 302
 303         LOG.info("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", dpnId , serviceId);
 304
 305         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(serviceId)}));
 306
 307         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 308                 getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5, String.format("%s:%d","TST Flow Entry ",serviceId),
 309                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)),mkMatches, customInstructions);
 310
 311         mdsalManager.installFlow(dpnId, terminatingServiceTableFlowEntity);
 312     }
 313
 314     private void makeLFibTableEntry(BigInteger dpId, long serviceId, long tableId) {
 315         List<MatchInfo> matches = new ArrayList<>();
 316         matches.add(new MatchInfo(MatchFieldType.eth_type,
 317                 new long[] { 0x8847L }));
 318         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(serviceId)}));
 319
 320         List<Instruction> instructions = new ArrayList<>();
 321         List<ActionInfo> actionsInfos = new ArrayList<>();
 322         actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 323         Instruction writeInstruction = new InstructionInfo(InstructionType.apply_actions, actionsInfos).buildInstruction(0);
 324         instructions.add(writeInstruction);
 325         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[]{tableId}).buildInstruction(1));
 326
 327         // Install the flow entry in L3_LFIB_TABLE
 328         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 329
 330         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 331                 10, flowRef, 0, 0,
 332                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 333
 334         mdsalManager.installFlow(dpId, flowEntity);
 335
 336         LOG.debug("LFIB Entry for dpID {} : label : {} modified successfully {}",dpId, serviceId );
 337     }
 338
 339     private void removeLFibTableEntry(BigInteger dpnId, long serviceId) {
 340         List<MatchInfo> matches = new ArrayList<>();
 341         matches.add(new MatchInfo(MatchFieldType.eth_type,
 342                                   new long[] { 0x8847L }));
 343         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(serviceId)}));
 344
 345         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 346
 347         LOG.debug("removing LFib entry with flow ref {}", flowRef);
 348
 349         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 350                                                10, flowRef, 0, 0,
 351                                                NwConstants.COOKIE_VM_LFIB_TABLE, matches, null);
 352
 353         mdsalManager.removeFlow(dpnId, flowEntity);
 354
 355         LOG.debug("LFIB Entry for dpID : {} label : {} removed successfully {}",dpnId, serviceId);
 356     }
 357
 358     private String getAssociatedVPN(Uuid networkId, String routerId) {
 359         String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId, LOG);
 360         return vpnName != null ? vpnName : routerId;
 361     }
 362
 363     private void sendGarpOnInterface(String interfaceName, final IpAddress floatingIpAddress, final String routerId) {
 364         if (floatingIpAddress.getIpv4Address() == null) {
 365             LOG.warn("Faild to send GARP for IP. recieved IPv6.");
 366             NatServiceCounters.garp_sent_ipv6.inc();
 367             return;
 368         }
 369
 370         try {
 371             Port floatingPort = NatUtil.getNeutronPortForFloatingIp(dataBroker, floatingIpAddress);
 372             PhysAddress floatingPortMac = new PhysAddress(floatingPort.getMacAddress().getValue());
 373             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 374             interfaceAddresses.add(new InterfaceAddressBuilder()
 375                     .setInterface(interfaceName)
 376                     .setIpAddress(floatingIpAddress)
 377                     .setMacaddress(floatingPortMac).build());
 378
 379             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 380                     .setInterfaceAddress(interfaceAddresses).build();
 381             arpUtilService.sendArpRequest(sendArpRequestInput);
 382             NatServiceCounters.garp_sent.inc();
 383         } catch (Exception e) {
 384             LOG.error("Failed to send GARP request for floating ip {} from interface {}",
 385                     floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 386             NatServiceCounters.garp_sent_failed.inc();
 387         }
 388     }
 389
 390 }