2 * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.netvirt.elan.l2gw.utils;
11 import static java.util.Collections.emptyList;
12 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
14 import com.google.common.collect.Lists;
15 import com.google.common.util.concurrent.FluentFuture;
16 import com.google.common.util.concurrent.FutureCallback;
17 import com.google.common.util.concurrent.Futures;
18 import com.google.common.util.concurrent.MoreExecutors;
19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.Objects;
23 import java.util.Optional;
25 import java.util.concurrent.CopyOnWriteArrayList;
26 import java.util.concurrent.ExecutionException;
27 import javax.annotation.PreDestroy;
28 import javax.inject.Inject;
29 import javax.inject.Singleton;
30 import org.eclipse.jdt.annotation.NonNull;
31 import org.eclipse.jdt.annotation.Nullable;
32 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
33 import org.opendaylight.genius.utils.hwvtep.HwvtepNodeHACache;
34 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
35 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
36 import org.opendaylight.mdsal.binding.api.DataBroker;
37 import org.opendaylight.mdsal.binding.util.ManagedNewTransactionRunner;
38 import org.opendaylight.mdsal.binding.util.RetryingManagedNewTransactionRunner;
39 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
40 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
41 import org.opendaylight.netvirt.elan.l2gw.ha.listeners.HAOpClusteredListener;
42 import org.opendaylight.netvirt.elan.l2gw.jobs.AssociateHwvtepToElanJob;
43 import org.opendaylight.netvirt.elan.l2gw.jobs.DisAssociateHwvtepFromElanJob;
44 import org.opendaylight.netvirt.elan.l2gw.listeners.LocalUcastMacListener;
45 import org.opendaylight.netvirt.elan.l2gw.recovery.impl.L2GatewayServiceRecoveryHandler;
46 import org.opendaylight.netvirt.elan.utils.ElanClusterUtils;
47 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
48 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
49 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
50 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateway.attributes.Devices;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateway.connections.attributes.L2gatewayConnections;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateway.connections.attributes.l2gatewayconnections.L2gatewayConnection;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateways.attributes.L2gateways;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateways.attributes.l2gateways.L2gateway;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateways.attributes.l2gateways.L2gatewayKey;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepGlobalAugmentation;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.LocalUcastMacs;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.LogicalSwitches;
63 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
64 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
65 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
70 public class L2GatewayConnectionUtils implements AutoCloseable {
71 private static final Logger LOG = LoggerFactory.getLogger(L2GatewayConnectionUtils.class);
73 private final DataBroker broker;
74 private final ElanL2GatewayUtils elanL2GatewayUtils;
75 private final ElanClusterUtils elanClusterUtils;
76 private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
77 private final JobCoordinator jobCoordinator;
78 private final L2GatewayCache l2GatewayCache;
79 private final ElanInstanceCache elanInstanceCache;
80 private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
81 private final HwvtepNodeHACache hwvtepNodeHACache;
82 private final HAOpClusteredListener haOpClusteredListener;
83 private final ElanRefUtil elanRefUtil;
84 private final L2GatewayServiceRecoveryHandler l2GatewayServiceRecoveryHandler;
85 private final ServiceRecoveryRegistry serviceRecoveryRegistry;
86 private final ManagedNewTransactionRunner txRunner;
89 public L2GatewayConnectionUtils(DataBroker dataBroker, ElanClusterUtils elanClusterUtils,
90 ElanL2GatewayUtils elanL2GatewayUtils, JobCoordinator jobCoordinator,
91 ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
92 L2GatewayCache l2GatewayCache, HAOpClusteredListener haOpClusteredListener,
93 ElanInstanceCache elanInstanceCache, HwvtepNodeHACache hwvtepNodeHACache,
94 ElanRefUtil elanRefUtil,
95 L2GatewayServiceRecoveryHandler l2GatewayServiceRecoveryHandler,
96 ServiceRecoveryRegistry serviceRecoveryRegistry) {
97 this.broker = dataBroker;
98 this.elanL2GatewayUtils = elanL2GatewayUtils;
99 this.elanClusterUtils = elanClusterUtils;
100 this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
101 this.jobCoordinator = jobCoordinator;
102 this.l2GatewayCache = l2GatewayCache;
103 this.haOpClusteredListener = haOpClusteredListener;
104 this.elanInstanceCache = elanInstanceCache;
105 this.hwvtepNodeHACache = hwvtepNodeHACache;
106 this.elanRefUtil = elanRefUtil;
107 this.l2GatewayServiceRecoveryHandler = l2GatewayServiceRecoveryHandler;
108 this.serviceRecoveryRegistry = serviceRecoveryRegistry;
109 this.txRunner = new RetryingManagedNewTransactionRunner(dataBroker);
114 @SuppressWarnings("checkstyle:IllegalCatch")
115 public void close() {
116 closeables.forEach(c -> {
119 } catch (Exception e) {
120 LOG.warn("Error closing {}", c, e);
125 public static boolean isGatewayAssociatedToL2Device(L2GatewayDevice l2GwDevice) {
126 return !l2GwDevice.getL2GatewayIds().isEmpty();
130 public static L2gateway getNeutronL2gateway(DataBroker broker, Uuid l2GatewayId) {
131 LOG.debug("getNeutronL2gateway for {}", l2GatewayId.getValue());
132 InstanceIdentifier<L2gateway> inst = InstanceIdentifier.create(Neutron.class).child(L2gateways.class)
133 .child(L2gateway.class, new L2gatewayKey(l2GatewayId));
135 return SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION, inst)
137 } catch (ExecutionException | InterruptedException e) {
138 LOG.error("getNeutronL2gateway: Exception while reading L2gateway DS for the ID {}", l2GatewayId, e);
144 public static List<L2gateway> getL2gatewayList(DataBroker broker) {
145 InstanceIdentifier<L2gateways> inst = InstanceIdentifier.create(Neutron.class).child(L2gateways.class);
147 return SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION, inst).map(
148 L2gateways::getL2gateway).orElse(emptyList());
149 } catch (ExecutionException | InterruptedException e) {
150 LOG.error("getNeutronL2gateway: Exception while reading L2gateway DS", e);
156 public static List<L2gatewayConnection> getAllL2gatewayConnections(DataBroker broker) {
157 InstanceIdentifier<L2gatewayConnections> inst = InstanceIdentifier.create(Neutron.class)
158 .child(L2gatewayConnections.class);
160 return SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION, inst).map(
161 L2gatewayConnections::getL2gatewayConnection).orElse(emptyList());
162 } catch (ExecutionException | InterruptedException e) {
163 LOG.error("getNeutronL2gateway: Exception while reading L2gateway DS", e);
169 * Gets the associated l2 gw connections.
173 * @param l2GatewayIds
175 * @return the associated l2 gw connections
178 public static List<L2gatewayConnection> getAssociatedL2GwConnections(DataBroker broker, Set<Uuid> l2GatewayIds) {
179 List<L2gatewayConnection> allL2GwConns = getAllL2gatewayConnections(broker);
180 List<L2gatewayConnection> l2GwConnections = new ArrayList<>();
181 for (Uuid l2GatewayId : l2GatewayIds) {
182 for (L2gatewayConnection l2GwConn : allL2GwConns) {
183 if (Objects.equals(l2GwConn.getL2gatewayId(), l2GatewayId)) {
184 l2GwConnections.add(l2GwConn);
188 return l2GwConnections;
192 * Gets the associated l2 gw connections.
198 * @return the associated l2 gw connection with elan
201 public static List<L2gatewayConnection> getL2GwConnectionsByElanName(DataBroker broker, String elanName) {
202 List<L2gatewayConnection> allL2GwConns = getAllL2gatewayConnections(broker);
203 List<L2gatewayConnection> elanL2GateWayConnections = new ArrayList<>();
204 for (L2gatewayConnection l2GwConn : allL2GwConns) {
205 if (l2GwConn.getNetworkId().getValue().equalsIgnoreCase(elanName)) {
206 elanL2GateWayConnections.add(l2GwConn);
209 return elanL2GateWayConnections;
212 public void addL2GatewayConnection(L2gatewayConnection input) {
213 addL2GatewayConnection(input, null/*deviceName*/, null);
216 public void addL2GatewayConnection(final L2gatewayConnection input,
217 final String l2GwDeviceName) {
218 addL2GatewayConnection(input, l2GwDeviceName, null);
221 public void addL2GatewayConnection(final L2gatewayConnection input,
222 @Nullable final String l2GwDeviceName ,
223 @Nullable L2gateway l2Gateway) {
224 LOG.info("Adding L2gateway Connection with ID: {}", input.key().getUuid());
226 Uuid networkUuid = input.getNetworkId();
228 // Taking cluster reboot scenario , if Elan instance is not available when l2GatewayConnection add events
229 // comes we need to wait for elaninstance to resolve. Hence updating the map with the runnable .
230 // When elanInstance add comes , it look in to the map and run the associated runnable associated with it.
231 ElanInstance elanInstance = elanInstanceCache.get(networkUuid.getValue(),
232 () -> addL2GatewayConnection(input, l2GwDeviceName)).orElse(null);
233 if (elanInstance == null) {
237 if (!isVxlanNetworkOrVxlanSegment(elanInstance)) {
238 LOG.error("Neutron network with id {} is not VxlanNetwork", networkUuid.getValue());
240 Uuid l2GatewayId = input.getL2gatewayId();
241 if (l2Gateway == null) {
242 l2Gateway = getNeutronL2gateway(broker, l2GatewayId);
244 if (l2Gateway == null) {
245 LOG.error("L2Gateway with id {} is not present", l2GatewayId.getValue());
247 associateHwvtepsToElan(elanInstance, l2Gateway, input, l2GwDeviceName);
252 public void deleteL2GatewayConnection(L2gatewayConnection input) {
253 LOG.info("Deleting L2gateway Connection with ID: {}", input.key().getUuid());
255 Uuid networkUuid = input.getNetworkId();
256 String elanName = networkUuid.getValue();
257 disAssociateHwvtepsFromElan(elanName, input);
260 private void disAssociateHwvtepsFromElan(String elanName, L2gatewayConnection input) {
261 Integer defaultVlan = input.getSegmentId();
262 List<L2GatewayDevice> l2Devices = ElanL2GwCacheUtils.getAllElanDevicesFromCache();
263 List<Devices> l2gwDevicesToBeDeleted = new ArrayList<>();
264 for (L2GatewayDevice elanL2gwDevice : l2Devices) {
265 if (elanL2gwDevice.getL2GatewayIds().contains(input.key().getUuid())) {
266 l2gwDevicesToBeDeleted.addAll(elanL2gwDevice.getDevicesForL2gwConnectionId(input.key().getUuid()));
269 if (l2gwDevicesToBeDeleted.isEmpty()) {
270 //delete logical switch
271 Uuid l2GatewayId = input.getL2gatewayId();
272 L2gateway l2Gateway = L2GatewayConnectionUtils.getNeutronL2gateway(broker, l2GatewayId);
273 if (l2Gateway == null) {
274 LOG.error("Failed to find the l2gateway for the connection {}", input.getUuid());
276 } else if (l2Gateway.getDevices() != null) {
277 l2gwDevicesToBeDeleted.addAll(l2Gateway.getDevices());
280 for (Devices l2Device : l2gwDevicesToBeDeleted) {
281 String l2DeviceName = l2Device.getDeviceName();
282 L2GatewayDevice l2GatewayDevice = l2GatewayCache.get(l2DeviceName);
283 String hwvtepNodeId = l2GatewayDevice.getHwvtepNodeId();
284 boolean isLastL2GwConnDeleted = false;
285 L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName, hwvtepNodeId);
286 if (elanL2GwDevice != null && isLastL2GwConnBeingDeleted(elanL2GwDevice)) {
287 // Delete L2 Gateway device from 'ElanL2GwDevice' cache
288 LOG.debug("Elan L2Gw Conn cache removed for id {}", hwvtepNodeId);
289 ElanL2GwCacheUtils.removeL2GatewayDeviceFromCache(elanName, hwvtepNodeId);
290 isLastL2GwConnDeleted = true;
292 Uuid l2GwConnId = input.key().getUuid();
293 LOG.debug("Elan L2Gw Conn cache with id {} is being referred by other L2Gw Conns; so only "
294 + "L2 Gw Conn {} reference is removed", hwvtepNodeId, l2GwConnId);
295 if (elanL2GwDevice != null) {
296 elanL2GwDevice.removeL2GatewayId(l2GwConnId);
298 isLastL2GwConnDeleted = true;
302 DisAssociateHwvtepFromElanJob disAssociateHwvtepToElanJob =
303 new DisAssociateHwvtepFromElanJob(elanL2GatewayUtils, elanL2GatewayMulticastUtils,
304 elanL2GwDevice, elanName,
305 l2Device, defaultVlan, hwvtepNodeId, isLastL2GwConnDeleted);
306 elanClusterUtils.runOnlyInOwnerNode(disAssociateHwvtepToElanJob.getJobKey(), "remove l2gw connection job",
307 disAssociateHwvtepToElanJob);
311 private void associateHwvtepsToElan(ElanInstance elanInstance,
312 L2gateway l2Gateway, L2gatewayConnection input, @Nullable String l2GwDeviceName) {
313 String elanName = elanInstance.getElanInstanceName();
314 Integer defaultVlan = input.getSegmentId();
315 Uuid l2GwConnId = input.key().getUuid();
316 List<Devices> l2Devices = l2Gateway.getDevices();
318 LOG.trace("Associating ELAN {} with L2Gw Conn Id {} having below L2Gw devices {}", elanName, l2GwConnId,
321 if (l2Devices == null) {
325 for (Devices l2Device : l2Devices) {
326 String l2DeviceName = l2Device.getDeviceName();
327 // L2gateway can have more than one L2 Gw devices. Configure Logical Switch, VLAN mappings,...
328 // only on the switch which has come up just now and exclude all other devices from
329 // preprovisioning/re-provisioning
330 if (l2GwDeviceName != null && !l2GwDeviceName.equals(l2DeviceName)) {
331 LOG.debug("Associating Hwvtep to ELAN is not been processed for {}; as only {} got connected now!",
332 l2DeviceName, l2GwDeviceName);
335 L2GatewayDevice l2GatewayDevice = l2GatewayCache.get(l2DeviceName);
336 if (isL2GwDeviceConnected(l2GatewayDevice)) {
337 NodeId hwvtepNodeId = new NodeId(l2GatewayDevice.getHwvtepNodeId());
339 // Delete pending delete logical switch task if scheduled
340 elanL2GatewayUtils.cancelDeleteLogicalSwitch(hwvtepNodeId,
341 ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName));
343 // Add L2 Gateway device to 'ElanL2GwDevice' cache
344 boolean createLogicalSwitch;
345 addL2DeviceToElanL2GwCache(elanName, l2GatewayDevice, l2GwConnId, l2Device);
347 AssociateHwvtepToElanJob associateHwvtepToElanJob = new AssociateHwvtepToElanJob(broker,
348 elanL2GatewayUtils, elanL2GatewayMulticastUtils, elanInstanceCache, l2GatewayDevice,
349 elanInstance, l2Device, defaultVlan, elanRefUtil);
351 elanClusterUtils.runOnlyInOwnerNode(associateHwvtepToElanJob.getJobKey(),
352 "create logical switch in hwvtep topo", associateHwvtepToElanJob);
355 LOG.info("L2GwConn create is not handled for device with id {} as it's not connected", l2DeviceName);
360 public L2GatewayDevice addL2DeviceToElanL2GwCache(String elanName, L2GatewayDevice l2GatewayDevice, Uuid l2GwConnId,
362 String l2gwDeviceNodeId = l2GatewayDevice.getHwvtepNodeId();
363 L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName, l2gwDeviceNodeId);
364 if (elanL2GwDevice == null) {
365 elanL2GwDevice = new L2GatewayDevice(l2GatewayDevice.getDeviceName());
366 elanL2GwDevice.setHwvtepNodeId(l2gwDeviceNodeId);
367 elanL2GwDevice.setTunnelIps(l2GatewayDevice.getTunnelIps());
368 ElanL2GwCacheUtils.addL2GatewayDeviceToCache(elanName, elanL2GwDevice);
369 LOG.debug("Elan L2GwConn cache created for hwvtep id {}", l2gwDeviceNodeId);
371 LOG.debug("Elan L2GwConn cache already exists for hwvtep id {}; updating L2GwConn id {} to it",
372 l2gwDeviceNodeId, l2GwConnId);
374 elanL2GwDevice.addL2GatewayId(l2GwConnId);
375 elanL2GwDevice.addL2gwConnectionIdToDevice(l2GwConnId, l2Device);
377 //incase of cluster reboot scenario southbound device would have added more macs
378 //while odl is down, pull them now
379 readAndCopyLocalUcastMacsToCache(elanName, l2GatewayDevice);
381 LOG.trace("Elan L2GwConn cache updated with below details: {}", elanL2GwDevice);
382 return elanL2GwDevice;
385 private static boolean isL2GwDeviceConnected(L2GatewayDevice l2GwDevice) {
386 return l2GwDevice != null && l2GwDevice.getHwvtepNodeId() != null;
389 protected static boolean isLastL2GwConnBeingDeleted(@NonNull L2GatewayDevice l2GwDevice) {
390 return l2GwDevice.getL2GatewayIds().size() == 1;
393 private void readAndCopyLocalUcastMacsToCache(final String elanName, final L2GatewayDevice l2GatewayDevice) {
394 final InstanceIdentifier<Node> nodeIid = HwvtepSouthboundUtils.createInstanceIdentifier(
395 new NodeId(l2GatewayDevice.getHwvtepNodeId()));
396 jobCoordinator.enqueueJob(elanName + ":" + l2GatewayDevice.getDeviceName(), () -> {
397 FluentFuture<Optional<Node>> fluentFuture = broker.newReadOnlyTransaction().read(
398 LogicalDatastoreType.OPERATIONAL, nodeIid);
399 Futures.addCallback(fluentFuture, new FutureCallback<Optional<Node>>() {
401 public void onSuccess(Optional<Node> nodeOptional) {
402 if (nodeOptional.isPresent()) {
403 Node node = nodeOptional.get();
404 if (node.augmentation(HwvtepGlobalAugmentation.class) != null) {
405 List<LocalUcastMacs> localUcastMacs =
406 node.augmentation(HwvtepGlobalAugmentation.class).getLocalUcastMacs();
407 if (localUcastMacs == null) {
410 LocalUcastMacListener localUcastMacListener =
411 new LocalUcastMacListener(broker, haOpClusteredListener,
412 elanL2GatewayUtils, jobCoordinator, elanInstanceCache, hwvtepNodeHACache,
413 l2GatewayServiceRecoveryHandler, serviceRecoveryRegistry);
414 localUcastMacs.stream()
415 .filter((mac) -> macBelongsToLogicalSwitch(mac, elanName))
417 InstanceIdentifier<LocalUcastMacs> macIid = getMacIid(nodeIid, mac);
418 localUcastMacListener.added(macIid, mac);
425 public void onFailure(Throwable throwable) {
427 }, MoreExecutors.directExecutor());
428 return Lists.newArrayList(fluentFuture);
433 * Gets the associated l2 gw connections.
435 * @param l2GatewayId the l2 gateway id
437 * @return the associated l2 gw connections
439 public List<L2gatewayConnection> getL2GwConnectionsByL2GatewayId(Uuid l2GatewayId) {
440 List<L2gatewayConnection> l2GwConnections = new ArrayList<>();
441 List<L2gatewayConnection> allL2GwConns = getAllL2gatewayConnections(broker);
442 for (L2gatewayConnection l2GwConn : allL2GwConns) {
443 if (Objects.equals(l2GwConn.getL2gatewayId(), l2GatewayId)) {
444 l2GwConnections.add(l2GwConn);
447 return l2GwConnections;
450 @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
451 justification = "https://github.com/spotbugs/spotbugs/issues/811")
452 private static boolean macBelongsToLogicalSwitch(LocalUcastMacs mac, String logicalSwitchName) {
453 InstanceIdentifier<LogicalSwitches> iid = (InstanceIdentifier<LogicalSwitches>)
454 mac.getLogicalSwitchRef().getValue();
455 return iid.firstKeyOf(LogicalSwitches.class).getHwvtepNodeName().getValue().equals(logicalSwitchName);
458 static InstanceIdentifier<LocalUcastMacs> getMacIid(InstanceIdentifier<Node> nodeIid, LocalUcastMacs mac) {
459 return nodeIid.augmentation(HwvtepGlobalAugmentation.class).child(LocalUcastMacs.class, mac.key());