2 * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.openflowplugin.applications.statistics.manager.impl;
11 import com.google.common.base.Optional;
12 import com.google.common.collect.BiMap;
13 import com.google.common.collect.HashBiMap;
14 import java.util.ArrayList;
15 import java.util.Collections;
16 import java.util.HashMap;
17 import java.util.Iterator;
18 import java.util.List;
20 import java.util.Map.Entry;
21 import java.util.concurrent.atomic.AtomicInteger;
22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
23 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
25 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
26 import org.opendaylight.controller.sal.binding.api.NotificationProviderService;
27 import org.opendaylight.openflowplugin.applications.statistics.manager.StatRpcMsgManager.TransactionCacheContainer;
28 import org.opendaylight.openflowplugin.applications.statistics.manager.StatisticsManager;
29 import org.opendaylight.openflowplugin.applications.statistics.manager.StatisticsManager.StatDataStoreOperation;
30 import org.opendaylight.openflowplugin.applications.statistics.manager.StatisticsManager.StatDataStoreOperation.StatsManagerOperationType;
31 import org.opendaylight.openflowplugin.applications.statistics.manager.impl.helper.FlowComparator;
32 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowHashIdMapping;
34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowHashIdMappingBuilder;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.nodes.node.table.FlowHashIdMap;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.nodes.node.table.FlowHashIdMapBuilder;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.nodes.node.table.FlowHashIdMapKey;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableBuilder;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.AggregateFlowStatisticsData;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.AggregateFlowStatisticsDataBuilder;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.AggregateFlowStatisticsUpdate;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.FlowStatisticsData;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.FlowStatisticsDataBuilder;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.FlowsStatisticsUpdate;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.OpendaylightFlowStatisticsListener;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.aggregate.flow.statistics.AggregateFlowStatisticsBuilder;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.flow.and.statistics.map.list.FlowAndStatisticsMapList;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.flow.and.statistics.map.list.FlowAndStatisticsMapListBuilder;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.statistics.rev130819.flow.statistics.FlowStatisticsBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.transaction.rev150304.TransactionAware;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.transaction.rev150304.TransactionId;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
62 import org.opendaylight.yangtools.yang.binding.DataObject;
63 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
64 import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier;
65 import org.slf4j.Logger;
66 import org.slf4j.LoggerFactory;
70 * org.opendaylight.openflowplugin.applications.statistics.manager.impl
72 * StatListenCommitFlow
73 * Class is a NotifyListener for FlowStatistics and DataChangeListener for Config/DataStore for Flow node.
74 * All expected (registered) FlowStatistics will be builded and commit to Operational/DataStore.
75 * DataChangeEven should call create/delete Flow in Operational/DS create process needs to pair
76 * Device Flow HashCode and FlowId from Config/DS
78 * @author <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
81 public class StatListenCommitFlow extends StatAbstractListenCommit<Flow, OpendaylightFlowStatisticsListener>
82 implements OpendaylightFlowStatisticsListener {
84 protected static final Logger LOG = LoggerFactory.getLogger(StatListenCommitFlow.class);
86 private static final String ALIEN_SYSTEM_FLOW_ID = "#UF$TABLE*";
88 private static final Integer REMOVE_AFTER_MISSING_COLLECTION = 1;
89 private static final int TRUNCATED_LOG_MESSAGE_LENGTH = 30;
91 private final AtomicInteger unaccountedFlowsCounter = new AtomicInteger(0);
93 public StatListenCommitFlow (final StatisticsManager manager, final DataBroker db,
94 final NotificationProviderService nps){
95 super(manager, db, nps, Flow.class);
99 protected OpendaylightFlowStatisticsListener getStatNotificationListener() {
104 protected InstanceIdentifier<Flow> getWildCardedRegistrationPath() {
105 return InstanceIdentifier.create(Nodes.class).child(Node.class)
106 .augmentation(FlowCapableNode.class).child(Table.class).child(Flow.class);
110 public void onAggregateFlowStatisticsUpdate(final AggregateFlowStatisticsUpdate notification) {
111 final TransactionId transId = notification.getTransactionId();
112 final NodeId nodeId = notification.getId();
113 if ( ! isExpectedStatistics(transId, nodeId)) {
114 LOG.debug("STAT-MANAGER - AggregateFlowStatisticsUpdate: unregistred notification detect TransactionId {}", transId);
117 manager.getRpcMsgManager().addNotification(notification, nodeId);
118 if (notification.isMoreReplies()) {
121 /* check flow Capable Node and write statistics */
122 manager.enqueue(new StatDataStoreOperation(StatsManagerOperationType.DATA_COMMIT_OPER_DS,nodeId) {
124 public void applyOperation(final ReadWriteTransaction tx) {
126 final Optional<TransactionCacheContainer<?>> txContainer = getTransactionCacheContainer(transId, nodeId);
127 if (( ! txContainer.isPresent()) || txContainer.get().getNotifications() == null) {
130 final Optional<? extends DataObject> inputObj = txContainer.get().getConfInput();
131 if (( ! inputObj.isPresent()) || ( ! (inputObj.get() instanceof Table))) {
134 final Table table = (Table) inputObj.get();
135 final List<? extends TransactionAware> cacheNotifs = txContainer.get().getNotifications();
136 for (final TransactionAware notif : cacheNotifs) {
137 if (notif instanceof AggregateFlowStatisticsUpdate) {
138 final AggregateFlowStatisticsData stats = new AggregateFlowStatisticsDataBuilder()
139 .setAggregateFlowStatistics(new AggregateFlowStatisticsBuilder(notification).build()).build();
140 final InstanceIdentifier<FlowCapableNode> fNodeIdent = InstanceIdentifier.create(Nodes.class)
141 .child(Node.class, new NodeKey(nodeId)).augmentation(FlowCapableNode.class);
142 final InstanceIdentifier<Table> tableRef = fNodeIdent.child(Table.class, table.getKey());
143 final InstanceIdentifier<AggregateFlowStatisticsData> tableStatRef = tableRef
144 .augmentation(AggregateFlowStatisticsData.class);
145 Optional<FlowCapableNode> fNode = Optional.absent();
147 fNode = tx.read(LogicalDatastoreType.OPERATIONAL, fNodeIdent).checkedGet();
148 } catch (final ReadFailedException e) {
149 LOG.debug("Read Operational/DS for FlowCapableNode fail! {}", fNodeIdent, e);
152 if (fNode.isPresent()) {
153 ensureTable(tx, table.getId(), tableRef);
154 tx.put(LogicalDatastoreType.OPERATIONAL, tableStatRef, stats);
162 public void ensureTable(final ReadWriteTransaction tx, final Short tableId, final InstanceIdentifier<Table> tableRef) {
163 final Table tableNew = new TableBuilder().setId(tableId).build();
164 tx.merge(LogicalDatastoreType.OPERATIONAL, tableRef, tableNew);
168 public void onFlowsStatisticsUpdate(final FlowsStatisticsUpdate notification) {
169 final TransactionId transId = notification.getTransactionId();
170 final NodeId nodeId = notification.getId();
171 if ( ! isExpectedStatistics(transId, nodeId)) {
172 LOG.debug("STAT-MANAGER - FlowsStatisticsUpdate: unregistred notification detect TransactionId {}", transId);
175 manager.getRpcMsgManager().addNotification(notification, nodeId);
176 if (notification.isMoreReplies()) {
177 LOG.trace("Next notification for join txId {}", transId);
180 /* add flow's statistics */
181 manager.enqueue(new StatDataStoreOperation(StatsManagerOperationType.DATA_COMMIT_OPER_DS,nodeId) {
183 public void applyOperation(final ReadWriteTransaction tx) {
184 final Optional<TransactionCacheContainer<?>> txContainer = getTransactionCacheContainer(transId, nodeId);
185 if (( ! txContainer.isPresent()) || txContainer.get().getNotifications() == null) {
188 final List<FlowAndStatisticsMapList> flowStats = new ArrayList<FlowAndStatisticsMapList>(10);
189 final InstanceIdentifier<Node> nodeIdent = InstanceIdentifier.create(Nodes.class)
190 .child(Node.class, new NodeKey(nodeId));
191 final List<? extends TransactionAware> cacheNotifs = txContainer.get().getNotifications();
192 for (final TransactionAware notif : cacheNotifs) {
193 if (notif instanceof FlowsStatisticsUpdate) {
194 final List<FlowAndStatisticsMapList> notifList =
195 ((FlowsStatisticsUpdate) notif).getFlowAndStatisticsMapList();
196 if (notifList != null) {
197 flowStats.addAll(notifList);
202 statsFlowCommitAll(flowStats, nodeIdent, tx);
203 /* cleaning all not cached hash collisions */
204 final Map<InstanceIdentifier<Flow>, Integer> listAliens = mapNodesForDelete.get(nodeIdent);
205 if (listAliens != null) {
206 for (final Entry<InstanceIdentifier<Flow>, Integer> nodeForDelete : listAliens.entrySet()) {
207 final Integer lifeIndex = nodeForDelete.getValue();
208 if (nodeForDelete.getValue() > 0) {
209 nodeForDelete.setValue(Integer.valueOf(lifeIndex.intValue() - 1));
211 final InstanceIdentifier<Flow> flowNodeIdent = nodeForDelete.getKey();
212 mapNodesForDelete.get(nodeIdent).remove(flowNodeIdent);
213 tx.delete(LogicalDatastoreType.OPERATIONAL, flowNodeIdent);
217 /* Notification for continue collecting statistics */
218 notifyToCollectNextStatistics(nodeIdent, transId);
224 private void statsFlowCommitAll(final List<FlowAndStatisticsMapList> list,
225 final InstanceIdentifier<Node> nodeIdent, final ReadWriteTransaction tx) {
227 final InstanceIdentifier<FlowCapableNode> fNodeIdent = nodeIdent.augmentation(FlowCapableNode.class);
229 final Optional<FlowCapableNode> fNode;
231 fNode = tx.read(LogicalDatastoreType.OPERATIONAL, fNodeIdent).checkedGet();
233 catch (final ReadFailedException e) {
234 LOG.debug("Read FlowCapableNode {} in Operational/DS fail! Statistic scan not be updated.", nodeIdent, e);
237 if ( ! fNode.isPresent()) {
238 LOG.trace("FlowCapableNode {} is not presented in Operational/DS. Statisticscan not be updated.", nodeIdent);
242 final NodeUpdateState nodeState = new NodeUpdateState(fNodeIdent,fNode.get());
244 for (final FlowAndStatisticsMapList flowStat : list) {
245 final TableKey tableKey = new TableKey(flowStat.getTableId());
246 final TableFlowUpdateState tableState = nodeState.getTable(tableKey, tx);
247 tableState.reportFlow(flowStat,tx);
250 for (final TableFlowUpdateState table : nodeState.getTables()) {
251 table.removeUnreportedFlows(tx);
256 * Method adds statistics to Flow
261 private void addStatistics(final FlowBuilder flowBuilder, final FlowAndStatisticsMapList deviceFlow) {
262 final FlowAndStatisticsMapListBuilder stats = new FlowAndStatisticsMapListBuilder(deviceFlow);
263 final FlowStatisticsBuilder flowStatisticsBuilder = new FlowStatisticsBuilder(stats.build());
264 final FlowStatisticsDataBuilder flowStatisticsData =new FlowStatisticsDataBuilder();
265 flowStatisticsData.setFlowStatistics(flowStatisticsBuilder.build());
266 flowBuilder.addAugmentation(FlowStatisticsData.class, flowStatisticsData.build());
270 * build pseudoUnique hashCode for flow in table
271 * for future easy identification
273 * FIXME: we expect same version for YANG models for all clusters and that has to be fix
274 * FIXME: CREATE BETTER KEY - for flow (MATCH is the problem)
276 static String buildFlowIdOperKey(final FlowAndStatisticsMapList deviceFlow) {
277 return new StringBuilder().append(deviceFlow.getMatch())
278 .append(deviceFlow.getPriority()).append(deviceFlow.getCookie().getValue()).toString();
281 private class NodeUpdateState {
282 private final InstanceIdentifier<FlowCapableNode> nodeIdentifier;
283 private final Map<TableKey,TableFlowUpdateState> tables = new HashMap<>();
285 public NodeUpdateState(final InstanceIdentifier<FlowCapableNode> fNodeIdent, final FlowCapableNode flowCapableNode) {
286 nodeIdentifier = fNodeIdent;
287 final List<Table> tableList = flowCapableNode.getTable();
288 if(tableList != null) {
289 for (final Table table : tableList) {
290 final TableKey tableKey = table.getKey();
291 tables.put(tableKey, new TableFlowUpdateState(nodeIdentifier.child(Table.class,tableKey),table));
296 public Iterable<TableFlowUpdateState> getTables() {
297 return tables.values();
300 TableFlowUpdateState getTable(final TableKey key,final ReadWriteTransaction tx) {
301 TableFlowUpdateState table = tables.get(key);
303 table = new TableFlowUpdateState(nodeIdentifier.child(Table.class, key), null);
304 tables.put(key, table);
310 private class TableFlowUpdateState {
312 private boolean tableEnsured = false;
313 final KeyedInstanceIdentifier<Table, TableKey> tableRef;
314 final TableKey tableKey;
315 final BiMap<FlowHashIdMapKey, FlowId> flowIdByHash;
316 List<Flow> configFlows;
318 public TableFlowUpdateState(final KeyedInstanceIdentifier<Table, TableKey> tablePath, final Table table) {
319 tableRef = tablePath;
320 tableKey = tablePath.getKey();
321 flowIdByHash = HashBiMap.create();
323 final FlowHashIdMapping flowHashMapping = table.getAugmentation(FlowHashIdMapping.class);
324 if (flowHashMapping != null) {
325 final List<FlowHashIdMap> flowHashMap = flowHashMapping.getFlowHashIdMap() != null
326 ? flowHashMapping.getFlowHashIdMap() : Collections.<FlowHashIdMap> emptyList();
327 for (final FlowHashIdMap flowHashId : flowHashMap) {
329 flowIdByHash.put(flowHashId.getKey(), flowHashId.getFlowId());
330 } catch (final Exception e) {
331 //flowHashId.getKey() too verbose for standard log.
332 if(LOG.isDebugEnabled()) {
333 final FlowId currData = flowIdByHash.get(flowHashId.getKey());
334 LOG.debug("flow hashing hit a duplicate for {} -> {}. Curr value: {} Equals:{}. Exception was raised:",
335 flowHashId.getKey(), flowHashId.getFlowId(), currData, flowHashId.getFlowId().equals(currData), e);
339 LOG.warn("flow hashing hit a duplicate {}. Exception was raised: {}. Enable DEBUG for more detail.",
340 flowHashId.getFlowId().toString().substring(0, Math.min(TRUNCATED_LOG_MESSAGE_LENGTH,flowHashId.getFlowId().toString().length())),
341 e.getMessage().substring(0,Math.min(TRUNCATED_LOG_MESSAGE_LENGTH,e.getMessage().length())));
349 private void ensureTableFowHashIdMapping(final ReadWriteTransaction tx) {
350 if( ! tableEnsured) {
351 ensureTable(tx, tableKey.getId(), tableRef);
352 final FlowHashIdMapping emptyMapping = new FlowHashIdMappingBuilder()
353 .setFlowHashIdMap(Collections.<FlowHashIdMap> emptyList()).build();
354 tx.merge(LogicalDatastoreType.OPERATIONAL, tableRef.augmentation(FlowHashIdMapping.class), emptyMapping);
359 private FlowKey searchInConfiguration(final FlowAndStatisticsMapList flowStat, final ReadWriteTransaction trans) {
361 final Iterator<Flow> it = configFlows.iterator();
362 while(it.hasNext()) {
363 final Flow cfgFlow = it.next();
364 final FlowKey cfgKey = cfgFlow.getKey();
365 final FlowId cfgFlowId = cfgKey.getId();
367 if(! flowIdByHash.inverse().containsKey(cfgFlowId)) {
368 if(FlowComparator.flowEquals(flowStat, cfgFlow)) {
376 private void initConfigFlows() {
377 final Optional<Table> table = readLatestConfiguration(tableRef);
378 List<Flow> localList = null;
379 if(table.isPresent()) {
380 localList = table.get().getFlow();
382 if(localList == null) {
383 configFlows = Collections.emptyList();
385 configFlows = new ArrayList<>(localList);
389 private FlowKey getFlowKeyAndRemoveHash(final FlowHashIdMapKey key) {
390 final FlowId ret = flowIdByHash.get(key);
392 flowIdByHash.remove(key);
393 return new FlowKey(ret);
398 /* Returns FlowKey which doesn't exist in any DataStore for now */
399 private FlowKey makeAlienFlowKey() {
400 final StringBuilder sBuilder = new StringBuilder(ALIEN_SYSTEM_FLOW_ID)
401 .append(tableKey.getId()).append("-").append(unaccountedFlowsCounter.incrementAndGet());
402 final FlowId flowId = new FlowId(sBuilder.toString());
403 return new FlowKey(flowId);
406 private Map<FlowHashIdMapKey, FlowId> getRemovalList() {
410 void reportFlow(final FlowAndStatisticsMapList flowStat, final ReadWriteTransaction trans) {
411 ensureTableFowHashIdMapping(trans);
412 final FlowHashIdMapKey hashingKey = new FlowHashIdMapKey(buildFlowIdOperKey(flowStat));
413 FlowKey flowKey = getFlowKeyAndRemoveHash(hashingKey);
414 if (flowKey == null) {
415 flowKey = searchInConfiguration(flowStat, trans);
416 if ( flowKey == null) {
417 flowKey = makeAlienFlowKey();
419 updateHashCache(trans,flowKey,hashingKey);
421 final FlowBuilder flowBuilder = new FlowBuilder(flowStat);
422 flowBuilder.setKey(flowKey);
423 addStatistics(flowBuilder, flowStat);
424 final InstanceIdentifier<Flow> flowIdent = tableRef.child(Flow.class, flowKey);
425 trans.put(LogicalDatastoreType.OPERATIONAL, flowIdent, flowBuilder.build());
426 /* check life for Alien flows */
427 if (flowKey.getId().getValue().startsWith(ALIEN_SYSTEM_FLOW_ID)) {
428 removeData(flowIdent, REMOVE_AFTER_MISSING_COLLECTION);
432 /* Build and deploy new FlowHashId map */
433 private void updateHashCache(final ReadWriteTransaction trans, final FlowKey flowKey, final FlowHashIdMapKey hashingKey) {
434 final FlowHashIdMapBuilder flHashIdMap = new FlowHashIdMapBuilder();
435 flHashIdMap.setFlowId(flowKey.getId());
436 flHashIdMap.setKey(hashingKey);
437 final KeyedInstanceIdentifier<FlowHashIdMap, FlowHashIdMapKey> flHashIdent = tableRef
438 .augmentation(FlowHashIdMapping.class).child(FlowHashIdMap.class, hashingKey);
439 /* Add new FlowHashIdMap */
440 trans.put(LogicalDatastoreType.OPERATIONAL, flHashIdent, flHashIdMap.build());
443 void removeUnreportedFlows(final ReadWriteTransaction tx) {
444 final InstanceIdentifier<Node> nodeIdent = tableRef.firstIdentifierOf(Node.class);
445 final List<InstanceIdentifier<Flow>> listMissingConfigFlows = notStatReportedConfigFlows();
446 final Map<InstanceIdentifier<Flow>, Integer> nodeDeleteMap = mapNodesForDelete.get(nodeIdent);
447 final Map<FlowHashIdMapKey, FlowId> listForRemove = getRemovalList();
448 for (final Entry<FlowHashIdMapKey, FlowId> entryForRemove : listForRemove.entrySet()) {
449 final FlowKey flowKey = new FlowKey(entryForRemove.getValue());
450 final InstanceIdentifier<Flow> flowRef = tableRef.child(Flow.class, flowKey);
451 if (nodeDeleteMap != null && flowKey.getId().getValue().startsWith(ALIEN_SYSTEM_FLOW_ID)) {
452 final Integer lifeIndex = nodeDeleteMap.get(flowRef);
456 nodeDeleteMap.remove(flowRef);
459 if (listMissingConfigFlows.remove(flowRef)) {
460 // it is probable that some multipart message was lost
464 final InstanceIdentifier<FlowHashIdMap> flHashIdent =
465 tableRef.augmentation(FlowHashIdMapping.class).child(FlowHashIdMap.class, entryForRemove.getKey());
466 tx.delete(LogicalDatastoreType.OPERATIONAL, flowRef);
467 tx.delete(LogicalDatastoreType.OPERATIONAL, flHashIdent);
471 List<InstanceIdentifier<Flow>> notStatReportedConfigFlows() {
472 if (configFlows != null) {
473 final List<InstanceIdentifier<Flow>> returnList = new ArrayList<>(configFlows.size());
474 for (final Flow confFlow : configFlows) {
475 final InstanceIdentifier<Flow> confFlowIdent = tableRef.child(Flow.class, confFlow.getKey());
476 returnList.add(confFlowIdent);
480 return Collections.emptyList();