Fix statistics race condition on big flows
[openflowplugin.git] / openflowplugin-impl / src / main / java / org / opendaylight / openflowplugin / impl / statistics / StatisticsGatheringUtils.java
index 4730e085bd6e552e7677e2ef29971aa7eaa22e4b..966608079424ad29c84e875d2eda890e1a989e8c 100644 (file)
@@ -11,29 +11,39 @@ package org.opendaylight.openflowplugin.impl.statistics;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.AsyncFunction;
 import com.google.common.util.concurrent.CheckedFuture;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.JdkFutureAdapters;
 import com.google.common.util.concurrent.ListenableFuture;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import javax.annotation.Nullable;
-import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
+import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionChainClosedException;
 import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext;
+import org.opendaylight.openflowplugin.api.openflow.device.DeviceInfo;
+import org.opendaylight.openflowplugin.api.openflow.device.DeviceRegistry;
+import org.opendaylight.openflowplugin.api.openflow.device.TxFacade;
+import org.opendaylight.openflowplugin.api.openflow.registry.flow.DeviceFlowRegistry;
 import org.opendaylight.openflowplugin.api.openflow.registry.flow.FlowRegistryKey;
+import org.opendaylight.openflowplugin.api.openflow.registry.group.DeviceGroupRegistry;
+import org.opendaylight.openflowplugin.api.openflow.registry.meter.DeviceMeterRegistry;
 import org.opendaylight.openflowplugin.api.openflow.statistics.ofpspecific.EventIdentifier;
 import org.opendaylight.openflowplugin.api.openflow.statistics.ofpspecific.StatisticsGatherer;
 import org.opendaylight.openflowplugin.impl.registry.flow.FlowRegistryKeyFactory;
 import org.opendaylight.openflowplugin.impl.statistics.ofpspecific.EventsTimeCounter;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.DateAndTime;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableStatisticsGatheringStatus;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableStatisticsGatheringStatusBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.Meter;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.MeterBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.MeterKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.snapshot.gathering.status.grouping.SnapshotGatheringStatusEnd;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.snapshot.gathering.status.grouping.SnapshotGatheringStatusEndBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.snapshot.gathering.status.grouping.SnapshotGatheringStatusStartBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
@@ -92,51 +102,80 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.queue.statistics.rev131216.
 import org.opendaylight.yang.gen.v1.urn.opendaylight.queue.statistics.rev131216.queue.id.and.statistics.map.QueueIdAndStatisticsMap;
 import org.opendaylight.yangtools.yang.binding.DataObject;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
-import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier;
 import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
 /**
- * Created by Martin Bobak <mbobak@cisco.com> on 2.4.2015.
+ * Utils for gathering statistics.
  */
 public final class StatisticsGatheringUtils {
 
+    private static String DATE_AND_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
+
     private static final Logger LOG = LoggerFactory.getLogger(StatisticsGatheringUtils.class);
-    private static final SinglePurposeMultipartReplyTranslator MULTIPART_REPLY_TRANSLATOR = new SinglePurposeMultipartReplyTranslator();
-    public static final String QUEUE2_REQCTX = "QUEUE2REQCTX-";
+    private static final String QUEUE2_REQCTX = "QUEUE2REQCTX-";
 
     private StatisticsGatheringUtils() {
         throw new IllegalStateException("This class should not be instantiated.");
     }
 
-
-    public static ListenableFuture<Boolean> gatherStatistics(final StatisticsGatherer statisticsGatheringService,
-                                                             final DeviceContext deviceContext,
-                                                             final MultipartType type) {
-        final String deviceId = deviceContext.getPrimaryConnectionContext().getNodeId().toString();
+    //TODO: Flow-,Group- and Meter- registry should be not in device context, consider move it in separate class
+    static ListenableFuture<Boolean> gatherStatistics(
+                        final StatisticsGatherer statisticsGatheringService,
+                        final DeviceInfo deviceInfo,
+                        final MultipartType type,
+                        final TxFacade txFacade,
+                        final DeviceRegistry registry,
+                        final Boolean initial,
+                        final SinglePurposeMultipartReplyTranslator multipartReplyTranslator) {
         EventIdentifier wholeProcessEventIdentifier = null;
         if (MultipartType.OFPMPFLOW.equals(type)) {
-            wholeProcessEventIdentifier = new EventIdentifier(type.toString(), deviceId);
+            wholeProcessEventIdentifier = new EventIdentifier(type.toString(), deviceInfo.getNodeId().getValue());
             EventsTimeCounter.markStart(wholeProcessEventIdentifier);
         }
-        EventIdentifier ofpQueuToRequestContextEventIdentifier = new EventIdentifier(QUEUE2_REQCTX + type.toString(), deviceId);
+        final EventIdentifier ofpQueueToRequestContextEventIdentifier
+                = new EventIdentifier(QUEUE2_REQCTX + type.toString(), deviceInfo.getNodeId().toString());
         final ListenableFuture<RpcResult<List<MultipartReply>>> statisticsDataInFuture =
                 JdkFutureAdapters.listenInPoolThread(statisticsGatheringService.getStatisticsOfType(
-                        ofpQueuToRequestContextEventIdentifier, type));
-        return transformAndStoreStatisticsData(statisticsDataInFuture, deviceContext, wholeProcessEventIdentifier, type);
+                        ofpQueueToRequestContextEventIdentifier, type));
+        return transformAndStoreStatisticsData(
+                statisticsDataInFuture,
+                deviceInfo,
+                wholeProcessEventIdentifier,
+                type,
+                txFacade,
+                registry,
+                initial,
+                multipartReplyTranslator);
     }
 
-    private static ListenableFuture<Boolean> transformAndStoreStatisticsData(final ListenableFuture<RpcResult<List<MultipartReply>>> statisticsDataInFuture,
-                                                                             final DeviceContext deviceContext,
-                                                                             final EventIdentifier eventIdentifier, final MultipartType type) {
-        return Futures.transform(statisticsDataInFuture, new Function<RpcResult<List<MultipartReply>>, Boolean>() {
+    private static ListenableFuture<Boolean> transformAndStoreStatisticsData(
+                        final ListenableFuture<RpcResult<List<MultipartReply>>> statisticsDataInFuture,
+                        final DeviceInfo deviceInfo,
+                        final EventIdentifier eventIdentifier,
+                        final MultipartType type,
+                        final TxFacade txFacade,
+                        final DeviceRegistry registry,
+                        final boolean initial,
+                        final SinglePurposeMultipartReplyTranslator multipartReplyTranslator) {
+        return Futures.transform(statisticsDataInFuture, new AsyncFunction<RpcResult<List<MultipartReply>>, Boolean>() {
             @Nullable
             @Override
-            public Boolean apply(final RpcResult<List<MultipartReply>> rpcResult) {
+            public ListenableFuture<Boolean> apply(final RpcResult<List<MultipartReply>> rpcResult) {
+                boolean isMultipartProcessed = Boolean.TRUE;
                 if (rpcResult.isSuccessful()) {
-                    LOG.debug("Stats reply successfully received for node {} of type {}", deviceContext.getDeviceState().getNodeId(), type);
-                    boolean isMultipartProcessed = Boolean.TRUE;
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Stats reply successfully received for node {} of type {}",
+                                deviceInfo.getNodeId(), type);
+                    }
 
                     // TODO: in case the result value is null then multipart data probably got processed on the fly -
                     // TODO: this contract should by clearly stated and enforced - now simple true value is returned
@@ -147,144 +186,256 @@ public final class StatisticsGatheringUtils {
 
                         try {
                             for (final MultipartReply singleReply : rpcResult.getResult()) {
-                                final List<? extends DataObject> multipartDataList = MULTIPART_REPLY_TRANSLATOR.translate(deviceContext, singleReply);
+                                final List<? extends DataObject> multipartDataList = multipartReplyTranslator.translate(
+                                        deviceInfo.getDatapathId(),
+                                        deviceInfo.getVersion(), singleReply);
                                 multipartData = multipartDataList.get(0);
                                 allMultipartData = Iterables.concat(allMultipartData, multipartDataList);
                             }
-                        } catch (Exception e) {
-                            LOG.warn("stats processing of type {} for node {} failed during transfomation step",
-                                    type, deviceContext.getDeviceState().getNodeId(), e);
-                            throw e;
+                        } catch (final TransactionChainClosedException e) {
+                            LOG.warn("stats processing of type {} for node {} failed during transformation step",
+                                    type, deviceInfo.getNodeId(), e);
+                            return Futures.immediateFailedFuture(e);
                         }
 
 
                         try {
                             if (multipartData instanceof GroupStatisticsUpdated) {
-                                processGroupStatistics((Iterable<GroupStatisticsUpdated>) allMultipartData, deviceContext);
+                                processGroupStatistics(
+                                        (Iterable<GroupStatisticsUpdated>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade);
                             } else if (multipartData instanceof MeterStatisticsUpdated) {
-                                processMetersStatistics((Iterable<MeterStatisticsUpdated>) allMultipartData, deviceContext);
+                                processMetersStatistics(
+                                        (Iterable<MeterStatisticsUpdated>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade);
                             } else if (multipartData instanceof NodeConnectorStatisticsUpdate) {
-                                processNodeConnectorStatistics((Iterable<NodeConnectorStatisticsUpdate>) allMultipartData, deviceContext);
+                                processNodeConnectorStatistics(
+                                        (Iterable<NodeConnectorStatisticsUpdate>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade);
                             } else if (multipartData instanceof FlowTableStatisticsUpdate) {
-                                processFlowTableStatistics((Iterable<FlowTableStatisticsUpdate>) allMultipartData, deviceContext);
+                                processFlowTableStatistics(
+                                        (Iterable<FlowTableStatisticsUpdate>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade);
                             } else if (multipartData instanceof QueueStatisticsUpdate) {
-                                processQueueStatistics((Iterable<QueueStatisticsUpdate>) allMultipartData, deviceContext);
+                                processQueueStatistics(
+                                        (Iterable<QueueStatisticsUpdate>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade);
                             } else if (multipartData instanceof FlowsStatisticsUpdate) {
-                                processFlowStatistics((Iterable<FlowsStatisticsUpdate>) allMultipartData, deviceContext);
-                                EventsTimeCounter.markEnd(eventIdentifier);
+                                /* FlowStat Processing is realized by NettyThread only by initPhase,
+                                 * otherwise it is realized
+                                 * by MD-SAL thread */
+                                return processFlowStatistics(
+                                        (Iterable<FlowsStatisticsUpdate>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade,
+                                        registry.getDeviceFlowRegistry(),
+                                        initial,
+                                        eventIdentifier);
+
                             } else if (multipartData instanceof GroupDescStatsUpdated) {
-                                processGroupDescStats((Iterable<GroupDescStatsUpdated>) allMultipartData, deviceContext);
+                                processGroupDescStats(
+                                        (Iterable<GroupDescStatsUpdated>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade,
+                                        registry.getDeviceGroupRegistry());
                             } else if (multipartData instanceof MeterConfigStatsUpdated) {
-                                processMeterConfigStatsUpdated((Iterable<MeterConfigStatsUpdated>) allMultipartData, deviceContext);
+                                processMeterConfigStatsUpdated(
+                                        (Iterable<MeterConfigStatsUpdated>) allMultipartData,
+                                        deviceInfo,
+                                        txFacade,
+                                        registry.getDeviceMeterRegistry());
                             } else {
                                 isMultipartProcessed = Boolean.FALSE;
                             }
-                        } catch (Exception e) {
+                        } catch (final TransactionChainClosedException e) {
                             LOG.warn("stats processing of type {} for node {} failed during write-to-tx step",
-                                    type, deviceContext.getDeviceState().getNodeId(), e);
-                            throw e;
+                                    type, deviceInfo.getNodeId(), e);
+                            return Futures.immediateFailedFuture(e);
                         }
 
-                        LOG.debug("Stats reply added to transaction for node {} of type {}", deviceContext.getDeviceState().getNodeId(), type);
+                        if (LOG.isDebugEnabled()) {
+                            LOG.debug("Stats reply added to transaction for node {} of type {}",
+                                    deviceInfo.getNodeId(), type);
+                        }
 
                         //TODO : implement experimenter
                     } else {
-                        LOG.debug("Stats reply was empty for node {} of type {}", deviceContext.getDeviceState().getNodeId(), type);
+                        if (LOG.isDebugEnabled()) {
+                            LOG.debug("Stats reply was empty for node {} of type {}",
+                                    deviceInfo.getNodeId(), type);
+                        }
                     }
 
-                    return isMultipartProcessed;
                 } else {
-                    LOG.debug("Stats reply FAILED for node {} of type {}: {}", deviceContext.getDeviceState().getNodeId(), type, rpcResult.getErrors());
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Stats reply FAILED for node {} of type {}: {}",
+                                deviceInfo.getNodeId(), type, rpcResult.getErrors());
+                    }
+                    isMultipartProcessed = Boolean.FALSE;
                 }
-                return Boolean.FALSE;
+                return Futures.immediateFuture(isMultipartProcessed);
             }
         });
     }
 
-    private static void processMeterConfigStatsUpdated(final Iterable<MeterConfigStatsUpdated> data, final DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
-        deleteAllKnownMeters(deviceContext, fNodeIdent);
+    private static void processMeterConfigStatsUpdated(final Iterable<MeterConfigStatsUpdated> data,
+                                                       final DeviceInfo deviceInfo,
+                                                       final TxFacade txFacade,
+                                                       final DeviceMeterRegistry meterRegistry)
+            throws TransactionChainClosedException {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
+        deleteAllKnownMeters(meterRegistry, fNodeIdent, txFacade);
         for (final MeterConfigStatsUpdated meterConfigStatsUpdated : data) {
             for (final MeterConfigStats meterConfigStats : meterConfigStatsUpdated.getMeterConfigStats()) {
                 final MeterId meterId = meterConfigStats.getMeterId();
-                final InstanceIdentifier<Meter> meterInstanceIdentifier = fNodeIdent.child(Meter.class, new MeterKey(meterId));
+                final InstanceIdentifier<Meter> meterInstanceIdentifier
+                        = fNodeIdent.child(Meter.class, new MeterKey(meterId));
 
                 final MeterBuilder meterBuilder = new MeterBuilder(meterConfigStats);
                 meterBuilder.setKey(new MeterKey(meterId));
                 meterBuilder.addAugmentation(NodeMeterStatistics.class, new NodeMeterStatisticsBuilder().build());
-                deviceContext.getDeviceMeterRegistry().store(meterId);
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, meterInstanceIdentifier, meterBuilder.build());
+                meterRegistry.store(meterId);
+                txFacade.writeToTransaction(
+                        LogicalDatastoreType.OPERATIONAL,
+                        meterInstanceIdentifier,
+                        meterBuilder.build());
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
-    private static void processFlowStatistics(final Iterable<FlowsStatisticsUpdate> data, final DeviceContext deviceContext) {
-        deleteAllKnownFlows(deviceContext);
-        writeFlowStatistics(data, deviceContext);
-        deviceContext.submitTransaction();
+    private static ListenableFuture<Boolean> processFlowStatistics(final Iterable<FlowsStatisticsUpdate> data,
+                                                                   final DeviceInfo deviceInfo,
+                                                                   final TxFacade txFacade,
+                                                                   final DeviceFlowRegistry flowRegistry,
+                                                                   final boolean initial,
+                                                                   final EventIdentifier eventIdentifier) {
+        final ListenableFuture<Void> deleteFuture
+                = initial ? Futures.immediateFuture(null) : deleteAllKnownFlows(deviceInfo, flowRegistry, txFacade);
+        return Futures.transform(deleteFuture, (Function<Void, Boolean>) input -> {
+            writeFlowStatistics(data, deviceInfo, flowRegistry, txFacade);
+            txFacade.submitTransaction();
+            EventsTimeCounter.markEnd(eventIdentifier);
+            flowRegistry.processMarks();
+            return Boolean.TRUE;
+        });
     }
 
-    public static void writeFlowStatistics(Iterable<FlowsStatisticsUpdate> data, DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
-        for (final FlowsStatisticsUpdate flowsStatistics : data) {
-            for (final FlowAndStatisticsMapList flowStat : flowsStatistics.getFlowAndStatisticsMapList()) {
-                final FlowBuilder flowBuilder = new FlowBuilder(flowStat);
-                flowBuilder.addAugmentation(FlowStatisticsData.class, refineFlowStatisticsAugmentation(flowStat).build());
-
-                final short tableId = flowStat.getTableId();
-                final FlowRegistryKey flowRegistryKey = FlowRegistryKeyFactory.create(flowBuilder.build());
-                final FlowId flowId = deviceContext.getDeviceFlowRegistry().storeIfNecessary(flowRegistryKey, tableId);
-
-                final FlowKey flowKey = new FlowKey(flowId);
-                flowBuilder.setKey(flowKey);
-                final TableKey tableKey = new TableKey(tableId);
-                final InstanceIdentifier<Flow> flowIdent = fNodeIdent.child(Table.class, tableKey).child(Flow.class, flowKey);
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, flowIdent, flowBuilder.build());
+    public static void writeFlowStatistics(final Iterable<FlowsStatisticsUpdate> data,
+                                           final DeviceInfo deviceInfo,
+                                           final DeviceFlowRegistry registry,
+                                           final TxFacade txFacade) {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
+        try {
+            for (final FlowsStatisticsUpdate flowsStatistics : data) {
+                for (final FlowAndStatisticsMapList flowStat : flowsStatistics.getFlowAndStatisticsMapList()) {
+                    final FlowBuilder flowBuilder = new FlowBuilder(flowStat);
+                    flowBuilder.addAugmentation(FlowStatisticsData.class,
+                            refineFlowStatisticsAugmentation(flowStat).build());
+
+                    final short tableId = flowStat.getTableId();
+                    final FlowRegistryKey flowRegistryKey = FlowRegistryKeyFactory.create(deviceInfo.getVersion(), flowBuilder.build());
+                    registry.store(flowRegistryKey);
+                    final FlowId flowId = registry.retrieveDescriptor(flowRegistryKey).getFlowId();
+
+                    final FlowKey flowKey = new FlowKey(flowId);
+                    flowBuilder.setKey(flowKey);
+                    final TableKey tableKey = new TableKey(tableId);
+                    final InstanceIdentifier<Flow> flowIdent
+                            = fNodeIdent.child(Table.class, tableKey).child(Flow.class, flowKey);
+                    txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, flowIdent, flowBuilder.build());
+                }
             }
+        } catch (TransactionChainClosedException e) {
+            LOG.warn("Not able to write to transaction chain: {}", e.getMessage());
         }
     }
 
     /**
-     * Method extracts flow statistics out of flowAndStatistics model
+     * Method extracts flow statistics out of flowAndStatistics model.
      *
-     * @param flowAndStats
+     * @param flowAndStats flow statistics map list
      */
-    private static FlowStatisticsDataBuilder refineFlowStatisticsAugmentation(final FlowAndStatisticsMapList flowAndStats) {
+    private static FlowStatisticsDataBuilder refineFlowStatisticsAugmentation(
+            final FlowAndStatisticsMapList flowAndStats) {
         final FlowStatisticsBuilder flowStatisticsBuilder = new FlowStatisticsBuilder(flowAndStats);
         final FlowStatisticsDataBuilder flowStatisticsDataBld = new FlowStatisticsDataBuilder();
         flowStatisticsDataBld.setFlowStatistics(flowStatisticsBuilder.build());
         return flowStatisticsDataBld;
     }
 
-    public static void deleteAllKnownFlows(final DeviceContext deviceContext) {
-        if (deviceContext.getDeviceState().deviceSynchronized()) {
-            InstanceIdentifier<FlowCapableNode> flowCapableNodePath = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
-            final Short numOfTablesOnDevice = deviceContext.getDeviceState().getFeatures().getTables();
-            for (short i = 0; i < numOfTablesOnDevice; i++) {
-                final KeyedInstanceIdentifier<Table, TableKey> iiToTable = flowCapableNodePath.child(Table.class, new TableKey(i));
-                final ReadTransaction readTx = deviceContext.getReadTransaction();
-                final CheckedFuture<Optional<Table>, ReadFailedException> tableDataFuture = readTx.read(LogicalDatastoreType.OPERATIONAL, iiToTable);
-                try {
-                    final Optional<Table> tableDataOpt = tableDataFuture.get();
-                    if (tableDataOpt.isPresent()) {
-                        final Table tableData = tableDataOpt.get();
-                        final Table table = new TableBuilder(tableData).setFlow(Collections.<Flow>emptyList()).build();
-                        deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, iiToTable, table);
+    public static ListenableFuture<Void> deleteAllKnownFlows(final DeviceInfo deviceInfo,
+                                                             final DeviceFlowRegistry registry,
+                                                             final TxFacade txFacade) {
+        final InstanceIdentifier<FlowCapableNode> flowCapableNodePath
+                = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
+        final ReadOnlyTransaction readTx = txFacade.getReadTransaction();
+        final CheckedFuture<Optional<FlowCapableNode>, ReadFailedException> flowCapableNodeFuture = readTx.read(
+                LogicalDatastoreType.OPERATIONAL, flowCapableNodePath);
+
+        /* we wish to close readTx for fallBack */
+        Futures.withFallback(flowCapableNodeFuture, t -> {
+            readTx.close();
+            return Futures.immediateFailedFuture(t);
+        });
+        /*
+         * we have to read actual tables with all information before we set empty Flow list, merge is expensive and
+         * not applicable for lists
+         */
+        return Futures.transform(
+                flowCapableNodeFuture, (AsyncFunction<Optional<FlowCapableNode>, Void>) flowCapNodeOpt -> {
+                    if (flowCapNodeOpt.isPresent()) {
+                        for (final Table tableData : flowCapNodeOpt.get().getTable()) {
+                            final Table table = new TableBuilder(tableData).setFlow(Collections.<Flow>emptyList()).build();
+                            final InstanceIdentifier<Table> iiToTable
+                                    = flowCapableNodePath.child(Table.class, tableData.getKey());
+                            txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, iiToTable, table);
+                        }
                     }
-                } catch (final InterruptedException e) {
-                    LOG.trace("Reading of table features for table wit ID {} was interrputed.", i);
-                } catch (final ExecutionException e) {
-                    LOG.trace("Reading of table features for table wit ID {} encountered execution exception {}.", i, e);
+                    readTx.close();
+                    return Futures.immediateFuture(null);
+                });
+    }
+
+    public static Optional<FlowCapableNode> deleteAllKnownFlows(final DeviceInfo deviceInfo,
+                                                             final TxFacade txFacade) {
+        final InstanceIdentifier<FlowCapableNode> flowCapableNodePath
+                = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
+        final ReadOnlyTransaction readTx = txFacade.getReadTransaction();
+        final CheckedFuture<Optional<FlowCapableNode>, ReadFailedException> flowCapableNodeFuture = readTx.read(
+                LogicalDatastoreType.OPERATIONAL, flowCapableNodePath);
+
+        try {
+            Optional<FlowCapableNode> fcNodeOpt = flowCapableNodeFuture.get();
+            if ( fcNodeOpt != null && fcNodeOpt.isPresent()){
+                for (final Table tableData : flowCapableNodeFuture.get().get().getTable()) {
+                    final Table table = new TableBuilder(tableData).setFlow(Collections.<Flow>emptyList()).build();
+                    final InstanceIdentifier<Table> iiToTable = flowCapableNodePath.child(Table.class, tableData.getKey());
+                    txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, iiToTable, table);
                 }
+                return fcNodeOpt;
+            }
+        } catch (InterruptedException|ExecutionException e) {
+            LOG.error("Failed to read current OpenFlow node {] operational data", deviceInfo.getNodeId());
+        } finally {
+            if (readTx != null){
+                readTx.close();
             }
-            deviceContext.getDeviceFlowRegistry().removeMarked();
         }
+        return Optional.absent();
     }
 
-    private static void processQueueStatistics(final Iterable<QueueStatisticsUpdate> data, final DeviceContext deviceContext) {
+    private static void processQueueStatistics(
+            final Iterable<QueueStatisticsUpdate> data,
+            final DeviceInfo deviceInfo,
+            final TxFacade txFacade) throws TransactionChainClosedException {
         // TODO: clean all queues of all node-connectors before writing up-to-date stats
-        final InstanceIdentifier<Node> nodeIdent = deviceContext.getDeviceState().getNodeInstanceIdentifier();
+        final InstanceIdentifier<Node> nodeIdent = deviceInfo.getNodeInstanceIdentifier();
         for (final QueueStatisticsUpdate queueStatisticsUpdate : data) {
             for (final QueueIdAndStatisticsMap queueStat : queueStatisticsUpdate.getQueueIdAndStatisticsMap()) {
                 if (queueStat.getQueueId() != null) {
@@ -301,75 +452,92 @@ public final class StatisticsGatheringUtils {
                     final QueueBuilder queueBuilder = new QueueBuilder()
                             .setKey(qKey)
                             .setQueueId(queueStat.getQueueId())
-                            // node-connector-id is already contained in parent node and the port-id here is of incompatible format
+                            // node-connector-id is already contained in parent node
+                            // and the port-id here is of incompatible format
                             .addAugmentation(FlowCapableNodeConnectorQueueStatisticsData.class, statBuild.build());
-                    deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, queueIdent, queueBuilder.build());
+                    txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, queueIdent, queueBuilder.build());
                 }
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
-    private static void processFlowTableStatistics(final Iterable<FlowTableStatisticsUpdate> data, final DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
+    private static void processFlowTableStatistics(
+            final Iterable<FlowTableStatisticsUpdate> data,
+            final DeviceInfo deviceInfo,
+            final TxFacade txFacade) throws TransactionChainClosedException {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
         for (final FlowTableStatisticsUpdate flowTableStatisticsUpdate : data) {
 
             for (final FlowTableAndStatisticsMap tableStat : flowTableStatisticsUpdate.getFlowTableAndStatisticsMap()) {
-                final InstanceIdentifier<FlowTableStatistics> tStatIdent = fNodeIdent.child(Table.class, new TableKey(tableStat.getTableId().getValue()))
+                final InstanceIdentifier<FlowTableStatistics> tStatIdent
+                        = fNodeIdent.child(Table.class, new TableKey(tableStat.getTableId().getValue()))
                         .augmentation(FlowTableStatisticsData.class).child(FlowTableStatistics.class);
                 final FlowTableStatistics stats = new FlowTableStatisticsBuilder(tableStat).build();
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, tStatIdent, stats);
+                txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, tStatIdent, stats);
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
-    private static void processNodeConnectorStatistics(final Iterable<NodeConnectorStatisticsUpdate> data, final DeviceContext deviceContext) {
-        final InstanceIdentifier<Node> nodeIdent = deviceContext.getDeviceState().getNodeInstanceIdentifier();
+    private static void processNodeConnectorStatistics(
+            final Iterable<NodeConnectorStatisticsUpdate> data,
+            final DeviceInfo deviceInfo,
+            final TxFacade txFacade) throws TransactionChainClosedException {
+        final InstanceIdentifier<Node> nodeIdent = deviceInfo.getNodeInstanceIdentifier();
         for (final NodeConnectorStatisticsUpdate nodeConnectorStatisticsUpdate : data) {
-            for (final NodeConnectorStatisticsAndPortNumberMap nConnectPort : nodeConnectorStatisticsUpdate.getNodeConnectorStatisticsAndPortNumberMap()) {
-                final FlowCapableNodeConnectorStatistics stats = new FlowCapableNodeConnectorStatisticsBuilder(nConnectPort).build();
-                final NodeConnectorKey key = new NodeConnectorKey(nConnectPort.getNodeConnectorId());
+            for (final NodeConnectorStatisticsAndPortNumberMap nodeConnectPort :
+                    nodeConnectorStatisticsUpdate.getNodeConnectorStatisticsAndPortNumberMap()) {
+                final FlowCapableNodeConnectorStatistics stats
+                        = new FlowCapableNodeConnectorStatisticsBuilder(nodeConnectPort).build();
+                final NodeConnectorKey key = new NodeConnectorKey(nodeConnectPort.getNodeConnectorId());
                 final InstanceIdentifier<NodeConnector> nodeConnectorIdent = nodeIdent.child(NodeConnector.class, key);
                 final InstanceIdentifier<FlowCapableNodeConnectorStatisticsData> nodeConnStatIdent = nodeConnectorIdent
                         .augmentation(FlowCapableNodeConnectorStatisticsData.class);
                 final InstanceIdentifier<FlowCapableNodeConnectorStatistics> flowCapNodeConnStatIdent =
                         nodeConnStatIdent.child(FlowCapableNodeConnectorStatistics.class);
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, flowCapNodeConnStatIdent, stats);
+                txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, flowCapNodeConnStatIdent, stats);
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
     private static void processMetersStatistics(final Iterable<MeterStatisticsUpdated> data,
-                                                final DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
+                                                final DeviceInfo deviceInfo,
+                                                final TxFacade txFacade) throws TransactionChainClosedException {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
         for (final MeterStatisticsUpdated meterStatisticsUpdated : data) {
-            for (final MeterStats mStat : meterStatisticsUpdated.getMeterStats()) {
-                final MeterStatistics stats = new MeterStatisticsBuilder(mStat).build();
-                final MeterId meterId = mStat.getMeterId();
+            for (final MeterStats meterStats : meterStatisticsUpdated.getMeterStats()) {
+                final MeterStatistics stats = new MeterStatisticsBuilder(meterStats).build();
+                final MeterId meterId = meterStats.getMeterId();
                 final InstanceIdentifier<Meter> meterIdent = fNodeIdent.child(Meter.class, new MeterKey(meterId));
                 final InstanceIdentifier<NodeMeterStatistics> nodeMeterStatIdent = meterIdent
                         .augmentation(NodeMeterStatistics.class);
                 final InstanceIdentifier<MeterStatistics> msIdent = nodeMeterStatIdent.child(MeterStatistics.class);
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, msIdent, stats);
+                txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, msIdent, stats);
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
-    private static void deleteAllKnownMeters(final DeviceContext deviceContext, final InstanceIdentifier<FlowCapableNode> fNodeIdent) {
-        for (final MeterId meterId : deviceContext.getDeviceMeterRegistry().getAllMeterIds()) {
-            final InstanceIdentifier<Meter> meterIdent = fNodeIdent.child(Meter.class, new MeterKey(meterId));
-            deviceContext.addDeleteToTxChain(LogicalDatastoreType.OPERATIONAL, meterIdent);
-        }
-        deviceContext.getDeviceMeterRegistry().removeMarked();
+    private static void deleteAllKnownMeters(
+            final DeviceMeterRegistry meterRegistry,
+            final InstanceIdentifier<FlowCapableNode> flowNodeIdent,
+            final TxFacade txFacade) throws TransactionChainClosedException {
+        meterRegistry.forEach(meterId -> {
+            final InstanceIdentifier<Meter> meterIdent = flowNodeIdent.child(Meter.class, new MeterKey(meterId));
+            txFacade.addDeleteToTxChain(LogicalDatastoreType.OPERATIONAL, meterIdent);
+        });
+        meterRegistry.processMarks();
     }
 
-    private static void processGroupDescStats(final Iterable<GroupDescStatsUpdated> data, final DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent =
-                deviceContext.getDeviceState().getNodeInstanceIdentifier().augmentation(FlowCapableNode.class);
-        deleteAllKnownGroups(deviceContext, fNodeIdent);
+    private static void processGroupDescStats(
+            final Iterable<GroupDescStatsUpdated> data,
+            final DeviceInfo deviceInfo,
+            final TxFacade txFacade,
+            final DeviceGroupRegistry groupRegistry) throws TransactionChainClosedException {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
+        deleteAllKnownGroups(txFacade, fNodeIdent, groupRegistry);
 
         for (final GroupDescStatsUpdated groupDescStatsUpdated : data) {
             for (final GroupDescStats groupDescStats : groupDescStatsUpdated.getGroupDescStats()) {
@@ -381,39 +549,99 @@ public final class StatisticsGatheringUtils {
 
                 final InstanceIdentifier<Group> groupIdent = fNodeIdent.child(Group.class, new GroupKey(groupId));
 
-                deviceContext.getDeviceGroupRegistry().store(groupId);
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, groupIdent, groupBuilder.build());
+                groupRegistry.store(groupId);
+                txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, groupIdent, groupBuilder.build());
             }
         }
-        deviceContext.submitTransaction();
+        txFacade.submitTransaction();
     }
 
-    private static void deleteAllKnownGroups(final DeviceContext deviceContext, final InstanceIdentifier<FlowCapableNode> fNodeIdent) {
-        for (final GroupId groupId : deviceContext.getDeviceGroupRegistry().getAllGroupIds()) {
-            final InstanceIdentifier<Group> groupIdent = fNodeIdent.child(Group.class, new GroupKey(groupId));
-            deviceContext.addDeleteToTxChain(LogicalDatastoreType.OPERATIONAL, groupIdent);
-        }
-        deviceContext.getDeviceGroupRegistry().removeMarked();
+    private static void deleteAllKnownGroups(
+            final TxFacade txFacade,
+            final InstanceIdentifier<FlowCapableNode> flowNodeIdent,
+            final DeviceGroupRegistry groupRegistry) throws TransactionChainClosedException {
+        groupRegistry.forEach(groupId -> {
+            final InstanceIdentifier<Group> groupIdent = flowNodeIdent.child(Group.class, new GroupKey(groupId));
+            txFacade.addDeleteToTxChain(LogicalDatastoreType.OPERATIONAL, groupIdent);
+        });
+        groupRegistry.processMarks();
     }
 
-    private static void processGroupStatistics(final Iterable<GroupStatisticsUpdated> data, final DeviceContext deviceContext) {
-        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceContext);
+    private static void processGroupStatistics(
+            final Iterable<GroupStatisticsUpdated> data,
+            final DeviceInfo deviceInfo,
+            final TxFacade txFacade) throws TransactionChainClosedException {
+        final InstanceIdentifier<FlowCapableNode> fNodeIdent = assembleFlowCapableNodeInstanceIdentifier(deviceInfo);
         for (final GroupStatisticsUpdated groupStatistics : data) {
             for (final GroupStats groupStats : groupStatistics.getGroupStats()) {
 
-                final InstanceIdentifier<Group> groupIdent = fNodeIdent.child(Group.class, new GroupKey(groupStats.getGroupId()));
+                final InstanceIdentifier<Group> groupIdent
+                        = fNodeIdent.child(Group.class, new GroupKey(groupStats.getGroupId()));
                 final InstanceIdentifier<NodeGroupStatistics> nGroupStatIdent = groupIdent
                         .augmentation(NodeGroupStatistics.class);
 
                 final InstanceIdentifier<GroupStatistics> gsIdent = nGroupStatIdent.child(GroupStatistics.class);
                 final GroupStatistics stats = new GroupStatisticsBuilder(groupStats).build();
-                deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, gsIdent, stats);
+                txFacade.writeToTransaction(LogicalDatastoreType.OPERATIONAL, gsIdent, stats);
             }
         }
+        txFacade.submitTransaction();
+    }
+
+    private static InstanceIdentifier<FlowCapableNode> assembleFlowCapableNodeInstanceIdentifier(
+            final DeviceInfo deviceInfo) {
+        return deviceInfo.getNodeInstanceIdentifier().augmentation(FlowCapableNode.class);
+    }
+
+    /**
+     * Writes snapshot gathering start timestamp + cleans end mark.
+     *
+     * @param deviceContext txManager + node path keeper
+     */
+    static void markDeviceStateSnapshotStart(final DeviceContext deviceContext) {
+        final InstanceIdentifier<FlowCapableStatisticsGatheringStatus> statusPath = deviceContext.getDeviceInfo()
+                .getNodeInstanceIdentifier().augmentation(FlowCapableStatisticsGatheringStatus.class);
+
+        final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATE_AND_TIME_FORMAT);
+        final FlowCapableStatisticsGatheringStatus gatheringStatus = new FlowCapableStatisticsGatheringStatusBuilder()
+                .setSnapshotGatheringStatusStart(new SnapshotGatheringStatusStartBuilder()
+                        .setBegin(new DateAndTime(simpleDateFormat.format(new Date())))
+                        .build())
+                .setSnapshotGatheringStatusEnd(null) // TODO: reconsider if really need to clean end mark here
+                .build();
+        try {
+            deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, statusPath, gatheringStatus);
+        } catch (final TransactionChainClosedException e) {
+            LOG.warn("Can't write to transaction, transaction chain probably closed.");
+            LOG.trace("Write to transaction exception: ", e);
+        }
+
         deviceContext.submitTransaction();
     }
 
-    private static InstanceIdentifier<FlowCapableNode> assembleFlowCapableNodeInstanceIdentifier(final DeviceContext deviceContext) {
-        return deviceContext.getDeviceState().getNodeInstanceIdentifier().augmentation(FlowCapableNode.class);
+    /**
+     * Writes snapshot gathering end timestamp + outcome.
+     *
+     * @param deviceContext txManager + node path keeper
+     * @param succeeded     outcome of currently finished gathering
+     */
+    static void markDeviceStateSnapshotEnd(final DeviceContext deviceContext, final boolean succeeded) {
+        final InstanceIdentifier<SnapshotGatheringStatusEnd> statusEndPath = deviceContext.getDeviceInfo()
+                .getNodeInstanceIdentifier().augmentation(FlowCapableStatisticsGatheringStatus.class)
+                .child(SnapshotGatheringStatusEnd.class);
+
+        final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATE_AND_TIME_FORMAT);
+        final SnapshotGatheringStatusEnd gatheringStatus = new SnapshotGatheringStatusEndBuilder()
+                .setEnd(new DateAndTime(simpleDateFormat.format(new Date())))
+                .setSucceeded(succeeded)
+                .build();
+        try {
+            deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, statusEndPath, gatheringStatus);
+        } catch (TransactionChainClosedException e) {
+            LOG.warn("Can't write to transaction, transaction chain probably closed.");
+            LOG.trace("Write to transaction exception: ", e);
+        }
+
+        deviceContext.submitTransaction();
     }
 }