From c65275bb09353d58610a644d1dfb70299143742c Mon Sep 17 00:00:00 2001 From: Jozef Gloncak Date: Tue, 18 Aug 2015 15:10:09 +0200 Subject: [PATCH] BUG-4084: Li:Save flows in operational based on barrier success - added ItemLifeCycleSource as the basic unit sourcing changes for DS/operational - this unit is provided by registered rpc services and if statistics are disabled then those units are listened to in order to keep device reflection in DS up to date - TODO: this covers flows only, groups and meters are on the line - from Jozef Gloncak: small fix of failing junit test - from Jozef Gloncak: change in test TimeCounterTest: Time marks are set in times 2 ms, 4 ms, 9 ms. Waited average time are 2/1 = 2 ms, 4/2 = 2 ms, 9/3 = 3 ms But this times are only theoretical if whole test is executed without latency and atomically. Therefore awaited average times can't be compared to exact values of awaited average time. (therefore == was replaced with >=) Change-Id: Ie52ad4421745b09db68dec5c9b1be1b64d04d697 Signed-off-by: Michal Rehak Signed-off-by: Jozef Gloncak --- .../api/openflow/device/DeviceContext.java | 6 + .../registry/ItemLifeCycleRegistry.java | 36 ++++++ .../api/openflow/rpc/ItemLifeCycleSource.java | 25 ++++ .../rpc/listener/ItemLifecycleListener.java | 36 ++++++ .../statistics/StatisticsContext.java | 6 + .../impl/device/DeviceContextImpl.java | 11 ++ .../device/ItemLifeCycleRegistryImpl.java | 51 ++++++++ .../impl/rpc/RpcContextImpl.java | 6 + .../listener/ItemLifecycleListenerImpl.java | 41 ++++++ .../impl/services/SalFlowServiceImpl.java | 66 +++++++++- .../statistics/StatisticsContextImpl.java | 10 ++ .../statistics/StatisticsManagerImpl.java | 7 ++ .../impl/rpc/RpcManagerImplTest.java | 5 + .../impl/services/SalFlowServiceImplTest.java | 117 ++++++++++++++++-- .../impl/statistics/TimeCounterTest.java | 21 +++- 15 files changed, 426 insertions(+), 18 deletions(-) create mode 100644 openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/registry/ItemLifeCycleRegistry.java create mode 100644 openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/ItemLifeCycleSource.java create mode 100644 openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/listener/ItemLifecycleListener.java create mode 100644 openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/ItemLifeCycleRegistryImpl.java create mode 100644 openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/listener/ItemLifecycleListenerImpl.java diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/device/DeviceContext.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/device/DeviceContext.java index e3ccbc465e..6eb8b322a8 100644 --- a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/device/DeviceContext.java +++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/device/DeviceContext.java @@ -22,6 +22,7 @@ import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceDiscon import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceReplyProcessor; import org.opendaylight.openflowplugin.api.openflow.device.handlers.MessageHandler; import org.opendaylight.openflowplugin.api.openflow.device.handlers.MultiMsgCollector; +import org.opendaylight.openflowplugin.api.openflow.registry.ItemLifeCycleRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.flow.DeviceFlowRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.group.DeviceGroupRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.meter.DeviceMeterRegistry; @@ -184,5 +185,10 @@ public interface DeviceContext extends AutoCloseable, * @param upperBound max amount of outstanding packetIns */ void updatePacketInRateLimit(long upperBound); + + /** + * @return registry point for item life cycle sources of device + */ + ItemLifeCycleRegistry getItemLifeCycleSourceRegistry(); } diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/registry/ItemLifeCycleRegistry.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/registry/ItemLifeCycleRegistry.java new file mode 100644 index 0000000000..ed51e0fecb --- /dev/null +++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/registry/ItemLifeCycleRegistry.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.openflowplugin.api.openflow.registry; + +import org.opendaylight.openflowplugin.api.openflow.rpc.ItemLifeCycleSource; +import org.opendaylight.yangtools.concepts.Registration; + +/** + * Registration point for any kind of lifecycle sources per device. + */ +public interface ItemLifeCycleRegistry { + + /** + * register given life cycle source to known sources of device + * + * @param lifeCycleSource life cycle changes provider + * @return closeable registration + */ + Registration registerLifeCycleSource(ItemLifeCycleSource lifeCycleSource); + + /** + * close all existing registrations + */ + void clear(); + + /** + * @return registered sources + */ + Iterable getLifeCycleSources(); +} diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/ItemLifeCycleSource.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/ItemLifeCycleSource.java new file mode 100644 index 0000000000..24395ad1f5 --- /dev/null +++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/ItemLifeCycleSource.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.openflowplugin.api.openflow.rpc; + +import javax.annotation.Nullable; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; + +/** + * A source of switch item lifecycle enables for injecting of + * a {@link org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener} + * in order to act upon item lifecycle changes. + */ +public interface ItemLifeCycleSource { + + /** + * @param itemLifecycleListener acts upon lifecycle changes + */ + void setItemLifecycleListener(@Nullable ItemLifecycleListener itemLifecycleListener); +} diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/listener/ItemLifecycleListener.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/listener/ItemLifecycleListener.java new file mode 100644 index 0000000000..f1dfaebdb2 --- /dev/null +++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/listener/ItemLifecycleListener.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.openflowplugin.api.openflow.rpc.listener; + +import org.opendaylight.yangtools.yang.binding.DataObject; +import org.opendaylight.yangtools.yang.binding.Identifiable; +import org.opendaylight.yangtools.yang.binding.Identifier; +import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier; + +/** + * Flow/group/meter lifecycle listener - aimed on rpc result approved by barrier message. + */ +public interface ItemLifecycleListener { + + /** + * react upon item added event + * + * @param itemPath keyed path in DS + * @param itemBody item body + */ + + & DataObject, K extends Identifier> void onAdded(KeyedInstanceIdentifier itemPath, I itemBody); + + /** + * react upon item removed event + * + * @param itemPath keyed path in DS + */ + & DataObject, K extends Identifier> void onRemoved(KeyedInstanceIdentifier itemPath); +} diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/statistics/StatisticsContext.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/statistics/StatisticsContext.java index 89aa9ecd59..e7c54940ff 100644 --- a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/statistics/StatisticsContext.java +++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/statistics/StatisticsContext.java @@ -12,6 +12,7 @@ import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; import io.netty.util.Timeout; import org.opendaylight.openflowplugin.api.openflow.device.RequestContextStack; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; /** * Created by Martin Bobak <mbobak@cisco.com> on 27.2.2015. @@ -29,4 +30,9 @@ public interface StatisticsContext extends RequestContextStack, AutoCloseable { * @return handle to currently scheduled statistics polling */ Optional getPollTimeout(); + + /** + * @return dedicated item life cycle change listener (per device) + */ + ItemLifecycleListener getItemLifeCycleListener(); } diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/DeviceContextImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/DeviceContextImpl.java index 92caca8e70..16ae98f599 100644 --- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/DeviceContextImpl.java +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/DeviceContextImpl.java @@ -41,6 +41,7 @@ import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceContex import org.opendaylight.openflowplugin.api.openflow.device.handlers.MultiMsgCollector; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.openflowplugin.api.openflow.md.core.TranslatorKey; +import org.opendaylight.openflowplugin.api.openflow.registry.ItemLifeCycleRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.flow.DeviceFlowRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.group.DeviceGroupRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.meter.DeviceMeterRegistry; @@ -111,6 +112,7 @@ public class DeviceContextImpl implements DeviceContext { private final MessageTranslator packetInTranslator; private final TranslatorLibrary translatorLibrary; private Map nodeConnectorCache; + private ItemLifeCycleRegistry itemLifeCycleSourceRegistry; @VisibleForTesting @@ -143,6 +145,8 @@ public class DeviceContextImpl implements DeviceContext { packetInTranslator = translatorLibrary.lookupTranslator( new TranslatorKey(deviceState.getVersion(), PacketIn.class.getName())); nodeConnectorCache = new ConcurrentHashMap<>(); + + itemLifeCycleSourceRegistry = new ItemLifeCycleRegistryImpl(); } /** @@ -352,6 +356,8 @@ public class DeviceContextImpl implements DeviceContext { deviceFlowRegistry.close(); deviceMeterRegistry.close(); + itemLifeCycleSourceRegistry.clear(); + for (final DeviceContextClosedHandler deviceContextClosedHandler : closeHandlers) { deviceContextClosedHandler.onDeviceContextClosed(this); @@ -436,4 +442,9 @@ public class DeviceContextImpl implements DeviceContext { public void updatePacketInRateLimit(long upperBound) { packetInLimiter.changeWaterMarks((int) (LOW_WATERMARK_FACTOR * upperBound), (int) (HIGH_WATERMARK_FACTOR * upperBound)); } + + @Override + public ItemLifeCycleRegistry getItemLifeCycleSourceRegistry() { + return itemLifeCycleSourceRegistry; + } } diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/ItemLifeCycleRegistryImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/ItemLifeCycleRegistryImpl.java new file mode 100644 index 0000000000..63272453b8 --- /dev/null +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/device/ItemLifeCycleRegistryImpl.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.openflowplugin.impl.device; + +import java.util.Collections; +import java.util.Set; +import org.apache.mina.util.ConcurrentHashSet; +import org.opendaylight.openflowplugin.api.openflow.registry.ItemLifeCycleRegistry; +import org.opendaylight.openflowplugin.api.openflow.rpc.ItemLifeCycleSource; +import org.opendaylight.yangtools.concepts.Registration; + +/** + * default implementation + */ +public class ItemLifeCycleRegistryImpl implements ItemLifeCycleRegistry { + + private final Set registry; + + public ItemLifeCycleRegistryImpl() { + registry = new ConcurrentHashSet<>(); + } + + + @Override + + public Registration registerLifeCycleSource(final ItemLifeCycleSource lifeCycleSource) { + registry.add(lifeCycleSource); + return new Registration() { + @Override + public void close() throws Exception { + registry.remove(lifeCycleSource); + } + }; + } + + @Override + public void clear() { + registry.clear(); + } + + @Override + public Iterable getLifeCycleSources() { + return Collections.unmodifiableCollection(registry); + } +} diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java index 60177e4732..502ac13724 100644 --- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java @@ -15,6 +15,7 @@ import org.opendaylight.controller.sal.binding.api.BindingAwareBroker.RoutedRpcR import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry; import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; import org.opendaylight.openflowplugin.api.openflow.device.RequestContext; +import org.opendaylight.openflowplugin.api.openflow.rpc.ItemLifeCycleSource; import org.opendaylight.openflowplugin.api.openflow.rpc.RpcContext; import org.opendaylight.openflowplugin.api.openflow.statistics.ofpspecific.MessageSpy; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeContext; @@ -50,6 +51,11 @@ public class RpcContextImpl implements RpcContext { routedRpcReg.registerPath(NodeContext.class, deviceContext.getDeviceState().getNodeInstanceIdentifier()); rpcRegistrations.add(routedRpcReg); LOG.debug("Registration of service {} for device {}.", serviceClass, deviceContext.getDeviceState().getNodeInstanceIdentifier()); + + if (serviceInstance instanceof ItemLifeCycleSource) { + // TODO: collect registration for selective unregistering in case of tearing down only one rpc + deviceContext.getItemLifeCycleSourceRegistry().registerLifeCycleSource((ItemLifeCycleSource) serviceInstance); + } } /** diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/listener/ItemLifecycleListenerImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/listener/ItemLifecycleListenerImpl.java new file mode 100644 index 0000000000..f35abc9e30 --- /dev/null +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/listener/ItemLifecycleListenerImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.openflowplugin.impl.rpc.listener; + +import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; +import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; +import org.opendaylight.yangtools.yang.binding.DataObject; +import org.opendaylight.yangtools.yang.binding.Identifiable; +import org.opendaylight.yangtools.yang.binding.Identifier; +import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier; + +/** + * General implementation of {@link ItemLifecycleListener} - keeping of DS/operational reflection up-to-date + */ +public class ItemLifecycleListenerImpl implements ItemLifecycleListener { + + private final DeviceContext deviceContext; + + public ItemLifecycleListenerImpl(DeviceContext deviceContext) { + this.deviceContext = deviceContext; + } + + @Override + public & DataObject, K extends Identifier> void onAdded(KeyedInstanceIdentifier itemPath, I itemBody) { + deviceContext.writeToTransaction(LogicalDatastoreType.OPERATIONAL, itemPath, itemBody); + deviceContext.submitTransaction(); + } + + @Override + public & DataObject, K extends Identifier> void onRemoved(KeyedInstanceIdentifier itemPath) { + deviceContext.addDeleteToTxChain(LogicalDatastoreType.OPERATIONAL, itemPath); + deviceContext.submitTransaction(); + } +} diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImpl.java index f206fb863b..4c16bf01c3 100644 --- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImpl.java +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImpl.java @@ -7,8 +7,7 @@ */ package org.opendaylight.openflowplugin.impl.services; -import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef; - +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -16,17 +15,23 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Future; +import javax.annotation.Nullable; import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; import org.opendaylight.openflowplugin.api.openflow.device.RequestContextStack; import org.opendaylight.openflowplugin.api.openflow.registry.flow.DeviceFlowRegistry; import org.opendaylight.openflowplugin.api.openflow.registry.flow.FlowDescriptor; import org.opendaylight.openflowplugin.api.openflow.registry.flow.FlowRegistryKey; +import org.opendaylight.openflowplugin.api.openflow.rpc.ItemLifeCycleSource; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; import org.opendaylight.openflowplugin.impl.registry.flow.FlowDescriptorFactory; import org.opendaylight.openflowplugin.impl.registry.flow.FlowRegistryKeyFactory; import org.opendaylight.openflowplugin.impl.util.FlowUtil; import org.opendaylight.openflowplugin.openflow.md.util.FlowCreatorUtil; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInputBuilder; @@ -39,24 +44,36 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.Upda import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.UpdateFlowOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.flow.update.OriginalFlow; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.flow.update.UpdatedFlow; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowModInputBuilder; +import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier; import org.opendaylight.yangtools.yang.common.RpcError; import org.opendaylight.yangtools.yang.common.RpcResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SalFlowServiceImpl implements SalFlowService { +public class SalFlowServiceImpl implements SalFlowService, ItemLifeCycleSource { private static final Logger LOG = LoggerFactory.getLogger(SalFlowServiceImpl.class); private final FlowService flowUpdate; private final FlowService flowAdd; private final FlowService flowRemove; + private final DeviceContext deviceContext; + private ItemLifecycleListener itemLifecycleListener; public SalFlowServiceImpl(final RequestContextStack requestContextStack, final DeviceContext deviceContext) { + this.deviceContext = deviceContext; flowRemove = new FlowService(requestContextStack, deviceContext, RemoveFlowOutput.class); flowAdd = new FlowService<>(requestContextStack, deviceContext, AddFlowOutput.class); flowUpdate = new FlowService<>(requestContextStack, deviceContext, UpdateFlowOutput.class); } + @Override + public void setItemLifecycleListener(@Nullable ItemLifecycleListener itemLifecycleListener) { + this.itemLifecycleListener = itemLifecycleListener; + } + @Override public Future> addFlow(final AddFlowInput input) { final FlowId flowId; @@ -66,7 +83,6 @@ public class SalFlowServiceImpl implements SalFlowService { flowId = FlowUtil.createAlienFlowId(input.getTableId()); } - final DeviceContext deviceContext = flowAdd.getDeviceContext(); final FlowRegistryKey flowRegistryKey = FlowRegistryKeyFactory.create(input); final FlowDescriptor flowDescriptor = FlowDescriptorFactory.create(input.getTableId(), flowId); deviceContext.getDeviceFlowRegistry().store(flowRegistryKey, flowDescriptor); @@ -76,6 +92,12 @@ public class SalFlowServiceImpl implements SalFlowService { public void onSuccess(final RpcResult rpcResult) { if (rpcResult.isSuccessful()) { LOG.debug("flow add finished without error, id={}", flowId.getValue()); + if (itemLifecycleListener != null) { + KeyedInstanceIdentifier flowPath = createFlowPath(flowDescriptor, + deviceContext.getDeviceState().getNodeInstanceIdentifier()); + final FlowBuilder flowBuilder = new FlowBuilder(input).setId(flowDescriptor.getFlowId()); + itemLifecycleListener.onAdded(flowPath, flowBuilder.build()); + } } else { LOG.debug("flow add failed with error, id={}", flowId.getValue()); } @@ -101,7 +123,15 @@ public class SalFlowServiceImpl implements SalFlowService { public void onSuccess(final RpcResult result) { if (result.isSuccessful()) { FlowRegistryKey flowRegistryKey = FlowRegistryKeyFactory.create(input); - flowRemove.getDeviceContext().getDeviceFlowRegistry().markToBeremoved(flowRegistryKey); + deviceContext.getDeviceFlowRegistry().markToBeremoved(flowRegistryKey); + if (itemLifecycleListener != null) { + final FlowDescriptor flowDescriptor = deviceContext.getDeviceFlowRegistry().retrieveIdForFlow(flowRegistryKey); + if (flowDescriptor != null) { + KeyedInstanceIdentifier flowPath = createFlowPath(flowDescriptor, + deviceContext.getDeviceState().getNodeInstanceIdentifier()); + itemLifecycleListener.onRemoved(flowPath); + } + } } else { if (LOG.isTraceEnabled()) { StringBuilder errors = new StringBuilder(); @@ -157,13 +187,29 @@ public class SalFlowServiceImpl implements SalFlowService { FlowRegistryKey updatedflowRegistryKey = FlowRegistryKeyFactory.create(updated); final FlowRef flowRef = input.getFlowRef(); - final DeviceFlowRegistry deviceFlowRegistry = flowUpdate.getDeviceContext().getDeviceFlowRegistry(); + final DeviceFlowRegistry deviceFlowRegistry = deviceContext.getDeviceFlowRegistry(); deviceFlowRegistry.markToBeremoved(flowRegistryKey); + + if (itemLifecycleListener != null) { + final FlowDescriptor flowDescriptor = deviceContext.getDeviceFlowRegistry().retrieveIdForFlow(flowRegistryKey); + if (flowDescriptor != null) { + KeyedInstanceIdentifier flowPath = createFlowPath(flowDescriptor, + deviceContext.getDeviceState().getNodeInstanceIdentifier()); + itemLifecycleListener.onRemoved(flowPath); + } + } //if provided, store flow id to flow registry if (flowRef != null) { final FlowId flowId = flowRef.getValue().firstKeyOf(Flow.class, FlowKey.class).getId(); final FlowDescriptor flowDescriptor = FlowDescriptorFactory.create(updated.getTableId(), flowId); deviceFlowRegistry.store(updatedflowRegistryKey, flowDescriptor); + + if (itemLifecycleListener != null) { + KeyedInstanceIdentifier flowPath = createFlowPath(flowDescriptor, + deviceContext.getDeviceState().getNodeInstanceIdentifier()); + final FlowBuilder flowBuilder = new FlowBuilder(input.getUpdatedFlow()).setId(flowDescriptor.getFlowId()); + itemLifecycleListener.onAdded(flowPath, flowBuilder.build()); + } } } @@ -174,4 +220,12 @@ public class SalFlowServiceImpl implements SalFlowService { }); return future; } + + @VisibleForTesting + static KeyedInstanceIdentifier createFlowPath(FlowDescriptor flowDescriptor, + KeyedInstanceIdentifier nodePath) { + return nodePath.augmentation(FlowCapableNode.class) + .child(Table.class, flowDescriptor.getTableKey()) + .child(Flow.class, new FlowKey(flowDescriptor.getFlowId())); + } } diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsContextImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsContextImpl.java index f09f18545e..301d0061fb 100644 --- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsContextImpl.java +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsContextImpl.java @@ -27,8 +27,10 @@ import org.opendaylight.openflowplugin.api.openflow.connection.ConnectionContext import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; import org.opendaylight.openflowplugin.api.openflow.device.DeviceState; import org.opendaylight.openflowplugin.api.openflow.device.RequestContext; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; import org.opendaylight.openflowplugin.api.openflow.statistics.StatisticsContext; import org.opendaylight.openflowplugin.impl.rpc.AbstractRequestContext; +import org.opendaylight.openflowplugin.impl.rpc.listener.ItemLifecycleListenerImpl; import org.opendaylight.openflowplugin.impl.services.RequestContextUtil; import org.opendaylight.openflowplugin.impl.statistics.services.dedicated.StatisticsGatheringOnTheFlyService; import org.opendaylight.openflowplugin.impl.statistics.services.dedicated.StatisticsGatheringService; @@ -43,6 +45,8 @@ public class StatisticsContextImpl implements StatisticsContext { private static final Logger LOG = LoggerFactory.getLogger(StatisticsContextImpl.class); private static final String CONNECTION_CLOSED = "Connection closed."; + + private final ItemLifecycleListener itemLifeCycleListener; private final Collection> requestContexts = new HashSet<>(); private final DeviceContext deviceContext; private final DeviceState devState; @@ -82,6 +86,7 @@ public class StatisticsContextImpl implements StatisticsContext { statListForCollecting.add(MultipartType.OFPMPQUEUE); } collectingStatType = ImmutableList.copyOf(statListForCollecting); + itemLifeCycleListener = new ItemLifecycleListenerImpl(deviceContext); } @Override @@ -245,4 +250,9 @@ public class StatisticsContextImpl implements StatisticsContext { statisticsGatheringOnTheFlyService) { this.statisticsGatheringOnTheFlyService = statisticsGatheringOnTheFlyService; } + + @Override + public ItemLifecycleListener getItemLifeCycleListener() { + return itemLifeCycleListener; + } } diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java index 52fa138c48..5f148257ba 100644 --- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java +++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java @@ -26,6 +26,7 @@ import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry; import org.opendaylight.openflowplugin.api.openflow.connection.ConnectionContext; import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitializationPhaseHandler; +import org.opendaylight.openflowplugin.api.openflow.rpc.ItemLifeCycleSource; import org.opendaylight.openflowplugin.api.openflow.statistics.StatisticsContext; import org.opendaylight.openflowplugin.api.openflow.statistics.StatisticsManager; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.openflowplugin.sm.control.rev150812.ChangeStatisticsWorkModeInput; @@ -221,12 +222,18 @@ public class StatisticsManagerImpl implements StatisticsManager, StatisticsManag switch (targetWorkMode) { case COLLECTALL: scheduleNextPolling(deviceContext, statisticsContext, new TimeCounter()); + for (ItemLifeCycleSource lifeCycleSource : deviceContext.getItemLifeCycleSourceRegistry().getLifeCycleSources()) { + lifeCycleSource.setItemLifecycleListener(null); + } break; case FULLYDISABLED: final Optional pollTimeout = statisticsContext.getPollTimeout(); if (pollTimeout.isPresent()) { pollTimeout.get().cancel(); } + for (ItemLifeCycleSource lifeCycleSource : deviceContext.getItemLifeCycleSourceRegistry().getLifeCycleSources()) { + lifeCycleSource.setItemLifecycleListener(statisticsContext.getItemLifeCycleListener()); + } break; default: LOG.warn("statistics work mode not supported: {}", targetWorkMode); diff --git a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImplTest.java b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImplTest.java index aacc8e2629..8923d4acf1 100644 --- a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImplTest.java +++ b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImplTest.java @@ -23,6 +23,7 @@ import org.opendaylight.openflowplugin.api.openflow.connection.ConnectionContext import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; import org.opendaylight.openflowplugin.api.openflow.device.DeviceState; import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitializationPhaseHandler; +import org.opendaylight.openflowplugin.api.openflow.registry.ItemLifeCycleRegistry; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeContext; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes; @@ -51,6 +52,9 @@ public class RpcManagerImplTest { private BindingAwareBroker.RoutedRpcRegistration routedRpcRegistration; @Mock private DeviceState deviceState; + @Mock + private ItemLifeCycleRegistry itemLifeCycleRegistry; + private KeyedInstanceIdentifier nodePath; @Before @@ -64,6 +68,7 @@ public class RpcManagerImplTest { Mockito.when(connectionContext.getFeatures()).thenReturn(features); Mockito.when(deviceContext.getPrimaryConnectionContext()).thenReturn(connectionContext); Mockito.when(deviceContext.getDeviceState()).thenReturn(deviceState); + Mockito.when(deviceContext.getItemLifeCycleSourceRegistry()).thenReturn(itemLifeCycleRegistry); Mockito.when(deviceState.getNodeInstanceIdentifier()).thenReturn(nodePath); } diff --git a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImplTest.java b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImplTest.java index 1be8251c86..d77aa5cd80 100644 --- a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImplTest.java +++ b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/services/SalFlowServiceImplTest.java @@ -11,27 +11,45 @@ import junit.framework.TestCase; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Matchers; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.opendaylight.openflowjava.protocol.api.connection.ConnectionAdapter; import org.opendaylight.openflowjava.protocol.api.connection.OutboundQueue; import org.opendaylight.openflowplugin.api.OFConstants; import org.opendaylight.openflowplugin.api.openflow.connection.ConnectionContext; import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext; +import org.opendaylight.openflowplugin.api.openflow.device.DeviceState; import org.opendaylight.openflowplugin.api.openflow.device.RequestContext; import org.opendaylight.openflowplugin.api.openflow.device.RequestContextStack; import org.opendaylight.openflowplugin.api.openflow.device.Xid; +import org.opendaylight.openflowplugin.api.openflow.registry.flow.DeviceFlowRegistry; +import org.opendaylight.openflowplugin.api.openflow.registry.flow.FlowDescriptor; +import org.opendaylight.openflowplugin.api.openflow.registry.flow.FlowRegistryKey; +import org.opendaylight.openflowplugin.api.openflow.rpc.listener.ItemLifecycleListener; import org.opendaylight.openflowplugin.api.openflow.statistics.ofpspecific.MessageSpy; -import org.opendaylight.openflowplugin.impl.registry.flow.DeviceFlowRegistryImpl; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.UpdateFlowInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.flow.update.OriginalFlow; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.flow.update.UpdatedFlow; -import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.Flow; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FeaturesReply; import org.opendaylight.yangtools.yang.binding.DataObject; +import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; +import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier; import org.opendaylight.yangtools.yang.common.RpcResult; import org.opendaylight.yangtools.yang.common.RpcResultBuilder; @@ -40,6 +58,15 @@ public class SalFlowServiceImplTest extends TestCase { private static final BigInteger DUMMY_DATAPATH_ID = new BigInteger("444"); private static final Short DUMMY_VERSION = OFConstants.OFP_VERSION_1_3; + private static final String DUMMY_NODE_ID = "dummyNodeID"; + private static final String DUMMY_FLOW_ID = "dummyFlowID"; + private static final Short DUMMY_TABLE_ID = (short) 0; + + private static final KeyedInstanceIdentifier NODE_II + = InstanceIdentifier.create(Nodes.class).child(Node.class, new NodeKey(new NodeId(DUMMY_NODE_ID))); + + private static final KeyedInstanceIdentifier TABLE_II + = NODE_II.augmentation(FlowCapableNode.class).child(Table.class, new TableKey(DUMMY_TABLE_ID)); @Mock private RequestContextStack mockedRequestContextStack; @@ -61,6 +88,10 @@ public class SalFlowServiceImplTest extends TestCase { private Match match; private SalFlowServiceImpl salFlowService; + @Mock + DeviceState mockedDeviceState; + @Mock + private DeviceFlowRegistry deviceFlowRegistry; @Before public void initialization() { @@ -74,40 +105,108 @@ public class SalFlowServiceImplTest extends TestCase { when(mockedDeviceContext.getPrimaryConnectionContext()).thenReturn(mockedPrimConnectionContext); when(mockedDeviceContext.getMessageSpy()).thenReturn(mockedMessagSpy); - when(mockedDeviceContext.getDeviceFlowRegistry()).thenReturn(new DeviceFlowRegistryImpl()); + when(mockedDeviceContext.getDeviceFlowRegistry()).thenReturn(deviceFlowRegistry); when(mockedRequestContextStack.createRequestContext()).thenReturn(requestContext); when(requestContext.getXid()).thenReturn(new Xid(84L)); when(requestContext.getFuture()).thenReturn(RpcResultBuilder.success().buildFuture()); salFlowService = new SalFlowServiceImpl(mockedRequestContextStack, mockedDeviceContext); + + + when(mockedDeviceState.getNodeInstanceIdentifier()).thenReturn(NODE_II); + when(mockedDeviceContext.getDeviceState()).thenReturn(mockedDeviceState); } @Test public void testAddFlow() throws Exception { - final AddFlowInput mockedAddFlowInput = createFlowMock(AddFlowInput.class); + addFlow(null); + } + + @Test + public void testAddFlowWithItemLifecycle() throws Exception { + addFlow(mock(ItemLifecycleListener.class)); + } + + private void addFlow(final ItemLifecycleListener itemLifecycleListener) throws ExecutionException, InterruptedException { + AddFlowInput mockedAddFlowInput = createFlowMock(AddFlowInput.class); + salFlowService.setItemLifecycleListener(itemLifecycleListener); verifyOutput(salFlowService.addFlow(mockedAddFlowInput)); + if (itemLifecycleListener != null) { + Mockito.verify(itemLifecycleListener).onAdded(Matchers.>any(), Matchers.any()); + } } @Test public void testRemoveFlow() throws Exception { - final RemoveFlowInput mockedRemoveFlowInput = createFlowMock(RemoveFlowInput.class); + removeFlow(null); + } + + @Test + public void testRemoveFlowWithItemLifecycle() throws Exception { + removeFlow(mock(ItemLifecycleListener.class)); + } + + private void removeFlow(final ItemLifecycleListener itemLifecycleListener) throws Exception { + RemoveFlowInput mockedRemoveFlowInput = createFlowMock(RemoveFlowInput.class); + + if (itemLifecycleListener != null) { + salFlowService.setItemLifecycleListener(itemLifecycleListener); + mockingFlowRegistryLookup(); + + } verifyOutput(salFlowService.removeFlow(mockedRemoveFlowInput)); + if (itemLifecycleListener != null) { + Mockito.verify(itemLifecycleListener).onRemoved(Matchers.>any()); + } + } @Test public void testUpdateFlow() throws Exception { - final UpdateFlowInput mockedUpdateFlowInput = mock(UpdateFlowInput.class); + updateFlow(null); + } + + @Test + public void testUpdateFlowWithItemLifecycle() throws Exception { + updateFlow(mock(ItemLifecycleListener.class)); + } + + private void updateFlow(final ItemLifecycleListener itemLifecycleListener) throws Exception { + UpdateFlowInput mockedUpdateFlowInput = mock(UpdateFlowInput.class); - final UpdatedFlow mockedUpdateFlow = createFlowMock(UpdatedFlow.class); + UpdatedFlow mockedUpdateFlow = createFlowMock(UpdatedFlow.class); when(mockedUpdateFlowInput.getUpdatedFlow()).thenReturn(mockedUpdateFlow); - final OriginalFlow mockedOriginalFlow = createFlowMock(OriginalFlow.class); + FlowRef mockedFlowRef = mock(FlowRef.class); + Mockito.doReturn(TABLE_II.child(Flow.class, new FlowKey(new FlowId(DUMMY_FLOW_ID)))).when(mockedFlowRef).getValue(); + when(mockedUpdateFlowInput.getFlowRef()).thenReturn(mockedFlowRef); + + OriginalFlow mockedOriginalFlow = createFlowMock(OriginalFlow.class); when(mockedUpdateFlowInput.getOriginalFlow()).thenReturn(mockedOriginalFlow); + if (itemLifecycleListener != null) { + salFlowService.setItemLifecycleListener(itemLifecycleListener); + mockingFlowRegistryLookup(); + } + verifyOutput(salFlowService.updateFlow(mockedUpdateFlowInput)); + + if (itemLifecycleListener != null) { + Mockito.verify(itemLifecycleListener).onAdded(Matchers.>any(), Matchers.any()); + Mockito.verify(itemLifecycleListener).onRemoved(Matchers.>any()); + } + + } + + private void mockingFlowRegistryLookup() { + FlowDescriptor mockedFlowDescriptor = mock(FlowDescriptor.class); + when(mockedFlowDescriptor.getFlowId()).thenReturn(new FlowId(DUMMY_FLOW_ID)); + when(mockedFlowDescriptor.getTableKey()).thenReturn(new TableKey(DUMMY_TABLE_ID)); + + when(deviceFlowRegistry.retrieveIdForFlow(Matchers.any(FlowRegistryKey.class))).thenReturn(mockedFlowDescriptor); } private void verifyOutput(Future> rpcResultFuture) throws ExecutionException, InterruptedException { @@ -117,7 +216,7 @@ public class SalFlowServiceImplTest extends TestCase { assertTrue(addFlowOutputRpcResult.isSuccessful()); } - private T createFlowMock(Class flowClazz) { + private T createFlowMock(Class flowClazz) { T mockedFlow = mock(flowClazz); when(mockedFlow.getMatch()).thenReturn(match); return mockedFlow; diff --git a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/statistics/TimeCounterTest.java b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/statistics/TimeCounterTest.java index b95c4acf05..0e37a63bc7 100644 --- a/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/statistics/TimeCounterTest.java +++ b/openflowplugin-impl/src/test/java/org/opendaylight/openflowplugin/impl/statistics/TimeCounterTest.java @@ -27,6 +27,21 @@ public class TimeCounterTest { timeCounter = new TimeCounter(); } + /** + * tm = time mark + * - tm1 at time 2 ms + * - tm2 at time 4 ms + * - tm3 at time 9 ms + * + * awaited average time: + * - tm1 = 2/1 = 2 ms + * - tm2 = 4/2 = 2 ms + * - tm3 = 9/3 = 3 ms + * + * But this times are only theoretical if whole test is executed without latency and atomically. Therefore awaited + * average times can't be compared to exact values of awaited average time (therefore == was replaced with >=) + * @throws Exception + */ @Test public void testGetAverageTimeBetweenMarks() throws Exception { Assert.assertEquals(0, timeCounter.getAverageTimeBetweenMarks()); @@ -35,15 +50,15 @@ public class TimeCounterTest { zzz(2L); timeCounter.addTimeMark(); - Assert.assertEquals(2, timeCounter.getAverageTimeBetweenMarks()); + Assert.assertTrue(timeCounter.getAverageTimeBetweenMarks() >= 2); zzz(2L); timeCounter.addTimeMark(); - Assert.assertEquals(2, timeCounter.getAverageTimeBetweenMarks()); + Assert.assertTrue(timeCounter.getAverageTimeBetweenMarks() >= 2); zzz(5L); timeCounter.addTimeMark(); - Assert.assertEquals(3, timeCounter.getAverageTimeBetweenMarks()); + Assert.assertTrue(timeCounter.getAverageTimeBetweenMarks() >= 3); } private void zzz(long length) { -- 2.36.6