From: Michal Rehak Date: Thu, 25 Sep 2014 16:32:18 +0000 (+0200) Subject: BUG-2091: notification NodeRemoved and processing queue X-Git-Tag: release/helium~6^2 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=commitdiff_plain;h=a9803346775508965175a091cb5c5980bc2c277d;hp=90cebf44394a15218b356d4ed35e3c934e78d0a3;p=openflowplugin.git BUG-2091: notification NodeRemoved and processing queue - NodeRemoved notification is now enqueued in the same way as other notifications so that original order will be preserved and "homeless" messages eliminated - NodeUpdated must use the same way of delivery to MD-SAL in order to preclude state where during reconnect nodeAdded would arrive before nodeRemoved Change-Id: Iede472870fac9ce12e7e5817c9b21967a2a43173 Signed-off-by: Michal Rehak --- diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/ConnectionConductorImpl.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/ConnectionConductorImpl.java index fe9dc4b4f8..285d5a80e6 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/ConnectionConductorImpl.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/ConnectionConductorImpl.java @@ -65,7 +65,7 @@ import com.google.common.util.concurrent.Futures; * @author mirehak */ public class ConnectionConductorImpl implements OpenflowProtocolListener, - SystemNotificationsListener, ConnectionConductor, ConnectionReadyListener, HandshakeListener { + SystemNotificationsListener, ConnectionConductor, ConnectionReadyListener, HandshakeListener, NotificationEnqueuer { /** ingress queue limit */ private static final int INGRESS_QUEUE_MAX_SIZE = 200; @@ -180,6 +180,11 @@ public class ConnectionConductorImpl implements OpenflowProtocolListener, private void enqueueMessage(OfHeader message) { enqueueMessage(message, QueueType.DEFAULT); } + + @Override + public void enqueueNotification(NotificationQueueWrapper notification) { + enqueueMessage(notification); + } /** * @param message diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/MDController.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/MDController.java index aef940b176..1b1e36df06 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/MDController.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/MDController.java @@ -25,7 +25,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.opendaylight.openflowjava.protocol.api.connection.ConnectionConfiguration; import org.opendaylight.openflowjava.protocol.spi.connection.SwitchConnectionProvider; import org.opendaylight.openflowplugin.api.OFConstants; import org.opendaylight.openflowplugin.api.openflow.md.core.TranslatorKey; @@ -40,6 +39,7 @@ import org.opendaylight.openflowplugin.openflow.md.core.translator.MultiPartMess import org.opendaylight.openflowplugin.openflow.md.core.translator.MultiPartReplyPortToNodeConnectorUpdatedTranslator; import org.opendaylight.openflowplugin.openflow.md.core.translator.MultipartReplyTableFeaturesToTableUpdatedTranslator; import org.opendaylight.openflowplugin.openflow.md.core.translator.MultipartReplyTranslator; +import org.opendaylight.openflowplugin.openflow.md.core.translator.NotificationPlainTranslator; import org.opendaylight.openflowplugin.openflow.md.core.translator.PacketInTranslator; import org.opendaylight.openflowplugin.openflow.md.core.translator.PacketInV10Translator; import org.opendaylight.openflowplugin.openflow.md.core.translator.PortStatusMessageToNodeConnectorUpdatedTranslator; @@ -56,6 +56,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.group.statistics.rev131111. import org.opendaylight.yang.gen.v1.urn.opendaylight.group.statistics.rev131111.GroupFeaturesUpdated; import org.opendaylight.yang.gen.v1.urn.opendaylight.group.statistics.rev131111.GroupStatisticsUpdated; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorUpdated; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRemoved; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeUpdated; import org.opendaylight.yang.gen.v1.urn.opendaylight.meter.statistics.rev131111.MeterConfigStatsUpdated; import org.opendaylight.yang.gen.v1.urn.opendaylight.meter.statistics.rev131111.MeterFeaturesUpdated; @@ -154,6 +155,8 @@ public class MDController implements IMDController, AutoCloseable { addMessageTranslator(MultipartReplyMessage.class,OF13, new MultipartReplyTranslator()); addMessageTranslator(MultipartReplyMessage.class,OF13,new MultipartReplyTableFeaturesToTableUpdatedTranslator()); addMessageTranslator(GetFeaturesOutput.class,OF10, new FeaturesV10ToNodeConnectorUpdatedTranslator()); + addMessageTranslator(NotificationQueueWrapper.class, OF10, new NotificationPlainTranslator()); + addMessageTranslator(NotificationQueueWrapper.class, OF13, new NotificationPlainTranslator()); NotificationPopListener notificationPopListener = new NotificationPopListener(); notificationPopListener.setNotificationProviderService( @@ -181,6 +184,7 @@ public class MDController implements IMDController, AutoCloseable { addMessagePopListener(PacketReceived.class,notificationPopListener); addMessagePopListener(TransmitPacketInput.class, notificationPopListener); addMessagePopListener(NodeUpdated.class, notificationPopListener); + addMessagePopListener(NodeRemoved.class, notificationPopListener); addMessagePopListener(SwitchFlowRemoved.class, notificationPopListener); addMessagePopListener(TableUpdated.class, notificationPopListener); diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationEnqueuer.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationEnqueuer.java new file mode 100644 index 0000000000..2729cef2d1 --- /dev/null +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationEnqueuer.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.openflowplugin.openflow.md.core; + +/** + * provider of wrapped notification enqueue + */ +public interface NotificationEnqueuer { + + /** + * enqueue given notification into standard message processing queue + * + * @param notification + */ + void enqueueNotification(NotificationQueueWrapper notification); + +} diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationQueueWrapper.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationQueueWrapper.java new file mode 100644 index 0000000000..508e28f3f4 --- /dev/null +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/NotificationQueueWrapper.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.openflowplugin.openflow.md.core; + +import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OfHeader; +import org.opendaylight.yangtools.yang.binding.DataContainer; +import org.opendaylight.yangtools.yang.binding.Notification; + +import com.google.common.base.Preconditions; + +/** + * + */ +public class NotificationQueueWrapper implements OfHeader { + + private final Notification notification; + private final Short version; + private Long xid = -1L; + + + /** + * @param notification + * @param version + */ + public NotificationQueueWrapper(final Notification notification, final Short version) { + Preconditions.checkArgument(notification != null, "wrapped notification must not be null"); + Preconditions.checkArgument(version != null, "message version of wrapped notification must not be null"); + this.notification = notification; + this.version = version; + } + + @Override + public Class getImplementedInterface() { + return NotificationQueueWrapper.class; + } + + @Override + public Short getVersion() { + return version; + } + + @Override + public Long getXid() { + return xid; + } + + /** + * @return the notification + */ + public Notification getNotification() { + return notification; + } + + /** + * @param xid the xid to set + */ + public void setXid(Long xid) { + this.xid = xid; + } +} diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManager.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManager.java index 7a2d0c7121..9f9940c099 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManager.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManager.java @@ -12,10 +12,12 @@ import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; + import org.opendaylight.controller.md.sal.binding.api.DataBroker; import org.opendaylight.controller.sal.binding.api.BindingAwareBroker.ProviderContext; import org.opendaylight.controller.sal.binding.api.NotificationProviderService; import org.opendaylight.openflowplugin.openflow.md.ModelDrivenSwitch; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationQueueWrapper; import org.opendaylight.openflowplugin.openflow.md.core.session.OFSessionUtil; import org.opendaylight.openflowplugin.openflow.md.core.session.SessionContext; import org.opendaylight.openflowplugin.openflow.md.core.session.SessionListener; @@ -100,7 +102,10 @@ public class SalRegistrationManager implements SessionListener, AutoCloseable { LOG.debug("ModelDrivenSwitch for {} registered to MD-SAL.", datapathId.toString()); - publishService.publish(nodeAdded(ofSwitch, features, nodeRef)); + NotificationQueueWrapper wrappedNotification = new NotificationQueueWrapper( + nodeAdded(ofSwitch, features, nodeRef), + context.getFeatures().getVersion()); + context.getNotificationEnqueuer().enqueueNotification(wrappedNotification); } @Override @@ -117,7 +122,10 @@ public class SalRegistrationManager implements SessionListener, AutoCloseable { } LOG.debug("ModelDrivenSwitch for {} unregistered from MD-SAL.", datapathId.toString()); - publishService.publish(nodeRemoved); + + NotificationQueueWrapper wrappedNotification = new NotificationQueueWrapper( + nodeRemoved, context.getFeatures().getVersion()); + context.getNotificationEnqueuer().enqueueNotification(wrappedNotification); } private NodeUpdated nodeAdded(ModelDrivenSwitch sw, GetFeaturesOutput features, NodeRef nodeRef) { diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/OFSessionUtil.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/OFSessionUtil.java index 697bd76903..83f1df12da 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/OFSessionUtil.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/OFSessionUtil.java @@ -14,7 +14,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductor; +import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductorImpl; import org.opendaylight.openflowplugin.openflow.md.core.IMDMessageTranslator; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.openflowplugin.api.openflow.md.core.TranslatorKey; @@ -39,7 +39,7 @@ public abstract class OFSessionUtil { * @param features * @param version */ - public static void registerSession(ConnectionConductor connectionConductor, + public static void registerSession(ConnectionConductorImpl connectionConductor, GetFeaturesOutput features, short version) { SwitchSessionKeyOF sessionKey = createSwitchSessionKey(features .getDatapathId()); @@ -58,6 +58,7 @@ public abstract class OFSessionUtil { // register new session context (based primary conductor) SessionContextOFImpl context = new SessionContextOFImpl(); context.setPrimaryConductor(connectionConductor); + context.setNotificationEnqueuer(connectionConductor); context.setFeatures(features); context.setSessionKey(sessionKey); context.setSeed((int) System.currentTimeMillis()); diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContext.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContext.java index 6992a6a26e..a623624871 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContext.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContext.java @@ -15,6 +15,8 @@ import java.util.Set; import org.opendaylight.openflowplugin.openflow.md.ModelDrivenSwitch; import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductor; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationEnqueuer; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationQueueWrapper; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortGrouping; @@ -159,4 +161,9 @@ public interface SessionContext { * @return seed value for random operations */ int getSeed(); + + /** + * @return (wrapped) notification enqueue service - {@link NotificationQueueWrapper} + */ + NotificationEnqueuer getNotificationEnqueuer(); } diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContextOFImpl.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContextOFImpl.java index 8a7b026ff0..678a0bb4e7 100644 --- a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContextOFImpl.java +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/session/SessionContextOFImpl.java @@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.opendaylight.openflowplugin.openflow.md.ModelDrivenSwitch; import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductor; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationEnqueuer; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortGrouping; @@ -32,6 +33,7 @@ public class SessionContextOFImpl implements SessionContext { private GetFeaturesOutput features; private ConnectionConductor primaryConductor; + private NotificationEnqueuer notificationEnqueuer; private ConcurrentHashMap auxiliaryConductors; private boolean valid; private SwitchSessionKeyOF sessionKey; @@ -218,4 +220,17 @@ public class SessionContextOFImpl implements SessionContext { public int getSeed() { return seed; } + + /** + * @param notificationEnqueuer the notificationEnqueuer to set + */ + public void setNotificationEnqueuer( + NotificationEnqueuer notificationEnqueuer) { + this.notificationEnqueuer = notificationEnqueuer; + } + + @Override + public NotificationEnqueuer getNotificationEnqueuer() { + return notificationEnqueuer; + } } diff --git a/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/translator/NotificationPlainTranslator.java b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/translator/NotificationPlainTranslator.java new file mode 100644 index 0000000000..0041828acf --- /dev/null +++ b/openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/translator/NotificationPlainTranslator.java @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.openflowplugin.openflow.md.core.translator; + +import java.math.BigInteger; +import java.util.Collections; +import java.util.List; + +import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; +import org.opendaylight.openflowplugin.openflow.md.core.IMDMessageTranslator; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationQueueWrapper; +import org.opendaylight.openflowplugin.openflow.md.core.session.SessionContext; +import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OfHeader; +import org.opendaylight.yangtools.yang.binding.DataObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * + */ +public class NotificationPlainTranslator implements IMDMessageTranslator> { + + private static final Logger LOG = LoggerFactory + .getLogger(NotificationPlainTranslator.class); + + @Override + public List translate(SwitchConnectionDistinguisher cookie, + SessionContext sc, OfHeader msg) { + List results = null; + + if(msg instanceof NotificationQueueWrapper) { + NotificationQueueWrapper wrappedNotification = (NotificationQueueWrapper) msg; + BigInteger datapathId = sc.getFeatures().getDatapathId(); + Short version = wrappedNotification.getVersion(); + LOG.debug("NotificationQueueWrapper: version {} dataPathId {} notification {}", version, datapathId, wrappedNotification.getImplementedInterface()); + results = Lists.newArrayList((DataObject) wrappedNotification.getNotification()); + } else { + // TODO - Do something smarter than returning null if translation fails... what Exception should we throw here? + results = Collections.emptyList(); + } + return results; + } + +} diff --git a/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManagerTest.java b/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManagerTest.java index d74f303b32..a0282a5fc9 100644 --- a/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManagerTest.java +++ b/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/sal/SalRegistrationManagerTest.java @@ -12,10 +12,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import com.google.common.util.concurrent.Futures; + import java.math.BigInteger; import java.util.Collections; import java.util.Set; import java.util.concurrent.ExecutorService; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,6 +32,7 @@ import org.opendaylight.controller.sal.common.util.Rpcs; import org.opendaylight.openflowplugin.openflow.md.ModelDrivenSwitch; import org.opendaylight.openflowplugin.api.OFConstants; import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductor; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationEnqueuer; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.openflowplugin.openflow.md.core.session.IMessageDispatchService; import org.opendaylight.openflowplugin.openflow.md.core.session.SessionContext; @@ -69,6 +72,8 @@ public class SalRegistrationManagerTest { private GetFeaturesOutput features; @Mock private BindingAwareBroker.ProviderContext providerContext; + @Mock + private NotificationEnqueuer notificationEnqueuer; private ModelDrivenSwitch mdSwitchOF13; @@ -77,7 +82,6 @@ public class SalRegistrationManagerTest { @Before public void setUp() { - Mockito.when(context.getPrimaryConductor()).thenReturn(conductor); Mockito.when(context.getMessageDispatchService()).thenReturn(messageDispatchService); Mockito.when(conductor.getVersion()).thenReturn(OFConstants.OFP_VERSION_1_0) @@ -88,6 +92,7 @@ public class SalRegistrationManagerTest { registration = new CompositeObjectRegistration<>(mdSwitchOF13, Collections.EMPTY_LIST); Mockito.when(context.getProviderRegistration()).thenReturn(registration); + Mockito.when(context.getNotificationEnqueuer()).thenReturn(notificationEnqueuer); Mockito.when(features.getDatapathId()).thenReturn(BigInteger.valueOf(1)); Mockito.when(features.getVersion()).thenReturn((short) 1); diff --git a/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/session/MessageDispatchServiceImplTest.java b/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/session/MessageDispatchServiceImplTest.java index 1133647557..4dfa9ae7d4 100644 --- a/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/session/MessageDispatchServiceImplTest.java +++ b/openflowplugin/src/test/java/org/opendaylight/openflowplugin/openflow/md/core/session/MessageDispatchServiceImplTest.java @@ -16,6 +16,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -25,6 +26,8 @@ import org.opendaylight.openflowplugin.openflow.md.ModelDrivenSwitch; import org.opendaylight.openflowplugin.api.OFConstants; import org.opendaylight.openflowplugin.openflow.md.core.ConnectionConductor; import org.opendaylight.openflowplugin.openflow.md.core.ErrorHandler; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationEnqueuer; +import org.opendaylight.openflowplugin.openflow.md.core.NotificationQueueWrapper; import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher; import org.opendaylight.openflowplugin.openflow.md.queue.QueueProcessor; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.BarrierInput; @@ -430,9 +433,14 @@ class MockSessionContext implements SessionContext { public void setSeed(int seed) { this.seed = seed; } + + @Override + public NotificationEnqueuer getNotificationEnqueuer() { + return conductor; + } } -class MockConnectionConductor implements ConnectionConductor { +class MockConnectionConductor implements ConnectionConductor, NotificationEnqueuer { private int conductorNum; private MockConnectionAdapter adapter; @@ -525,6 +533,11 @@ class MockConnectionConductor implements ConnectionConductor { public void setId(int conductorId) { // NOOP } + + @Override + public void enqueueNotification(NotificationQueueWrapper notification) { + // NOOP + } } enum MessageType {