openflowplugin/src/main/java/org/opendaylight/openflowplugin/openflow/md/core/ConnectionConductorImpl.java

   1 /**
   2  * Copyright (c) 2013-2014 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.openflowplugin.openflow.md.core;
  10
  11 import java.util.concurrent.Future;
  12 import java.util.concurrent.LinkedBlockingQueue;
  13 import java.util.concurrent.ThreadPoolExecutor;
  14 import java.util.concurrent.TimeUnit;
  15
  16 import org.opendaylight.openflowjava.protocol.api.connection.ConnectionAdapter;
  17 import org.opendaylight.openflowjava.protocol.api.connection.ConnectionReadyListener;
  18 import org.opendaylight.openflowplugin.api.OFConstants;
  19 import org.opendaylight.openflowplugin.api.openflow.md.core.ConnectionConductor;
  20 import org.opendaylight.openflowplugin.api.openflow.md.core.ErrorHandler;
  21 import org.opendaylight.openflowplugin.api.openflow.md.core.HandshakeListener;
  22 import org.opendaylight.openflowplugin.api.openflow.md.core.NotificationEnqueuer;
  23 import org.opendaylight.openflowplugin.api.openflow.md.core.NotificationQueueWrapper;
  24 import org.opendaylight.openflowplugin.api.openflow.md.core.SwitchConnectionDistinguisher;
  25 import org.opendaylight.openflowplugin.openflow.md.core.session.OFSessionUtil;
  26 import org.opendaylight.openflowplugin.openflow.md.core.session.PortFeaturesUtil;
  27 import org.opendaylight.openflowplugin.api.openflow.md.core.session.SessionContext;
  28 import org.opendaylight.openflowplugin.api.openflow.md.core.session.SessionManager;
  29 import org.opendaylight.openflowplugin.api.openflow.md.queue.QueueKeeper;
  30 import org.opendaylight.openflowplugin.api.openflow.md.queue.QueueKeeper.QueueType;
  31 import org.opendaylight.openflowplugin.openflow.md.queue.QueueKeeperFactory;
  32 import org.opendaylight.openflowplugin.api.openflow.md.queue.QueueProcessor;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.common.types.rev130731.MultipartRequestFlags;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.common.types.rev130731.MultipartType;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.common.types.rev130731.SwitchConfigFlag;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoInputBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoOutput;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoReplyInputBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoRequestMessage;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ErrorMessage;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ExperimenterMessage;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowRemovedMessage;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesOutput;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.HelloMessage;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartReplyMessage;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartRequestInputBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OfHeader;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OpenflowProtocolListener;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PacketInMessage;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortGrouping;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortStatus;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortStatusMessage;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.SetConfigInputBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.multipart.request.multipart.request.body.MultipartRequestDescCaseBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.multipart.request.multipart.request.body.MultipartRequestGroupFeaturesCaseBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.multipart.request.multipart.request.body.MultipartRequestMeterFeaturesCaseBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.multipart.request.multipart.request.body.MultipartRequestPortDescCaseBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.DisconnectEvent;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SwitchIdleEvent;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SystemNotificationsListener;
  61 import org.opendaylight.yangtools.yang.binding.DataObject;
  62 import org.opendaylight.yangtools.yang.common.RpcError;
  63 import org.opendaylight.yangtools.yang.common.RpcResult;
  64 import org.slf4j.Logger;
  65 import org.slf4j.LoggerFactory;
  66
  67 import com.google.common.util.concurrent.Futures;
  68
  69 /**
  70  * @author mirehak
  71  */
  72 public class ConnectionConductorImpl implements OpenflowProtocolListener,
  73         SystemNotificationsListener, ConnectionConductor, ConnectionReadyListener, HandshakeListener, NotificationEnqueuer {
  74
  75     /** ingress queue limit */
  76     private static final int INGRESS_QUEUE_MAX_SIZE = 200;
  77
  78     protected static final Logger LOG = LoggerFactory
  79             .getLogger(ConnectionConductorImpl.class);
  80
  81     /* variable to make BitMap-based negotiation enabled / disabled.
  82      * it will help while testing and isolating issues related to processing of
  83      * BitMaps from switches.
  84      */
  85     private boolean isBitmapNegotiationEnable = true;
  86     protected ErrorHandler errorHandler;
  87
  88     private final ConnectionAdapter connectionAdapter;
  89     private ConnectionConductor.CONDUCTOR_STATE conductorState;
  90     private Short version;
  91
  92     protected SwitchConnectionDistinguisher auxiliaryKey;
  93
  94     protected SessionContext sessionContext;
  95
  96     private QueueProcessor<OfHeader, DataObject> queueProcessor;
  97     private QueueKeeper<OfHeader> queue;
  98     private ThreadPoolExecutor hsPool;
  99     private HandshakeManager handshakeManager;
 100
 101     private boolean firstHelloProcessed;
 102
 103     private PortFeaturesUtil portFeaturesUtils;
 104
 105     private int conductorId;
 106
 107     private int ingressMaxQueueSize;
 108
 109
 110     /**
 111      * @param connectionAdapter
 112      */
 113     public ConnectionConductorImpl(ConnectionAdapter connectionAdapter) {
 114         this(connectionAdapter, INGRESS_QUEUE_MAX_SIZE);
 115     }
 116
 117     /**
 118      * @param connectionAdapter
 119      * @param ingressMaxQueueSize ingress queue limit (blocking)
 120      */
 121     public ConnectionConductorImpl(ConnectionAdapter connectionAdapter, int ingressMaxQueueSize) {
 122         this.connectionAdapter = connectionAdapter;
 123         this.ingressMaxQueueSize = ingressMaxQueueSize;
 124         conductorState = CONDUCTOR_STATE.HANDSHAKING;
 125         firstHelloProcessed = false;
 126         handshakeManager = new HandshakeManagerImpl(connectionAdapter,
 127                 ConnectionConductor.versionOrder.get(0), ConnectionConductor.versionOrder);
 128         handshakeManager.setUseVersionBitmap(isBitmapNegotiationEnable);
 129         handshakeManager.setHandshakeListener(this);
 130         portFeaturesUtils = PortFeaturesUtil.getInstance();
 131     }
 132
 133     @Override
 134     public void init() {
 135         int handshakeThreadLimit = 1;
 136         hsPool = new ThreadPoolLoggingExecutor(handshakeThreadLimit , handshakeThreadLimit, 0L,
 137                 TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
 138                 "OFHandshake-"+conductorId);
 139
 140         connectionAdapter.setMessageListener(this);
 141         connectionAdapter.setSystemListener(this);
 142         connectionAdapter.setConnectionReadyListener(this);
 143         queue = QueueKeeperFactory.createFairQueueKeeper(queueProcessor, ingressMaxQueueSize);
 144     }
 145
 146     @Override
 147     public void setQueueProcessor(QueueProcessor<OfHeader, DataObject> queueProcessor) {
 148         this.queueProcessor = queueProcessor;
 149     }
 150
 151     /**
 152      * @param errorHandler the errorHandler to set
 153      */
 154     @Override
 155     public void setErrorHandler(ErrorHandler errorHandler) {
 156         this.errorHandler = errorHandler;
 157         handshakeManager.setErrorHandler(errorHandler);
 158     }
 159
 160     @Override
 161     public void onEchoRequestMessage(final EchoRequestMessage echoRequestMessage) {
 162         new Thread(new Runnable() {
 163             @Override
 164             public void run() {
 165                 LOG.debug("echo request received: " + echoRequestMessage.getXid());
 166                 EchoReplyInputBuilder builder = new EchoReplyInputBuilder();
 167                 builder.setVersion(echoRequestMessage.getVersion());
 168                 builder.setXid(echoRequestMessage.getXid());
 169                 builder.setData(echoRequestMessage.getData());
 170
 171                 getConnectionAdapter().echoReply(builder.build());
 172             }
 173         }).start();
 174     }
 175
 176     @Override
 177     public void onErrorMessage(ErrorMessage errorMessage) {
 178         enqueueMessage(errorMessage);
 179     }
 180
 181
 182     /**
 183      * @param message
 184      */
 185     private void enqueueMessage(OfHeader message) {
 186         enqueueMessage(message, QueueType.DEFAULT);
 187     }
 188
 189     @Override
 190     public void enqueueNotification(NotificationQueueWrapper notification) {
 191         enqueueMessage(notification);
 192     }
 193
 194     /**
 195      * @param message
 196      * @param queueType enqueue type
 197      */
 198     private void enqueueMessage(OfHeader message, QueueType queueType) {
 199         queue.push(message, this, queueType);
 200     }
 201
 202     @Override
 203     public void onExperimenterMessage(ExperimenterMessage experimenterMessage) {
 204         enqueueMessage(experimenterMessage);
 205     }
 206
 207     @Override
 208     public void onFlowRemovedMessage(FlowRemovedMessage message) {
 209         enqueueMessage(message);
 210     }
 211
 212
 213     /**
 214      * version negotiation happened as per following steps:
 215      * 1. If HelloMessage version field has same version, continue connection processing.
 216      *    If HelloMessage version is lower than supported versions, just disconnect.
 217      * 2. If HelloMessage contains bitmap and common version found in bitmap
 218      *    then continue connection processing. if no common version found, just disconnect.
 219      * 3. If HelloMessage version is not supported, send HelloMessage with lower supported version.
 220      * 4. If Hello message received again with not supported version, just disconnect.
 221      */
 222     @Override
 223     public void onHelloMessage(final HelloMessage hello) {
 224         LOG.debug("processing HELLO.xid: {}", hello.getXid());
 225         firstHelloProcessed = true;
 226         checkState(CONDUCTOR_STATE.HANDSHAKING);
 227         HandshakeStepWrapper handshakeStepWrapper = new HandshakeStepWrapper(
 228                 hello, handshakeManager, connectionAdapter);
 229         hsPool.submit(handshakeStepWrapper);
 230     }
 231
 232     /**
 233      * @return rpc-response timeout in [ms]
 234      */
 235     protected long getMaxTimeout() {
 236         // TODO:: get from configuration
 237         return 2000;
 238     }
 239
 240     /**
 241      * @return milliseconds
 242      */
 243     protected TimeUnit getMaxTimeoutUnit() {
 244         // TODO:: get from configuration
 245         return TimeUnit.MILLISECONDS;
 246     }
 247
 248     @Override
 249     public void onMultipartReplyMessage(MultipartReplyMessage message) {
 250         enqueueMessage(message);
 251     }
 252
 253     @Override
 254     public void onPacketInMessage(PacketInMessage message) {
 255         enqueueMessage(message, QueueKeeper.QueueType.UNORDERED);
 256     }
 257
 258     @Override
 259     public void onPortStatusMessage(PortStatusMessage message) {
 260         processPortStatusMsg(message);
 261         enqueueMessage(message);
 262     }
 263
 264     protected void processPortStatusMsg(PortStatus msg) {
 265         if (msg.getReason().getIntValue() == 2) {
 266             updatePort(msg);
 267         } else if (msg.getReason().getIntValue() == 0) {
 268             updatePort(msg);
 269         } else if (msg.getReason().getIntValue() == 1) {
 270             deletePort(msg);
 271         }
 272     }
 273
 274     protected void updatePort(PortStatus msg) {
 275         Long portNumber = msg.getPortNo();
 276         Boolean portBandwidth = portFeaturesUtils.getPortBandwidth(msg);
 277
 278         if(portBandwidth == null) {
 279             LOG.debug("can't get bandwidth info from port: {}, aborting port update", msg.toString());
 280         } else {
 281             this.getSessionContext().getPhysicalPorts().put(portNumber, msg);
 282             this.getSessionContext().getPortsBandwidth().put(portNumber, portBandwidth);
 283         }
 284     }
 285
 286     protected void deletePort(PortGrouping port) {
 287         Long portNumber = port.getPortNo();
 288
 289         this.getSessionContext().getPhysicalPorts().remove(portNumber);
 290         this.getSessionContext().getPortsBandwidth().remove(portNumber);
 291     }
 292
 293     @Override
 294     public void onSwitchIdleEvent(SwitchIdleEvent notification) {
 295         new Thread(new Runnable() {
 296             @Override
 297             public void run() {
 298                 if (!CONDUCTOR_STATE.WORKING.equals(getConductorState())) {
 299                     // idle state in any other conductorState than WORKING means real
 300                     // problem and wont be handled by echoReply, but disconnection
 301                     disconnect();
 302                     OFSessionUtil.getSessionManager().invalidateOnDisconnect(ConnectionConductorImpl.this);
 303                 } else {
 304                     LOG.debug("first idle state occured, sessionCtx={}|auxId={}", sessionContext, auxiliaryKey);
 305                     EchoInputBuilder builder = new EchoInputBuilder();
 306                     builder.setVersion(getVersion());
 307                     builder.setXid(getSessionContext().getNextXid());
 308
 309                     Future<RpcResult<EchoOutput>> echoReplyFuture = getConnectionAdapter()
 310                             .echo(builder.build());
 311
 312                     try {
 313                         RpcResult<EchoOutput> echoReplyValue = echoReplyFuture.get(getMaxTimeout(),
 314                                 getMaxTimeoutUnit());
 315                         if (echoReplyValue.isSuccessful()) {
 316                             setConductorState(CONDUCTOR_STATE.WORKING);
 317                         } else {
 318                             for (RpcError replyError : echoReplyValue.getErrors()) {
 319                                 Throwable cause = replyError.getCause();
 320                                 LOG.error(
 321                                         "while receiving echoReply in TIMEOUTING state: "
 322                                                 + cause.getMessage(), cause);
 323                             }
 324                             //switch issue occurred
 325                             throw new Exception("switch issue occurred");
 326                         }
 327                     } catch (Exception e) {
 328                         LOG.error("while waiting for echoReply in TIMEOUTING state: "
 329                                 + e.getMessage());
 330                         errorHandler.handleException(e, sessionContext);
 331                         //switch is not responding
 332                         disconnect();
 333                         OFSessionUtil.getSessionManager().invalidateOnDisconnect(ConnectionConductorImpl.this);
 334                     }
 335                 }
 336             }
 337
 338         }).start();
 339     }
 340
 341     /**
 342      * @param conductorState
 343      *            the connectionState to set
 344      */
 345     @Override
 346     public void setConductorState(CONDUCTOR_STATE conductorState) {
 347         this.conductorState = conductorState;
 348     }
 349
 350     @Override
 351     public CONDUCTOR_STATE getConductorState() {
 352         return conductorState;
 353     }
 354
 355     /**
 356      * @param expectedState
 357      */
 358     protected void checkState(CONDUCTOR_STATE expectedState) {
 359         if (!conductorState.equals(expectedState)) {
 360             throw new IllegalStateException("Expected state: " + expectedState
 361                     + ", actual state:" + conductorState);
 362         }
 363     }
 364
 365     @Override
 366     public void onDisconnectEvent(DisconnectEvent arg0) {
 367         SessionManager sessionManager = OFSessionUtil.getSessionManager();
 368         sessionManager.invalidateOnDisconnect(this);
 369     }
 370
 371     @Override
 372     public Short getVersion() {
 373         return version;
 374     }
 375
 376     @Override
 377     public Future<Boolean> disconnect() {
 378         LOG.trace("disconnecting: sessionCtx={}|auxId={}", sessionContext, auxiliaryKey);
 379
 380         Future<Boolean> result = null;
 381         if (connectionAdapter.isAlive()) {
 382             result = connectionAdapter.disconnect();
 383         } else {
 384             LOG.debug("connection already disconnected");
 385             result = Futures.immediateFuture(true);
 386         }
 387
 388         return result;
 389     }
 390
 391     @Override
 392     public void setConnectionCookie(SwitchConnectionDistinguisher auxiliaryKey) {
 393         this.auxiliaryKey = auxiliaryKey;
 394     }
 395
 396     @Override
 397     public void setSessionContext(SessionContext sessionContext) {
 398         this.sessionContext = sessionContext;
 399     }
 400
 401     @Override
 402     public SwitchConnectionDistinguisher getAuxiliaryKey() {
 403         return auxiliaryKey;
 404     }
 405
 406     @Override
 407     public SessionContext getSessionContext() {
 408         return sessionContext;
 409     }
 410
 411     @Override
 412     public ConnectionAdapter getConnectionAdapter() {
 413         return connectionAdapter;
 414     }
 415
 416     @Override
 417     public void onConnectionReady() {
 418         LOG.debug("connection is ready-to-use");
 419         if (! firstHelloProcessed) {
 420             HandshakeStepWrapper handshakeStepWrapper = new HandshakeStepWrapper(
 421                     null, handshakeManager, connectionAdapter);
 422             hsPool.execute(handshakeStepWrapper);
 423             firstHelloProcessed = true;
 424         } else {
 425             LOG.debug("already touched by hello message");
 426         }
 427     }
 428
 429     @Override
 430     public void onHandshakeSuccessfull(GetFeaturesOutput featureOutput,
 431             Short negotiatedVersion) {
 432         postHandshakeBasic(featureOutput, negotiatedVersion);
 433
 434         // post-handshake actions
 435         if(version == OFConstants.OFP_VERSION_1_3){
 436             requestPorts();
 437             requestGroupFeatures();
 438             requestMeterFeatures();
 439         }
 440
 441         requestDesc();
 442     }
 443
 444     /**
 445      * used by tests
 446      * @param featureOutput
 447      * @param negotiatedVersion
 448      */
 449     protected void postHandshakeBasic(GetFeaturesOutput featureOutput,
 450             Short negotiatedVersion) {
 451         version = negotiatedVersion;
 452         if (version == OFConstants.OFP_VERSION_1_0) {
 453             //  Because the GetFeaturesOutput contains information about the port
 454             //  in OF1.0 (that we would otherwise get from the PortDesc) we have to pass
 455             //  it up for parsing to convert into a NodeConnectorUpdate
 456             //
 457             // BUG-1988 - this must be the first item in queue in order not to get behind link-up message
 458             enqueueMessage(featureOutput);
 459         }
 460
 461         OFSessionUtil.registerSession(this, featureOutput, negotiatedVersion);
 462         hsPool.shutdown();
 463         hsPool.purge();
 464         conductorState = CONDUCTOR_STATE.WORKING;
 465         QueueKeeperFactory.plugQueue(queueProcessor, queue);
 466     }
 467
 468     /*
 469      *  Send an OFPMP_DESC request message to the switch
 470      */
 471     private void requestDesc() {
 472         MultipartRequestInputBuilder builder = new MultipartRequestInputBuilder();
 473         builder.setType(MultipartType.OFPMPDESC);
 474         builder.setVersion(getVersion());
 475         builder.setFlags(new MultipartRequestFlags(false));
 476         builder.setMultipartRequestBody(new MultipartRequestDescCaseBuilder().build());
 477         builder.setXid(getSessionContext().getNextXid());
 478         getConnectionAdapter().multipartRequest(builder.build());
 479     }
 480
 481     private void requestPorts() {
 482         MultipartRequestInputBuilder builder = new MultipartRequestInputBuilder();
 483         builder.setType(MultipartType.OFPMPPORTDESC);
 484         builder.setVersion(getVersion());
 485         builder.setFlags(new MultipartRequestFlags(false));
 486         builder.setMultipartRequestBody(new MultipartRequestPortDescCaseBuilder().build());
 487         builder.setXid(getSessionContext().getNextXid());
 488         getConnectionAdapter().multipartRequest(builder.build());
 489     }
 490     private void requestGroupFeatures(){
 491         MultipartRequestInputBuilder mprInput = new MultipartRequestInputBuilder();
 492         mprInput.setType(MultipartType.OFPMPGROUPFEATURES);
 493         mprInput.setVersion(getVersion());
 494         mprInput.setFlags(new MultipartRequestFlags(false));
 495         mprInput.setXid(getSessionContext().getNextXid());
 496
 497         MultipartRequestGroupFeaturesCaseBuilder mprGroupFeaturesBuild =
 498                 new MultipartRequestGroupFeaturesCaseBuilder();
 499         mprInput.setMultipartRequestBody(mprGroupFeaturesBuild.build());
 500
 501         LOG.debug("Send group features statistics request :{}",mprGroupFeaturesBuild);
 502         getConnectionAdapter().multipartRequest(mprInput.build());
 503
 504     }
 505     private void requestMeterFeatures(){
 506         MultipartRequestInputBuilder mprInput = new MultipartRequestInputBuilder();
 507         mprInput.setType(MultipartType.OFPMPMETERFEATURES);
 508         mprInput.setVersion(getVersion());
 509         mprInput.setFlags(new MultipartRequestFlags(false));
 510         mprInput.setXid(getSessionContext().getNextXid());
 511
 512         MultipartRequestMeterFeaturesCaseBuilder mprMeterFeaturesBuild =
 513                 new MultipartRequestMeterFeaturesCaseBuilder();
 514         mprInput.setMultipartRequestBody(mprMeterFeaturesBuild.build());
 515
 516         LOG.debug("Send meter features statistics request :{}",mprMeterFeaturesBuild);
 517         getConnectionAdapter().multipartRequest(mprInput.build());
 518
 519     }
 520     /**
 521      * @param isBitmapNegotiationEnable the isBitmapNegotiationEnable to set
 522      */
 523     public void setBitmapNegotiationEnable(
 524             boolean isBitmapNegotiationEnable) {
 525         this.isBitmapNegotiationEnable = isBitmapNegotiationEnable;
 526     }
 527
 528     protected void shutdownPool() {
 529         hsPool.shutdownNow();
 530         LOG.debug("pool is terminated: {}", hsPool.isTerminated());
 531     }
 532
 533     @Override
 534     public void setId(int conductorId) {
 535         this.conductorId = conductorId;
 536     }
 537 }