netconf/sal-netconf-connector/src/test/java/org/opendaylight/netconf/sal/connect/netconf/listener/NetconfDeviceCommunicatorTest.java

   1 /*
   2  * Copyright (c) 2014 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.sal.connect.netconf.listener;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertFalse;
  13 import static org.junit.Assert.assertNotNull;
  14 import static org.junit.Assert.assertTrue;
  15 import static org.mockito.Matchers.any;
  16 import static org.mockito.Matchers.eq;
  17 import static org.mockito.Matchers.same;
  18 import static org.mockito.Mockito.doNothing;
  19 import static org.mockito.Mockito.doReturn;
  20 import static org.mockito.Mockito.mock;
  21 import static org.mockito.Mockito.never;
  22 import static org.mockito.Mockito.reset;
  23 import static org.mockito.Mockito.spy;
  24 import static org.mockito.Mockito.timeout;
  25 import static org.mockito.Mockito.verify;
  26 import static org.opendaylight.netconf.api.xml.XmlNetconfConstants.URN_IETF_PARAMS_XML_NS_NETCONF_BASE_1_0;
  27
  28 import com.google.common.base.CharMatcher;
  29 import com.google.common.base.Strings;
  30 import com.google.common.collect.Sets;
  31 import com.google.common.util.concurrent.ListenableFuture;
  32 import io.netty.channel.ChannelFuture;
  33 import io.netty.channel.EventLoopGroup;
  34 import io.netty.channel.nio.NioEventLoopGroup;
  35 import io.netty.util.HashedWheelTimer;
  36 import io.netty.util.Timer;
  37 import io.netty.util.concurrent.Future;
  38 import io.netty.util.concurrent.GenericFutureListener;
  39 import io.netty.util.concurrent.GlobalEventExecutor;
  40 import java.io.ByteArrayInputStream;
  41 import java.net.InetSocketAddress;
  42 import java.util.ArrayList;
  43 import java.util.Collection;
  44 import java.util.Collections;
  45 import java.util.UUID;
  46 import java.util.concurrent.TimeUnit;
  47 import java.util.concurrent.TimeoutException;
  48 import javax.xml.parsers.ParserConfigurationException;
  49 import org.junit.Before;
  50 import org.junit.Test;
  51 import org.mockito.ArgumentCaptor;
  52 import org.mockito.Mock;
  53 import org.mockito.MockitoAnnotations;
  54 import org.opendaylight.controller.config.util.xml.XmlMappingConstants;
  55 import org.opendaylight.netconf.api.NetconfMessage;
  56 import org.opendaylight.netconf.api.NetconfTerminationReason;
  57 import org.opendaylight.netconf.client.NetconfClientDispatcherImpl;
  58 import org.opendaylight.netconf.client.NetconfClientSession;
  59 import org.opendaylight.netconf.client.conf.NetconfClientConfiguration;
  60 import org.opendaylight.netconf.client.conf.NetconfReconnectingClientConfiguration;
  61 import org.opendaylight.netconf.client.conf.NetconfReconnectingClientConfigurationBuilder;
  62 import org.opendaylight.netconf.nettyutil.handler.ssh.authentication.LoginPasswordHandler;
  63 import org.opendaylight.netconf.sal.connect.api.RemoteDevice;
  64 import org.opendaylight.netconf.sal.connect.netconf.util.NetconfMessageTransformUtil;
  65 import org.opendaylight.netconf.sal.connect.util.RemoteDeviceId;
  66 import org.opendaylight.protocol.framework.ReconnectStrategy;
  67 import org.opendaylight.protocol.framework.TimedReconnectStrategy;
  68 import org.opendaylight.yangtools.util.xml.UntrustedXML;
  69 import org.opendaylight.yangtools.yang.common.QName;
  70 import org.opendaylight.yangtools.yang.common.RpcError;
  71 import org.opendaylight.yangtools.yang.common.RpcResult;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74 import org.w3c.dom.Document;
  75 import org.w3c.dom.Element;
  76
  77 public class NetconfDeviceCommunicatorTest {
  78
  79     private static final Logger LOG = LoggerFactory.getLogger(NetconfDeviceCommunicatorTest.class);
  80
  81     @Mock
  82     NetconfClientSession mockSession;
  83
  84     @Mock
  85     RemoteDevice<NetconfSessionPreferences, NetconfMessage, NetconfDeviceCommunicator> mockDevice;
  86
  87     NetconfDeviceCommunicator communicator;
  88
  89     @Before
  90     public void setUp() throws Exception {
  91         MockitoAnnotations.initMocks(this);
  92
  93         communicator = new NetconfDeviceCommunicator(
  94                 new RemoteDeviceId("test", InetSocketAddress.createUnresolved("localhost", 22)), mockDevice, 10);
  95     }
  96
  97     void setupSession() {
  98         doReturn(Collections.<String>emptySet()).when(mockSession).getServerCapabilities();
  99         doNothing().when(mockDevice).onRemoteSessionUp(any(NetconfSessionPreferences.class),
 100                 any(NetconfDeviceCommunicator.class));
 101         communicator.onSessionUp(mockSession);
 102     }
 103
 104     private ListenableFuture<RpcResult<NetconfMessage>> sendRequest() throws Exception {
 105         return sendRequest(UUID.randomUUID().toString(), true);
 106     }
 107
 108     @SuppressWarnings("unchecked")
 109     private ListenableFuture<RpcResult<NetconfMessage>> sendRequest(final String messageID,
 110                                                                     final boolean doLastTest) throws Exception {
 111         Document doc = UntrustedXML.newDocumentBuilder().newDocument();
 112         Element element = doc.createElement("request");
 113         element.setAttribute("message-id", messageID);
 114         doc.appendChild(element);
 115         NetconfMessage message = new NetconfMessage(doc);
 116
 117         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 118         doReturn(mockChannelFuture).when(mockChannelFuture)
 119                 .addListener(any(GenericFutureListener.class));
 120         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 121
 122         ListenableFuture<RpcResult<NetconfMessage>> resultFuture =
 123                 communicator.sendRequest(message, QName.create("", "mockRpc"));
 124         if (doLastTest) {
 125             assertNotNull("ListenableFuture is null", resultFuture);
 126         }
 127         return resultFuture;
 128     }
 129
 130     @Test
 131     public void testOnSessionUp() {
 132         String testCapability = "urn:opendaylight:params:xml:ns:test?module=test-module&revision=2014-06-02";
 133         Collection<String> serverCapabilities =
 134                 Sets.newHashSet(NetconfMessageTransformUtil.NETCONF_ROLLBACK_ON_ERROR_URI.toString(),
 135                         NetconfMessageTransformUtil.IETF_NETCONF_MONITORING.getNamespace().toString(),
 136                         testCapability);
 137         doReturn(serverCapabilities).when(mockSession).getServerCapabilities();
 138
 139         ArgumentCaptor<NetconfSessionPreferences> netconfSessionPreferences =
 140                 ArgumentCaptor.forClass(NetconfSessionPreferences.class);
 141         doNothing().when(mockDevice).onRemoteSessionUp(netconfSessionPreferences.capture(), eq(communicator));
 142
 143         communicator.onSessionUp(mockSession);
 144
 145         verify(mockSession).getServerCapabilities();
 146         verify(mockDevice).onRemoteSessionUp(netconfSessionPreferences.capture(), eq(communicator));
 147
 148         NetconfSessionPreferences actualCapabilites = netconfSessionPreferences.getValue();
 149         assertEquals("containsModuleCapability", true, actualCapabilites.containsNonModuleCapability(
 150                 NetconfMessageTransformUtil.NETCONF_ROLLBACK_ON_ERROR_URI.toString()));
 151         assertEquals("containsModuleCapability", false, actualCapabilites.containsNonModuleCapability(testCapability));
 152         assertEquals("getModuleBasedCaps", Sets.newHashSet(
 153                 QName.create("urn:opendaylight:params:xml:ns:test", "2014-06-02", "test-module")),
 154                 actualCapabilites.getModuleBasedCaps());
 155         assertEquals("isRollbackSupported", true, actualCapabilites.isRollbackSupported());
 156         assertEquals("isMonitoringSupported", true, actualCapabilites.isMonitoringSupported());
 157     }
 158
 159     @SuppressWarnings("unchecked")
 160     @Test(timeout = 5000)
 161     public void testOnSessionDown() throws Exception {
 162         setupSession();
 163
 164         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest();
 165         final ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest();
 166
 167         doNothing().when(mockDevice).onRemoteSessionDown();
 168
 169         communicator.onSessionDown(mockSession, new Exception("mock ex"));
 170
 171         verifyErrorRpcResult(resultFuture1.get(), RpcError.ErrorType.TRANSPORT, "operation-failed");
 172         verifyErrorRpcResult(resultFuture2.get(), RpcError.ErrorType.TRANSPORT, "operation-failed");
 173
 174         verify(mockDevice).onRemoteSessionDown();
 175
 176         reset(mockDevice);
 177
 178         communicator.onSessionDown(mockSession, new Exception("mock ex"));
 179
 180         verify(mockDevice, never()).onRemoteSessionDown();
 181     }
 182
 183     @Test
 184     public void testOnSessionTerminated() throws Exception {
 185         setupSession();
 186
 187         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest();
 188
 189         doNothing().when(mockDevice).onRemoteSessionDown();
 190
 191         String reasonText = "testing terminate";
 192         NetconfTerminationReason reason = new NetconfTerminationReason(reasonText);
 193         communicator.onSessionTerminated(mockSession, reason);
 194
 195         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.TRANSPORT,
 196                 "operation-failed");
 197         assertEquals("RpcError message", reasonText, rpcError.getMessage());
 198
 199         verify(mockDevice).onRemoteSessionDown();
 200     }
 201
 202     @Test
 203     public void testClose() throws Exception {
 204         communicator.close();
 205         verify(mockDevice, never()).onRemoteSessionDown();
 206     }
 207
 208     @SuppressWarnings({"rawtypes", "unchecked"})
 209     @Test
 210     public void testSendRequest() throws Exception {
 211         setupSession();
 212
 213         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 214         QName rpc = QName.create("", "mockRpc");
 215
 216         ArgumentCaptor<GenericFutureListener> futureListener =
 217                 ArgumentCaptor.forClass(GenericFutureListener.class);
 218
 219         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 220         doReturn(mockChannelFuture).when(mockChannelFuture).addListener(futureListener.capture());
 221         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 222
 223         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 224
 225         verify(mockSession).sendMessage(same(message));
 226
 227         assertNotNull("ListenableFuture is null", resultFuture);
 228
 229         verify(mockChannelFuture).addListener(futureListener.capture());
 230         Future<Void> operationFuture = mock(Future.class);
 231         doReturn(true).when(operationFuture).isSuccess();
 232         doReturn(true).when(operationFuture).isDone();
 233         futureListener.getValue().operationComplete(operationFuture);
 234
 235         try {
 236             resultFuture.get(1, TimeUnit.MILLISECONDS); // verify it's not cancelled or has an error set
 237         } catch (TimeoutException e) {
 238             LOG.info("Operation failed due timeout.");
 239         } // expected
 240     }
 241
 242     @Test
 243     public void testSendRequestWithNoSession() throws Exception {
 244         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 245         QName rpc = QName.create("", "mockRpc");
 246
 247         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 248
 249         assertNotNull("ListenableFuture is null", resultFuture);
 250
 251         // Should have an immediate result
 252         RpcResult<NetconfMessage> rpcResult = resultFuture.get(3, TimeUnit.MILLISECONDS);
 253
 254         verifyErrorRpcResult(rpcResult, RpcError.ErrorType.TRANSPORT, "operation-failed");
 255     }
 256
 257     private static NetconfMessage createSuccessResponseMessage(final String messageID)
 258             throws ParserConfigurationException {
 259         Document doc = UntrustedXML.newDocumentBuilder().newDocument();
 260         Element rpcReply =
 261                 doc.createElementNS(URN_IETF_PARAMS_XML_NS_NETCONF_BASE_1_0, XmlMappingConstants.RPC_REPLY_KEY);
 262         rpcReply.setAttribute("message-id", messageID);
 263         Element element = doc.createElementNS("ns", "data");
 264         element.setTextContent(messageID);
 265         rpcReply.appendChild(element);
 266         doc.appendChild(rpcReply);
 267
 268         return new NetconfMessage(doc);
 269     }
 270
 271     @SuppressWarnings({ "rawtypes", "unchecked" })
 272     @Test
 273     public void testSendRequestWithWithSendFailure() throws Exception {
 274         setupSession();
 275
 276         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 277         QName rpc = QName.create("", "mockRpc");
 278
 279         ArgumentCaptor<GenericFutureListener> futureListener =
 280                 ArgumentCaptor.forClass(GenericFutureListener.class);
 281
 282         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 283         doReturn(mockChannelFuture).when(mockChannelFuture).addListener(futureListener.capture());
 284         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 285
 286         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 287
 288         assertNotNull("ListenableFuture is null", resultFuture);
 289
 290         verify(mockChannelFuture).addListener(futureListener.capture());
 291
 292         Future<Void> operationFuture = mock(Future.class);
 293         doReturn(false).when(operationFuture).isSuccess();
 294         doReturn(true).when(operationFuture).isDone();
 295         doReturn(new Exception("mock error")).when(operationFuture).cause();
 296         futureListener.getValue().operationComplete(operationFuture);
 297
 298         // Should have an immediate result
 299         RpcResult<NetconfMessage> rpcResult = resultFuture.get(3, TimeUnit.MILLISECONDS);
 300
 301         RpcError rpcError = verifyErrorRpcResult(rpcResult, RpcError.ErrorType.TRANSPORT, "operation-failed");
 302         assertEquals("RpcError message contains \"mock error\"", true,
 303                 rpcError.getMessage().contains("mock error"));
 304     }
 305
 306     //Test scenario verifying whether missing message is handled
 307     @Test
 308     public void testOnMissingResponseMessage() throws Exception {
 309
 310         setupSession();
 311
 312         String messageID1 = UUID.randomUUID().toString();
 313         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest(messageID1, true);
 314
 315         String messageID2 = UUID.randomUUID().toString();
 316         ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest(messageID2, true);
 317
 318         String messageID3 = UUID.randomUUID().toString();
 319         ListenableFuture<RpcResult<NetconfMessage>> resultFuture3 = sendRequest(messageID3, true);
 320
 321         //response messages 1,2 are omitted
 322         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID3));
 323
 324         verifyResponseMessage(resultFuture3.get(), messageID3);
 325     }
 326
 327     @Test
 328     public void testOnSuccessfulResponseMessage() throws Exception {
 329         setupSession();
 330
 331         String messageID1 = UUID.randomUUID().toString();
 332         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest(messageID1, true);
 333
 334         String messageID2 = UUID.randomUUID().toString();
 335         final ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest(messageID2, true);
 336
 337         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID1));
 338         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID2));
 339
 340         verifyResponseMessage(resultFuture1.get(), messageID1);
 341         verifyResponseMessage(resultFuture2.get(), messageID2);
 342     }
 343
 344     @Test
 345     public void testOnResponseMessageWithError() throws Exception {
 346         setupSession();
 347
 348         String messageID = UUID.randomUUID().toString();
 349         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID, true);
 350
 351         communicator.onMessage(mockSession, createErrorResponseMessage(messageID));
 352
 353         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.RPC,
 354                 "missing-attribute");
 355         assertEquals("RpcError message", "Missing attribute", rpcError.getMessage());
 356
 357         String errorInfo = rpcError.getInfo();
 358         assertNotNull("RpcError info is null", errorInfo);
 359         assertTrue("Error info contains \"foo\"", errorInfo.contains("<bad-attribute>foo</bad-attribute>"));
 360         assertTrue("Error info contains \"bar\"", errorInfo.contains("<bad-element>bar</bad-element>"));
 361     }
 362
 363     /**
 364      * Test whether reconnect is scheduled properly.
 365      */
 366     @Test
 367     public void testNetconfDeviceReconnectInCommunicator() throws Exception {
 368         final RemoteDevice<NetconfSessionPreferences, NetconfMessage, NetconfDeviceCommunicator> device =
 369                 mock(RemoteDevice.class);
 370
 371         final TimedReconnectStrategy timedReconnectStrategy =
 372                 new TimedReconnectStrategy(GlobalEventExecutor.INSTANCE, 10000, 0, 1.0, null, 100L, null);
 373         final ReconnectStrategy reconnectStrategy = spy(new ReconnectStrategy() {
 374             @Override
 375             public int getConnectTimeout() throws Exception {
 376                 return timedReconnectStrategy.getConnectTimeout();
 377             }
 378
 379             @Override
 380             public Future<Void> scheduleReconnect(final Throwable cause) {
 381                 return timedReconnectStrategy.scheduleReconnect(cause);
 382             }
 383
 384             @Override
 385             public void reconnectSuccessful() {
 386                 timedReconnectStrategy.reconnectSuccessful();
 387             }
 388         });
 389
 390         final EventLoopGroup group = new NioEventLoopGroup();
 391         final Timer time = new HashedWheelTimer();
 392         try {
 393             final NetconfDeviceCommunicator listener = new NetconfDeviceCommunicator(
 394                     new RemoteDeviceId("test", InetSocketAddress.createUnresolved("localhost", 22)), device, 10);
 395             final NetconfReconnectingClientConfiguration cfg = NetconfReconnectingClientConfigurationBuilder.create()
 396                     .withAddress(new InetSocketAddress("localhost", 65000))
 397                     .withReconnectStrategy(reconnectStrategy)
 398                     .withConnectStrategyFactory(() -> reconnectStrategy)
 399                     .withAuthHandler(new LoginPasswordHandler("admin", "admin"))
 400                     .withConnectionTimeoutMillis(10000)
 401                     .withProtocol(NetconfClientConfiguration.NetconfClientProtocol.SSH)
 402                     .withSessionListener(listener)
 403                     .build();
 404
 405             listener.initializeRemoteConnection(new NetconfClientDispatcherImpl(group, group, time), cfg);
 406
 407             verify(reconnectStrategy,
 408                     timeout((int) TimeUnit.MINUTES.toMillis(3)).times(101)).scheduleReconnect(any(Throwable.class));
 409         } finally {
 410             time.stop();
 411             group.shutdownGracefully();
 412         }
 413     }
 414
 415     @Test
 416     public void testOnResponseMessageWithWrongMessageID() throws Exception {
 417         setupSession();
 418
 419         String messageID = UUID.randomUUID().toString();
 420         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID, true);
 421
 422         communicator.onMessage(mockSession, createSuccessResponseMessage(UUID.randomUUID().toString()));
 423
 424         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.PROTOCOL,
 425                 "bad-attribute");
 426         assertFalse("RpcError message non-empty", Strings.isNullOrEmpty(rpcError.getMessage()));
 427
 428         String errorInfo = rpcError.getInfo();
 429         assertNotNull("RpcError info is null", errorInfo);
 430         assertTrue("Error info contains \"actual-message-id\"", errorInfo.contains("actual-message-id"));
 431         assertTrue("Error info contains \"expected-message-id\"", errorInfo.contains("expected-message-id"));
 432     }
 433
 434     @Test
 435     public void testConcurrentMessageLimit() throws Exception {
 436         setupSession();
 437         ArrayList<String> messageID = new ArrayList<>();
 438
 439         for (int i = 0; i < 10; i++) {
 440             messageID.add(UUID.randomUUID().toString());
 441             ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID.get(i), false);
 442             assertEquals("ListenableFuture is null", true, resultFuture instanceof UncancellableFuture);
 443         }
 444
 445         final String notWorkingMessageID = UUID.randomUUID().toString();
 446         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(notWorkingMessageID, false);
 447         assertEquals("ListenableFuture is null", false, resultFuture instanceof UncancellableFuture);
 448
 449         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID.get(0)));
 450
 451         resultFuture = sendRequest(messageID.get(0), false);
 452         assertNotNull("ListenableFuture is null", resultFuture);
 453     }
 454
 455     private static NetconfMessage createErrorResponseMessage(final String messageID) throws Exception {
 456         String xmlStr = "<rpc-reply xmlns=\"urn:ietf:params:xml:ns:netconf:base:1.0\""
 457                 + "           message-id=\"" + messageID + "\">"
 458                 + "  <rpc-error>"
 459                 + "    <error-type>rpc</error-type>"
 460                 + "    <error-tag>missing-attribute</error-tag>"
 461                 + "    <error-severity>error</error-severity>"
 462                 + "    <error-message>Missing attribute</error-message>"
 463                 + "    <error-info>"
 464                 + "      <bad-attribute>foo</bad-attribute>"
 465                 + "      <bad-element>bar</bad-element>"
 466                 + "    </error-info>"
 467                 + "  </rpc-error>"
 468                 + "</rpc-reply>";
 469
 470         ByteArrayInputStream bis = new ByteArrayInputStream(xmlStr.getBytes());
 471         Document doc = UntrustedXML.newDocumentBuilder().parse(bis);
 472         return new NetconfMessage(doc);
 473     }
 474
 475     private static void verifyResponseMessage(final RpcResult<NetconfMessage> rpcResult, final String dataText) {
 476         assertNotNull("RpcResult is null", rpcResult);
 477         assertTrue("isSuccessful", rpcResult.isSuccessful());
 478         NetconfMessage messageResult = rpcResult.getResult();
 479         assertNotNull("getResult", messageResult);
 480 //        List<SimpleNode<?>> nodes = messageResult.getSimpleNodesByName(
 481 //                                         QName.create( URI.create( "ns" ), null, "data" ) );
 482 //        assertNotNull( "getSimpleNodesByName", nodes );
 483 //        assertEquals( "List<SimpleNode<?>> size", 1, nodes.size() );
 484 //        assertEquals( "SimpleNode value", dataText, nodes.iterator().next().getValue() );
 485     }
 486
 487     private static RpcError verifyErrorRpcResult(final RpcResult<NetconfMessage> rpcResult,
 488                                                  final RpcError.ErrorType expErrorType, final String expErrorTag) {
 489         assertNotNull("RpcResult is null", rpcResult);
 490         assertFalse("isSuccessful", rpcResult.isSuccessful());
 491         assertNotNull("RpcResult errors is null", rpcResult.getErrors());
 492         assertEquals("Errors size", 1, rpcResult.getErrors().size());
 493         RpcError rpcError = rpcResult.getErrors().iterator().next();
 494         assertEquals("getErrorSeverity", RpcError.ErrorSeverity.ERROR, rpcError.getSeverity());
 495         assertEquals("getErrorType", expErrorType, rpcError.getErrorType());
 496         assertEquals("getErrorTag", expErrorTag, rpcError.getTag());
 497
 498         final String msg = rpcError.getMessage();
 499         assertNotNull("getMessage is null", msg);
 500         assertFalse("getMessage is empty", msg.isEmpty());
 501         assertFalse("getMessage is blank", CharMatcher.whitespace().matchesAllOf(msg));
 502         return rpcError;
 503     }
 504 }