opendaylight/netconf/config-persister-impl/src/main/java/org/opendaylight/controller/netconf/persist/impl/ConfigPusher.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.netconf.persist.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import io.netty.channel.EventLoopGroup;
  13 import org.opendaylight.controller.config.api.ConflictingVersionException;
  14 import org.opendaylight.controller.config.persist.api.ConfigSnapshotHolder;
  15 import org.opendaylight.controller.netconf.api.NetconfMessage;
  16 import org.opendaylight.controller.netconf.client.NetconfClient;
  17 import org.opendaylight.controller.netconf.client.NetconfClientDispatcher;
  18 import org.opendaylight.controller.netconf.util.NetconfUtil;
  19 import org.opendaylight.controller.netconf.util.messages.NetconfHelloMessageAdditionalHeader;
  20 import org.opendaylight.controller.netconf.util.xml.XmlElement;
  21 import org.opendaylight.controller.netconf.util.xml.XmlNetconfConstants;
  22 import org.opendaylight.controller.netconf.util.xml.XmlUtil;
  23 import org.slf4j.Logger;
  24 import org.slf4j.LoggerFactory;
  25 import org.w3c.dom.Document;
  26 import org.w3c.dom.Element;
  27 import org.xml.sax.SAXException;
  28
  29 import javax.annotation.concurrent.Immutable;
  30 import java.io.IOException;
  31 import java.io.InputStream;
  32 import java.net.InetSocketAddress;
  33 import java.util.Collections;
  34 import java.util.HashSet;
  35 import java.util.LinkedHashMap;
  36 import java.util.List;
  37 import java.util.Set;
  38 import java.util.concurrent.ExecutionException;
  39 import java.util.concurrent.TimeUnit;
  40 import java.util.concurrent.TimeoutException;
  41
  42 @Immutable
  43 public class ConfigPusher {
  44     private static final Logger logger = LoggerFactory.getLogger(ConfigPersisterNotificationHandler.class);
  45     private static final int NETCONF_SEND_ATTEMPT_MS_DELAY = 1000;
  46     private static final int NETCONF_SEND_ATTEMPTS = 20;
  47
  48     private final InetSocketAddress address;
  49     private final EventLoopGroup nettyThreadGroup;
  50
  51     // Default timeout for netconf becoming stable
  52     public static final long DEFAULT_MAX_WAIT_FOR_CAPABILITIES_MILLIS = TimeUnit.MINUTES.toMillis(2);
  53     public static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = 5000;
  54     private final int delayMillis = 5000;
  55     private final long maxWaitForCapabilitiesMillis;
  56     private final long connectionTimeoutMillis;
  57
  58     public ConfigPusher(InetSocketAddress address, EventLoopGroup nettyThreadGroup) {
  59         this(address, nettyThreadGroup, DEFAULT_MAX_WAIT_FOR_CAPABILITIES_MILLIS, DEFAULT_CONNECTION_TIMEOUT_MILLIS);
  60     }
  61
  62     public ConfigPusher(InetSocketAddress address, EventLoopGroup nettyThreadGroup,
  63                         long maxWaitForCapabilitiesMillis, long connectionTimeoutMillis) {
  64         this.address = address;
  65         this.nettyThreadGroup = nettyThreadGroup;
  66         this.maxWaitForCapabilitiesMillis = maxWaitForCapabilitiesMillis;
  67         this.connectionTimeoutMillis = connectionTimeoutMillis;
  68     }
  69
  70     public synchronized LinkedHashMap<ConfigSnapshotHolder, EditAndCommitResponseWithRetries> pushConfigs(
  71             List<ConfigSnapshotHolder> configs) throws InterruptedException {
  72         logger.debug("Last config snapshots to be pushed to netconf: {}", configs);
  73
  74         // first just make sure we can connect to netconf, even if nothing is being pushed
  75         {
  76             NetconfClient netconfClient = makeNetconfConnection(Collections.<String>emptySet());
  77             Util.closeClientAndDispatcher(netconfClient);
  78         }
  79         LinkedHashMap<ConfigSnapshotHolder, EditAndCommitResponseWithRetries> result = new LinkedHashMap<>();
  80         // start pushing snapshots:
  81         for (ConfigSnapshotHolder configSnapshotHolder : configs) {
  82             EditAndCommitResponseWithRetries editAndCommitResponseWithRetries = pushSnapshotWithRetries(configSnapshotHolder);
  83             logger.debug("Config snapshot pushed successfully: {}, result: {}", configSnapshotHolder, result);
  84             result.put(configSnapshotHolder, editAndCommitResponseWithRetries);
  85         }
  86         logger.debug("All configuration snapshots have been pushed successfully.");
  87         return result;
  88     }
  89
  90     /**
  91      * Checks for ConflictingVersionException and retries until optimistic lock succeeds or maximal
  92      * number of attempts is reached.
  93      */
  94     private synchronized EditAndCommitResponseWithRetries pushSnapshotWithRetries(ConfigSnapshotHolder configSnapshotHolder)
  95             throws InterruptedException {
  96
  97         ConflictingVersionException lastException = null;
  98         int maxAttempts = 30;
  99
 100         for (int retryAttempt = 1; retryAttempt <= maxAttempts; retryAttempt++) {
 101             NetconfClient netconfClient = makeNetconfConnection(configSnapshotHolder.getCapabilities());
 102             logger.trace("Pushing following xml to netconf {}", configSnapshotHolder);
 103             try {
 104                 EditAndCommitResponse editAndCommitResponse = pushLastConfig(configSnapshotHolder, netconfClient);
 105                 return new EditAndCommitResponseWithRetries(editAndCommitResponse, retryAttempt);
 106             } catch (ConflictingVersionException e) {
 107                 logger.debug("Conflicting version detected, will retry after timeout");
 108                 lastException = e;
 109                 Thread.sleep(1000);
 110             } catch (RuntimeException e) {
 111                 throw new IllegalStateException("Unable to load " + configSnapshotHolder, e);
 112             } finally {
 113                 Util.closeClientAndDispatcher(netconfClient);
 114             }
 115         }
 116         throw new IllegalStateException("Maximum attempt count has been reached for pushing " + configSnapshotHolder,
 117                 lastException);
 118     }
 119
 120     /**
 121      * @param expectedCaps capabilities that server hello must contain. Will retry until all are found or throws RuntimeException.
 122      *                     If empty set is provided, will only make sure netconf client successfuly connected to the server.
 123      * @return NetconfClient that has all required capabilities from server.
 124      */
 125     private synchronized NetconfClient makeNetconfConnection(Set<String> expectedCaps) throws InterruptedException {
 126
 127         // TODO think about moving capability subset check to netconf client
 128         // could be utilized by integration tests
 129
 130         final long pollingStartNanos = System.nanoTime();
 131         final long deadlineNanos = pollingStartNanos + TimeUnit.MILLISECONDS.toNanos(maxWaitForCapabilitiesMillis);
 132         int attempt = 0;
 133
 134         NetconfHelloMessageAdditionalHeader additionalHeader = new NetconfHelloMessageAdditionalHeader("unknown", address.getAddress().getHostAddress(),
 135                 Integer.toString(address.getPort()), "tcp", "persister");
 136
 137         Set<String> latestCapabilities = null;
 138         while (System.nanoTime() < deadlineNanos) {
 139             attempt++;
 140             NetconfClientDispatcher netconfClientDispatcher = new NetconfClientDispatcher(nettyThreadGroup,
 141                     nettyThreadGroup, additionalHeader, connectionTimeoutMillis);
 142             NetconfClient netconfClient;
 143             try {
 144                 netconfClient = new NetconfClient(this.toString(), address, delayMillis, netconfClientDispatcher);
 145             } catch (IllegalStateException e) {
 146                 logger.debug("Netconf {} was not initialized or is not stable, attempt {}", address, attempt, e);
 147                 netconfClientDispatcher.close();
 148                 Thread.sleep(delayMillis);
 149                 continue;
 150             }
 151             latestCapabilities = netconfClient.getCapabilities();
 152             if (Util.isSubset(netconfClient, expectedCaps)) {
 153                 logger.debug("Hello from netconf stable with {} capabilities", latestCapabilities);
 154                 logger.trace("Session id received from netconf server: {}", netconfClient.getClientSession());
 155                 return netconfClient;
 156             }
 157             Set<String> allNotFound = computeNotFoundCapabilities(expectedCaps, latestCapabilities);
 158             logger.debug("Netconf server did not provide required capabilities. Attempt {}. " +
 159                     "Expected but not found: {}, all expected {}, current {}",
 160                     attempt, allNotFound, expectedCaps, latestCapabilities);
 161             Util.closeClientAndDispatcher(netconfClient);
 162             Thread.sleep(delayMillis);
 163         }
 164         if (latestCapabilities == null) {
 165             logger.error("Could not connect to the server in {} ms", maxWaitForCapabilitiesMillis);
 166             throw new RuntimeException("Could not connect to netconf server");
 167         }
 168         Set<String> allNotFound = computeNotFoundCapabilities(expectedCaps, latestCapabilities);
 169         logger.error("Netconf server did not provide required capabilities. Expected but not found: {}, all expected {}, current {}",
 170                 allNotFound, expectedCaps, latestCapabilities);
 171         throw new RuntimeException("Netconf server did not provide required capabilities. Expected but not found:" + allNotFound);
 172     }
 173
 174     private static Set<String> computeNotFoundCapabilities(Set<String> expectedCaps, Set<String> latestCapabilities) {
 175         Set<String> allNotFound = new HashSet<>(expectedCaps);
 176         allNotFound.removeAll(latestCapabilities);
 177         return allNotFound;
 178     }
 179
 180
 181     /**
 182      * Sends two RPCs to the netconf server: edit-config and commit.
 183      *
 184      * @param configSnapshotHolder
 185      * @param netconfClient
 186      * @throws ConflictingVersionException if commit fails on optimistic lock failure inside of config-manager
 187      * @throws java.lang.RuntimeException  if edit-config or commit fails otherwise
 188      */
 189     private synchronized EditAndCommitResponse pushLastConfig(ConfigSnapshotHolder configSnapshotHolder, NetconfClient netconfClient)
 190             throws ConflictingVersionException {
 191
 192         Element xmlToBePersisted;
 193         try {
 194             xmlToBePersisted = XmlUtil.readXmlToElement(configSnapshotHolder.getConfigSnapshot());
 195         } catch (SAXException | IOException e) {
 196             throw new IllegalStateException("Cannot parse " + configSnapshotHolder);
 197         }
 198         logger.trace("Pushing last configuration to netconf: {}", configSnapshotHolder);
 199
 200         NetconfMessage editConfigMessage = createEditConfigMessage(xmlToBePersisted);
 201
 202         // sending message to netconf
 203         NetconfMessage editResponseMessage;
 204         try {
 205             editResponseMessage = sendRequestGetResponseCheckIsOK(editConfigMessage, netconfClient);
 206         } catch (IOException e) {
 207             throw new IllegalStateException("Edit-config failed on " + configSnapshotHolder, e);
 208         }
 209
 210         // commit
 211         NetconfMessage commitResponseMessage;
 212         try {
 213             commitResponseMessage = sendRequestGetResponseCheckIsOK(getCommitMessage(), netconfClient);
 214         } catch (IOException e) {
 215             throw new IllegalStateException("Edit commit succeeded, but commit failed on " + configSnapshotHolder, e);
 216         }
 217
 218         if (logger.isTraceEnabled()) {
 219             StringBuilder response = new StringBuilder("editConfig response = {");
 220             response.append(XmlUtil.toString(editResponseMessage.getDocument()));
 221             response.append("}");
 222             response.append("commit response = {");
 223             response.append(XmlUtil.toString(commitResponseMessage.getDocument()));
 224             response.append("}");
 225             logger.trace("Last configuration loaded successfully");
 226             logger.trace("Detailed message {}", response);
 227         }
 228         return new EditAndCommitResponse(editResponseMessage, commitResponseMessage);
 229     }
 230
 231
 232     private static NetconfMessage sendRequestGetResponseCheckIsOK(NetconfMessage request, NetconfClient netconfClient)
 233             throws ConflictingVersionException, IOException {
 234         try {
 235             NetconfMessage netconfMessage = netconfClient.sendMessage(request, NETCONF_SEND_ATTEMPTS, NETCONF_SEND_ATTEMPT_MS_DELAY);
 236             NetconfUtil.checkIsMessageOk(netconfMessage);
 237             return netconfMessage;
 238         }catch(ConflictingVersionException e) {
 239             logger.trace("conflicting version detected: {}", e.toString());
 240             throw e;
 241         } catch (RuntimeException | ExecutionException | InterruptedException | TimeoutException e) { // TODO: change NetconfClient#sendMessage to throw checked exceptions
 242             logger.debug("Error while executing netconf transaction {} to {}", request, netconfClient, e);
 243             throw new IOException("Failed to execute netconf transaction", e);
 244         }
 245     }
 246
 247
 248     // load editConfig.xml template, populate /rpc/edit-config/config with parameter
 249     private static NetconfMessage createEditConfigMessage(Element dataElement) {
 250         String editConfigResourcePath = "/netconfOp/editConfig.xml";
 251         try (InputStream stream = ConfigPersisterNotificationHandler.class.getResourceAsStream(editConfigResourcePath)) {
 252             Preconditions.checkNotNull(stream, "Unable to load resource " + editConfigResourcePath);
 253
 254             Document doc = XmlUtil.readXmlToDocument(stream);
 255
 256             XmlElement editConfigElement = XmlElement.fromDomDocument(doc).getOnlyChildElement();
 257             XmlElement configWrapper = editConfigElement.getOnlyChildElement(XmlNetconfConstants.CONFIG_KEY);
 258             editConfigElement.getDomElement().removeChild(configWrapper.getDomElement());
 259             for (XmlElement el : XmlElement.fromDomElement(dataElement).getChildElements()) {
 260                 boolean deep = true;
 261                 configWrapper.appendChild((Element) doc.importNode(el.getDomElement(), deep));
 262             }
 263             editConfigElement.appendChild(configWrapper.getDomElement());
 264             return new NetconfMessage(doc);
 265         } catch (IOException | SAXException e) {
 266             // error reading the xml file bundled into the jar
 267             throw new RuntimeException("Error while opening local resource " + editConfigResourcePath, e);
 268         }
 269     }
 270
 271     private static NetconfMessage getCommitMessage() {
 272         String resource = "/netconfOp/commit.xml";
 273         try (InputStream stream = ConfigPusher.class.getResourceAsStream(resource)) {
 274             Preconditions.checkNotNull(stream, "Unable to load resource " + resource);
 275             return new NetconfMessage(XmlUtil.readXmlToDocument(stream));
 276         } catch (SAXException | IOException e) {
 277             // error reading the xml file bundled into the jar
 278             throw new RuntimeException("Error while opening local resource " + resource, e);
 279         }
 280     }
 281
 282     static class EditAndCommitResponse {
 283         private final NetconfMessage editResponse, commitResponse;
 284
 285         EditAndCommitResponse(NetconfMessage editResponse, NetconfMessage commitResponse) {
 286             this.editResponse = editResponse;
 287             this.commitResponse = commitResponse;
 288         }
 289
 290         public NetconfMessage getEditResponse() {
 291             return editResponse;
 292         }
 293
 294         public NetconfMessage getCommitResponse() {
 295             return commitResponse;
 296         }
 297
 298         @Override
 299         public String toString() {
 300             return "EditAndCommitResponse{" +
 301                     "editResponse=" + editResponse +
 302                     ", commitResponse=" + commitResponse +
 303                     '}';
 304         }
 305     }
 306
 307
 308     static class EditAndCommitResponseWithRetries {
 309         private final EditAndCommitResponse editAndCommitResponse;
 310         private final int retries;
 311
 312         EditAndCommitResponseWithRetries(EditAndCommitResponse editAndCommitResponse, int retries) {
 313             this.editAndCommitResponse = editAndCommitResponse;
 314             this.retries = retries;
 315         }
 316
 317         public int getRetries() {
 318             return retries;
 319         }
 320
 321         public EditAndCommitResponse getEditAndCommitResponse() {
 322             return editAndCommitResponse;
 323         }
 324
 325         @Override
 326         public String toString() {
 327             return "EditAndCommitResponseWithRetries{" +
 328                     "editAndCommitResponse=" + editAndCommitResponse +
 329                     ", retries=" + retries +
 330                     '}';
 331         }
 332     }
 333 }