FRM event sync developement 35/1035/2
authorGiovanni Meo <gmeo@cisco.com>
Thu, 22 Aug 2013 21:00:35 +0000 (23:00 +0200)
committerGiovanni Meo <gmeo@cisco.com>
Thu, 29 Aug 2013 13:49:39 +0000 (15:49 +0200)
- Created necessary logic for distributing entry programming among the
FRM instances in the cluster.
- Made serializable Status so it can be distributed.
- Hooked FRM with connection manager, such that the FRM programming
can be distributed if the node is not local
- In case of non-local node someone picks it and process it, then the
status is returned back to caller.
- At the moment the whole logic to timeout entries that don't get an
answer is missing, will come along.
- HostTracker ages out the ARP entries in clustered environment
because when refreshed the timeout was not being refreshed
clusterwide, probably need a better way to deal with the timing out
-Added license in files missing it.
- Fixed integration test failing, now in fact we can no longer use
fake values else the connection manager will not report them as
attached to local controller and will stuck the test case

Signed-off-by: Giovanni Meo <gmeo@cisco.com>
Change-Id: I74b7535371f61759e89883c66ba9b22596d7dddc

opendaylight/forwardingrulesmanager/implementation/pom.xml
opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/implementation/data/FlowEntryDistributionOrder.java [new file with mode: 0644]
opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/Activator.java
opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/FlowEntryDistributionOrderFutureTask.java [new file with mode: 0644]
opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManager.java
opendaylight/forwardingrulesmanager/integrationtest/pom.xml
opendaylight/forwardingrulesmanager/integrationtest/src/test/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManagerIT.java
opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/hostAware/HostNodeConnector.java
opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java
opendaylight/sal/api/src/main/java/org/opendaylight/controller/sal/utils/Status.java

index 9170fd303b25ae89179db1d1fe1d0d1b0a5963c8..ea55ac8b08951f14191cbcd900a6309c7154fcfc 100644 (file)
@@ -48,6 +48,7 @@
             <Include-Resource>
             </Include-Resource>
             <Export-Package>
             <Include-Resource>
             </Include-Resource>
             <Export-Package>
+              org.opendaylight.controller.forwardingrulesmanager.implementation.data
             </Export-Package>
             <Import-Package>
               org.opendaylight.controller.clustering.services,
             </Export-Package>
             <Import-Package>
               org.opendaylight.controller.clustering.services,
@@ -62,6 +63,7 @@
               org.opendaylight.controller.sal.utils,
               org.opendaylight.controller.sal.packet,
               org.opendaylight.controller.forwardingrulesmanager,
               org.opendaylight.controller.sal.utils,
               org.opendaylight.controller.sal.packet,
               org.opendaylight.controller.forwardingrulesmanager,
+              org.opendaylight.controller.connectionmanager,
               javax.xml.bind.annotation,
               javax.xml.bind,
               org.apache.felix.dm,
               javax.xml.bind.annotation,
               javax.xml.bind,
               org.apache.felix.dm,
       <version>4.8.1</version>
       <scope>test</scope>
     </dependency>
       <version>4.8.1</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>connectionmanager</artifactId>
+      <version>0.1.0-SNAPSHOT</version>
+    </dependency>
   </dependencies>
 </project>
   </dependencies>
 </project>
diff --git a/opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/implementation/data/FlowEntryDistributionOrder.java b/opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/implementation/data/FlowEntryDistributionOrder.java
new file mode 100644 (file)
index 0000000..5220428
--- /dev/null
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+/**
+ * Class used by the FRM to distribute the forwarding rules programming in the
+ * cluster and to collect back the results of the programming
+ */
+package org.opendaylight.controller.forwardingrulesmanager.implementation.data;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
+import org.opendaylight.controller.sal.core.UpdateType;
+
+/**
+ * Class used by the FRM to distribute the forwarding rules programming in the
+ * cluster and to collect back the results of the programming
+ */
+public final class FlowEntryDistributionOrder implements Serializable {
+    /**
+     * Serialization UID
+     */
+    private static final long serialVersionUID = 416280377113255147L;
+    final private FlowEntryInstall entry;
+    final private UpdateType upType;
+    final private InetAddress requestorController;
+
+    /**
+     * @return the entry
+     */
+    public FlowEntryInstall getEntry() {
+        return entry;
+    }
+
+    /**
+     * @return the upType
+     */
+    public UpdateType getUpType() {
+        return upType;
+    }
+
+    /**
+     * @return the requestorController
+     */
+    public InetAddress getRequestorController() {
+        return requestorController;
+    }
+
+    /**
+     * @param entry
+     *            FlowEntryInstall key value
+     * @param upType
+     *            UpdateType key value
+     * @param requestorController
+     *            identifier of the controller that initiated the request
+     */
+
+    public FlowEntryDistributionOrder(FlowEntryInstall entry, UpdateType upType, InetAddress requestorController) {
+        super();
+        this.entry = entry;
+        this.upType = upType;
+        this.requestorController = requestorController;
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#hashCode()
+     */
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = (prime * result) + ((entry == null) ? 0 : entry.hashCode());
+        result = (prime * result) + ((requestorController == null) ? 0 : requestorController.hashCode());
+        result = (prime * result) + ((upType == null) ? 0 : upType.hashCode());
+        return result;
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#equals(java.lang.Object)
+     */
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof FlowEntryDistributionOrder)) {
+            return false;
+        }
+        FlowEntryDistributionOrder other = (FlowEntryDistributionOrder) obj;
+        if (entry == null) {
+            if (other.entry != null) {
+                return false;
+            }
+        } else if (!entry.equals(other.entry)) {
+            return false;
+        }
+        if (requestorController == null) {
+            if (other.requestorController != null) {
+                return false;
+            }
+        } else if (!requestorController.equals(other.requestorController)) {
+            return false;
+        }
+        if (upType != other.upType) {
+            return false;
+        }
+        return true;
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("FlowEntryDistributionOrder [");
+        if (entry != null) {
+            builder.append("entry=")
+                    .append(entry)
+                    .append(", ");
+        }
+        if (upType != null) {
+            builder.append("upType=")
+                    .append(upType)
+                    .append(", ");
+        }
+        if (requestorController != null) {
+            builder.append("requestorController=")
+                    .append(requestorController);
+        }
+        builder.append("]");
+        return builder.toString();
+    }
+}
index bbb9ad7b7385fe8f7fe09d17e1f9433d9382a417..cc0a0444b218a624abac6365b8f45561cfa057ef 100644 (file)
@@ -8,6 +8,7 @@
 
 package org.opendaylight.controller.forwardingrulesmanager.internal;
 
 
 package org.opendaylight.controller.forwardingrulesmanager.internal;
 
+import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
 import java.util.Dictionary;
 import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.Dictionary;
 import java.util.HashSet;
 import java.util.Hashtable;
@@ -29,6 +30,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
 import org.slf4j.LoggerFactory;
 
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
+import org.opendaylight.controller.connectionmanager.IConnectionManager;
 
 public class Activator extends ComponentActivatorAbstractBase {
     protected static final Logger logger = LoggerFactory.getLogger(Activator.class);
 
 public class Activator extends ComponentActivatorAbstractBase {
     protected static final Logger logger = LoggerFactory.getLogger(Activator.class);
@@ -87,14 +89,19 @@ public class Activator extends ComponentActivatorAbstractBase {
     public void configureInstance(Component c, Object imp, String containerName) {
         if (imp.equals(ForwardingRulesManager.class)) {
             String interfaces[] = null;
     public void configureInstance(Component c, Object imp, String containerName) {
         if (imp.equals(ForwardingRulesManager.class)) {
             String interfaces[] = null;
+            Dictionary<String, Object> props = new Hashtable<String, Object>();
+            Set<String> propSet = new HashSet<String>();
+            propSet.add(ForwardingRulesManager.WORKSTATUSCACHE);
+            propSet.add(ForwardingRulesManager.WORKORDERCACHE);
+            props.put("cachenames", propSet);
 
             // export the service
             interfaces = new String[] { IContainerListener.class.getName(), ISwitchManagerAware.class.getName(),
                     IForwardingRulesManager.class.getName(), IInventoryListener.class.getName(),
 
             // export the service
             interfaces = new String[] { IContainerListener.class.getName(), ISwitchManagerAware.class.getName(),
                     IForwardingRulesManager.class.getName(), IInventoryListener.class.getName(),
-                    IConfigurationContainerAware.class.getName(),
+                    IConfigurationContainerAware.class.getName(), ICacheUpdateAware.class.getName(),
                     IFlowProgrammerListener.class.getName() };
 
                     IFlowProgrammerListener.class.getName() };
 
-            c.setInterface(interfaces, null);
+            c.setInterface(interfaces, props);
 
             c.add(createContainerServiceDependency(containerName).setService(IFlowProgrammerService.class)
                     .setCallbacks("setFlowProgrammerService", "unsetFlowProgrammerService").setRequired(true));
 
             c.add(createContainerServiceDependency(containerName).setService(IFlowProgrammerService.class)
                     .setCallbacks("setFlowProgrammerService", "unsetFlowProgrammerService").setRequired(true));
@@ -106,6 +113,9 @@ public class Activator extends ComponentActivatorAbstractBase {
                     .setCallbacks("setFrmAware", "unsetFrmAware").setRequired(false));
             c.add(createContainerServiceDependency(containerName).setService(IContainer.class)
                     .setCallbacks("setIContainer", "unsetIContainer").setRequired(true));
                     .setCallbacks("setFrmAware", "unsetFrmAware").setRequired(false));
             c.add(createContainerServiceDependency(containerName).setService(IContainer.class)
                     .setCallbacks("setIContainer", "unsetIContainer").setRequired(true));
+            c.add(createServiceDependency().setService(IConnectionManager.class)
+                    .setCallbacks("setIConnectionManager", "unsetIConnectionManager")
+                    .setRequired(true));
         }
     }
 }
         }
     }
 }
diff --git a/opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/FlowEntryDistributionOrderFutureTask.java b/opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/FlowEntryDistributionOrderFutureTask.java
new file mode 100644 (file)
index 0000000..9761a43
--- /dev/null
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+/**
+ * Class which will monitor the completion of a FlowEntryDistributionOrder it
+ * implements a Future interface so it can be inspected by who is waiting for
+ * it.
+ */
+package org.opendaylight.controller.forwardingrulesmanager.internal;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
+import org.opendaylight.controller.sal.utils.Status;
+import org.opendaylight.controller.sal.utils.StatusCode;
+
+/**
+ * Class which will monitor the completion of a FlowEntryDistributionOrder it
+ * implements a Future interface so it can be inspected by who is waiting for
+ * it.
+ */
+final class FlowEntryDistributionOrderFutureTask implements Future<Status> {
+    private final FlowEntryDistributionOrder order;
+    private boolean amICancelled;
+    private CountDownLatch waitingLatch;
+    private Status retStatus;
+
+    /**
+     * @param order
+     *            for which we are monitoring the execution
+     */
+    FlowEntryDistributionOrderFutureTask(FlowEntryDistributionOrder order) {
+        // Order being monitored
+        this.order = order;
+        this.amICancelled = false;
+        // We need to wait for one completion to happen
+        this.waitingLatch = new CountDownLatch(1);
+        // No return status yet!
+        this.retStatus = new Status(StatusCode.UNDEFINED);
+    }
+
+    @Override
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        return false;
+    }
+
+    @Override
+    public Status get() throws InterruptedException, ExecutionException {
+        // If i'm done lets return the status as many times as caller wants
+        if (this.waitingLatch.getCount() == 0L) {
+            return retStatus;
+        }
+
+        // Wait till someone signal that we are done
+        this.waitingLatch.await();
+
+        // Return the known status
+        return retStatus;
+    }
+
+    @Override
+    public Status get(long timeout, TimeUnit unit) throws InterruptedException,
+            ExecutionException, TimeoutException {
+        // If i'm done lets return the status as many times as caller wants
+        if (this.waitingLatch.getCount() == 0L) {
+            return retStatus;
+        }
+
+        // Wait till someone signal that we are done
+        this.waitingLatch.await(timeout, unit);
+
+        // Return the known status, could also be null if didn't return
+        return retStatus;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return this.amICancelled;
+    }
+
+    @Override
+    public boolean isDone() {
+        return (this.waitingLatch.getCount() == 0L);
+    }
+
+    /**
+     * Used by the thread that gets back the status for the order so can unblock
+     * an eventual caller waiting on the result to comes back
+     *
+     * @param order
+     * @param retStatus
+     */
+    void gotStatus(FlowEntryDistributionOrder order, Status retStatus) {
+        if (order != this.order) {
+            // Weird we got a call for an order we didn't make
+            return;
+        }
+        this.retStatus = retStatus;
+        // Now we are not waiting any longer
+        this.waitingLatch.countDown();
+    }
+}
index 19b045b217a3a5877a4d3e00cb58840e23f13f7e..0d0874990e88a5902e510a62ca31aa1294c3d67d 100644 (file)
@@ -24,15 +24,19 @@ import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.eclipse.osgi.framework.console.CommandInterpreter;
 import org.eclipse.osgi.framework.console.CommandProvider;
 import org.opendaylight.controller.clustering.services.CacheConfigException;
 import org.opendaylight.controller.clustering.services.CacheExistException;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.eclipse.osgi.framework.console.CommandInterpreter;
 import org.eclipse.osgi.framework.console.CommandProvider;
 import org.opendaylight.controller.clustering.services.CacheConfigException;
 import org.opendaylight.controller.clustering.services.CacheExistException;
+import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
 import org.opendaylight.controller.clustering.services.IClusterServices;
 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
 import org.opendaylight.controller.clustering.services.IClusterServices;
 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
+import org.opendaylight.controller.connectionmanager.IConnectionManager;
 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
@@ -42,6 +46,7 @@ import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
+import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
 import org.opendaylight.controller.sal.action.Action;
 import org.opendaylight.controller.sal.action.ActionType;
 import org.opendaylight.controller.sal.action.Controller;
 import org.opendaylight.controller.sal.action.Action;
 import org.opendaylight.controller.sal.action.ActionType;
 import org.opendaylight.controller.sal.action.Controller;
@@ -85,13 +90,22 @@ import org.slf4j.LoggerFactory;
  * the network. It also maintains the central repository of all the forwarding
  * rules installed on the network nodes.
  */
  * the network. It also maintains the central repository of all the forwarding
  * rules installed on the network nodes.
  */
-public class ForwardingRulesManager implements IForwardingRulesManager, PortGroupChangeListener,
-        IContainerListener, ISwitchManagerAware, IConfigurationContainerAware, IInventoryListener, IObjectReader,
-        CommandProvider, IFlowProgrammerListener {
+public class ForwardingRulesManager implements
+        IForwardingRulesManager,
+        PortGroupChangeListener,
+        IContainerListener,
+        ISwitchManagerAware,
+        IConfigurationContainerAware,
+        IInventoryListener,
+        IObjectReader,
+        ICacheUpdateAware,
+        CommandProvider,
+        IFlowProgrammerListener {
     private static final String NODEDOWN = "Node is Down";
     private static final String SUCCESS = StatusCode.SUCCESS.toString();
     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
     private static final String PORTREMOVED = "Port removed";
     private static final String NODEDOWN = "Node is Down";
     private static final String SUCCESS = StatusCode.SUCCESS.toString();
     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
     private static final String PORTREMOVED = "Port removed";
+    private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
     private String frmFileName;
     private String portGroupFileName;
     private ConcurrentMap<Integer, FlowConfig> staticFlows;
     private String frmFileName;
     private String portGroupFileName;
     private ConcurrentMap<Integer, FlowConfig> staticFlows;
@@ -136,6 +150,98 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
     private Thread frmEventHandler;
     protected BlockingQueue<FRMEvent> pendingEvents;
 
     private Thread frmEventHandler;
     protected BlockingQueue<FRMEvent> pendingEvents;
 
+    // Distributes FRM programming in the cluster
+    private IConnectionManager connectionManager;
+
+    /*
+     * Name clustered caches used to support FRM entry distribution these are by
+     * necessity non-transactional as long as need to be able to synchronize
+     * states also while a transaction is in progress
+     */
+    static final String WORKORDERCACHE = "frm.workOrder";
+    static final String WORKSTATUSCACHE = "frm.workStatus";
+
+    /*
+     * Data structure responsible for distributing the FlowEntryInstall requests
+     * in the cluster. The key value is entry that is being either Installed or
+     * Updated or Delete. The value field is the same of the key value in case
+     * of Installation or Deletion, it's the new entry in case of Modification,
+     * this because the clustering caches don't allow null values.
+     *
+     * The logic behind this data structure is that the controller that initiate
+     * the request will place the order here, someone will pick it and then will
+     * remove from this data structure because is being served.
+     *
+     * TODO: We need to have a way to cleanup this data structure if entries are
+     * not picked by anyone, which is always a case can happen especially on
+     * Node disconnect cases.
+     */
+    private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
+
+    /*
+     * Data structure responsible for retrieving the results of the workOrder
+     * submitted to the cluster.
+     *
+     * The logic behind this data structure is that the controller that has
+     * executed the order will then place the result in workStatus signaling
+     * that there was a success or a failure.
+     *
+     * TODO: The workStatus entries need to have a lifetime associated in case
+     * of requestor controller leaving the cluster.
+     */
+    private ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
+
+    /*
+     * Local Map used to hold the Future which a caller can use to monitor for
+     * completion
+     */
+    private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
+            new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
+
+    /**
+     * @param e
+     *            Entry being installed/updated/removed
+     * @param u
+     *            New entry will be placed after the update operation. Valid
+     *            only for UpdateType.CHANGED, null for all the other cases
+     * @param t
+     *            Type of update
+     * @return a Future object for monitoring the progress of the result, or
+     *         null in case the processing should take place locally
+     */
+    private Future<Status> distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u, UpdateType t) {
+        // A null entry it's an unexpected condition, anyway it's safe to keep
+        // the handling local
+        if (e == null) {
+            return null;
+        }
+
+        Node n = e.getNode();
+        if (!connectionManager.isLocal(n)) {
+            // Create the work order and distribute it
+            FlowEntryDistributionOrder fe =
+                    new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
+            // First create the monitor job
+            FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
+            logsync.trace("Node {} not local so sending fe {}", n, fe);
+            workMonitor.put(fe, ret);
+            if (t.equals(UpdateType.CHANGED)) {
+                // Then distribute the work
+                workOrder.put(fe, u);
+            } else {
+                // Then distribute the work
+                workOrder.put(fe, e);
+            }
+            logsync.trace("WorkOrder requested");
+            // Now create an Handle to monitor the execution of the operation
+            return ret;
+        }
+
+        logsync.trace("LOCAL Node {} so processing Entry:{} UpdateType:{}", n, e, t);
+
+        return null;
+    }
+
     /**
      * Adds a flow entry onto the network node It runs various validity checks
      * and derive the final container flows merged entries that will be
     /**
      * Adds a flow entry onto the network node It runs various validity checks
      * and derive the final container flows merged entries that will be
@@ -434,25 +540,40 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
      *         contain the unique id assigned to this request
      */
     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
      *         contain the unique id assigned to this request
      */
     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
-        // Modify the flow on the network node
-        Status status = (async) ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
-                .getFlow(), newEntries.getInstall().getFlow()) : programmer.modifyFlow(currentEntries.getNode(),
-                currentEntries.getInstall().getFlow(), newEntries.getInstall().getFlow());
+        Future<Status> futureStatus = distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
+        if (futureStatus != null) {
+            Status retStatus = new Status(StatusCode.UNDEFINED);
+            try {
+                retStatus = futureStatus.get();
+            } catch (InterruptedException e) {
+                log.error("", e);
+            } catch (ExecutionException e) {
+                log.error("", e);
+            }
+            return retStatus;
+        } else {
+            // Modify the flow on the network node
+            Status status = async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
+                    .getFlow(), newEntries.getInstall()
+                    .getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries.getInstall()
+                    .getFlow(), newEntries.getInstall()
+                    .getFlow());
 
 
-        if (!status.isSuccess()) {
-            log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
-                    status.getDescription());
-            return status;
-        }
+            if (!status.isSuccess()) {
+                log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
+                        status.getDescription());
+                return status;
+            }
 
 
-        log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
+            log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
 
 
-        // Update DB
-        newEntries.setRequestId(status.getRequestId());
-        updateLocalDatabase(currentEntries, false);
-        updateLocalDatabase(newEntries, true);
+            // Update DB
+            newEntries.setRequestId(status.getRequestId());
+            updateLocalDatabase(currentEntries, false);
+            updateLocalDatabase(newEntries, true);
 
 
-        return status;
+            return status;
+        }
     }
 
     /**
     }
 
     /**
@@ -531,24 +652,38 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
      *         contain the unique id assigned to this request
      */
     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
      *         contain the unique id assigned to this request
      */
     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
-        // Mark the entry to be deleted (for CC just in case we fail)
-        entry.toBeDeleted();
+        Future<Status> futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
+        if (futureStatus != null) {
+            Status retStatus = new Status(StatusCode.UNDEFINED);
+            try {
+                retStatus = futureStatus.get();
+            } catch (InterruptedException e) {
+                log.error("", e);
+            } catch (ExecutionException e) {
+                log.error("", e);
+            }
+            return retStatus;
+        } else {
+            // Mark the entry to be deleted (for CC just in case we fail)
+            entry.toBeDeleted();
 
 
-        // Remove from node
-        Status status = (async) ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall().getFlow())
-                : programmer.removeFlow(entry.getNode(), entry.getInstall().getFlow());
+            // Remove from node
+            Status status = async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall()
+                    .getFlow()) : programmer.removeFlow(entry.getNode(), entry.getInstall()
+                    .getFlow());
 
 
-        if (!status.isSuccess()) {
-            log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
-                    status.getDescription());
-            return status;
-        }
-        log.trace("Removed  {}", entry.getInstall());
+            if (!status.isSuccess()) {
+                log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
+                        status.getDescription());
+                return status;
+            }
+            log.trace("Removed  {}", entry.getInstall());
 
 
-        // Update DB
-        updateLocalDatabase(entry, false);
+            // Update DB
+            updateLocalDatabase(entry, false);
 
 
-        return status;
+            return status;
+        }
     }
 
     /**
     }
 
     /**
@@ -565,23 +700,37 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
      *         contain the unique id assigned to this request
      */
     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
      *         contain the unique id assigned to this request
      */
     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
-        // Install the flow on the network node
-        Status status = (async) ? programmer.addFlowAsync(entry.getNode(), entry.getInstall().getFlow()) : programmer
-                .addFlow(entry.getNode(), entry.getInstall().getFlow());
+        Future<Status> futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
+        if (futureStatus != null) {
+            Status retStatus = new Status(StatusCode.UNDEFINED);
+            try {
+                retStatus = futureStatus.get();
+            } catch (InterruptedException e) {
+                log.error("", e);
+            } catch (ExecutionException e) {
+                log.error("", e);
+            }
+            return retStatus;
+        } else {
+            // Install the flow on the network node
+            Status status = async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall()
+                    .getFlow()) : programmer.addFlow(entry.getNode(), entry.getInstall()
+                    .getFlow());
 
 
-        if (!status.isSuccess()) {
-            log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
-                    status.getDescription());
-            return status;
-        }
+            if (!status.isSuccess()) {
+                log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
+                        status.getDescription());
+                return status;
+            }
 
 
-        log.trace("Added    {}", entry.getInstall());
+            log.trace("Added    {}", entry.getInstall());
 
 
-        // Update DB
-        entry.setRequestId(status.getRequestId());
-        updateLocalDatabase(entry, true);
+            // Update DB
+            entry.setRequestId(status.getRequestId());
+            updateLocalDatabase(entry, true);
 
 
-        return status;
+            return status;
+        }
     }
 
     /**
     }
 
     /**
@@ -1203,6 +1352,12 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
             clusterContainerService.createCache("frm.TSPolicies",
                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
 
             clusterContainerService.createCache("frm.TSPolicies",
                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
 
+            clusterContainerService.createCache(WORKSTATUSCACHE,
+                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
+
+            clusterContainerService.createCache(WORKORDERCACHE,
+                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
+
         } catch (CacheConfigException cce) {
             log.error("CacheConfigException");
         } catch (CacheExistException cce) {
         } catch (CacheConfigException cce) {
             log.error("CacheConfigException");
         } catch (CacheExistException cce) {
@@ -1292,6 +1447,19 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
         }
 
             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
         }
 
+        map = clusterContainerService.getCache(WORKORDERCACHE);
+        if (map != null) {
+            workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
+        } else {
+            log.error("Retrieval of " + WORKORDERCACHE + " cache failed for Container {}", container.getName());
+        }
+
+        map = clusterContainerService.getCache(WORKSTATUSCACHE);
+        if (map != null) {
+            workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
+        } else {
+            log.error("Retrieval of " + WORKSTATUSCACHE + " cache failed for Container {}", container.getName());
+        }
     }
 
     private boolean flowConfigExists(FlowConfig config) {
     }
 
     private boolean flowConfigExists(FlowConfig config) {
@@ -2201,11 +2369,60 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
                         } else if (event instanceof ErrorReportedEvent) {
                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
                             processErrorEvent(errEvent);
                         } else if (event instanceof ErrorReportedEvent) {
                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
                             processErrorEvent(errEvent);
+                        } else if (event instanceof WorkOrderEvent) {
+                            /*
+                             * Take care of handling the remote Work request
+                             */
+                            WorkOrderEvent work = (WorkOrderEvent) event;
+                            FlowEntryDistributionOrder fe = work.getFe();
+                            if (fe != null) {
+                                logsync.trace("Executing the workOrder {}", fe);
+                                Status gotStatus = null;
+                                FlowEntryInstall feiCurrent = fe.getEntry();
+                                FlowEntryInstall feiNew = workOrder.get(fe.getEntry());
+                                switch (fe.getUpType()) {
+                                case ADDED:
+                                    /*
+                                     * TODO: Not still sure how to handle the
+                                     * sync entries
+                                     */
+                                    gotStatus = addEntriesInternal(feiCurrent, true);
+                                    break;
+                                case CHANGED:
+                                    gotStatus = modifyEntryInternal(feiCurrent, feiNew, true);
+                                    break;
+                                case REMOVED:
+                                    gotStatus = removeEntryInternal(feiCurrent, true);
+                                    break;
+                                }
+                                // Remove the Order
+                                workOrder.remove(fe);
+                                logsync.trace(
+                                        "The workOrder has been executed and now the status is being returned {}", fe);
+                                // Place the status
+                                workStatus.put(fe, gotStatus);
+                            } else {
+                                log.warn("Not expected null WorkOrder", work);
+                            }
+                        } else if (event instanceof WorkStatusCleanup) {
+                            /*
+                             * Take care of handling the remote Work request
+                             */
+                            WorkStatusCleanup work = (WorkStatusCleanup) event;
+                            FlowEntryDistributionOrder fe = work.getFe();
+                            if (fe != null) {
+                                logsync.trace("The workStatus {} is being removed", fe);
+                                workStatus.remove(fe);
+                            } else {
+                                log.warn("Not expected null WorkStatus", work);
+                            }
                         } else {
                         } else {
-                            log.warn("Dequeued unknown event {}", event.getClass().getSimpleName());
+                            log.warn("Dequeued unknown event {}", event.getClass()
+                                    .getSimpleName());
                         }
                     } catch (InterruptedException e) {
                         }
                     } catch (InterruptedException e) {
-                        log.warn("FRM EventHandler thread interrupted", e);
+                        // clear pending events
+                        pendingEvents.clear();
                     }
                 }
             }
                     }
                 }
             }
@@ -2219,7 +2436,12 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
      *
      */
     void destroy() {
      *
      */
     void destroy() {
+        // Interrupt the thread
+        frmEventHandler.interrupt();
+        // Clear the pendingEvents queue
+        pendingEvents.clear();
         frmAware.clear();
         frmAware.clear();
+        workMonitor.clear();
     }
 
     /**
     }
 
     /**
@@ -2426,6 +2648,52 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
         }
     }
 
         }
     }
 
+    private class WorkOrderEvent extends FRMEvent {
+        private FlowEntryDistributionOrder fe;
+        private FlowEntryInstall newEntry;
+
+        /**
+         * @param fe
+         * @param newEntry
+         */
+        WorkOrderEvent(FlowEntryDistributionOrder fe, FlowEntryInstall newEntry) {
+            this.fe = fe;
+            this.newEntry = newEntry;
+        }
+
+        /**
+         * @return the fe
+         */
+        public FlowEntryDistributionOrder getFe() {
+            return fe;
+        }
+
+        /**
+         * @return the newEntry
+         */
+        public FlowEntryInstall getNewEntry() {
+            return newEntry;
+        }
+    }
+
+    private class WorkStatusCleanup extends FRMEvent {
+        private FlowEntryDistributionOrder fe;
+
+        /**
+         * @param fe
+         */
+        WorkStatusCleanup(FlowEntryDistributionOrder fe) {
+            this.fe = fe;
+        }
+
+        /**
+         * @return the fe
+         */
+        public FlowEntryDistributionOrder getFe() {
+            return fe;
+        }
+    }
+
     /*
      * OSGI COMMANDS
      */
     /*
      * OSGI COMMANDS
      */
@@ -2696,4 +2964,78 @@ public class ForwardingRulesManager implements IForwardingRulesManager, PortGrou
 
         return rv;
     }
 
         return rv;
     }
+
+    public void unsetIConnectionManager(IConnectionManager s) {
+        if (s == this.connectionManager) {
+            this.connectionManager = null;
+        }
+    }
+
+    public void setIConnectionManager(IConnectionManager s) {
+        this.connectionManager = s;
+    }
+
+    @Override
+    public void entryCreated(Object key, String cacheName, boolean originLocal) {
+        /*
+         * Do nothing
+         */
+    }
+
+    @Override
+    public void entryUpdated(Object key, Object new_value, String cacheName, boolean originLocal) {
+        if (originLocal) {
+            /*
+             * Local updates are of no interest
+             */
+            return;
+        }
+        if (cacheName.equals(WORKORDERCACHE)) {
+            logsync.trace("Got a WorkOrderCacheUpdate for {}", key);
+            /*
+             * This is the case of one workOrder becoming available, so we need
+             * to dispatch the work to the appropriate handler
+             */
+            FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
+            FlowEntryInstall fei = fe.getEntry();
+            if (fei == null) {
+                return;
+            }
+            Node n = fei.getNode();
+            if (connectionManager.isLocal(n)) {
+                logsync.trace("workOrder for fe {} processed locally", fe);
+                // I'm the controller in charge for the request, queue it for
+                // processing
+                pendingEvents.offer(new WorkOrderEvent(fe, (FlowEntryInstall) new_value));
+            }
+        } else if (cacheName.equals(WORKSTATUSCACHE)) {
+            logsync.trace("Got a WorkStatusCacheUpdate for {}", key);
+            /*
+             * This is the case of one workOrder being completed and a status
+             * returned
+             */
+            FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
+            /*
+             * Check if the order was initiated by this controller in that case
+             * we need to actually look at the status returned
+             */
+            if (fe.getRequestorController()
+                    .equals(clusterContainerService.getMyAddress())) {
+                FlowEntryDistributionOrderFutureTask fet = workMonitor.get(fe);
+                if (fet != null) {
+                    logsync.trace("workStatus response is for us {}", fe);
+                    // Signal we got the status
+                    fet.gotStatus(fe, workStatus.get(fe));
+                    pendingEvents.offer(new WorkStatusCleanup(fe));
+                }
+            }
+        }
+    }
+
+    @Override
+    public void entryDeleted(Object key, String cacheName, boolean originLocal) {
+        /*
+         * Do nothing
+         */
+    }
 }
 }
index fbb5bbd37e2305c4f58347f1ed320fc3581c903d..b0ffaacec864aecfe65c20da8a80cc2b3fa3c34a 100644 (file)
       <artifactId>hosttracker.implementation</artifactId>
       <version>0.4.0-SNAPSHOT</version>
     </dependency>
       <artifactId>hosttracker.implementation</artifactId>
       <version>0.4.0-SNAPSHOT</version>
     </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>connectionmanager</artifactId>
+      <version>0.1.0-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>connectionmanager.implementation</artifactId>
+      <version>0.1.0-SNAPSHOT</version>
+    </dependency>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>configuration</artifactId>
       <version>0.4.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>configuration</artifactId>
       <version>0.4.0-SNAPSHOT</version>
     </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>sal.connection</artifactId>
+      <version>0.1.0-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>sal.connection.implementation</artifactId>
+      <version>0.1.0-SNAPSHOT</version>
+    </dependency>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>configuration.implementation</artifactId>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>configuration.implementation</artifactId>
index 6e8d9ab55b3173240567ec818ea9264bef059d77..67377c1ccadacf52e3841b0d9ac40046e2eeb233 100644 (file)
@@ -23,6 +23,7 @@ import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
 import org.opendaylight.controller.sal.action.Action;
 import org.opendaylight.controller.sal.action.Drop;
 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
 import org.opendaylight.controller.sal.action.Action;
 import org.opendaylight.controller.sal.action.Drop;
+import org.opendaylight.controller.sal.core.ConstructionException;
 import org.opendaylight.controller.sal.core.Node;
 import org.opendaylight.controller.sal.flowprogrammer.Flow;
 import org.opendaylight.controller.sal.match.Match;
 import org.opendaylight.controller.sal.core.Node;
 import org.opendaylight.controller.sal.flowprogrammer.Flow;
 import org.opendaylight.controller.sal.match.Match;
@@ -122,6 +123,14 @@ public class ForwardingRulesManagerIT {
                         .versionAsInProject(),
                 mavenBundle("org.opendaylight.controller",
                         "hosttracker.implementation").versionAsInProject(),
                         .versionAsInProject(),
                 mavenBundle("org.opendaylight.controller",
                         "hosttracker.implementation").versionAsInProject(),
+                mavenBundle("org.opendaylight.controller",
+                        "connectionmanager.implementation").versionAsInProject(),
+                mavenBundle("org.opendaylight.controller",
+                        "connectionmanager").versionAsInProject(),
+                mavenBundle("org.opendaylight.controller",
+                        "sal.connection").versionAsInProject(),
+                mavenBundle("org.opendaylight.controller",
+                        "sal.connection.implementation").versionAsInProject(),
 
                 // needed by hosttracker
                 mavenBundle("org.opendaylight.controller", "topologymanager")
 
                 // needed by hosttracker
                 mavenBundle("org.opendaylight.controller", "topologymanager")
@@ -196,14 +205,18 @@ public class ForwardingRulesManagerIT {
         List<Action> actions = new ArrayList<Action>();
         actions.add(action);
         flow.setActions(actions);
         List<Action> actions = new ArrayList<Action>();
         actions.add(action);
         flow.setActions(actions);
-
-        Node node = NodeCreator.createOFNode(1L);
-        FlowEntry fe = new FlowEntry("g1", "f1", flow, node);
-
-        Status stat = manager.installFlowEntry(fe);
-
-        // OF plugin is not there in integration testing mode
-        Assert.assertTrue(stat.getCode() == StatusCode.NOSERVICE);
+        Node node;
+        try {
+            // Must use a node published by the stub protocol plugin else
+            // connection manager will not report it as a local node
+            node = new Node("STUB", 51966);
+            FlowEntry fe = new FlowEntry("g1", "f1", flow, node);
+            Status stat = manager.installFlowEntry(fe);
+
+            Assert.assertTrue(stat.getCode() == StatusCode.SUCCESS);
+        } catch (ConstructionException e) {
+            // Got a failure while allocating the node
+            Assert.assertTrue(false);
+        }
     }
     }
-
 }
 }
index 206124b93bf67a2fc995e960e36bc0bdae21942a..5d52e24c16f3dd82e998459f328689b64b8a6944 100644 (file)
@@ -172,10 +172,28 @@ public class HostNodeConnector extends Host implements Serializable {
         return !Arrays.equals(emptyArray, macaddr);
     }
 
         return !Arrays.equals(emptyArray, macaddr);
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
     @Override
     public String toString() {
     @Override
     public String toString() {
-        return "HostNodeConnector [nodeConnector=" + nodeConnector + ", vlan="
-                + vlan + ", staticHost=" + staticHost + "]";
+        StringBuilder builder = new StringBuilder();
+        builder.append("HostNodeConnector [");
+        if (nodeConnector != null) {
+            builder.append("nodeConnector=")
+                    .append(nodeConnector)
+                    .append(", ");
+        }
+        builder.append("vlan=")
+                .append(vlan)
+                .append(", staticHost=")
+                .append(staticHost)
+                .append(", arpSendCountDown=")
+                .append(arpSendCountDown)
+                .append("]");
+        return builder.toString();
     }
 
     public boolean isV4Host() {
     }
 
     public boolean isV4Host() {
index 367df5ebb05e87e7b93737f9a9db03bd43075e9f..2fd81cbcd7b652e604f63c8f691a16b7dd29b185 100644 (file)
@@ -526,6 +526,8 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
         if (hostExists(host)) {
             HostNodeConnector existinghost = hostsDB.get(host.getNetworkAddress());
             existinghost.initArpSendCountDown();
         if (hostExists(host)) {
             HostNodeConnector existinghost = hostsDB.get(host.getNetworkAddress());
             existinghost.initArpSendCountDown();
+            // Update the host
+            hostsDB.put(host.getNetworkAddress(), existinghost);
             return;
         }
         new NotifyHostThread(host).start();
             return;
         }
         new NotifyHostThread(host).start();
index 1d7ce224b5620f438947eeb29139309cd865b7f2..a7a44456cc3011ec6e1957395f65dbcf2fccd9af 100644 (file)
@@ -8,12 +8,14 @@
 
 package org.opendaylight.controller.sal.utils;
 
 
 package org.opendaylight.controller.sal.utils;
 
+import java.io.Serializable;
+
 /**
  * Represents the return object of the osgi service interfaces function calls.
  * It contains a code {@code StatusCode} representing the result of the call and
  * a string which describes a failure reason (if any) in human readable form.
  */
 /**
  * Represents the return object of the osgi service interfaces function calls.
  * It contains a code {@code StatusCode} representing the result of the call and
  * a string which describes a failure reason (if any) in human readable form.
  */
-public class Status {
+public class Status implements Serializable {
     private StatusCode code;
     private String description;
     private long requestId;
     private StatusCode code;
     private String description;
     private long requestId;