Initial Configuration for Clustering 77/9677/9
authorMoiz Raja <moraja@cisco.com>
Mon, 4 Aug 2014 22:44:35 +0000 (15:44 -0700)
committerMoiz Raja <moraja@cisco.com>
Thu, 7 Aug 2014 16:02:13 +0000 (09:02 -0700)
To try out clustering do the following,

1. Rename configuration/initial/05-clustering.xml.conf to configuration/initial/05-clustering.xml
2. Run the controller with these options

    ./run.sh -XX:MaxPermSize=512m

Change-Id: I61ede6cc083ce5fc722c44f0c127f4a03e2869d0
Signed-off-by: Moiz Raja <moraja@cisco.com>
13 files changed:
opendaylight/commons/opendaylight/pom.xml
opendaylight/distribution/opendaylight/pom.xml
opendaylight/md-sal/pom.xml
opendaylight/md-sal/sal-clustering-config/pom.xml [new file with mode: 0644]
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf [new file with mode: 0644]
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf [new file with mode: 0644]
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf [new file with mode: 0644]
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ActorSystemFactory.java
opendaylight/md-sal/sal-distributed-datastore/src/main/resources/application.conf
opendaylight/md-sal/sal-distributed-datastore/src/main/resources/modules.conf
opendaylight/md-sal/sal-remoterpc-connector/src/main/java/org/opendaylight/controller/remote/rpc/ActorSystemFactory.java
opendaylight/md-sal/sal-remoterpc-connector/src/main/resources/application.conf

index 6b17011099254e3a96bd0ead702f29698919865c..3c2e5accfa948589005c1ed50fbb60edfa0d557d 100644 (file)
             <artifactId>sal-clustering-commons</artifactId>
             <version>${mdsal.version}</version>
         </dependency>
+      <dependency>
+        <groupId>org.opendaylight.controller</groupId>
+        <artifactId>sal-clustering-config</artifactId>
+        <version>${mdsal.version}</version>
+      </dependency>
 
       <dependency>
         <groupId>org.opendaylight.controller</groupId>
index b02835e845cf1f5f16464bf41c10161532565a42..7ab56e6d03d63071216189c716f3022b6e834967 100644 (file)
             <phase>generate-resources</phase>
             <configuration>
                <outputDirectory>${project.build.directory}/configuration</outputDirectory>
-               <includeArtifactIds>sal-rest-connector-config,config-netty-config,md-sal-config,netconf-config,toaster-config,netconf-connector-config</includeArtifactIds>
-               <includes>**\/*.xml</includes>
+               <includeArtifactIds>sal-rest-connector-config,config-netty-config,md-sal-config,netconf-config,toaster-config,netconf-connector-config,sal-clustering-config</includeArtifactIds>
+               <includes>**\/*.xml,**/*.conf</includes>
                <excludeTransitive>true</excludeTransitive>
                <ignorePermissions>false</ignorePermissions>
             </configuration>
           <artifactId>jeromq</artifactId>
           <version>0.3.1</version>
         </dependency>
-          <dependency>
-              <groupId>org.opendaylight.controller</groupId>
-              <artifactId>sal-distributed-datastore</artifactId>
-          </dependency>
+        <dependency>
+          <groupId>org.opendaylight.controller</groupId>
+          <artifactId>sal-distributed-datastore</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.opendaylight.controller</groupId>
+          <artifactId>sal-clustering-config</artifactId>
+        </dependency>
       </dependencies>
     </profile>
     <profile>
index fd828dc05a92cfd2070825a1edc4fb4dddab8af5..6c6760d789a086157bdb35a0a66d620db8057d3c 100644 (file)
@@ -69,6 +69,9 @@
     <!--sal-protocolbuffer-encoding is now part of sal-clutering-commons-->
     <module>sal-clustering-commons</module>
 
+    <!-- sal clustering configuration -->
+    <module>sal-clustering-config</module>
+
     <!-- sal-distributed-datastore -->
     <module>sal-distributed-datastore</module>
 
diff --git a/opendaylight/md-sal/sal-clustering-config/pom.xml b/opendaylight/md-sal/sal-clustering-config/pom.xml
new file mode 100644 (file)
index 0000000..d726823
--- /dev/null
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
+
+ This program and the accompanying materials are made available under the
+ terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ and is available at http://www.eclipse.org/legal/epl-v10.html
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.opendaylight.controller</groupId>
+    <artifactId>sal-parent</artifactId>
+    <version>1.1-SNAPSHOT</version>
+  </parent>
+  <artifactId>sal-clustering-config</artifactId>
+  <description>Configuration files for md-sal clustering</description>
+  <packaging>jar</packaging>
+  <build>
+    <plugins>
+        <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>attach-artifacts</id>
+            <goals>
+              <goal>attach-artifact</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <artifacts>
+                <artifact>
+                  <file>${project.build.directory}/classes/initial/*.conf</file>
+                  <type>xml</type>
+                  <classifier>config</classifier>
+                </artifact>
+              </artifacts>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf
new file mode 100644 (file)
index 0000000..7891ee2
--- /dev/null
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- vi: set et smarttab sw=4 tabstop=4: -->
+<!--
+ Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
+
+ This program and the accompanying materials are made available under the
+ terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ and is available at http://www.eclipse.org/legal/epl-v10.html
+-->
+<snapshot>
+    <configuration>
+        <data xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
+            <modules xmlns="urn:opendaylight:params:xml:ns:yang:controller:config">
+                <module>
+                    <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom:impl">prefix:dom-inmemory-data-broker</type>
+                    <name>inmemory-data-broker</name>
+
+                    <schema-service>
+                        <type xmlns:dom="urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom">dom:schema-service</type>
+                        <name>yang-schema-service</name>
+                    </schema-service>
+
+                    <config-data-store>
+                        <type xmlns:config-dom-store-spi="urn:opendaylight:params:xml:ns:yang:controller:md:sal:core:spi:config-dom-store">config-dom-store-spi:config-dom-datastore</type>
+                        <name>distributed-config-store-service</name>
+                    </config-data-store>
+
+                    <operational-data-store>
+                        <type xmlns:operational-dom-store-spi="urn:opendaylight:params:xml:ns:yang:controller:md:sal:core:spi:operational-dom-store">operational-dom-store-spi:operational-dom-datastore</type>
+                        <name>distributed-operational-store-service</name>
+                    </operational-data-store>
+                </module>
+
+                <module>
+                    <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:config:distributed-datastore-provider">prefix:distributed-operational-datastore-provider</type>
+                    <name>distributed-operational-store-module</name>
+                    <schema-service>
+                        <type xmlns:dom="urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom">dom:schema-service</type>
+                        <name>yang-schema-service</name>
+                    </schema-service>
+                </module>
+
+                <module>
+                    <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:config:distributed-datastore-provider">prefix:distributed-config-datastore-provider</type>
+                    <name>distributed-config-store-module</name>
+                    <schema-service>
+                        <type xmlns:dom="urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom">dom:schema-service</type>
+                        <name>yang-schema-service</name>
+                    </schema-service>
+                </module>
+
+                <module>
+                    <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:config:remote-rpc-connector">prefix:remote-rpc-connector</type>
+                    <name>remote-rpc-connector</name>
+                    <dom-broker xmlns="urn:opendaylight:params:xml:ns:yang:controller:config:remote-rpc-connector">
+                        <type xmlns:dom="urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom">dom:dom-broker-osgi-registry</type>
+                        <name>dom-broker</name>
+                    </dom-broker>
+                </module>
+
+            </modules>
+            <services xmlns="urn:opendaylight:params:xml:ns:yang:controller:config">
+
+                <service>
+                    <type xmlns:config-dom-store-spi="urn:opendaylight:params:xml:ns:yang:controller:md:sal:core:spi:config-dom-store">config-dom-store-spi:config-dom-datastore</type>
+                    <instance>
+                        <name>distributed-config-store-service</name>
+                        <provider>/modules/module[type='distributed-config-datastore-provider'][name='distributed-config-store-module']</provider>
+                    </instance>
+                </service>
+                <service>
+                    <type xmlns:operational-dom-store-spi="urn:opendaylight:params:xml:ns:yang:controller:md:sal:core:spi:operational-dom-store">operational-dom-store-spi:operational-dom-datastore</type>
+                    <instance>
+                        <name>distributed-operational-store-service</name>
+                        <provider>/modules/module[type='distributed-operational-datastore-provider'][name='distributed-operational-store-module']</provider>
+                    </instance>
+                </service>
+
+            </services>
+        </data>
+    </configuration>
+    <required-capabilities>
+        <capability>urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom:impl?module=opendaylight-sal-dom-broker-impl&amp;revision=2013-10-28</capability>
+    </required-capabilities>
+</snapshot>
diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf
new file mode 100644 (file)
index 0000000..9749ae2
--- /dev/null
@@ -0,0 +1,55 @@
+
+odl-cluster-data {
+  akka {
+    actor {
+      provider = "akka.cluster.ClusterActorRefProvider"
+      serializers {
+                java = "akka.serialization.JavaSerializer"
+                proto = "akka.remote.serialization.ProtobufSerializer"
+              }
+
+              serialization-bindings {
+                  "com.google.protobuf.Message" = proto
+
+              }
+    }
+    remote {
+      log-remote-lifecycle-events = off
+      netty.tcp {
+        hostname = "<CHANGE_ME>"
+        port = 2550
+           maximum-frame-size = 2097152
+           send-buffer-size = 52428800
+           receive-buffer-size = 52428800
+      }
+    }
+
+    cluster {
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@<CHANGE_ME>:2550"]
+
+      auto-down-unreachable-after = 10s
+    }
+  }
+}
+
+odl-cluster-rpc {
+  akka {
+    actor {
+      provider = "akka.cluster.ClusterActorRefProvider"
+
+    }
+    remote {
+      log-remote-lifecycle-events = off
+      netty.tcp {
+        hostname = "<CHANGE_ME>"
+        port = 2551
+      }
+    }
+
+    cluster {
+      seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@<CHANGE_ME>:2551"]
+
+      auto-down-unreachable-after = 10s
+    }
+  }
+}
diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf
new file mode 100644 (file)
index 0000000..8449abb
--- /dev/null
@@ -0,0 +1,70 @@
+# This file describes which shards live on which members
+# The format for a module-shards is as follows,
+# {
+#    name = "<friendly_name_of_the_module>"
+#    shards = [
+#        {
+#            name="<any_name_that_is_unique_for_the_module>"
+#            replicas = [
+#                "<name_of_member_on_which_to_run>"
+#            ]
+#     ]
+# }
+#
+# For Helium we support only one shard per module. Beyond Helium
+# we will support more than 1
+# The replicas section is a collection of member names. This information
+# will be used to decide on which members replicas of a particular shard will be
+# located. Once replication is integrated with the distributed data store then
+# this section can have multiple entries.
+#
+#
+
+
+module-shards = [
+    {
+        name = "default"
+        shards = [
+            {
+                name="default"
+                replicas = [
+                    "member-1"
+                ]
+            }
+        ]
+    },
+    {
+        name = "topology"
+        shards = [
+            {
+                name="topology"
+                replicas = [
+                    "member-1"
+                ]
+            }
+        ]
+    },
+    {
+        name = "inventory"
+        shards = [
+            {
+                name="inventory"
+                replicas = [
+                    "member-1"
+                ]
+            }
+        ]
+    },
+         {
+             name = "toaster"
+             shards = [
+                 {
+                     name="toaster"
+                     replicas = [
+                         "member-1"
+                     ]
+                 }
+             ]
+         }
+
+]
diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf
new file mode 100644 (file)
index 0000000..68347ee
--- /dev/null
@@ -0,0 +1,32 @@
+# This file should describe all the modules that need to be placed in a separate shard
+# The format of the configuration is as follows
+# {
+#    name = "<friendly_name_of_module>"
+#    namespace = "<the yang namespace of the module>"
+#    shard-strategy = "module"
+# }
+#
+# Note that at this time the only shard-strategy we support is module which basically
+# will put all the data of a single module in two shards (one for config and one for
+# operational data)
+
+modules = [
+    {
+        name = "inventory"
+        namespace = "urn:opendaylight:inventory"
+        shard-strategy = "module"
+    },
+
+    {
+        name = "topology"
+        namespace = "urn:TBD:params:xml:ns:yang:network-topology"
+        shard-strategy = "module"
+    },
+
+    {
+        name = "toaster"
+        namespace = "http://netconfcentral.org/ns/toaster"
+        shard-strategy = "module"
+    }
+
+]
index baf04fe43b771fddc1c009d11aed056dea6bbe24..15c0548761ecd96afee865025fc573d5a2d38c6e 100644 (file)
@@ -20,8 +20,8 @@ public class ActorSystemFactory {
 
         @Nullable @Override public ActorSystem apply(@Nullable Void aVoid) {
                 ActorSystem system =
-                    ActorSystem.create("opendaylight-cluster", ConfigFactory
-                        .load().getConfig("ODLCluster"));
+                    ActorSystem.create("opendaylight-cluster-data", ConfigFactory
+                        .load().getConfig("odl-cluster-data"));
                 system.actorOf(Props.create(TerminationMonitor.class), "termination-monitor");
                 return system;
         }
index 76914c2c84f020268377b2a5d191cf831bb96f6d..daac89c4c8adaef39047034138a8318e639f54c8 100644 (file)
@@ -1,15 +1,60 @@
-ODLCluster{
 
-actor {
-        serializers {
-          java = "akka.serialization.JavaSerializer"
-          proto = "akka.remote.serialization.ProtobufSerializer"
-        }
+odl-cluster-data {
+  akka {
+    cluster {
+        roles = [
+          "member-1"
+        ]
+    }
+    actor {
+      provider = "akka.cluster.ClusterActorRefProvider"
+      serializers {
+                java = "akka.serialization.JavaSerializer"
+                proto = "akka.remote.serialization.ProtobufSerializer"
+              }
+
+              serialization-bindings {
+                  "com.google.protobuf.Message" = proto
+
+              }
+    }
+    remote {
+      log-remote-lifecycle-events = off
+      netty.tcp {
+        hostname = "127.0.0.1"
+        port = 2550
+           maximum-frame-size = 2097152
+           send-buffer-size = 52428800
+           receive-buffer-size = 52428800
+      }
+    }
 
-        serialization-bindings {
-            "com.google.protobuf.Message" = proto
+    cluster {
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"]
 
-        }
+      auto-down-unreachable-after = 10s
     }
+  }
+}
+
+odl-cluster-rpc {
+  akka {
+    actor {
+      provider = "akka.cluster.ClusterActorRefProvider"
 
-}
\ No newline at end of file
+    }
+    remote {
+      log-remote-lifecycle-events = off
+      netty.tcp {
+        hostname = "127.0.0.1"
+        port = 2551
+      }
+    }
+
+    cluster {
+      seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@127.0.0.1:2551"]
+
+      auto-down-unreachable-after = 10s
+    }
+  }
+}
index 05ef33f759009b480dd156dab49af8d7a3a674fd..e820703eeb5fa0e5e6806afc9145620716153ce4 100644 (file)
@@ -1,7 +1,7 @@
 modules = [
     {
         name = "inventory"
-        namespace = "urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom:store:test:people"
+        namespace = "urn:opendaylight:inventory"
         shard-strategy = "module"
     }
 ]
index bd49b6239c443ccd04f70da3dabd964da00c504f..f1ca3ccd505e1b1732ad906f75f3662368fdc4a2 100644 (file)
@@ -27,7 +27,6 @@ public class ActorSystemFactory {
    * @param bundleContext
    */
   public static final void createInstance(final BundleContext bundleContext) {
-
     if(actorSystem == null) {
       // Create an OSGi bundle classloader for actor system
       BundleDelegatingClassLoader classLoader = new BundleDelegatingClassLoader(bundleContext.getBundle(),
@@ -35,8 +34,8 @@ public class ActorSystemFactory {
       synchronized (ActorSystemFactory.class) {
         // Double check
         if (actorSystem == null) {
-          ActorSystem system = ActorSystem.create("opendaylight-rpc",
-              ConfigFactory.load().getConfig("odl-cluster"), classLoader);
+          ActorSystem system = ActorSystem.create("opendaylight-cluster-rpc",
+              ConfigFactory.load().getConfig("odl-cluster-rpc"), classLoader);
           actorSystem = system;
         }
       }
index 6088dd0e0ea517c27bd74e79272a92f4b5b7ef1d..daac89c4c8adaef39047034138a8318e639f54c8 100644 (file)
@@ -1,4 +1,43 @@
-odl-cluster{
+
+odl-cluster-data {
+  akka {
+    cluster {
+        roles = [
+          "member-1"
+        ]
+    }
+    actor {
+      provider = "akka.cluster.ClusterActorRefProvider"
+      serializers {
+                java = "akka.serialization.JavaSerializer"
+                proto = "akka.remote.serialization.ProtobufSerializer"
+              }
+
+              serialization-bindings {
+                  "com.google.protobuf.Message" = proto
+
+              }
+    }
+    remote {
+      log-remote-lifecycle-events = off
+      netty.tcp {
+        hostname = "127.0.0.1"
+        port = 2550
+           maximum-frame-size = 2097152
+           send-buffer-size = 52428800
+           receive-buffer-size = 52428800
+      }
+    }
+
+    cluster {
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"]
+
+      auto-down-unreachable-after = 10s
+    }
+  }
+}
+
+odl-cluster-rpc {
   akka {
     actor {
       provider = "akka.cluster.ClusterActorRefProvider"
@@ -7,15 +46,15 @@ odl-cluster{
     remote {
       log-remote-lifecycle-events = off
       netty.tcp {
-        hostname = "192.168.141.141"
+        hostname = "127.0.0.1"
         port = 2551
       }
     }
 
     cluster {
-      seed-nodes = ["akka.tcp://opendaylight-rpc@192.168.141.141:2551"]
+      seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@127.0.0.1:2551"]
 
       auto-down-unreachable-after = 10s
     }
   }
-}
\ No newline at end of file
+}