From 9a92d23e5687608789af35125036c97c84c6d92e Mon Sep 17 00:00:00 2001 From: Moiz Raja Date: Mon, 4 Aug 2014 15:44:35 -0700 Subject: [PATCH] Initial Configuration for Clustering To try out clustering do the following, 1. Rename configuration/initial/05-clustering.xml.conf to configuration/initial/05-clustering.xml 2. Run the controller with these options ./run.sh -XX:MaxPermSize=512m Change-Id: I61ede6cc083ce5fc722c44f0c127f4a03e2869d0 Signed-off-by: Moiz Raja --- opendaylight/commons/opendaylight/pom.xml | 5 ++ .../distribution/opendaylight/pom.xml | 16 ++-- opendaylight/md-sal/pom.xml | 3 + .../md-sal/sal-clustering-config/pom.xml | 46 ++++++++++ .../resources/initial/05-clustering.xml.conf | 85 +++++++++++++++++++ .../src/main/resources/initial/akka.conf | 55 ++++++++++++ .../main/resources/initial/module-shards.conf | 70 +++++++++++++++ .../src/main/resources/initial/modules.conf | 32 +++++++ .../cluster/datastore/ActorSystemFactory.java | 4 +- .../src/main/resources/application.conf | 65 +++++++++++--- .../src/main/resources/modules.conf | 2 +- .../remote/rpc/ActorSystemFactory.java | 5 +- .../src/main/resources/application.conf | 47 +++++++++- 13 files changed, 409 insertions(+), 26 deletions(-) create mode 100644 opendaylight/md-sal/sal-clustering-config/pom.xml create mode 100644 opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf create mode 100644 opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf create mode 100644 opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf create mode 100644 opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf diff --git a/opendaylight/commons/opendaylight/pom.xml b/opendaylight/commons/opendaylight/pom.xml index 6b17011099..3c2e5accfa 100644 --- a/opendaylight/commons/opendaylight/pom.xml +++ b/opendaylight/commons/opendaylight/pom.xml @@ -1290,6 +1290,11 @@ sal-clustering-commons ${mdsal.version} + + org.opendaylight.controller + sal-clustering-config + ${mdsal.version} + org.opendaylight.controller diff --git a/opendaylight/distribution/opendaylight/pom.xml b/opendaylight/distribution/opendaylight/pom.xml index b02835e845..7ab56e6d03 100644 --- a/opendaylight/distribution/opendaylight/pom.xml +++ b/opendaylight/distribution/opendaylight/pom.xml @@ -745,8 +745,8 @@ generate-resources ${project.build.directory}/configuration - sal-rest-connector-config,config-netty-config,md-sal-config,netconf-config,toaster-config,netconf-connector-config - **\/*.xml + sal-rest-connector-config,config-netty-config,md-sal-config,netconf-config,toaster-config,netconf-connector-config,sal-clustering-config + **\/*.xml,**/*.conf true false @@ -1296,10 +1296,14 @@ jeromq 0.3.1 - - org.opendaylight.controller - sal-distributed-datastore - + + org.opendaylight.controller + sal-distributed-datastore + + + org.opendaylight.controller + sal-clustering-config + diff --git a/opendaylight/md-sal/pom.xml b/opendaylight/md-sal/pom.xml index fd828dc05a..6c6760d789 100644 --- a/opendaylight/md-sal/pom.xml +++ b/opendaylight/md-sal/pom.xml @@ -69,6 +69,9 @@ sal-clustering-commons + + sal-clustering-config + sal-distributed-datastore diff --git a/opendaylight/md-sal/sal-clustering-config/pom.xml b/opendaylight/md-sal/sal-clustering-config/pom.xml new file mode 100644 index 0000000000..d726823b98 --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-config/pom.xml @@ -0,0 +1,46 @@ + + + + + 4.0.0 + + org.opendaylight.controller + sal-parent + 1.1-SNAPSHOT + + sal-clustering-config + Configuration files for md-sal clustering + jar + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-artifacts + + attach-artifact + + package + + + + ${project.build.directory}/classes/initial/*.conf + xml + config + + + + + + + + + diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf new file mode 100644 index 0000000000..7891ee2088 --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/05-clustering.xml.conf @@ -0,0 +1,85 @@ + + + + + + + + + prefix:dom-inmemory-data-broker + inmemory-data-broker + + + dom:schema-service + yang-schema-service + + + + config-dom-store-spi:config-dom-datastore + distributed-config-store-service + + + + operational-dom-store-spi:operational-dom-datastore + distributed-operational-store-service + + + + + prefix:distributed-operational-datastore-provider + distributed-operational-store-module + + dom:schema-service + yang-schema-service + + + + + prefix:distributed-config-datastore-provider + distributed-config-store-module + + dom:schema-service + yang-schema-service + + + + + prefix:remote-rpc-connector + remote-rpc-connector + + dom:dom-broker-osgi-registry + dom-broker + + + + + + + + config-dom-store-spi:config-dom-datastore + + distributed-config-store-service + /modules/module[type='distributed-config-datastore-provider'][name='distributed-config-store-module'] + + + + operational-dom-store-spi:operational-dom-datastore + + distributed-operational-store-service + /modules/module[type='distributed-operational-datastore-provider'][name='distributed-operational-store-module'] + + + + + + + + urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom:impl?module=opendaylight-sal-dom-broker-impl&revision=2013-10-28 + + diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf new file mode 100644 index 0000000000..9749ae27ae --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf @@ -0,0 +1,55 @@ + +odl-cluster-data { + akka { + actor { + provider = "akka.cluster.ClusterActorRefProvider" + serializers { + java = "akka.serialization.JavaSerializer" + proto = "akka.remote.serialization.ProtobufSerializer" + } + + serialization-bindings { + "com.google.protobuf.Message" = proto + + } + } + remote { + log-remote-lifecycle-events = off + netty.tcp { + hostname = "" + port = 2550 + maximum-frame-size = 2097152 + send-buffer-size = 52428800 + receive-buffer-size = 52428800 + } + } + + cluster { + seed-nodes = ["akka.tcp://opendaylight-cluster-data@:2550"] + + auto-down-unreachable-after = 10s + } + } +} + +odl-cluster-rpc { + akka { + actor { + provider = "akka.cluster.ClusterActorRefProvider" + + } + remote { + log-remote-lifecycle-events = off + netty.tcp { + hostname = "" + port = 2551 + } + } + + cluster { + seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@:2551"] + + auto-down-unreachable-after = 10s + } + } +} diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf new file mode 100644 index 0000000000..8449abb7b0 --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/module-shards.conf @@ -0,0 +1,70 @@ +# This file describes which shards live on which members +# The format for a module-shards is as follows, +# { +# name = "" +# shards = [ +# { +# name="" +# replicas = [ +# "" +# ] +# ] +# } +# +# For Helium we support only one shard per module. Beyond Helium +# we will support more than 1 +# The replicas section is a collection of member names. This information +# will be used to decide on which members replicas of a particular shard will be +# located. Once replication is integrated with the distributed data store then +# this section can have multiple entries. +# +# + + +module-shards = [ + { + name = "default" + shards = [ + { + name="default" + replicas = [ + "member-1" + ] + } + ] + }, + { + name = "topology" + shards = [ + { + name="topology" + replicas = [ + "member-1" + ] + } + ] + }, + { + name = "inventory" + shards = [ + { + name="inventory" + replicas = [ + "member-1" + ] + } + ] + }, + { + name = "toaster" + shards = [ + { + name="toaster" + replicas = [ + "member-1" + ] + } + ] + } + +] diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf new file mode 100644 index 0000000000..68347eeda9 --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/modules.conf @@ -0,0 +1,32 @@ +# This file should describe all the modules that need to be placed in a separate shard +# The format of the configuration is as follows +# { +# name = "" +# namespace = "" +# shard-strategy = "module" +# } +# +# Note that at this time the only shard-strategy we support is module which basically +# will put all the data of a single module in two shards (one for config and one for +# operational data) + +modules = [ + { + name = "inventory" + namespace = "urn:opendaylight:inventory" + shard-strategy = "module" + }, + + { + name = "topology" + namespace = "urn:TBD:params:xml:ns:yang:network-topology" + shard-strategy = "module" + }, + + { + name = "toaster" + namespace = "http://netconfcentral.org/ns/toaster" + shard-strategy = "module" + } + +] diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ActorSystemFactory.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ActorSystemFactory.java index baf04fe43b..15c0548761 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ActorSystemFactory.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ActorSystemFactory.java @@ -20,8 +20,8 @@ public class ActorSystemFactory { @Nullable @Override public ActorSystem apply(@Nullable Void aVoid) { ActorSystem system = - ActorSystem.create("opendaylight-cluster", ConfigFactory - .load().getConfig("ODLCluster")); + ActorSystem.create("opendaylight-cluster-data", ConfigFactory + .load().getConfig("odl-cluster-data")); system.actorOf(Props.create(TerminationMonitor.class), "termination-monitor"); return system; } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/application.conf b/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/application.conf index 76914c2c84..daac89c4c8 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/application.conf +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/application.conf @@ -1,15 +1,60 @@ -ODLCluster{ -actor { - serializers { - java = "akka.serialization.JavaSerializer" - proto = "akka.remote.serialization.ProtobufSerializer" - } +odl-cluster-data { + akka { + cluster { + roles = [ + "member-1" + ] + } + actor { + provider = "akka.cluster.ClusterActorRefProvider" + serializers { + java = "akka.serialization.JavaSerializer" + proto = "akka.remote.serialization.ProtobufSerializer" + } + + serialization-bindings { + "com.google.protobuf.Message" = proto + + } + } + remote { + log-remote-lifecycle-events = off + netty.tcp { + hostname = "127.0.0.1" + port = 2550 + maximum-frame-size = 2097152 + send-buffer-size = 52428800 + receive-buffer-size = 52428800 + } + } - serialization-bindings { - "com.google.protobuf.Message" = proto + cluster { + seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"] - } + auto-down-unreachable-after = 10s } + } +} + +odl-cluster-rpc { + akka { + actor { + provider = "akka.cluster.ClusterActorRefProvider" -} \ No newline at end of file + } + remote { + log-remote-lifecycle-events = off + netty.tcp { + hostname = "127.0.0.1" + port = 2551 + } + } + + cluster { + seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@127.0.0.1:2551"] + + auto-down-unreachable-after = 10s + } + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/modules.conf b/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/modules.conf index 05ef33f759..e820703eeb 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/modules.conf +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/resources/modules.conf @@ -1,7 +1,7 @@ modules = [ { name = "inventory" - namespace = "urn:opendaylight:params:xml:ns:yang:controller:md:sal:dom:store:test:people" + namespace = "urn:opendaylight:inventory" shard-strategy = "module" } ] diff --git a/opendaylight/md-sal/sal-remoterpc-connector/src/main/java/org/opendaylight/controller/remote/rpc/ActorSystemFactory.java b/opendaylight/md-sal/sal-remoterpc-connector/src/main/java/org/opendaylight/controller/remote/rpc/ActorSystemFactory.java index bd49b6239c..f1ca3ccd50 100644 --- a/opendaylight/md-sal/sal-remoterpc-connector/src/main/java/org/opendaylight/controller/remote/rpc/ActorSystemFactory.java +++ b/opendaylight/md-sal/sal-remoterpc-connector/src/main/java/org/opendaylight/controller/remote/rpc/ActorSystemFactory.java @@ -27,7 +27,6 @@ public class ActorSystemFactory { * @param bundleContext */ public static final void createInstance(final BundleContext bundleContext) { - if(actorSystem == null) { // Create an OSGi bundle classloader for actor system BundleDelegatingClassLoader classLoader = new BundleDelegatingClassLoader(bundleContext.getBundle(), @@ -35,8 +34,8 @@ public class ActorSystemFactory { synchronized (ActorSystemFactory.class) { // Double check if (actorSystem == null) { - ActorSystem system = ActorSystem.create("opendaylight-rpc", - ConfigFactory.load().getConfig("odl-cluster"), classLoader); + ActorSystem system = ActorSystem.create("opendaylight-cluster-rpc", + ConfigFactory.load().getConfig("odl-cluster-rpc"), classLoader); actorSystem = system; } } diff --git a/opendaylight/md-sal/sal-remoterpc-connector/src/main/resources/application.conf b/opendaylight/md-sal/sal-remoterpc-connector/src/main/resources/application.conf index 6088dd0e0e..daac89c4c8 100644 --- a/opendaylight/md-sal/sal-remoterpc-connector/src/main/resources/application.conf +++ b/opendaylight/md-sal/sal-remoterpc-connector/src/main/resources/application.conf @@ -1,4 +1,43 @@ -odl-cluster{ + +odl-cluster-data { + akka { + cluster { + roles = [ + "member-1" + ] + } + actor { + provider = "akka.cluster.ClusterActorRefProvider" + serializers { + java = "akka.serialization.JavaSerializer" + proto = "akka.remote.serialization.ProtobufSerializer" + } + + serialization-bindings { + "com.google.protobuf.Message" = proto + + } + } + remote { + log-remote-lifecycle-events = off + netty.tcp { + hostname = "127.0.0.1" + port = 2550 + maximum-frame-size = 2097152 + send-buffer-size = 52428800 + receive-buffer-size = 52428800 + } + } + + cluster { + seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"] + + auto-down-unreachable-after = 10s + } + } +} + +odl-cluster-rpc { akka { actor { provider = "akka.cluster.ClusterActorRefProvider" @@ -7,15 +46,15 @@ odl-cluster{ remote { log-remote-lifecycle-events = off netty.tcp { - hostname = "192.168.141.141" + hostname = "127.0.0.1" port = 2551 } } cluster { - seed-nodes = ["akka.tcp://opendaylight-rpc@192.168.141.141:2551"] + seed-nodes = ["akka.tcp://opendaylight-cluster-rpc@127.0.0.1:2551"] auto-down-unreachable-after = 10s } } -} \ No newline at end of file +} -- 2.36.6