From: Moiz Raja Date: Wed, 22 Jul 2015 02:11:10 +0000 (-0700) Subject: BUG 2185 : Make the Custom Raft Policy externally configurable X-Git-Tag: release/beryllium~333 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=e08568ddef3a5455b6b477e6672b8629f6935c20 BUG 2185 : Make the Custom Raft Policy externally configurable A class which implements RaftPolicy needs to be set to customize the behavior. I am hoping that this will be a barrier to people unintentionally breaking the default raft policy as they would need to know the internals of clustering in order to supply a valid class. I added two configurable raft policy classes for the two known use cases, - org.opendaylight.controller.cluster.datastore.policy.TestOnlyRaftPolicy - org.opendaylight.controller.cluster.datastore.policy.TwoNodeClusterRaftPolicy. Change-Id: Ic3cc2f27754c37e85c3be8a863764fc88ec84399 Signed-off-by: Moiz Raja (cherry picked from commit 1ef2f7321202ef77b5cfd0ca430e6a58cb07331e) --- diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java index fd49737cac..a2d30f7ae6 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java @@ -7,6 +7,7 @@ */ package org.opendaylight.controller.cluster.raft; +import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; import scala.concurrent.duration.FiniteDuration; /** @@ -84,4 +85,11 @@ public interface ConfigParams { */ long getElectionTimeoutFactor(); + + /** + * + * @return An instance of org.opendaylight.controller.cluster.raft.policy.RaftPolicy or an instance of the + * DefaultRaftPolicy + */ + RaftPolicy getRaftPolicy(); } diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java index 1b87f44f84..35dee10ba4 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java @@ -7,7 +7,14 @@ */ package org.opendaylight.controller.cluster.raft; +import com.google.common.base.Strings; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import java.util.concurrent.TimeUnit; +import org.opendaylight.controller.cluster.raft.policy.DefaultRaftPolicy; +import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import scala.concurrent.duration.FiniteDuration; /** @@ -17,6 +24,8 @@ import scala.concurrent.duration.FiniteDuration; */ public class DefaultConfigParamsImpl implements ConfigParams { + private static final Logger LOG = LoggerFactory.getLogger(DefaultConfigParamsImpl.class); + private static final int SNAPSHOT_BATCH_COUNT = 20000; private static final int JOURNAL_RECOVERY_LOG_BATCH_SIZE = 1000; @@ -52,6 +61,9 @@ public class DefaultConfigParamsImpl implements ConfigParams { private int snaphotChunkSize = SNAPSHOT_CHUNK_SIZE; private long electionTimeoutFactor = 2; + private String customRaftPolicyImplementationClass; + + Supplier policySupplier = Suppliers.memoize(new PolicySupplier()); public void setHeartBeatInterval(FiniteDuration heartBeatInterval) { this.heartBeatInterval = heartBeatInterval; @@ -83,6 +95,10 @@ public class DefaultConfigParamsImpl implements ConfigParams { electionTimeOutInterval = null; } + public void setCustomRaftPolicyImplementationClass(String customRaftPolicyImplementationClass){ + this.customRaftPolicyImplementationClass = customRaftPolicyImplementationClass; + } + @Override public long getSnapshotBatchCount() { return snapshotBatchCount; @@ -132,4 +148,31 @@ public class DefaultConfigParamsImpl implements ConfigParams { public long getElectionTimeoutFactor() { return electionTimeoutFactor; } + + @Override + public RaftPolicy getRaftPolicy() { + return policySupplier.get(); + } + + private class PolicySupplier implements Supplier{ + @Override + public RaftPolicy get() { + if(Strings.isNullOrEmpty(DefaultConfigParamsImpl.this.customRaftPolicyImplementationClass)){ + return DefaultRaftPolicy.INSTANCE; + } + try { + String className = DefaultConfigParamsImpl.this.customRaftPolicyImplementationClass; + Class c = Class.forName(className); + RaftPolicy obj = (RaftPolicy)c.newInstance(); + return obj; + } catch (Exception e) { + if(LOG.isDebugEnabled()) { + LOG.error("Could not create custom raft policy, will stick with default", e); + } else { + LOG.error("Could not create custom raft policy, will stick with default : cause = {}", e.getMessage()); + } + } + return DefaultRaftPolicy.INSTANCE; + } + } } diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorContextImpl.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorContextImpl.java index 815c554b9c..f14ee7dfa8 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorContextImpl.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorContextImpl.java @@ -18,7 +18,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import java.util.Map; import org.opendaylight.controller.cluster.DataPersistenceProvider; -import org.opendaylight.controller.cluster.raft.policy.DefaultRaftPolicy; import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; import org.slf4j.Logger; @@ -208,6 +207,6 @@ public class RaftActorContextImpl implements RaftActorContext { @Override public RaftPolicy getRaftPolicy() { - return DefaultRaftPolicy.INSTANCE; + return configParams.getRaftPolicy(); } } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImplTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImplTest.java new file mode 100644 index 0000000000..e86b377cb6 --- /dev/null +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImplTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.raft; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.opendaylight.controller.cluster.raft.policy.DefaultRaftPolicy; +import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; +import org.opendaylight.controller.cluster.raft.policy.TestRaftPolicy; + +public class DefaultConfigParamsImplTest { + + @Test + public void testGetRaftPolicyWithDefault(){ + DefaultConfigParamsImpl params = new DefaultConfigParamsImpl(); + + assertEquals("Default instance", DefaultRaftPolicy.INSTANCE, params.getRaftPolicy()); + } + + @Test + public void testGetRaftPolicyInvalidClassName(){ + DefaultConfigParamsImpl params = new DefaultConfigParamsImpl(); + params.setCustomRaftPolicyImplementationClass("foobar"); + + assertEquals("Default instance", DefaultRaftPolicy.INSTANCE, params.getRaftPolicy()); + } + + @Test + public void testGetRaftPolicyValidClassNameButInvalidType(){ + DefaultConfigParamsImpl params = new DefaultConfigParamsImpl(); + params.setCustomRaftPolicyImplementationClass("java.lang.String"); + + assertEquals("Default instance", DefaultRaftPolicy.INSTANCE, params.getRaftPolicy()); + } + + @Test + public void testGetRaftPolicyValidClass(){ + DefaultConfigParamsImpl params1 = new DefaultConfigParamsImpl(); + params1.setCustomRaftPolicyImplementationClass("org.opendaylight.controller.cluster.raft.policy.TestRaftPolicy"); + RaftPolicy behavior1 = params1.getRaftPolicy(); + + assertEquals("TestCustomBehavior", TestRaftPolicy.class, behavior1.getClass()); + assertEquals("Same instance returned", behavior1, params1.getRaftPolicy()); + + DefaultConfigParamsImpl params2 = new DefaultConfigParamsImpl(); + RaftPolicy behavior2 = params2.getRaftPolicy(); + params1.setCustomRaftPolicyImplementationClass("org.opendaylight.controller.cluster.raft.policy.TestRaftPolicy"); + + assertEquals("Default instance", DefaultRaftPolicy.INSTANCE, behavior2); + assertEquals("Default instance", DefaultRaftPolicy.INSTANCE, params2.getRaftPolicy()); + + } + + +} \ No newline at end of file diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/policy/TestRaftPolicy.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/policy/TestRaftPolicy.java new file mode 100644 index 0000000000..ca8fd0c90a --- /dev/null +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/policy/TestRaftPolicy.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.raft.policy; + +public class TestRaftPolicy implements RaftPolicy { + + @Override + public boolean automaticElectionsEnabled() { + return false; + } + + @Override + public boolean applyModificationToStateBeforeConsensus() { + return false; + } +} diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/datastore.cfg b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/datastore.cfg index e27376290e..13ccf93f5e 100644 --- a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/datastore.cfg +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/datastore.cfg @@ -73,3 +73,11 @@ operational.persistent=false # The maximum queue size for each shard's data store executor. #max-shard-data-store-executor-queue-size=5000 +# A fully qualified java class name. The class should implement +# org.opendaylight.controller.cluster.raft.policy.RaftPolicy. This java class should be +# accessible to the distributed data store OSGi module so that it can be dynamically loaded via +# reflection. For now let's assume that these classes to customize raft behaviors should be +# present in the distributed data store module itself. If this property is set to a class which +# cannot be found then the default raft policy will be applied +#custom-raft-policy-implementation= + diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java index a7e21e8ae7..26d1a6eb97 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java @@ -67,6 +67,7 @@ public class DatastoreContext { private boolean writeOnlyTransactionOptimizationsEnabled = true; private long shardCommitQueueExpiryTimeoutInMillis = DEFAULT_SHARD_COMMIT_QUEUE_EXPIRY_TIMEOUT_IN_MS; private boolean transactionDebugContextEnabled = false; + private String customRaftPolicyImplementation = ""; public static Set getGlobalDatastoreTypes() { return globalDatastoreTypes; @@ -98,6 +99,7 @@ public class DatastoreContext { this.writeOnlyTransactionOptimizationsEnabled = other.writeOnlyTransactionOptimizationsEnabled; this.shardCommitQueueExpiryTimeoutInMillis = other.shardCommitQueueExpiryTimeoutInMillis; this.transactionDebugContextEnabled = other.transactionDebugContextEnabled; + this.customRaftPolicyImplementation = other.customRaftPolicyImplementation; setShardJournalRecoveryLogBatchSize(other.raftConfig.getJournalRecoveryLogBatchSize()); setSnapshotBatchCount(other.raftConfig.getSnapshotBatchCount()); @@ -105,6 +107,8 @@ public class DatastoreContext { setIsolatedLeaderCheckInterval(other.raftConfig.getIsolatedCheckIntervalInMillis()); setSnapshotDataThresholdPercentage(other.raftConfig.getSnapshotDataThresholdPercentage()); setElectionTimeoutFactor(other.raftConfig.getElectionTimeoutFactor()); + setCustomRaftPolicyImplementation(other.customRaftPolicyImplementation); + } public static Builder newBuilder() { @@ -190,6 +194,11 @@ public class DatastoreContext { raftConfig.setElectionTimeoutFactor(shardElectionTimeoutFactor); } + private void setCustomRaftPolicyImplementation(String customRaftPolicyImplementation) { + raftConfig.setCustomRaftPolicyImplementationClass(customRaftPolicyImplementation); + } + + private void setSnapshotDataThresholdPercentage(int shardSnapshotDataThresholdPercentage) { raftConfig.setSnapshotDataThresholdPercentage(shardSnapshotDataThresholdPercentage); } @@ -411,5 +420,10 @@ public class DatastoreContext { return datastoreContext; } + + public Builder customRaftPolicyImplementation(String customRaftPolicyImplementation) { + datastoreContext.setCustomRaftPolicyImplementation(customRaftPolicyImplementation); + return this; + } } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TestOnlyRaftPolicy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TestOnlyRaftPolicy.java new file mode 100644 index 0000000000..83f9d3d277 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TestOnlyRaftPolicy.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.datastore.policy; + + +import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; + +/** + * A RaftPolicy that disables elections so that we can then specify exactly which Shard Replica should be + * Leader. Once a Leader is assigned it will behave as per Raft. + */ +public class TestOnlyRaftPolicy implements RaftPolicy { + @Override + public boolean automaticElectionsEnabled() { + return false; + } + + @Override + public boolean applyModificationToStateBeforeConsensus() { + return false; + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TwoNodeClusterRaftPolicy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TwoNodeClusterRaftPolicy.java new file mode 100644 index 0000000000..695f4f7d40 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/policy/TwoNodeClusterRaftPolicy.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.datastore.policy; + +import org.opendaylight.controller.cluster.raft.policy.RaftPolicy; + +/** + * The TwoNodeClusterRaftPolicy is intended to be used in a two node deployment where when one instance + * of the controller goes down the other instance is to take over and move the state forward. + * When a TwoNodeClusterRaftPolicy is used Raft elections are disabled. This is primarily because we would + * need to specify the leader externally. Also since we want one node to continue to function while the other + * node is down we would need to apply a modification to the state before consensus occurs. + */ +public class TwoNodeClusterRaftPolicy implements RaftPolicy { + @Override + public boolean automaticElectionsEnabled() { + return false; + } + + @Override + public boolean applyModificationToStateBeforeConsensus() { + return true; + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedConfigDataStoreProviderModule.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedConfigDataStoreProviderModule.java index cd27d1df79..878c0351a2 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedConfigDataStoreProviderModule.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedConfigDataStoreProviderModule.java @@ -67,6 +67,7 @@ public class DistributedConfigDataStoreProviderModule extends .shardCommitQueueExpiryTimeoutInSeconds( props.getShardCommitQueueExpiryTimeoutInSeconds().getValue().intValue()) .transactionDebugContextEnabled(props.getTransactionDebugContextEnabled()) + .customRaftPolicyImplementation(props.getCustomRaftPolicyImplementation()) .build(); return DistributedDataStoreFactory.createInstance(getConfigSchemaServiceDependency(), diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedOperationalDataStoreProviderModule.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedOperationalDataStoreProviderModule.java index 1c94acfca1..2db784a918 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedOperationalDataStoreProviderModule.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/config/yang/config/distributed_datastore_provider/DistributedOperationalDataStoreProviderModule.java @@ -61,13 +61,14 @@ public class DistributedOperationalDataStoreProviderModule extends props.getShardTransactionCommitQueueCapacity().getValue().intValue()) .persistent(props.getPersistent().booleanValue()) .shardIsolatedLeaderCheckIntervalInMillis( - props.getShardIsolatedLeaderCheckIntervalInMillis().getValue()) + props.getShardIsolatedLeaderCheckIntervalInMillis().getValue()) .shardElectionTimeoutFactor(props.getShardElectionTimeoutFactor().getValue()) .transactionCreationInitialRateLimit(props.getTransactionCreationInitialRateLimit().getValue()) .shardBatchedModificationCount(props.getShardBatchedModificationCount().getValue().intValue()) .shardCommitQueueExpiryTimeoutInSeconds( props.getShardCommitQueueExpiryTimeoutInSeconds().getValue().intValue()) .transactionDebugContextEnabled(props.getTransactionDebugContextEnabled()) + .customRaftPolicyImplementation(props.getCustomRaftPolicyImplementation()) .build(); return DistributedDataStoreFactory.createInstance(getOperationalSchemaServiceDependency(), diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang b/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang index 71c440eece..fcd4a301db 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang @@ -214,6 +214,17 @@ module distributed-datastore-provider { description "Enable or disable transaction context debug. This will log the call site trace for transactions that fail"; } + + leaf custom-raft-policy-implementation { + default ""; + type string; + description "A fully qualified java class name. The class should implement + org.opendaylight.controller.cluster.raft.policy.RaftPolicy. This java class should be + accessible to the distributed data store OSGi module so that it can be dynamically loaded via + reflection. For now let's assume that these classes to customize raft behaviors should be + present in the distributed data store module itself. If this property is set to a class which + cannot be found then the default raft behavior will be applied"; + } } // Augments the 'configuration' choice node under modules/module.