import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
-import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
import org.opendaylight.controller.cluster.datastore.config.Configuration;
import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
import org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterCandidateLocal;
import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
import org.opendaylight.controller.cluster.datastore.messages.GetShardDataTree;
import org.opendaylight.controller.cluster.datastore.shardstrategy.ModuleShardStrategy;
+import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
import org.opendaylight.controller.md.sal.common.api.clustering.CandidateAlreadyRegisteredException;
import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipCandidateRegistration;
* @author Thomas Pantelis
*/
public class DistributedEntityOwnershipService implements EntityOwnershipService, AutoCloseable {
- private static final Logger LOG = LoggerFactory.getLogger(DistributedEntityOwnershipService.class);
+ @VisibleForTesting
static final String ENTITY_OWNERSHIP_SHARD_NAME = "entity-ownership";
+
+ private static final Logger LOG = LoggerFactory.getLogger(DistributedEntityOwnershipService.class);
private static final Timeout MESSAGE_TIMEOUT = new Timeout(1, TimeUnit.MINUTES);
- private final DistributedDataStore datastore;
- private final EntityOwnerSelectionStrategyConfig strategyConfig;
private final ConcurrentMap<Entity, Entity> registeredEntities = new ConcurrentHashMap<>();
+ private final ActorContext context;
+
private volatile ActorRef localEntityOwnershipShard;
private volatile DataTree localEntityOwnershipShardDataTree;
- public DistributedEntityOwnershipService(DistributedDataStore datastore, EntityOwnerSelectionStrategyConfig strategyConfig) {
- this.datastore = Preconditions.checkNotNull(datastore);
- this.strategyConfig = Preconditions.checkNotNull(strategyConfig);
+ private DistributedEntityOwnershipService(final ActorContext context) {
+ this.context = Preconditions.checkNotNull(context);
}
- public void start() {
- ActorRef shardManagerActor = datastore.getActorContext().getShardManager();
+ public static DistributedEntityOwnershipService start(final ActorContext context,
+ final EntityOwnerSelectionStrategyConfig strategyConfig) {
+ ActorRef shardManagerActor = context.getShardManager();
- Configuration configuration = datastore.getActorContext().getConfiguration();
+ Configuration configuration = context.getConfiguration();
Collection<String> entityOwnersMemberNames = configuration.getUniqueMemberNamesForAllShards();
CreateShard createShard = new CreateShard(new ModuleShardConfiguration(EntityOwners.QNAME.getNamespace(),
"entity-owners", ENTITY_OWNERSHIP_SHARD_NAME, ModuleShardStrategy.NAME, entityOwnersMemberNames),
- newShardBuilder(), null);
+ newShardBuilder(context, strategyConfig), null);
- Future<Object> createFuture = datastore.getActorContext().executeOperationAsync(shardManagerActor,
+ Future<Object> createFuture = context.executeOperationAsync(shardManagerActor,
createShard, MESSAGE_TIMEOUT);
createFuture.onComplete(new OnComplete<Object>() {
@Override
- public void onComplete(Throwable failure, Object response) {
- if(failure != null) {
+ public void onComplete(final Throwable failure, final Object response) {
+ if (failure != null) {
LOG.error("Failed to create {} shard", ENTITY_OWNERSHIP_SHARD_NAME, failure);
} else {
LOG.info("Successfully created {} shard", ENTITY_OWNERSHIP_SHARD_NAME);
}
}
- }, datastore.getActorContext().getClientDispatcher());
+ }, context.getClientDispatcher());
+
+ return new DistributedEntityOwnershipService(context);
}
private void executeEntityOwnershipShardOperation(final ActorRef shardActor, final Object message) {
- Future<Object> future = datastore.getActorContext().executeOperationAsync(shardActor, message, MESSAGE_TIMEOUT);
+ Future<Object> future = context.executeOperationAsync(shardActor, message, MESSAGE_TIMEOUT);
future.onComplete(new OnComplete<Object>() {
@Override
- public void onComplete(Throwable failure, Object response) {
+ public void onComplete(final Throwable failure, final Object response) {
if(failure != null) {
LOG.debug("Error sending message {} to {}", message, shardActor, failure);
} else {
LOG.debug("{} message to {} succeeded", message, shardActor, failure);
}
}
- }, datastore.getActorContext().getClientDispatcher());
+ }, context.getClientDispatcher());
}
- private void executeLocalEntityOwnershipShardOperation(final Object message) {
+ @VisibleForTesting
+ void executeLocalEntityOwnershipShardOperation(final Object message) {
if(localEntityOwnershipShard == null) {
- Future<ActorRef> future = datastore.getActorContext().findLocalShardAsync(ENTITY_OWNERSHIP_SHARD_NAME);
+ Future<ActorRef> future = context.findLocalShardAsync(ENTITY_OWNERSHIP_SHARD_NAME);
future.onComplete(new OnComplete<ActorRef>() {
@Override
- public void onComplete(Throwable failure, ActorRef shardActor) {
+ public void onComplete(final Throwable failure, final ActorRef shardActor) {
if(failure != null) {
LOG.error("Failed to find local {} shard", ENTITY_OWNERSHIP_SHARD_NAME, failure);
} else {
executeEntityOwnershipShardOperation(localEntityOwnershipShard, message);
}
}
- }, datastore.getActorContext().getClientDispatcher());
+ }, context.getClientDispatcher());
} else {
executeEntityOwnershipShardOperation(localEntityOwnershipShard, message);
}
@Override
- public EntityOwnershipCandidateRegistration registerCandidate(Entity entity)
+ public EntityOwnershipCandidateRegistration registerCandidate(final Entity entity)
throws CandidateAlreadyRegisteredException {
Preconditions.checkNotNull(entity, "entity cannot be null");
return new DistributedEntityOwnershipCandidateRegistration(entity, this);
}
- void unregisterCandidate(Entity entity) {
+ void unregisterCandidate(final Entity entity) {
LOG.debug("Unregistering candidate for {}", entity);
executeLocalEntityOwnershipShardOperation(new UnregisterCandidateLocal(entity));
}
@Override
- public EntityOwnershipListenerRegistration registerListener(String entityType, EntityOwnershipListener listener) {
+ public EntityOwnershipListenerRegistration registerListener(final String entityType, final EntityOwnershipListener listener) {
Preconditions.checkNotNull(entityType, "entityType cannot be null");
Preconditions.checkNotNull(listener, "listener cannot be null");
}
@Override
- public Optional<EntityOwnershipState> getOwnershipState(Entity forEntity) {
+ public Optional<EntityOwnershipState> getOwnershipState(final Entity forEntity) {
Preconditions.checkNotNull(forEntity, "forEntity cannot be null");
DataTree dataTree = getLocalEntityOwnershipShardDataTree();
return Optional.absent();
}
- String localMemberName = datastore.getActorContext().getCurrentMemberName();
+ String localMemberName = context.getCurrentMemberName();
Optional<DataContainerChild<? extends PathArgument, ?>> ownerLeaf = entity.getChild(ENTITY_OWNER_NODE_ID);
String owner = ownerLeaf.isPresent() ? ownerLeaf.get().getValue().toString() : null;
boolean hasOwner = !Strings.isNullOrEmpty(owner);
}
@Override
- public boolean isCandidateRegistered(@Nonnull Entity entity) {
+ public boolean isCandidateRegistered(@Nonnull final Entity entity) {
return registeredEntities.get(entity) != null;
}
- private DataTree getLocalEntityOwnershipShardDataTree() {
- if(localEntityOwnershipShardDataTree == null) {
+ @VisibleForTesting
+ DataTree getLocalEntityOwnershipShardDataTree() {
+ if (localEntityOwnershipShardDataTree == null) {
try {
if(localEntityOwnershipShard == null) {
- localEntityOwnershipShard = Await.result(datastore.getActorContext().findLocalShardAsync(
+ localEntityOwnershipShard = Await.result(context.findLocalShardAsync(
ENTITY_OWNERSHIP_SHARD_NAME), Duration.Inf());
}
return localEntityOwnershipShardDataTree;
}
- void unregisterListener(String entityType, EntityOwnershipListener listener) {
+ void unregisterListener(final String entityType, final EntityOwnershipListener listener) {
LOG.debug("Unregistering listener {} for entity type {}", listener, entityType);
executeLocalEntityOwnershipShardOperation(new UnregisterListenerLocal(listener, entityType));
public void close() {
}
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return EntityOwnershipShard.newBuilder().localMemberName(datastore.getActorContext().getCurrentMemberName())
- .ownerSelectionStrategyConfig(this.strategyConfig);
+ private static EntityOwnershipShard.Builder newShardBuilder(final ActorContext context,
+ final EntityOwnerSelectionStrategyConfig strategyConfig) {
+ return EntityOwnershipShard.newBuilder().localMemberName(context.getCurrentMemberName())
+ .ownerSelectionStrategyConfig(strategyConfig);
}
@VisibleForTesting
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class EntityOwnerSelectionStrategyConfig {
+/**
+ * FIXME: this is simple registry service, except it also loads classes.
+ */
+public final class EntityOwnerSelectionStrategyConfig {
private static final Logger LOG = LoggerFactory.getLogger(EntityOwnerSelectionStrategyConfig.class);
private final Map<String, StrategyInfo> entityTypeToStrategyInfo = new HashMap<>();
private final Map<String, EntityOwnerSelectionStrategy> entityTypeToOwnerSelectionStrategy = new HashMap<>();
- private EntityOwnerSelectionStrategyConfig(){
+ private EntityOwnerSelectionStrategyConfig() {
}
- public boolean isStrategyConfigured(String entityType){
+ public boolean isStrategyConfigured(final String entityType) {
return entityTypeToStrategyInfo.get(entityType) != null;
}
- public EntityOwnerSelectionStrategy createStrategy(String entityType){
+ public EntityOwnerSelectionStrategy createStrategy(final String entityType) {
final EntityOwnerSelectionStrategy strategy;
final EntityOwnerSelectionStrategy existingStrategy = entityTypeToOwnerSelectionStrategy.get(entityType);
if(existingStrategy != null){
entityTypeToOwnerSelectionStrategy.put(entityType, strategy);
}
return strategy;
-
}
+ /**
+ * @deprecated FIXME: THIS IS CONFIGURATION FOR A CUSTOM-LOADED CLASS CONSTRUCTOR
+ *
+ * This class should not exist. It contains a single long, which is passed to the constructor (via reflection).
+ * We are getting that information from a BundleContext. We are running in OSGi environment, hence this class
+ * needs to be deployed in its own bundle, with its own configuration.
+ *
+ * If this is used internally, it needs to be relocated into a separate package along with the implementation
+ * using it.
+ */
+ @Deprecated
private static final class StrategyInfo {
private final Class<? extends EntityOwnerSelectionStrategy> strategyClass;
private final long delay;
- private StrategyInfo(Class<? extends EntityOwnerSelectionStrategy> strategyClass, long delay) {
+ private StrategyInfo(final Class<? extends EntityOwnerSelectionStrategy> strategyClass, final long delay) {
this.strategyClass = strategyClass;
this.delay = delay;
}
- public EntityOwnerSelectionStrategy createStrategy(){
+ public EntityOwnerSelectionStrategy createStrategy() {
try {
return strategyClass.getDeclaredConstructor(long.class).newInstance(delay);
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
}
}
- public static Builder newBuilder(){
+ public static Builder newBuilder() {
return new Builder(new EntityOwnerSelectionStrategyConfig());
}
public static class Builder {
private final EntityOwnerSelectionStrategyConfig config;
- private Builder(EntityOwnerSelectionStrategyConfig config){
+ private Builder(final EntityOwnerSelectionStrategyConfig config){
this.config = config;
}
- public Builder addStrategy(String entityType, Class<? extends EntityOwnerSelectionStrategy> strategy, long delay){
+ public Builder addStrategy(final String entityType, final Class<? extends EntityOwnerSelectionStrategy> strategy, final long delay){
config.entityTypeToStrategyInfo.put(entityType, new StrategyInfo(strategy, delay));
return this;
}
return this.config;
}
}
-
}
import java.io.IOException;
import java.util.Dictionary;
import java.util.Enumeration;
-import javax.annotation.Nullable;
+import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig.Builder;
import org.osgi.framework.BundleContext;
import org.osgi.framework.ServiceReference;
import org.osgi.service.cm.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class EntityOwnerSelectionStrategyConfigReader {
+/**
+ * @deprecated FIXME: Service injection class. This class needs to be eliminated in favor of proper service injection,
+ * which can be any of OSGi (which this class uses internally), java.util.ServiceLoader, or config
+ * subsystem.
+ */
+@Deprecated
+public final class EntityOwnerSelectionStrategyConfigReader {
public static final String CONFIG_ID = "org.opendaylight.controller.cluster.entity.owner.selection.strategies";
private static final Logger LOG = LoggerFactory.getLogger(EntityOwnerSelectionStrategyConfigReader.class);
private static final String ENTITY_TYPE_PREFIX = "entity.type.";
- private final BundleContext bundleContext;
- private final EntityOwnerSelectionStrategyConfig config;
+ private EntityOwnerSelectionStrategyConfigReader() {
+ // Hidden on purpose
+ }
+
+ public static EntityOwnerSelectionStrategyConfig loadStrategyWithConfig(final BundleContext bundleContext) {
+ final EntityOwnerSelectionStrategyConfig.Builder builder = EntityOwnerSelectionStrategyConfig.newBuilder();
- public EntityOwnerSelectionStrategyConfigReader(BundleContext bundleContext) {
- this.bundleContext = Preconditions.checkNotNull(bundleContext);
- ServiceReference<ConfigurationAdmin> configAdminServiceReference =
+ final ServiceReference<ConfigurationAdmin> configAdminServiceReference =
bundleContext.getServiceReference(ConfigurationAdmin.class);
- if(configAdminServiceReference == null) {
+ if (configAdminServiceReference == null) {
LOG.warn("No ConfigurationAdmin service found");
- this.config = EntityOwnerSelectionStrategyConfig.newBuilder().build();
- } else {
- this.config = readConfiguration(configAdminServiceReference);
+ return builder.build();
+ }
+
+ final ConfigurationAdmin configAdmin = bundleContext.getService(configAdminServiceReference);
+ if (configAdmin == null) {
+ LOG.warn("Failed to get ConfigurationAdmin service");
+ return builder.build();
}
- }
- private EntityOwnerSelectionStrategyConfig readConfiguration(ServiceReference<ConfigurationAdmin> configAdminServiceReference) {
- EntityOwnerSelectionStrategyConfig.Builder builder = EntityOwnerSelectionStrategyConfig.newBuilder();
- ConfigurationAdmin configAdmin = null;
+ final Configuration config;
try {
- configAdmin = bundleContext.getService(configAdminServiceReference);
- Dictionary<String, Object> properties = getProperties(configAdmin);
- if(properties != null) {
- Enumeration<String> keys = properties.keys();
- while (keys.hasMoreElements()) {
- String key = keys.nextElement();
- String strategyProps = (String) properties.get(key);
- String[] strategyClassAndDelay = strategyProps.split(",");
- if(key.startsWith(ENTITY_TYPE_PREFIX)) {
- @SuppressWarnings("unchecked")
- Class<? extends EntityOwnerSelectionStrategy> aClass
- = (Class<? extends EntityOwnerSelectionStrategy>) getClass().getClassLoader().loadClass(strategyClassAndDelay[0]);
- long delay = 0;
- if(strategyClassAndDelay.length > 1){
- delay = Long.parseLong(strategyClassAndDelay[1]);
- }
- String entityType = key.substring(key.lastIndexOf(".") + 1);
- builder.addStrategy(entityType, aClass, delay);
- } else {
- LOG.debug("Ignoring non-conforming property key : {}, value : {}", key, strategyProps);
- }
- }
- } else {
- LOG.error("Could not read strategy configuration file, will use default configuration");
+ config = configAdmin.getConfiguration(CONFIG_ID);
+ if (config != null) {
+ return parseConfiguration(builder, config);
}
- } catch(Exception e){
- LOG.warn("Failed to read selection strategy configuration file. All configuration will be ignored.", e);
+
+ LOG.debug("Could not read strategy configuration file, will use default configuration");
+ } catch (IOException e1) {
+ LOG.warn("Failed to get configuration for {}, starting up empty", CONFIG_ID);
+ return builder.build();
} finally {
- if(configAdmin != null) {
- try {
- bundleContext.ungetService(configAdminServiceReference);
- } catch (Exception e) {
- LOG.debug("Error from ungetService", e);
- }
+ try {
+ bundleContext.ungetService(configAdminServiceReference);
+ } catch (Exception e) {
+ LOG.debug("Error from ungetService", e);
}
}
return builder.build();
}
- @Nullable
- private static Dictionary<String, Object> getProperties(ConfigurationAdmin configAdmin) throws IOException {
- Configuration config = configAdmin.getConfiguration(CONFIG_ID);
- return config != null ? config.getProperties() : null;
+ private static EntityOwnerSelectionStrategyConfig parseConfiguration(final Builder builder, final Configuration config) {
+ // Historic note: java.util.Dictionary since introduction of java.util.Map in Java 1.2
+ final Dictionary<String, Object> properties = config.getProperties();
+ if (properties == null) {
+ LOG.debug("Empty strategy configuration {}, using defaults", config);
+ return builder.build();
+ }
+
+ // No java.util.Iterable: Wheeey, pre-Java 5 world!!!
+ final Enumeration<String> keys = properties.keys();
+ while (keys.hasMoreElements()) {
+ final String key = keys.nextElement();
+ if (!key.startsWith(ENTITY_TYPE_PREFIX)) {
+ LOG.debug("Ignoring non-conforming property key : {}");
+ continue;
+ }
+
+ final String[] strategyClassAndDelay = ((String) properties.get(key)).split(",");
+ final Class<? extends EntityOwnerSelectionStrategy> aClass;
+ try {
+ aClass = loadClass(strategyClassAndDelay[0]);
+ } catch (ClassNotFoundException e) {
+ LOG.error("Failed to load class {}, ignoring it", strategyClassAndDelay[0], e);
+ continue;
+ }
+
+ final long delay;
+ if (strategyClassAndDelay.length > 1) {
+ delay = Long.parseLong(strategyClassAndDelay[1]);
+ } else {
+ delay = 0;
+ }
+
+ String entityType = key.substring(key.lastIndexOf(".") + 1);
+ builder.addStrategy(entityType, aClass, delay);
+ LOG.debug("Entity Type '{}' using strategy {} delay {}", entityType, aClass, delay);
+ }
+
+ return builder.build();
}
- public EntityOwnerSelectionStrategyConfig getConfig() {
- return config;
+ @Deprecated
+ @SuppressWarnings("unchecked")
+ private static Class<? extends EntityOwnerSelectionStrategy> loadClass(final String strategyClassAndDelay)
+ throws ClassNotFoundException {
+ final Class<?> clazz;
+ try {
+ clazz = EntityOwnerSelectionStrategyConfigReader.class.getClassLoader().loadClass(strategyClassAndDelay);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("Failed to load strategy " + strategyClassAndDelay);
+ }
+
+ Preconditions.checkArgument(EntityOwnerSelectionStrategy.class.isAssignableFrom(clazz),
+ "Selected implementation %s must implement EntityOwnerSelectionStrategy, clazz");
+
+ return (Class<? extends EntityOwnerSelectionStrategy>) clazz;
}
}
import org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService;
import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfigReader;
+import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
+import org.opendaylight.controller.config.api.DependencyResolver;
+import org.opendaylight.controller.config.api.ModuleIdentifier;
import org.opendaylight.controller.sal.core.spi.data.DOMStore;
import org.osgi.framework.BundleContext;
-
-public class DistributedEntityOwnershipServiceProviderModule extends org.opendaylight.controller.config.yang.config.distributed_entity_ownership_service.AbstractDistributedEntityOwnershipServiceProviderModule {
+public class DistributedEntityOwnershipServiceProviderModule extends AbstractDistributedEntityOwnershipServiceProviderModule {
+ private EntityOwnerSelectionStrategyConfig strategyConfig;
private BundleContext bundleContext;
- public DistributedEntityOwnershipServiceProviderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
+ public DistributedEntityOwnershipServiceProviderModule(final ModuleIdentifier identifier,
+ final DependencyResolver dependencyResolver) {
super(identifier, dependencyResolver);
}
- public DistributedEntityOwnershipServiceProviderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver, org.opendaylight.controller.config.yang.config.distributed_entity_ownership_service.DistributedEntityOwnershipServiceProviderModule oldModule, java.lang.AutoCloseable oldInstance) {
+ public DistributedEntityOwnershipServiceProviderModule(final ModuleIdentifier identifier,
+ final DependencyResolver dependencyResolver,
+ final DistributedEntityOwnershipServiceProviderModule oldModule, final AutoCloseable oldInstance) {
super(identifier, dependencyResolver, oldModule, oldInstance);
}
@Override
public void customValidation() {
- // add custom validation form module attributes here.
+ strategyConfig = EntityOwnerSelectionStrategyConfigReader.loadStrategyWithConfig(bundleContext);
}
@Override
- public boolean canReuseInstance(AbstractDistributedEntityOwnershipServiceProviderModule oldModule) {
+ public boolean canReuseInstance(final AbstractDistributedEntityOwnershipServiceProviderModule oldModule) {
return true;
}
@Override
- public java.lang.AutoCloseable createInstance() {
+ public AutoCloseable createInstance() {
+ // FIXME: EntityOwnership needs only the ActorContext, not the entire datastore
DOMStore dataStore = getDataStoreDependency();
Preconditions.checkArgument(dataStore instanceof DistributedDataStore,
"Injected DOMStore must be an instance of DistributedDataStore");
- EntityOwnerSelectionStrategyConfig strategyConfig = new EntityOwnerSelectionStrategyConfigReader(bundleContext).getConfig();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService((DistributedDataStore)dataStore, strategyConfig);
- service.start();
- return service;
+
+ final ActorContext context = ((DistributedDataStore)dataStore).getActorContext();
+ return DistributedEntityOwnershipService.start(context, strategyConfig);
}
- public void setBundleContext(BundleContext bundleContext) {
+ public void setBundleContext(final BundleContext bundleContext) {
+ // What do we need from the bundle context?
this.bundleContext = bundleContext;
}
}
import akka.actor.Status.Success;
import akka.cluster.Cluster;
import akka.testkit.JavaTestKit;
-import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
import org.opendaylight.controller.cluster.datastore.MemberNode;
-import org.opendaylight.controller.cluster.datastore.MemberNode.RaftStateVerifier;
import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
import org.opendaylight.controller.cluster.raft.RaftState;
-import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState;
import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipState;
import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.entity.type.entity.Candidate;
-import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
}
}
- private static DistributedEntityOwnershipService newOwnershipService(DistributedDataStore datastore) {
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(datastore,
+ private static DistributedEntityOwnershipService newOwnershipService(final DistributedDataStore datastore) {
+ return DistributedEntityOwnershipService.start(datastore.getActorContext(),
EntityOwnerSelectionStrategyConfig.newBuilder().build());
- service.start();
- return service;
}
@Test
follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).
customRaftPolicyImplementation(null).build(), ActorRef.noSender());
- MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, new RaftStateVerifier() {
- @Override
- public void verify(OnDemandRaftState raftState) {
- assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState());
- }
- });
+ MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
+ raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
// Verify the prior leader's candidates are removed
follower1EntityOwnershipService.registerCandidate(ENTITY1);
verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
- verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, new RaftStateVerifier() {
- @Override
- public void verify(OnDemandRaftState raftState) {
- assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
- assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
- assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
- org.hamcrest.CoreMatchers.containsString("member-1"));
- }
+ verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
+ assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
+ assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
+ assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
+ org.hamcrest.CoreMatchers.containsString("member-1"));
});
}
- private static void verifyGetOwnershipState(EntityOwnershipService service, Entity entity,
- boolean isOwner, boolean hasOwner) {
+ private static void verifyGetOwnershipState(final EntityOwnershipService service, final Entity entity,
+ final boolean isOwner, final boolean hasOwner) {
Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
assertEquals("getOwnershipState present", true, state.isPresent());
assertEquals("isOwner", isOwner, state.get().isOwner());
assertEquals("hasOwner", hasOwner, state.get().hasOwner());
}
- private static void verifyCandidates(DistributedDataStore dataStore, Entity entity, String... expCandidates) throws Exception {
+ private static void verifyCandidates(final DistributedDataStore dataStore, final Entity entity, final String... expCandidates) throws Exception {
AssertionError lastError = null;
Stopwatch sw = Stopwatch.createStarted();
while(sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) {
throw lastError;
}
- private static void verifyOwner(final DistributedDataStore dataStore, Entity entity, String expOwner) {
+ private static void verifyOwner(final DistributedDataStore dataStore, final Entity entity, final String expOwner) {
AbstractEntityOwnershipTest.verifyOwner(expOwner, entity.getType(), entity.getId(),
- new Function<YangInstanceIdentifier, NormalizedNode<?,?>>() {
- @Override
- public NormalizedNode<?, ?> apply(YangInstanceIdentifier path) {
- try {
- return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
- } catch (Exception e) {
- return null;
- }
+ path -> {
+ try {
+ return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
+ } catch (Exception e) {
+ return null;
}
});
}
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_ID_QNAME;
import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME;
import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityTypeEntryWithEntityEntry;
import akka.actor.ActorRef;
import akka.actor.PoisonPill;
-import akka.actor.Props;
-import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Uninterruptibles;
import java.util.Collection;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.opendaylight.controller.cluster.datastore.DatastoreContext;
import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipState;
import org.opendaylight.yangtools.yang.common.QName;
import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
-import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
import org.opendaylight.yangtools.yang.data.api.schema.tree.TreeType;
import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
* @author Thomas Pantelis
*/
public class DistributedEntityOwnershipServiceTest extends AbstractEntityOwnershipTest {
- static String ENTITY_TYPE = "test";
- static String ENTITY_TYPE2 = "test2";
- static int ID_COUNTER = 1;
+ static final String ENTITY_TYPE = "test";
+ static final String ENTITY_TYPE2 = "test2";
static final QName QNAME = QName.create("test", "2015-08-11", "foo");
+ static int ID_COUNTER = 1;
private final String dataStoreName = "config" + ID_COUNTER++;
private DistributedDataStore dataStore;
}
};
- DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
+ DatastoreContextFactory mockContextFactory = mock(DatastoreContextFactory.class);
Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
dataStore.getActorContext().getShardManager().tell(PoisonPill.getInstance(), ActorRef.noSender());
}
+ private static <T> T verifyMessage(final DistributedEntityOwnershipService mock, final Class<T> type) {
+ final ArgumentCaptor<T> message = ArgumentCaptor.forClass(type);
+ verify(mock).executeLocalEntityOwnershipShardOperation(message.capture());
+ return message.getValue();
+ }
+
@Test
public void testEntityOwnershipShardCreated() throws Exception {
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
+ DistributedEntityOwnershipService service = DistributedEntityOwnershipService.start(dataStore.getActorContext(),
EntityOwnerSelectionStrategyConfig.newBuilder().build());
- service.start();
Future<ActorRef> future = dataStore.getActorContext().findLocalShardAsync(
DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME);
@Test
public void testRegisterCandidate() throws Exception {
- final TestShardBuilder shardBuilder = new TestShardBuilder();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
- EntityOwnerSelectionStrategyConfig.newBuilder().build()) {
- @Override
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return shardBuilder;
- }
- };
-
- service.start();
-
- shardBuilder.expectShardMessage(RegisterCandidateLocal.class);
+ DistributedEntityOwnershipService service = spy(DistributedEntityOwnershipService.start(
+ dataStore.getActorContext(), EntityOwnerSelectionStrategyConfig.newBuilder().build()));
YangInstanceIdentifier entityId = YangInstanceIdentifier.of(QNAME);
Entity entity = new Entity(ENTITY_TYPE, entityId);
EntityOwnershipCandidateRegistration reg = service.registerCandidate(entity);
-
+ verifyRegisterCandidateLocal(service, entity);
verifyEntityOwnershipCandidateRegistration(entity, reg);
- verifyRegisterCandidateLocal(shardBuilder, entity);
verifyEntityCandidate(service.getLocalEntityOwnershipShard(), ENTITY_TYPE, entityId,
dataStore.getActorContext().getCurrentMemberName());
}
// Register a different entity - should succeed
+ reset(service);
Entity entity2 = new Entity(ENTITY_TYPE2, entityId);
- shardBuilder.expectShardMessage(RegisterCandidateLocal.class);
-
EntityOwnershipCandidateRegistration reg2 = service.registerCandidate(entity2);
-
+ verifyRegisterCandidateLocal(service, entity2);
verifyEntityOwnershipCandidateRegistration(entity2, reg2);
- verifyRegisterCandidateLocal(shardBuilder, entity2);
verifyEntityCandidate(service.getLocalEntityOwnershipShard(), ENTITY_TYPE2, entityId,
dataStore.getActorContext().getCurrentMemberName());
@Test
public void testCloseCandidateRegistration() throws Exception {
- final TestShardBuilder shardBuilder = new TestShardBuilder();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
- EntityOwnerSelectionStrategyConfig.newBuilder().build()) {
- @Override
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return shardBuilder;
- }
- };
-
- service.start();
-
- shardBuilder.expectShardMessage(RegisterCandidateLocal.class);
+ DistributedEntityOwnershipService service = spy(DistributedEntityOwnershipService.start(
+ dataStore.getActorContext(), EntityOwnerSelectionStrategyConfig.newBuilder().build()));
Entity entity = new Entity(ENTITY_TYPE, YangInstanceIdentifier.of(QNAME));
-
EntityOwnershipCandidateRegistration reg = service.registerCandidate(entity);
verifyEntityOwnershipCandidateRegistration(entity, reg);
- verifyRegisterCandidateLocal(shardBuilder, entity);
-
- shardBuilder.expectShardMessage(UnregisterCandidateLocal.class);
+ verifyRegisterCandidateLocal(service, entity);
+ reset(service);
reg.close();
-
- UnregisterCandidateLocal unregCandidate = shardBuilder.waitForShardMessage();
+ UnregisterCandidateLocal unregCandidate = verifyMessage(service, UnregisterCandidateLocal.class);
assertEquals("getEntity", entity, unregCandidate.getEntity());
// Re-register - should succeed.
-
- shardBuilder.expectShardMessage(RegisterCandidateLocal.class);
-
+ reset(service);
service.registerCandidate(entity);
-
- verifyRegisterCandidateLocal(shardBuilder, entity);
+ verifyRegisterCandidateLocal(service, entity);
service.close();
}
@Test
public void testListenerRegistration() {
- final TestShardBuilder shardBuilder = new TestShardBuilder();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
- EntityOwnerSelectionStrategyConfig.newBuilder().build()) {
- @Override
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return shardBuilder;
- }
- };
-
- service.start();
-
- shardBuilder.expectShardMessage(RegisterListenerLocal.class);
+ DistributedEntityOwnershipService service = spy(DistributedEntityOwnershipService.start(
+ dataStore.getActorContext(), EntityOwnerSelectionStrategyConfig.newBuilder().build()));
YangInstanceIdentifier entityId = YangInstanceIdentifier.of(QNAME);
Entity entity = new Entity(ENTITY_TYPE, entityId);
assertEquals("getEntityType", entity.getType(), reg.getEntityType());
assertEquals("getInstance", listener, reg.getInstance());
- RegisterListenerLocal regListener = shardBuilder.waitForShardMessage();
+ RegisterListenerLocal regListener = verifyMessage(service, RegisterListenerLocal.class);
assertSame("getListener", listener, regListener.getListener());
assertEquals("getEntityType", entity.getType(), regListener.getEntityType());
- shardBuilder.expectShardMessage(UnregisterListenerLocal.class);
-
+ reset(service);
reg.close();
-
- UnregisterListenerLocal unregListener = shardBuilder.waitForShardMessage();
+ UnregisterListenerLocal unregListener = verifyMessage(service, UnregisterListenerLocal.class);
assertEquals("getEntityType", entity.getType(), unregListener.getEntityType());
assertSame("getListener", listener, unregListener.getListener());
@Test
public void testGetOwnershipState() throws Exception {
- final TestShardBuilder shardBuilder = new TestShardBuilder();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
- EntityOwnerSelectionStrategyConfig.newBuilder().build()) {
- @Override
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return shardBuilder;
- }
- };
-
- service.start();
+ DistributedEntityOwnershipService service = spy(DistributedEntityOwnershipService.start(
+ dataStore.getActorContext(), EntityOwnerSelectionStrategyConfig.newBuilder().build()));
ShardDataTree shardDataTree = new ShardDataTree(SchemaContextHelper.entityOwners(), TreeType.OPERATIONAL);
- shardBuilder.setDataTree(shardDataTree.getDataTree());
+
+ when(service.getLocalEntityOwnershipShardDataTree()).thenReturn(shardDataTree.getDataTree());
Entity entity1 = new Entity(ENTITY_TYPE, "one");
writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, entity1.getId(), "member-1"), shardDataTree);
@Test
public void testIsCandidateRegistered() throws CandidateAlreadyRegisteredException {
- final TestShardBuilder shardBuilder = new TestShardBuilder();
- DistributedEntityOwnershipService service = new DistributedEntityOwnershipService(dataStore,
- EntityOwnerSelectionStrategyConfig.newBuilder().build()) {
- @Override
- protected EntityOwnershipShard.Builder newShardBuilder() {
- return shardBuilder;
- }
- };
-
- service.start();
+ DistributedEntityOwnershipService service = DistributedEntityOwnershipService.start(dataStore.getActorContext(),
+ EntityOwnerSelectionStrategyConfig.newBuilder().build());
final Entity test = new Entity("test-type", "test");
service.close();
}
- private static void verifyGetOwnershipState(DistributedEntityOwnershipService service, Entity entity,
- boolean isOwner, boolean hasOwner) {
+ private static void verifyGetOwnershipState(final DistributedEntityOwnershipService service, final Entity entity,
+ final boolean isOwner, final boolean hasOwner) {
Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
assertEquals("getOwnershipState present", true, state.isPresent());
assertEquals("isOwner", isOwner, state.get().isOwner());
assertEquals("hasOwner", hasOwner, state.get().hasOwner());
}
- private void verifyEntityCandidate(ActorRef entityOwnershipShard, String entityType,
- YangInstanceIdentifier entityId, String candidateName) {
+ private void verifyEntityCandidate(final ActorRef entityOwnershipShard, final String entityType,
+ final YangInstanceIdentifier entityId, final String candidateName) {
verifyEntityCandidate(entityType, entityId, candidateName,
- new Function<YangInstanceIdentifier, NormalizedNode<?,?>>() {
- @Override
- public NormalizedNode<?, ?> apply(YangInstanceIdentifier path) {
- try {
- return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
- } catch (Exception e) {
- return null;
- }
+ path -> {
+ try {
+ return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
+ } catch (Exception e) {
+ return null;
}
});
}
- private static void verifyRegisterCandidateLocal(final TestShardBuilder shardBuilder, Entity entity) {
- RegisterCandidateLocal regCandidate = shardBuilder.waitForShardMessage();
+ private static void verifyRegisterCandidateLocal(final DistributedEntityOwnershipService service, final Entity entity) {
+ RegisterCandidateLocal regCandidate = verifyMessage(service, RegisterCandidateLocal.class);
assertEquals("getEntity", entity, regCandidate.getEntity());
}
- private static void verifyEntityOwnershipCandidateRegistration(Entity entity, EntityOwnershipCandidateRegistration reg) {
+ private static void verifyEntityOwnershipCandidateRegistration(final Entity entity, final EntityOwnershipCandidateRegistration reg) {
assertNotNull("EntityOwnershipCandidateRegistration null", reg);
assertEquals("getInstance", entity, reg.getInstance());
}
- static class TestShardBuilder extends EntityOwnershipShard.Builder {
- TestShardBuilder() {
- localMemberName("member-1").ownerSelectionStrategyConfig(
- EntityOwnerSelectionStrategyConfig.newBuilder().build());
- }
-
- private final AtomicReference<CountDownLatch> messageReceived = new AtomicReference<>();
- private final AtomicReference<Object> receivedMessage = new AtomicReference<>();
- private final AtomicReference<Class<?>> messageClass = new AtomicReference<>();
- private final AtomicReference<DataTree> dataTree = new AtomicReference<>();
-
- @Override
- public Props props() {
- verify();
- return Props.create(TestEntityOwnershipShard.class,this, messageClass, messageReceived,
- receivedMessage, dataTree);
- }
-
- @SuppressWarnings("unchecked")
- <T> T waitForShardMessage() {
- assertTrue("Message " + messageClass.get().getSimpleName() + " was not received",
- Uninterruptibles.awaitUninterruptibly(messageReceived.get(), 5, TimeUnit.SECONDS));
- assertEquals("Message type", messageClass.get(), receivedMessage.get().getClass());
- return (T) receivedMessage.get();
- }
-
- void expectShardMessage(Class<?> ofType) {
- messageReceived.set(new CountDownLatch(1));
- receivedMessage.set(null);
- messageClass.set(ofType);
- }
-
- void setDataTree(DataTree tree) {
- this.dataTree.set(tree);
- }
- }
-
static class TestEntityOwnershipShard extends EntityOwnershipShard {
private final AtomicReference<CountDownLatch> messageReceived;
private final AtomicReference<Object> receivedMessage;
private final AtomicReference<Class<?>> messageClass;
private final AtomicReference<DataTree> dataTree;
- protected TestEntityOwnershipShard(EntityOwnershipShard.Builder builder,
- AtomicReference<Class<?>> messageClass, AtomicReference<CountDownLatch> messageReceived,
- AtomicReference<Object> receivedMessage, AtomicReference<DataTree> dataTree) {
+ protected TestEntityOwnershipShard(final EntityOwnershipShard.Builder builder,
+ final AtomicReference<Class<?>> messageClass, final AtomicReference<CountDownLatch> messageReceived,
+ final AtomicReference<Object> receivedMessage, final AtomicReference<DataTree> dataTree) {
super(builder);
this.messageClass = messageClass;
this.messageReceived = messageReceived;
doReturn(mockConfig).when(mockConfigAdmin).getConfiguration(EntityOwnerSelectionStrategyConfigReader.CONFIG_ID);
}
+ private EntityOwnerSelectionStrategyConfig loadStrategyConfig() {
+ return EntityOwnerSelectionStrategyConfigReader.loadStrategyWithConfig(mockBundleContext);
+ }
+
@Test
public void testReadStrategies(){
Hashtable<String, Object> props = new Hashtable<>();
doReturn(props).when(mockConfig).getProperties();
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
+ EntityOwnerSelectionStrategyConfig config = loadStrategyConfig();
assertTrue(config.isStrategyConfigured("test"));
public void testReadStrategiesWithIOException() throws IOException {
doThrow(IOException.class).when(mockConfigAdmin).getConfiguration(EntityOwnerSelectionStrategyConfigReader.CONFIG_ID);
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
+ EntityOwnerSelectionStrategyConfig config = loadStrategyConfig();
assertFalse(config.isStrategyConfigured("test"));
}
public void testReadStrategiesWithNullConfiguration() throws IOException {
doReturn(null).when(mockConfigAdmin).getConfiguration(EntityOwnerSelectionStrategyConfigReader.CONFIG_ID);
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
+ EntityOwnerSelectionStrategyConfig config = loadStrategyConfig();
assertFalse(config.isStrategyConfigured("test"));
}
public void testReadStrategiesWithNullConfigurationProperties() throws IOException {
doReturn(null).when(mockConfig).getProperties();
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
+ EntityOwnerSelectionStrategyConfig config = loadStrategyConfig();
assertFalse(config.isStrategyConfigured("test"));
}
- @Test
+ @Test(expected = IllegalArgumentException.class)
public void testReadStrategiesInvalidDelay(){
Hashtable<String, Object> props = new Hashtable<>();
props.put("entity.type.test", "org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.LastCandidateSelectionStrategy,foo");
doReturn(props).when(mockConfig).getProperties();
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
-
- assertFalse(config.isStrategyConfigured("test"));
+ loadStrategyConfig();
}
- @Test
+ @Test(expected = IllegalArgumentException.class)
public void testReadStrategiesInvalidClassType(){
Hashtable<String, Object> props = new Hashtable<>();
props.put("entity.type.test", "String,100");
doReturn(props).when(mockConfig).getProperties();
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
-
- assertFalse(config.isStrategyConfigured("test"));
+ loadStrategyConfig();
}
@Test
doReturn(props).when(mockConfig).getProperties();
- EntityOwnerSelectionStrategyConfig config = new EntityOwnerSelectionStrategyConfigReader(mockBundleContext).getConfig();
+ EntityOwnerSelectionStrategyConfig config = loadStrategyConfig();
assertEquals(100, config.createStrategy("test").getSelectionDelayInMillis());
assertEquals(0, config.createStrategy("test2").getSelectionDelayInMillis());
Code Review / controller.git / commitdiff