2 * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.bgpcep.programming.impl;
10 import com.google.common.base.Preconditions;
11 import com.google.common.util.concurrent.CheckedFuture;
12 import com.google.common.util.concurrent.FutureCallback;
13 import com.google.common.util.concurrent.Futures;
14 import com.google.common.util.concurrent.ListenableFuture;
15 import com.google.common.util.concurrent.ListeningExecutorService;
16 import com.google.common.util.concurrent.SettableFuture;
17 import io.netty.util.Timeout;
18 import io.netty.util.Timer;
19 import java.math.BigInteger;
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.HashMap;
23 import java.util.Iterator;
24 import java.util.List;
26 import java.util.concurrent.TimeUnit;
27 import org.opendaylight.bgpcep.programming.NanotimeUtil;
28 import org.opendaylight.bgpcep.programming.impl.InstructionDeployerImpl.WriteConfiguration;
29 import org.opendaylight.bgpcep.programming.spi.ExecutionResult;
30 import org.opendaylight.bgpcep.programming.spi.Instruction;
31 import org.opendaylight.bgpcep.programming.spi.InstructionScheduler;
32 import org.opendaylight.bgpcep.programming.spi.SchedulerException;
33 import org.opendaylight.bgpcep.programming.spi.SuccessfulRpcResult;
34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
37 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
39 import org.opendaylight.controller.sal.binding.api.BindingAwareBroker.RpcRegistration;
40 import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry;
41 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
42 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceProvider;
43 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
44 import org.opendaylight.mdsal.singleton.common.api.ServiceGroupIdentifier;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CancelInstructionInput;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CancelInstructionOutput;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CancelInstructionOutputBuilder;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CleanInstructionsInput;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CleanInstructionsOutput;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.CleanInstructionsOutputBuilder;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.DeadOnArrival;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.DuplicateInstructionId;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionId;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionStatus;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionStatusChangedBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionsQueue;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionsQueueBuilder;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.InstructionsQueueKey;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.Nanotime;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.ProgrammingService;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.SubmitInstructionInput;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.UnknownInstruction;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.UnknownPreconditionId;
64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.instruction.queue.InstructionBuilder;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.instruction.queue.InstructionKey;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.instruction.status.changed.Details;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.submit.instruction.output.result.failure._case.FailureBuilder;
68 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
69 import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier;
70 import org.opendaylight.yangtools.yang.common.RpcResult;
71 import org.osgi.framework.ServiceRegistration;
72 import org.slf4j.Logger;
73 import org.slf4j.LoggerFactory;
75 public final class ProgrammingServiceImpl implements AutoCloseable, ClusterSingletonService, InstructionScheduler,
77 private static final Logger LOG = LoggerFactory.getLogger(ProgrammingServiceImpl.class);
79 private final Map<InstructionId, InstructionImpl> insns = new HashMap<>();
80 private final InstanceIdentifier<InstructionsQueue> qid;
81 private final NotificationPublishService notifs;
82 private final ListeningExecutorService executor;
83 private final DataBroker dataProvider;
84 private final Timer timer;
85 private final String instructionId;
86 private final ServiceGroupIdentifier sgi;
87 private final ClusterSingletonServiceRegistration csspReg;
88 private final RpcProviderRegistry rpcProviderRegistry;
89 private final WriteConfiguration writeConfiguration;
90 private RpcRegistration<ProgrammingService> reg;
91 private ServiceRegistration<?> serviceRegistration;
93 private final class InstructionPusher implements QueueInstruction {
94 private final InstructionBuilder builder = new InstructionBuilder();
96 InstructionPusher(final InstructionId id, final Nanotime deadline) {
97 this.builder.setDeadline(deadline);
98 this.builder.setId(id);
99 this.builder.setKey(new InstructionKey(id));
100 this.builder.setStatus(InstructionStatus.Queued);
104 public void instructionUpdated(final InstructionStatus status, final Details details) {
105 if (!status.equals(this.builder.getStatus())) {
106 this.builder.setStatus(status);
108 final WriteTransaction t = ProgrammingServiceImpl.this.dataProvider.newWriteOnlyTransaction();
109 t.put(LogicalDatastoreType.OPERATIONAL,
110 ProgrammingServiceImpl.this.qid.child(
111 org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.instruction.queue.Instruction.class,
112 new InstructionKey(this.builder.getId())), this.builder.build());
113 Futures.addCallback(t.submit(), new FutureCallback<Void>() {
115 public void onSuccess(final Void result) {
116 LOG.debug("Instruction Queue {} updated", ProgrammingServiceImpl.this.qid);
120 public void onFailure(final Throwable t) {
121 LOG.error("Failed to update Instruction Queue {}", ProgrammingServiceImpl.this.qid, t);
127 ProgrammingServiceImpl.this.notifs.putNotification(new InstructionStatusChangedBuilder()
128 .setId(this.builder.getId()).setStatus(status).setDetails(details).build());
129 } catch (final InterruptedException e) {
130 LOG.debug("Failed to publish notification", e);
135 public void instructionRemoved() {
136 final WriteTransaction t = ProgrammingServiceImpl.this.dataProvider.newWriteOnlyTransaction();
137 t.delete(LogicalDatastoreType.OPERATIONAL, ProgrammingServiceImpl.this.qid.child(
138 org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.programming.rev150720.instruction.queue.Instruction.class,
139 new InstructionKey(this.builder.getId())));
140 Futures.addCallback(t.submit(), new FutureCallback<Void>() {
142 public void onSuccess(final Void result) {
143 LOG.debug("Instruction Queue {} removed", ProgrammingServiceImpl.this.qid);
147 public void onFailure(final Throwable t) {
148 LOG.error("Failed to remove Instruction Queue {}", ProgrammingServiceImpl.this.qid, t);
154 ProgrammingServiceImpl(final DataBroker dataProvider, final NotificationPublishService notifs,
155 final ListeningExecutorService executor, final RpcProviderRegistry rpcProviderRegistry,
156 final ClusterSingletonServiceProvider cssp, final Timer timer, final String instructionId,
157 final WriteConfiguration writeConfiguration) {
158 this.dataProvider = Preconditions.checkNotNull(dataProvider);
159 this.instructionId = Preconditions.checkNotNull(instructionId);
160 this.notifs = Preconditions.checkNotNull(notifs);
161 this.executor = Preconditions.checkNotNull(executor);
162 this.rpcProviderRegistry = Preconditions.checkNotNull(rpcProviderRegistry);
163 this.timer = Preconditions.checkNotNull(timer);
164 this.qid = KeyedInstanceIdentifier.builder(InstructionsQueue.class, new InstructionsQueueKey(this.instructionId)).build();
165 this.writeConfiguration = writeConfiguration;
166 this.sgi = ServiceGroupIdentifier.create("programming-"+ this.instructionId + "-service-group");
167 this.csspReg = cssp.registerClusterSingletonService(this);
171 public void instantiateServiceInstance() {
172 LOG.info("Instruction Queue service {} instantiated", this.sgi.getValue());
174 if (this.writeConfiguration != null) {
175 this.writeConfiguration.create();
177 this.reg = this.rpcProviderRegistry.addRpcImplementation(ProgrammingService.class, this);
179 final WriteTransaction t = this.dataProvider.newWriteOnlyTransaction();
180 t.put(LogicalDatastoreType.OPERATIONAL, this.qid, new InstructionsQueueBuilder()
181 .setKey(new InstructionsQueueKey(this.instructionId)).setInstruction(Collections.emptyList()).build());
182 Futures.addCallback(t.submit(), new FutureCallback<Void>() {
184 public void onSuccess(final Void result) {
185 LOG.debug("Instruction Queue {} added", ProgrammingServiceImpl.this.qid);
189 public void onFailure(final Throwable t) {
190 LOG.error("Failed to add Instruction Queue {}", ProgrammingServiceImpl.this.qid, t);
196 public ServiceGroupIdentifier getIdentifier() {
201 public ListenableFuture<RpcResult<CancelInstructionOutput>> cancelInstruction(final CancelInstructionInput input) {
202 return this.executor.submit(() -> realCancelInstruction(input));
206 public ListenableFuture<RpcResult<CleanInstructionsOutput>> cleanInstructions(final CleanInstructionsInput input) {
207 return this.executor.submit(() -> realCleanInstructions(input));
210 private synchronized RpcResult<CancelInstructionOutput> realCancelInstruction(final CancelInstructionInput input) {
211 final InstructionImpl i = this.insns.get(input.getId());
213 LOG.debug("Instruction {} not present in the graph", input.getId());
215 final CancelInstructionOutput out = new CancelInstructionOutputBuilder().setFailure(UnknownInstruction.class).build();
216 return SuccessfulRpcResult.create(out);
219 return SuccessfulRpcResult.create(new CancelInstructionOutputBuilder().setFailure(i.tryCancel(null)).build());
222 private synchronized RpcResult<CleanInstructionsOutput> realCleanInstructions(final CleanInstructionsInput input) {
223 final List<InstructionId> failed = new ArrayList<>();
225 for (final InstructionId id : input.getId()) {
226 // Find the instruction
227 final InstructionImpl i = this.insns.get(id);
229 LOG.debug("Instruction {} not present in the graph", input.getId());
235 switch (i.getStatus()) {
244 LOG.debug("Instruction {} cannot be cleaned because of it's in state {}", id, i.getStatus());
251 // The instruction is in a terminal state, we need to just unlink
252 // it from its dependencies and dependents
255 this.insns.remove(id);
256 LOG.debug("Instruction {} cleaned successfully", id);
259 final CleanInstructionsOutputBuilder ob = new CleanInstructionsOutputBuilder();
260 ob.setUnflushed(failed);
262 return SuccessfulRpcResult.create(ob.build());
265 private List<InstructionImpl> checkDependencies(final SubmitInstructionInput input) throws SchedulerException {
266 final List<InstructionImpl> dependencies = collectDependencies(input);
267 // Check if all dependencies are non-failed
268 final List<InstructionId> unmet = checkIfUnfailed(dependencies);
270 * Some dependencies have failed, declare the request dead-on-arrival
271 * and fail the operation.
273 if (!unmet.isEmpty()) {
274 throw new SchedulerException("Instruction's dependencies are already unsuccessful", new FailureBuilder().setType(
275 DeadOnArrival.class).setFailedPreconditions(unmet).build());
280 private List<InstructionImpl> collectDependencies(final SubmitInstructionInput input) throws SchedulerException {
281 final List<InstructionImpl> dependencies = new ArrayList<>();
282 for (final InstructionId pid : input.getPreconditions()) {
283 final InstructionImpl i = this.insns.get(pid);
285 LOG.info("Instruction {} depends on {}, which is not a known instruction", input.getId(), pid);
286 throw new SchedulerException("Unknown dependency ID specified", new FailureBuilder().setType(UnknownPreconditionId.class).build());
293 private List<InstructionId> checkIfUnfailed(final List<InstructionImpl> dependencies) {
294 final List<InstructionId> unmet = new ArrayList<>();
295 for (final InstructionImpl d : dependencies) {
296 switch (d.getStatus()) {
300 unmet.add(d.getId());
315 public synchronized ListenableFuture<Instruction> scheduleInstruction(final SubmitInstructionInput input) throws SchedulerException {
316 final InstructionId id = input.getId();
317 if (this.insns.get(id) != null) {
318 LOG.info("Instruction ID {} already present", id);
319 throw new SchedulerException("Instruction ID currently in use", new FailureBuilder().setType(DuplicateInstructionId.class).build());
322 // First things first: check the deadline
323 final Nanotime now = NanotimeUtil.currentTime();
324 final BigInteger left = input.getDeadline().getValue().subtract(now.getValue());
326 if (left.compareTo(BigInteger.ZERO) <= 0) {
327 LOG.debug("Instruction {} deadline has already passed by {}ns", id, left);
328 throw new SchedulerException("Instruction arrived after specified deadline", new FailureBuilder().setType(DeadOnArrival.class).build());
331 // Resolve dependencies
332 final List<InstructionImpl> dependencies = checkDependencies(input);
335 * All pre-flight checks done are at this point, the following
336 * steps can only fail in catastrophic scenarios (OOM and the
340 // Schedule a timeout for the instruction
341 final Timeout t = this.timer.newTimeout(timeout -> timeoutInstruction(input.getId()), left.longValue(), TimeUnit.NANOSECONDS);
343 // Put it into the instruction list
344 final SettableFuture<Instruction> ret = SettableFuture.create();
345 final InstructionImpl i = new InstructionImpl(new InstructionPusher(id, input.getDeadline()), ret, id, dependencies, t);
346 this.insns.put(id, i);
348 // Attach it into its dependencies
349 for (final InstructionImpl d : dependencies) {
354 * All done. The next part is checking whether the instruction can
355 * run, which we can figure out after sending out the acknowledgement.
356 * This task should be ingress-weighed, so we reinsert it into the
357 * same execution service.
359 this.executor.submit(() -> tryScheduleInstruction(i));
364 public String getInstructionID() {
365 return this.instructionId;
368 private synchronized void timeoutInstruction(final InstructionId id) {
369 final InstructionImpl i = this.insns.get(id);
371 LOG.warn("Instruction {} timed out, but not found in the queue", id);
378 private synchronized void tryScheduleDependants(final InstructionImpl i) {
379 // Walk all dependants and try to schedule them
380 final Iterator<InstructionImpl> it = i.getDependants();
381 while (it.hasNext()) {
382 tryScheduleInstruction(it.next());
386 private synchronized void tryScheduleInstruction(final InstructionImpl i) {
387 final ListenableFuture<ExecutionResult<Details>> f = i.ready();
389 Futures.addCallback(f, new FutureCallback<ExecutionResult<Details>>() {
391 public void onSuccess(final ExecutionResult<Details> result) {
392 tryScheduleDependants(i);
396 public void onFailure(final Throwable t) {
397 LOG.error("Instruction {} failed to execute", i.getId(), t);
405 public ListenableFuture<Void> closeServiceInstance() {
406 LOG.info("Closing Instruction Queue service {}", this.sgi.getValue());
408 if (this.writeConfiguration != null) {
409 this.writeConfiguration.remove();
412 for (final InstructionImpl i : this.insns.values()) {
415 // Workaround for BUG-2283
416 final WriteTransaction t = this.dataProvider.newWriteOnlyTransaction();
417 t.delete(LogicalDatastoreType.OPERATIONAL, this.qid);
418 final CheckedFuture<Void, TransactionCommitFailedException> future = t.submit();
419 Futures.addCallback(future, new FutureCallback<Void>() {
421 public void onSuccess(final Void result) {
422 LOG.debug("Instruction Queue {} removed", ProgrammingServiceImpl.this.qid);
426 public void onFailure(final Throwable t) {
427 LOG.error("Failed to shutdown Instruction Queue {}", ProgrammingServiceImpl.this.qid, t);
434 public synchronized void close() {
435 if (this.csspReg != null) {
437 this.csspReg.close();
438 } catch (final Exception e) {
439 LOG.debug("Failed to close Instruction Scheduler service", e);
442 if (this.serviceRegistration != null) {
443 this.serviceRegistration.unregister();
444 this.serviceRegistration = null;
448 void setServiceRegistration(final ServiceRegistration<?> serviceRegistration) {
449 this.serviceRegistration = serviceRegistration;