support for bandwidth profiles in ovs-driver support for updating bandwidth profile... 10/66610/5
authorBartosz Michalik <bartosz.michalik@amartus.com>
Tue, 7 Nov 2017 08:22:57 +0000 (09:22 +0100)
committerBartosz Michalik <bartosz.michalik@amartus.com>
Thu, 4 Jan 2018 11:24:53 +0000 (11:24 +0000)
Change-Id: Ieb43893d4fca49cee1fa662add9595bcae1833ee
Signed-off-by: Bartosz Michalik <Bartosz.Michalik@amartus.com>
Signed-off-by: Donald Hunter <donaldh@cisco.com>
15 files changed:
impl/src/main/java/org/opendaylight/unimgr/mef/nrp/api/ActivationDriver.java
impl/src/main/java/org/opendaylight/unimgr/mef/nrp/impl/ActivationTransaction.java
impl/src/main/java/org/opendaylight/unimgr/mef/nrp/impl/connectivityservice/TapiConnectivityServiceImpl.java
impl/src/main/java/org/opendaylight/unimgr/mef/nrp/impl/connectivityservice/UpdateConnectivityAction.java [new file with mode: 0644]
impl/src/test/java/org/opendaylight/unimgr/mef/nrp/impl/connectivityservice/TapiConnectivityServiceInplIntTest.java
nrp-api/src/main/yang/nrp-interface.yang
ovs-driver/pom.xml
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/activator/OvsActivator.java
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/activator/OvsActivatorHelper.java
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/driver/OvsDriver.java
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/util/ActionUtils.java
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/util/OpenFlowUtils.java
ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/util/OvsdbUtils.java [new file with mode: 0644]
ovs-driver/src/test/java/org/opendaylight/unimgr/mef/nrp/ovs/OvsdbTopologyTestUtils.java
ovs-driver/src/test/java/org/opendaylight/unimgr/mef/nrp/ovs/activator/OvsActivatorTest.java

index f5428afc5885a0e84b046ff9ac0ea7c443c4c3f9..89a95e8cd0fc1beb9ef2b9f7d9fca92ca3e1ced6 100644 (file)
@@ -52,6 +52,16 @@ public interface ActivationDriver {
      */
     void activate() throws TransactionCommitFailedException, ResourceActivatorException;
 
+    /**
+     * Performs the update action.
+     * @throws TransactionCommitFailedException
+     * @throws ResourceActivatorException
+     */
+    default void update() throws TransactionCommitFailedException, ResourceActivatorException {
+        deactivate();
+        activate();
+    }
+
     /**
      * Performs the deactivation action.
      * @throws TransactionCommitFailedException
index 3f559d5534af0be5c75f54f882b8b835550bc292..209380104b6a2097f347da7801e488932e314b4c 100644 (file)
@@ -58,6 +58,32 @@ public class ActivationTransaction {
         }
     }
 
+    /**
+     * Update the contents of this transaction.
+     * @return result
+     */
+    public Result update() {
+        if (drivers.isEmpty()) {
+            throw new IllegalStateException("at least one driver required");
+        }
+        sortDrivers();
+        try {
+            for (ActivationDriver d: drivers) {
+                d.update();
+            }
+            commit();
+            LOG.info("Update transaction successful");
+
+            return Result.success();
+        } catch (Exception e) {
+            //XXX add transaction identification ???
+            LOG.warn("Rolling back update transaction ", e);
+            rollback();
+
+            return Result.fail(e.getMessage(), e);
+        }
+    }
+
     /**
      * Deactivate the contents of this transaction.
      * @return result
index b8573ef3b52b42b5860a4d78e322af47b76a679d..a0b03eb6d6ad2576db70e2d22f9777a68c96b38d 100644 (file)
@@ -75,7 +75,7 @@ public class TapiConnectivityServiceImpl implements TapiConnectivityService, Aut
 
     @Override
     public Future<RpcResult<UpdateConnectivityServiceOutput>> updateConnectivityService(UpdateConnectivityServiceInput input) {
-        return null;
+       return executor.submit(new UpdateConnectivityAction(this, input));
     }
 
     @Override
diff --git a/impl/src/main/java/org/opendaylight/unimgr/mef/nrp/impl/connectivityservice/UpdateConnectivityAction.java b/impl/src/main/java/org/opendaylight/unimgr/mef/nrp/impl/connectivityservice/UpdateConnectivityAction.java
new file mode 100644 (file)
index 0000000..7de03f9
--- /dev/null
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2017 Cisco Systems Inc and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.unimgr.mef.nrp.impl.connectivityservice;
+
+import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
+import org.opendaylight.unimgr.mef.nrp.api.ActivationDriver;
+import org.opendaylight.unimgr.mef.nrp.api.EndPoint;
+import org.opendaylight.unimgr.mef.nrp.api.FailureResult;
+import org.opendaylight.unimgr.mef.nrp.api.TapiConstants;
+import org.opendaylight.unimgr.mef.nrp.common.NrpDao;
+import org.opendaylight.unimgr.mef.nrp.impl.ActivationTransaction;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.nrp._interface.rev170712.EndPoint5;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.common.rev170712.Uuid;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.UpdateConnectivityServiceInput;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.UpdateConnectivityServiceOutput;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.UpdateConnectivityServiceOutputBuilder;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.connectivity.context.ConnectivityService;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.update.connectivity.service.output.ServiceBuilder;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.topology.rev170712.Node;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.topology.rev170712.topology.context.Topology;
+import org.opendaylight.yangtools.yang.common.RpcError.ErrorType;
+import org.opendaylight.yangtools.yang.common.RpcResult;
+import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+public class UpdateConnectivityAction implements Callable<RpcResult<UpdateConnectivityServiceOutput>> {
+
+       private static final Logger LOG = LoggerFactory.getLogger(UpdateConnectivityServiceOutput.class);
+
+       private TapiConnectivityServiceImpl service;
+       private final UpdateConnectivityServiceInput input;
+       private EndPoint endpoint;
+
+       public UpdateConnectivityAction(TapiConnectivityServiceImpl tapiConnectivityService,
+                       UpdateConnectivityServiceInput input) {
+
+               Objects.requireNonNull(tapiConnectivityService);
+               Objects.requireNonNull(input);
+               this.service = tapiConnectivityService;
+               this.input = input;
+       }
+
+       @Override
+       public RpcResult<UpdateConnectivityServiceOutput> call() throws Exception {
+
+               LOG.debug("running UpdateConnectivityService task");
+
+               NrpDao nrpDao = new NrpDao(service.getBroker().newReadOnlyTransaction());
+               try {
+                       // TODO validate input
+                       // RequestValidator.ValidationResult validationResult =
+                       // validateInput();
+                       // if (!validationResult.isValid()) {
+                       // RpcResultBuilder<UpdateConnectivityServiceOutput> res =
+                       // RpcResultBuilder.failed();
+                       // validationResult.getProblems().forEach(p ->
+                       // res.withError(RpcError.ErrorType.APPLICATION, p));
+                       // return res.build();
+                       //
+                       // }
+
+                       endpoint = new EndPoint(input.getEndPoint(), input.getEndPoint().getAugmentation(EndPoint5.class));
+
+                       String serviceId = input.getServiceIdOrName();
+
+                       ActivationTransaction tx = prepareTransaction(nrpDao, serviceId);
+                       if (tx != null) {
+                               ActivationTransaction.Result txResult = tx.update();
+                               if (txResult.isSuccessful()) {
+                                       LOG.info("ConnectivityService construct updated successfully, request = {} ", input);
+
+                                       ConnectivityService service = nrpDao.getConnectivityService(serviceId);
+                                       UpdateConnectivityServiceOutput result = new UpdateConnectivityServiceOutputBuilder()
+                                                       .setService(new ServiceBuilder(service).build()).build();
+                                       return RpcResultBuilder.success(result).build();
+                               } else {
+                                       LOG.warn("UpdateConnectivityService failed, reason = {}, request = {}", txResult.getMessage(),
+                                                       input);
+                               }
+                       }
+                       throw new IllegalStateException("no transaction created for update connectivity request");
+
+               } catch (Exception e) {
+                       LOG.warn("Exception in update connectivity service", e);
+                       return RpcResultBuilder.<UpdateConnectivityServiceOutput>failed()
+                                       .withError(ErrorType.APPLICATION, e.getMessage()).build();
+               }
+
+       }
+
+       private ActivationTransaction prepareTransaction(NrpDao nrpDao, String serviceId) throws FailureResult {
+               ActivationTransaction tx = new ActivationTransaction();
+
+               Optional<Uuid> nodeUuid = getNodeUuid(nrpDao);
+               if (nodeUuid.isPresent()) {
+                       Optional<ActivationDriver> driver = service.getDriverRepo().getDriver(nodeUuid.get());
+                       if (!driver.isPresent()) {
+                               throw new IllegalStateException(MessageFormat.format("driver {} cannot be created", nodeUuid.get()));
+                       }
+                       driver.get().initialize(Arrays.asList(endpoint), serviceId, null);
+                       tx.addDriver(driver.get());
+               }
+               return tx;
+
+       }
+
+       private Optional<Uuid> getNodeUuid(NrpDao nrpDao) throws FailureResult {
+               Optional<Uuid> result = Optional.empty();
+               try {
+                       Topology prestoTopo = nrpDao.getTopology(TapiConstants.PRESTO_SYSTEM_TOPO);
+                       if (prestoTopo.getNode() == null) {
+                               throw new FailureResult("There are no nodes in {0} topology", TapiConstants.PRESTO_SYSTEM_TOPO);
+                       }
+                       for (Node node : prestoTopo.getNode()) {
+                               if (node.getOwnedNodeEdgePoint().stream().filter(nep -> nep.getMappedServiceInterfacePoint() != null).flatMap(nep -> nep.getMappedServiceInterfacePoint().stream())
+                                               .anyMatch(sipUuid -> sipUuid.equals(endpoint.getEndpoint().getServiceInterfacePoint()))) {
+                                       return Optional.of(node.getUuid());
+                               }
+                       }
+               } catch (ReadFailedException e) {
+                       throw new FailureResult("Cannot read {0} topology", TapiConstants.PRESTO_SYSTEM_TOPO);
+               }
+               return result;
+       }
+}
index cadd10c9749b66ad36ec8bbe91d0422224152eee..7d32cbc3b06ddda9dde940a82bf01d39d5c6045e 100644 (file)
@@ -131,7 +131,6 @@ public class TapiConnectivityServiceInplIntTest extends AbstractTestWithTopo {
         verify(ad1).activate();
         verify(ad1).commit();
         verifyZeroInteractions(ad2);
-        verifyZeroInteractions(ad2);
 
         ReadOnlyTransaction tx2 = dataBroker.newReadOnlyTransaction();
         Context1 connCtx = tx2.read(LogicalDatastoreType.OPERATIONAL, TapiConnectivityServiceImpl.connectivityCtx).checkedGet().get();
@@ -342,7 +341,7 @@ public class TapiConnectivityServiceInplIntTest extends AbstractTestWithTopo {
     private org.opendaylight.yang.gen.v1.urn.mef.yang.tapi.connectivity.rev170712.connectivity.context.ConnectivityService cs(String csId, Uuid connectionId) {
         return new ConnectivityServiceBuilder()
                 .setUuid(new Uuid(csId))
-                .setConnection(Arrays.asList(connectionId))
+                .setConnection(Collections.singletonList(connectionId))
                 .build();
     }
 
index 92b44adadef5cd8c025784b3b72218457a07a17d..f3e6fc9e79bd1ee73f1b5cb828be64e25b920bc0 100644 (file)
@@ -32,6 +32,12 @@ module nrp-interface {
         uses nrp-connectivity-service-end-point-attrs;
         description "none";
     }
+
+    augment "/tapi-connectivity:update-connectivity-service/tapi-connectivity:input/tapi-connectivity:end-point" {
+        uses nrp-connectivity-service-end-point-attrs;
+        description "none";
+    }
+
     augment "/tapi-common:context/tapi-connectivity:connectivity-service/tapi-connectivity:end-point" {
         uses nrp-connectivity-service-end-point-attrs;
         description "none";
index 9dbb55c6dcec20e324ff510b479ca8bf178c82d1..2b01c5eccc3e3edf00a9b3b176ea2c3ba7174576 100644 (file)
@@ -14,6 +14,7 @@
         <checkstyle.skip>true</checkstyle.skip>
         <powermock.version>1.6.4</powermock.version>
         <openflow.version>0.6.0-SNAPSHOT</openflow.version>
+        <ovsdb.version>1.6.0-SNAPSHOT</ovsdb.version>
     </properties>
 
     <modelVersion>4.0.0</modelVersion>
             <version>${project.version}</version>
         </dependency>
 
+       <dependency>
+               <groupId>org.opendaylight.ovsdb</groupId>
+               <artifactId>utils.mdsal-utils</artifactId>
+               <version>${ovsdb.version}</version>
+       </dependency>
+
         <!-- Testing Dependencies -->
         <dependency>
             <groupId>org.opendaylight.controller</groupId>
index 0462143fd45287236cac09386f5687fbee3952f4..0dad56e45f1d917fa369ca365bdc87cee86e3e41 100644 (file)
@@ -7,6 +7,12 @@
  */
 package org.opendaylight.unimgr.mef.nrp.ovs.activator;
 
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
 import org.opendaylight.unimgr.mef.nrp.api.EndPoint;
@@ -15,6 +21,7 @@ import org.opendaylight.unimgr.mef.nrp.common.ResourceNotAvailableException;
 import org.opendaylight.unimgr.mef.nrp.ovs.transaction.TableTransaction;
 import org.opendaylight.unimgr.mef.nrp.ovs.transaction.TopologyTransaction;
 import org.opendaylight.unimgr.mef.nrp.ovs.util.OpenFlowUtils;
+import org.opendaylight.unimgr.mef.nrp.ovs.util.OvsdbUtils;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
@@ -22,18 +29,17 @@ import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * @author marek.ryznar@amartus.com
  */
 public class OvsActivator implements ResourceActivator {
 
     private final DataBroker dataBroker;
-    private String serviceName;
     private static final Logger LOG = LoggerFactory.getLogger(OvsActivator.class);
 
+    //TODO introduce poll synced with ovsdb config
+    private static final AtomicInteger queueNumberGenerator = new AtomicInteger(200);
+
     public OvsActivator(DataBroker dataBroker) {
         this.dataBroker = dataBroker;
     }
@@ -44,12 +50,12 @@ public class OvsActivator implements ResourceActivator {
      */
     @Override
     public void activate(List<EndPoint> endPoints, String serviceName) throws ResourceNotAvailableException, TransactionCommitFailedException {
-        this.serviceName = serviceName;
         for (EndPoint endPoint:endPoints)
-            activateEndpoint(endPoint);
+            activateEndpoint(endPoint, serviceName);
     }
 
-    private void activateEndpoint(EndPoint endPoint) throws ResourceNotAvailableException, TransactionCommitFailedException {
+
+    private void activateEndpoint(EndPoint endPoint, String serviceName) throws ResourceNotAvailableException, TransactionCommitFailedException {
         // Transaction - Get Open vSwitch node and its flow table
         String portName = OvsActivatorHelper.getPortName(endPoint.getEndpoint().getServiceInterfacePoint().getValue());
         TopologyTransaction topologyTransaction = new TopologyTransaction(dataBroker);
@@ -68,35 +74,94 @@ public class OvsActivator implements ResourceActivator {
 
         OvsActivatorHelper ovsActivatorHelper = new OvsActivatorHelper(topologyTransaction, endPoint);
         String openFlowPortName = ovsActivatorHelper.getOpenFlowPortName();
-        int externalVlanId = ovsActivatorHelper.getCeVlanId();
+        long queueNumber = queueNumberGenerator.getAndIncrement();
         int internalVlanId = ovsActivatorHelper.getInternalVlanId();
-        flowsToWrite.addAll(OpenFlowUtils.getVlanFlows(openFlowPortName, externalVlanId, internalVlanId, interswitchLinks, serviceName));
+        flowsToWrite.addAll(OpenFlowUtils.getVlanFlows(openFlowPortName, internalVlanId, interswitchLinks, serviceName, queueNumber));
 
         // Transaction - Add flows related to service to table and remove unnecessary flows
         TableTransaction tableTransaction = new TableTransaction(dataBroker, node, table);
         tableTransaction.deleteFlows(flowsToDelete, true);
         tableTransaction.writeFlows(flowsToWrite);
+
+               List<String> outputPortNames = interswitchLinks.stream()
+                               .map(link -> ovsActivatorHelper.getTpNameFromOpenFlowPortName(link.getLinkId().getValue()))
+                               .collect(Collectors.toList());
+
+        //Create egress qos
+               OvsdbUtils.createEgressQos(dataBroker, portName, outputPortNames, ovsActivatorHelper.getQosMinRate(),
+                               ovsActivatorHelper.getQosMaxRate(), serviceName, queueNumber);
+
     }
 
     @Override
     public void deactivate(List<EndPoint> endPoints, String serviceName) throws TransactionCommitFailedException, ResourceNotAvailableException {
         for (EndPoint endPoint:endPoints)
-            deactivateEndpoint(endPoint);
+            deactivateEndpoint(endPoint, serviceName);
     }
 
-    private void deactivateEndpoint(EndPoint endPoint) throws ResourceNotAvailableException, TransactionCommitFailedException {
+    private void deactivateEndpoint(EndPoint endPoint, String serviceName) throws ResourceNotAvailableException, TransactionCommitFailedException {
+
         // Transaction - Get Open vSwitch node and its flow table
         TopologyTransaction topologyTransaction = new TopologyTransaction(dataBroker);
-        OvsActivatorHelper ovsActivatorHelper = new OvsActivatorHelper(topologyTransaction,endPoint);
+        OvsActivatorHelper ovsActivatorHelper = new OvsActivatorHelper(topologyTransaction, endPoint);
 
-        Node node = topologyTransaction.readNodeOF(ovsActivatorHelper.getOpenFlowPortName());
-        Table table = OpenFlowUtils.getTable(node);
+        Node openFlowNode = topologyTransaction.readNodeOF(ovsActivatorHelper.getOpenFlowPortName());
+        Table table = OpenFlowUtils.getTable(openFlowNode);
         // Get list of flows to be removed
         List<Flow> flowsToDelete = OpenFlowUtils.getServiceFlows(table, serviceName);
 
         // Transaction - Remove flows related to service from table
-        TableTransaction tableTransaction = new TableTransaction(dataBroker, node, table);
+        TableTransaction tableTransaction = new TableTransaction(dataBroker, openFlowNode, table);
         tableTransaction.deleteFlows(flowsToDelete, false);
+
+        String portName = OvsActivatorHelper.getPortName(endPoint.getEndpoint().getServiceInterfacePoint().getValue());
+        Node node = topologyTransaction.readNode(portName);
+
+        //list with endpoint + all interswitch ports
+       List<String> tpsWithQos = topologyTransaction.readInterswitchLinks(node).stream()
+                               .map(link -> ovsActivatorHelper.getTpNameFromOpenFlowPortName(link.getLinkId().getValue()))
+                               .collect(Collectors.toList());
+       tpsWithQos.add(portName);
+
+        OvsdbUtils.removeQosEntryFromTerminationPoints(dataBroker, serviceName, tpsWithQos);
     }
 
+       public void update(List<EndPoint> endPoints, String serviceName) throws ResourceNotAvailableException, TransactionCommitFailedException {
+        for (EndPoint endPoint:endPoints) {
+            updateEndpoint(endPoint, serviceName);
+        }
+       }
+
+       private void updateEndpoint(EndPoint endPoint, String serviceName) throws ResourceNotAvailableException, TransactionCommitFailedException{
+
+               TopologyTransaction topologyTransaction = new TopologyTransaction(dataBroker);
+           OvsActivatorHelper ovsActivatorHelper = new OvsActivatorHelper(topologyTransaction, endPoint);
+
+               String portName = OvsActivatorHelper.getPortName(endPoint.getEndpoint().getServiceInterfacePoint().getValue());
+               Node node = topologyTransaction.readNode(portName);
+
+               //list with endpoint + all interswitch ports
+               List<String> interswitchPorts = topologyTransaction.readInterswitchLinks(node).stream()
+                               .map(link -> ovsActivatorHelper.getTpNameFromOpenFlowPortName(link.getLinkId().getValue()))
+                               .collect(Collectors.toList());
+
+               List<String> tpsWithQos = new LinkedList<>(interswitchPorts);
+               tpsWithQos.add(portName);
+
+               //remove old egress qos
+               OvsdbUtils.removeQosEntryFromTerminationPoints(dataBroker, serviceName, tpsWithQos);
+
+
+               long queueNumber = queueNumberGenerator.getAndIncrement();
+        //Create egress qos
+               OvsdbUtils.createEgressQos(dataBroker, portName, interswitchPorts, ovsActivatorHelper.getQosMinRate(),
+                               ovsActivatorHelper.getQosMaxRate(), serviceName, queueNumber);
+
+               //modify flow with new queue number
+                Table table = OpenFlowUtils.getTable(node);
+        TableTransaction tableTransaction = new TableTransaction(dataBroker, node, table);
+               tableTransaction.writeFlow(OpenFlowUtils.createVlanIngressFlow(ovsActivatorHelper.getOpenFlowPortName(), ovsActivatorHelper.getInternalVlanId(),
+                               serviceName, topologyTransaction.readInterswitchLinks(node), queueNumber));
+       }
+
 }
index 0eb86cd812fc4f097ca70ab047093c7f93cb71f3..4c276cc8c27ca74a8efa06ad7c8fd9cbd954983a 100644 (file)
@@ -7,16 +7,14 @@
  */
 package org.opendaylight.unimgr.mef.nrp.ovs.activator;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import org.opendaylight.unimgr.mef.nrp.api.EndPoint;
 import org.opendaylight.unimgr.mef.nrp.common.ResourceNotAvailableException;
 import org.opendaylight.unimgr.mef.nrp.ovs.exception.VlanNotSetException;
 import org.opendaylight.unimgr.mef.nrp.ovs.transaction.TopologyTransaction;
-import org.opendaylight.unimgr.mef.nrp.ovs.util.VlanUtils;
 import org.opendaylight.unimgr.utils.NullAwareDatastoreGetter;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.nrm.connectivity.rev170712.carrier.eth.connectivity.end.point.resource.IngressBwpFlow;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.nrp._interface.rev170712.nrp.connectivity.service.end.point.attrs.NrpCarrierEthConnectivityEndPointResource;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
@@ -24,6 +22,9 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.N
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+
 /**
  * Helper class for OvsDriver activation.
  *
@@ -33,9 +34,10 @@ class OvsActivatorHelper {
     private List<NullAwareDatastoreGetter<Node>> nodes;
     private EndPoint endPoint;
     private String tpName;
-    private Map<String, String> portMap;
+    private BiMap<String, String> portMap;
 
     private static final String CTAG_VLAN_ID_NOT_SET_ERROR_MESSAGE = "C-Tag VLAN Id not set for End Point '%s'.";
+    private static final String INGRESS_BWP_FLOW_NOT_SET_ERROR_MESSAGE = "Ingress bwp flow is not set for End Point '%s'.";
     private static final String ATTRS_NOT_SET_ERROR_MESSAGE = "End Point '%s' does not have '%s' set.";
 
 
@@ -77,29 +79,39 @@ class OvsActivatorHelper {
      * @return Integer with VLAN Id
      */
     int getInternalVlanId() throws ResourceNotAvailableException {
-        VlanUtils vlanUtils = new VlanUtils(nodes);
-        int serviceVlanId = getCeVlanId();
 
-        if (vlanUtils.isVlanInUse(serviceVlanId)) {
-            LOG.debug("VLAN ID = '" + serviceVlanId + "' already in use.");
-            return vlanUtils.generateVlanID();
-        } else {
-            LOG.debug("VLAN ID = '" + serviceVlanId + "' not in use.");
-            return serviceVlanId;
-        }
+        return getCeVlanId();
+//             VlanUtils vlanUtils = new VlanUtils(nodes);
+//             Disable VLAN pool, refactor in the future
+//        if (vlanUtils.isVlanInUse(serviceVlanId)) {
+//            LOG.debug("VLAN ID = '" + serviceVlanId + "' already in use.");
+//            return vlanUtils.generateVlanID();
+//        } else {
+//            LOG.debug("VLAN ID = '" + serviceVlanId + "' not in use.");
+//            return serviceVlanId;
+//        }
     }
 
     /**
-     * Returns port name in openflow plugin convention (e. openflow:1:4)
+     * Returns port name in openflow plugin naming convention (e.g. openflow:1:4)
      *
-     * @return String with port name
+     * @return String with openflow port name
      */
     String getOpenFlowPortName() {
         return portMap.get(tpName);
     }
 
-    private Map<String, String> createPortMap(List<NullAwareDatastoreGetter<Node>> nodes) {
-        Map<String, String> portMap = new HashMap<>();
+    /**
+     * Returns port name for specifiec port name in openflow convention
+     * @param openFlowPortName port in openflow plugin naming convention
+     * @return String with port name
+     */
+    String getTpNameFromOpenFlowPortName(String openFlowPortName) {
+       return portMap.inverse().get(openFlowPortName);
+    }
+
+    private BiMap<String, String> createPortMap(List<NullAwareDatastoreGetter<Node>> nodes) {
+       BiMap<String, String> portMap = HashBiMap.create();
         for (NullAwareDatastoreGetter<Node> node : nodes) {
             if (node.get().isPresent()) {
                 for (NodeConnector nodeConnector : node.get().get().getNodeConnector()) {
@@ -117,4 +129,34 @@ class OvsActivatorHelper {
         String[] tab = sip.split(":");
         return tab[tab.length-1];
     }
+
+       public long getQosMinRate() throws ResourceNotAvailableException {
+               if ( (endPoint.getAttrs() != null) && (endPoint.getAttrs().getNrpCarrierEthConnectivityEndPointResource() != null) ) {
+                       NrpCarrierEthConnectivityEndPointResource attr = endPoint.getAttrs().getNrpCarrierEthConnectivityEndPointResource();
+                       IngressBwpFlow ingressBwpFlow = attr.getIngressBwpFlow();
+                       if(ingressBwpFlow != null) {
+                               //TODO add validation
+                               return ingressBwpFlow.getCir().getValue();
+                       } else {
+                LOG.warn(String.format(INGRESS_BWP_FLOW_NOT_SET_ERROR_MESSAGE, tpName));
+                throw new ResourceNotAvailableException(String.format(INGRESS_BWP_FLOW_NOT_SET_ERROR_MESSAGE, tpName));
+                       }
+               }
+               return 0;
+       }
+
+       public long getQosMaxRate() throws ResourceNotAvailableException {
+               if ( (endPoint.getAttrs() != null) && (endPoint.getAttrs().getNrpCarrierEthConnectivityEndPointResource() != null) ) {
+                       NrpCarrierEthConnectivityEndPointResource attr = endPoint.getAttrs().getNrpCarrierEthConnectivityEndPointResource();
+                       IngressBwpFlow ingressBwpFlow = attr.getIngressBwpFlow();
+                       if(ingressBwpFlow != null) {
+                               //TODO add validation
+                               return ingressBwpFlow.getCir().getValue() + ingressBwpFlow.getEir().getValue();
+                       } else {
+                LOG.warn(String.format(INGRESS_BWP_FLOW_NOT_SET_ERROR_MESSAGE, tpName));
+                throw new ResourceNotAvailableException(String.format(INGRESS_BWP_FLOW_NOT_SET_ERROR_MESSAGE, tpName));
+                       }
+               }
+               return 0;
+       }
 }
index da98150342d1af100080cef3fd3b54cf2b008801..3cc8bb4c0ba44d94a9a6363850589f7b7098880c 100644 (file)
@@ -15,6 +15,7 @@ import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFaile
 import org.opendaylight.unimgr.mef.nrp.api.ActivationDriver;
 import org.opendaylight.unimgr.mef.nrp.api.ActivationDriverBuilder;
 import org.opendaylight.unimgr.mef.nrp.api.EndPoint;
+import org.opendaylight.unimgr.mef.nrp.common.ResourceActivatorException;
 import org.opendaylight.unimgr.mef.nrp.common.ResourceNotAvailableException;
 import org.opendaylight.unimgr.mef.nrp.ovs.activator.OvsActivator;
 import org.opendaylight.unimgr.mef.nrp.ovs.tapi.TopologyDataHandler;
@@ -59,6 +60,11 @@ public class OvsDriver implements ActivationDriverBuilder {
                 activator.activate(endPoints,serviceId);
             }
 
+                       @Override
+                       public void update() throws TransactionCommitFailedException, ResourceActivatorException {
+                               activator.update(endPoints,serviceId);
+                       }
+
             @Override
             public void deactivate() throws TransactionCommitFailedException, ResourceNotAvailableException {
                 activator.deactivate(endPoints,serviceId);
index 02fcf12857b2bfbecd2aa3e36bc3b03ae611c935..0c3aaed21a7acfb220e7c195b4f9bc0c58ddd509 100644 (file)
@@ -7,14 +7,23 @@
  */
 package org.opendaylight.unimgr.mef.nrp.ovs.util;
 
+import java.util.LinkedList;
+import java.util.List;
+
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Uri;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.*;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.DropActionCaseBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCaseBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PopVlanActionCaseBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCaseBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCaseBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetQueueActionCaseBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.drop.action._case.DropActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.output.action._case.OutputActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.pop.vlan.action._case.PopVlanActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.push.vlan.action._case.PushVlanAction;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.push.vlan.action._case.PushVlanActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.set.field._case.SetFieldBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.set.queue.action._case.SetQueueActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Instructions;
@@ -27,9 +36,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.l2.types.rev130827.VlanId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.VlanMatchBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.vlan.match.fields.VlanIdBuilder;
 
-import java.util.LinkedList;
-import java.util.List;
-
 /**
  * Utility class providing common operations for Instruction and Action objects.
  *
@@ -162,4 +168,18 @@ class ActionUtils {
             throw new IllegalArgumentException(String.format(INVALID_PORT_NAME_FORMAT_ERROR_MESSAGE, portName));
         }
     }
+
+       static Action createSetQueueNumberAction(long queueNumber, int order) {
+        ActionBuilder actionBuilder = new ActionBuilder();
+
+        SetQueueActionBuilder setQueueActionBuilder = new SetQueueActionBuilder();
+        setQueueActionBuilder.setQueueId(queueNumber);
+
+        SetQueueActionCaseBuilder setQueueActionCaseBuilder = new SetQueueActionCaseBuilder();
+        setQueueActionCaseBuilder.setSetQueueAction(setQueueActionBuilder.build());
+
+        actionBuilder.setAction(setQueueActionCaseBuilder.build());
+        actionBuilder.setOrder(order);
+        return actionBuilder.build();
+       }
 }
index dde3067da47cdf192d04c665d36e4155c715ee52..be8f1279980c3156b247dfc3405c2bd24dbac1ba 100644 (file)
@@ -92,19 +92,19 @@ public class OpenFlowUtils {
     }
 
     /**
-     * Returns list of flows for passing traffic with given VLAN ID
+     * Returns list of flows for passing traffic using decorated S-VLAN ID via qos queue numer
      *
      * @param servicePort port on which service is activated (format: openflow:[node]:[port])
-     * @param externalVlanId VLAN ID used outside OvSwitch network
      * @param internalVlanId VLAN ID used internally in OvSwitch network
      * @param interswitchLinks list of interswitch links for the node on which service is activated
      * @param serviceName service name (used as prefix for flow IDs)
+     * @param queueNumber qos queue number
      * @return list of flows
      */
-    public static List<Flow> getVlanFlows(String servicePort, int externalVlanId, int internalVlanId, List<Link> interswitchLinks, String serviceName) {
+    public static List<Flow> getVlanFlows(String servicePort, int internalVlanId, List<Link> interswitchLinks, String serviceName, long queueNumber) {
         List<Flow> flows = new ArrayList<>();
-        flows.addAll(createVlanPassingFlows(servicePort, externalVlanId, internalVlanId, serviceName, interswitchLinks));
-        flows.add(createVlanIngressFlow(servicePort, externalVlanId, internalVlanId, serviceName, interswitchLinks));
+        flows.addAll(createVlanPassingFlows(servicePort, internalVlanId, serviceName, interswitchLinks));
+        flows.add(createVlanIngressFlow(servicePort, internalVlanId, serviceName, interswitchLinks, queueNumber));
 
         return flows;
     }
@@ -184,25 +184,20 @@ public class OpenFlowUtils {
                                 .build();
     }
 
-    private static List<Flow> createVlanPassingFlows(String outputPort, int externalVlanId, int internalVlanId, String serviceName, List<Link> interswitchLinks) {
+    private static List<Flow> createVlanPassingFlows(String outputPort, int internalVlanId, String serviceName, List<Link> interswitchLinks) {
         return interswitchLinks.stream()
-                               .map(link -> createVlanPassingFlow(outputPort, link.getLinkId().getValue(), externalVlanId, internalVlanId, serviceName))
+                               .map(link -> createVlanPassingFlow(outputPort, link.getLinkId().getValue(), internalVlanId, serviceName))
                                .collect(Collectors.toList());
     }
 
-    private static Flow createVlanPassingFlow(String outputPort, String inputPort, int externalVlanId, int internalVlanId, String serviceName) {
+    private static Flow createVlanPassingFlow(String outputPort, String inputPort, int internalVlanId, String serviceName) {
         // Create list of actions and VLAN match
         List<Action> actions = new ArrayList<>();
         Match vlanMatch;
         int actionOrder = 0;
-        if (externalVlanId == internalVlanId) {
-            vlanMatch = MatchUtils.createVlanMatch(externalVlanId, inputPort);
-        } else {
-            vlanMatch = MatchUtils.createVlanMatch(internalVlanId, inputPort);
-            actions.add(ActionUtils.createPopVlanAction(actionOrder++));
-            actions.add(ActionUtils.createPushVlanAction(actionOrder++));
-            actions.add(ActionUtils.createSetVlanIdAction(externalVlanId, actionOrder++));
-        }
+
+        vlanMatch = MatchUtils.createVlanMatch(internalVlanId, inputPort);
+        actions.add(ActionUtils.createPopVlanAction(actionOrder++));
         actions.add(ActionUtils.createOutputAction(outputPort, actionOrder));
 
         FlowId flowId = new FlowId(getVlanFlowId(serviceName, inputPort));
@@ -215,7 +210,7 @@ public class OpenFlowUtils {
                 .build();
     }
 
-    private static Flow createVlanIngressFlow(String inputPort, int externalVlanId, int internalVlanId, String serviceName, List<Link> interswitchLinks) {
+    public static Flow createVlanIngressFlow(String inputPort, int internalVlanId, String serviceName, List<Link> interswitchLinks, long queueNumber) {
         // Create list of output port IDs
         List<String> outputPortIds = interswitchLinks.stream()
                                                      .map(link -> link.getLinkId().getValue())
@@ -226,11 +221,10 @@ public class OpenFlowUtils {
         List<Action> actions = new ArrayList<>();
         int actionOrder = 0;
         // 1. Create VLAN actions performing VLAN translation when service VLAN is already used in OvSwitch network
-        if (externalVlanId != internalVlanId) {
-            actions.add(ActionUtils.createPopVlanAction(actionOrder++));
-            actions.add(ActionUtils.createPushVlanAction(actionOrder++));
-            actions.add(ActionUtils.createSetVlanIdAction(internalVlanId, actionOrder++));
-        }
+        actions.add(ActionUtils.createPushVlanAction(actionOrder++));
+        actions.add(ActionUtils.createSetVlanIdAction(internalVlanId, actionOrder++));
+        actions.add(ActionUtils.createSetQueueNumberAction(queueNumber, actionOrder++));
+
         // 2. Create output actions
         final int outputActionOrder = actionOrder;
         actions.addAll(outputPortIds.stream()
@@ -243,7 +237,8 @@ public class OpenFlowUtils {
                                 .setKey(new FlowKey(flowId))
                                 .setTableId(FLOW_TABLE_ID)
                                 .setPriority(VLAN_FLOW_PRIORITY)
-                                .setMatch(MatchUtils.createVlanMatch(externalVlanId, inputPort))
+//                                .setMatch(MatchUtils.createVlanMatch(externalVlanId, inputPort))
+                                .setMatch(MatchUtils.createInPortMatch(inputPort))
                                 .setInstructions(ActionUtils.createInstructions(actions))
                                 .build();
     }
diff --git a/ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/util/OvsdbUtils.java b/ovs-driver/src/main/java/org/opendaylight/unimgr/mef/nrp/ovs/util/OvsdbUtils.java
new file mode 100644 (file)
index 0000000..3d154fb
--- /dev/null
@@ -0,0 +1,486 @@
+/*
+ * Copyright (c) 2017 Cisco Systems Inc and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.unimgr.mef.nrp.ovs.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.ovsdb.southbound.SouthboundConstants;
+import org.opendaylight.ovsdb.utils.mdsal.utils.NotifyingDataChangeListener;
+import org.opendaylight.unimgr.utils.MdsalUtils;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Uri;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbNodeAugmentation;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbQosRef;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbQueueRef;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentationBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.QosTypeLinuxHtb;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.QosEntries;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.QosEntriesBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.Queues;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.QueuesBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.qos.entries.QueueList;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.qos.entries.QueueListBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.queues.QueuesOtherConfig;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.queues.QueuesOtherConfigBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.QosEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.QosEntryBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.QosEntryKey;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.TopologyId;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.TopologyKey;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.NodeKey;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
+import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointBuilder;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.CheckedFuture;
+
+/**
+ * Class responsible for managing OVSDB Nodes
+ *
+ * @author marek.ryznar@amartus.com
+ */
+public class OvsdbUtils {
+    private static final TopologyId ovsdbTopoId = new TopologyId(new Uri("ovsdb:1"));
+    private static final NodeId odlNodeId = new NodeId(new Uri("odl"));
+    private static final Logger LOG = LoggerFactory.getLogger(OvsdbUtils.class);
+    
+    /**
+        * Configures ovs egress qos shaping with following steps:
+        * <ul>
+        * 
+        * <li>Create qos queue on all port (tp + output ports)</li>
+        * <li>Create qos entry with queues created in previous step</li>
+        * <li>Add created qos entries to termination point</li> 
+        * </ul>
+        * 
+        * @param dataBroker
+        *            access to data tree store
+        * @param tpId
+        *            termination point id
+        * @param outputPorts
+        *            list of output aka interswitch/internal ports
+        * @param minRate
+        *            qos min-rate
+        * @param maxRate
+        *            qos max-rate
+        * @param serviceName
+        *            name of service
+        * @param queueNumber
+        *            qos queue number
+        */
+    public static void createEgressQos(DataBroker dataBroker, String tpId, List<String> outputPorts, Long minRate, Long maxRate, String serviceName, Long queueNumber){
+       
+       String qosQueueId = serviceName + "_queue_" + queueNumber;
+       String qosEntryId = serviceName + "_qos_" + queueNumber;
+       
+       //create queue
+       Queues queue = createQueue(minRate, maxRate, qosQueueId);
+        InstanceIdentifier<Queues> queueInstanceIdentifier = getQueueInstanceIdentifier(queue);
+       addQueueToConfigDatastore(dataBroker, queue, queueInstanceIdentifier);
+
+               // create qos entry that contains ref to queue uuid, it has to be
+               // present in operational config, that's why it happens separate tx
+       QosEntries qosEntry = createQosEntry(qosEntryId, queueNumber, queueInstanceIdentifier);
+        InstanceIdentifier<QosEntries> qosInstanceIdentifier = getQosEntryInstanceIdentifier(qosEntry);
+        addQosEntryToConfigDatastore(dataBroker, qosEntry, qosInstanceIdentifier);
+
+
+               //bind created qos to endpoint + all interswitchLinks on this this node
+               Optional<TerminationPoint> otp = findTerminationPoint(dataBroker, tpId);
+               Optional<Node> optNode = findBridgeNode(dataBroker, tpId);
+               
+               addQosEntryToTerminationPoint(dataBroker, qosInstanceIdentifier, otp, optNode);
+               
+               for (String outputTpId : outputPorts) {
+                       otp = findTerminationPoint(dataBroker, outputTpId);
+                       addQosEntryToTerminationPoint(dataBroker, qosInstanceIdentifier, otp, optNode);
+               }
+    }    
+
+       private static void deleteQosFromConfigDatastore(DataBroker dataBroker, InstanceIdentifier<?> qosId) {
+               WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+               tx.delete(LogicalDatastoreType.CONFIGURATION, qosId);
+               final CheckedFuture<Void, TransactionCommitFailedException> future = tx.submit();
+               try {
+                       future.checkedGet();
+                       LOG.info("Succesfully removed Qos entry from Config datastore: {}", qosId.toString());
+               } catch (final TransactionCommitFailedException e) {
+                       LOG.warn("Failed to remove Qos entry from Config datastore: {}", qosId.toString(), e);
+               }
+       }
+       
+
+       private static void deleteQueueFromConfigDatastore(DataBroker dataBroker, InstanceIdentifier<?> queueId) {
+               WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+               tx.delete(LogicalDatastoreType.CONFIGURATION, queueId);
+               final CheckedFuture<Void, TransactionCommitFailedException> future = tx.submit();
+               try {
+                       future.checkedGet();
+                       LOG.info("Succesfully removed Qos Queue from Config datastore: {}", queueId.toString());
+               } catch (final TransactionCommitFailedException e) {
+                       LOG.warn("Failed to remove Qos Queue from Config datastore: {}", queueId.toString(), e);
+               }
+       }
+       
+       private static void deleteTerminationPointQosEntryFromConfigDatastore(DataBroker dataBroker, InstanceIdentifier<QosEntry> qosEntryId) {
+               WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+               tx.delete(LogicalDatastoreType.CONFIGURATION, qosEntryId);
+               final CheckedFuture<Void, TransactionCommitFailedException> future = tx.submit();
+               try {
+                       future.checkedGet();
+                       LOG.info("Succesfully removed Termination Point Qos entry from  Config datastore: {}", qosEntryId.toString());
+               } catch (final TransactionCommitFailedException e) {
+                       LOG.warn("Failed to remove Termination Point Qos entry from Config datastore: {}", qosEntryId.toString(), e);
+               }
+       }
+
+       private static void addQosEntryToTerminationPoint(DataBroker dataBroker,
+                       InstanceIdentifier<QosEntries> qosInstanceIdentifier, Optional<TerminationPoint> otp,
+                       Optional<Node> optNode) {
+               if (otp.isPresent() && optNode.isPresent()) {
+                       TerminationPoint tp = otp.get();
+                       tp = buildTerminationPoint(tp, qosInstanceIdentifier);
+                       InstanceIdentifier<TerminationPoint> tpIid = getTerminationPointInstanceIdentifier(optNode.get(), tp);
+                       
+               final NotifyingDataChangeListener qosEntryToTpOperationalListener = new NotifyingDataChangeListener(
+                                       LogicalDatastoreType.OPERATIONAL, tpIid, null);
+               qosEntryToTpOperationalListener.registerDataChangeListener(dataBroker);
+                       
+                       WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                       tx.merge(LogicalDatastoreType.CONFIGURATION, tpIid, tp, true);
+                       
+                       
+                       final CheckedFuture<Void, TransactionCommitFailedException> future = tx.submit();
+                       try {
+                               future.checkedGet();
+                               LOG.info("Succesfully added Qos Uiid to termination point: {}", tp.getTpId().getValue());
+                       } catch (final TransactionCommitFailedException e) {
+                               LOG.warn("Failed to add Qos Uiid to termination point: {} ", tp.getTpId().getValue(), e);
+                       }
+                       
+                       try {
+                               qosEntryToTpOperationalListener.waitForUpdate();
+                       } catch (InterruptedException e) {
+                               LOG.warn("Sleep interrupted while waiting for Qos entry to Termination Point update {}", qosInstanceIdentifier, e);
+                       } finally {
+                               try {
+                                       qosEntryToTpOperationalListener.close();
+                               } catch (Exception e) {
+                                       LOG.debug(
+                                                       "Failed to properly close qosEntryToTpOperationalListener while waiting for qos entry to TerminationPoint update oper {}",
+                                                       qosInstanceIdentifier, e);
+                               }
+                       }
+               }
+
+       }
+
+    /**
+        * Removes egress qos shaping from ovsdb for specified service:
+        * <ul>
+        * <li>Remove all qos entries to TerminationPoints assignments for specified
+        * service from config datastore</li>
+        * <li>Remove all qos entries assinged to specified TerminationPoints and
+        * service from config datastore</li>
+        * <li>Remove all queues assigned to qos entries for specified TerminationPoints
+        * and service from config datastore</li>
+        * </ul>
+        * 
+        * @param dataBroker
+        *            access to data tree store
+        * @param serviceName
+        *            name of the service
+        * @param tpsWithQos
+        *            list of TerminationPoints that might contain QoS
+        */
+       public static void removeQosEntryFromTerminationPoints(DataBroker dataBroker, String serviceName,
+                       List<String> tpsWithQos) {
+               Optional<Node> optNode = findBridgeNode(dataBroker, tpsWithQos.get(0));
+               Set<InstanceIdentifier<?>> qosEntriesTodelete = new HashSet<>();
+               if (optNode.isPresent()) {
+                       for (String tpId : tpsWithQos) {
+                               Optional<TerminationPoint> otp = findTerminationPoint(dataBroker, tpId);
+                               if (otp.isPresent()) {
+                                       TerminationPoint tp = otp.get();
+                                       OvsdbTerminationPointAugmentation ovsdbTpAug = tp
+                                                       .getAugmentation(OvsdbTerminationPointAugmentation.class);
+                                       if (ovsdbTpAug != null && ovsdbTpAug.getQosEntry() != null) {
+                                               for (QosEntry qosEntry : ovsdbTpAug.getQosEntry()) {
+                                                       OvsdbQosRef qosRef = qosEntry.getQosRef();
+                                                       if (qosRef.getValue().toString().contains(serviceName)) {
+                                                               deleteTerminationPointQosEntryFromConfigDatastore(dataBroker,
+                                                                               getTerminationPointQosEntryInstanceIdentifier(optNode.get(), tp, qosEntry));
+                                                               qosEntriesTodelete.add(qosRef.getValue());
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+               }
+               for (InstanceIdentifier<?> qosEntryToDeleteId : qosEntriesTodelete) {
+                       QosEntries qosEntries = (QosEntries) MdsalUtils.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
+                                       qosEntryToDeleteId);
+                       if (qosEntries != null) {
+                               List<QueueList> queueList = qosEntries.getQueueList();
+                               deleteQosFromConfigDatastore(dataBroker, qosEntryToDeleteId);
+                               queueList.stream()
+                                               .forEach(ql -> deleteQueueFromConfigDatastore(dataBroker, ql.getQueueRef().getValue()));
+                       }
+               }
+       }
+
+       private static void addQueueToConfigDatastore(DataBroker dataBroker, Queues queue,
+                       InstanceIdentifier<Queues> queueInstanceIdentifier) {
+               
+       final NotifyingDataChangeListener queueCreateOperationalListener = new NotifyingDataChangeListener(
+                               LogicalDatastoreType.OPERATIONAL, queueInstanceIdentifier, null);
+               queueCreateOperationalListener.registerDataChangeListener(dataBroker);
+               
+               WriteTransaction createQoSQueueTx = dataBroker.newWriteOnlyTransaction();
+               createQoSQueueTx.merge(LogicalDatastoreType.CONFIGURATION, queueInstanceIdentifier, queue, true);
+
+               final CheckedFuture<Void, TransactionCommitFailedException> futureCreateQoSQueueTx = createQoSQueueTx.submit();
+               try {
+                       futureCreateQoSQueueTx.checkedGet();
+                       LOG.info("Succesfully created QoS queue :{}", queueInstanceIdentifier);
+               } catch (final TransactionCommitFailedException e) {
+                       LOG.warn("Failed to create new QoS queue: {} ", queueInstanceIdentifier, e);
+               }
+
+               try {
+                       queueCreateOperationalListener.waitForCreation();
+               } catch (InterruptedException e) {
+                       LOG.warn("Sleep interrupted while waiting for Qos queue creation {}", queueInstanceIdentifier, e);
+               } finally {
+                       try {
+                               queueCreateOperationalListener.close();
+                       } catch (Exception e) {
+                               LOG.debug(
+                                               "Failed to properly close queueCreateOperationalListener while waiting for qos queue creation {}",
+                                               queueInstanceIdentifier, e);
+                       }
+               }
+
+       }
+
+       public static void addQosEntryToConfigDatastore(DataBroker dataBroker, QosEntries qosEntry, InstanceIdentifier<QosEntries> qosInstanceIdentifier) {
+       final NotifyingDataChangeListener qosEntryOperationalListener = new NotifyingDataChangeListener(
+                               LogicalDatastoreType.OPERATIONAL, qosInstanceIdentifier, null);
+               qosEntryOperationalListener.registerDataChangeListener(dataBroker);
+               
+       WriteTransaction createQosEntryTx = dataBroker.newWriteOnlyTransaction();
+       createQosEntryTx.merge(LogicalDatastoreType.CONFIGURATION, qosInstanceIdentifier, qosEntry, true);
+       
+        final CheckedFuture<Void, TransactionCommitFailedException> futureCreateQosEntryTx = createQosEntryTx.submit();
+        try {
+               futureCreateQosEntryTx.checkedGet();
+               LOG.info("Succesfully created QoS entry: {}", qosInstanceIdentifier);
+        } catch (final TransactionCommitFailedException e) {
+            LOG.warn("Failed to create new QoS entry: {} ", qosInstanceIdentifier, e);
+               }
+        
+               try {
+                       qosEntryOperationalListener.waitForCreation();
+               } catch (InterruptedException e) {
+                       LOG.warn("Sleep interrupted while waiting for qos entry creation {}", qosInstanceIdentifier, e);
+               } finally {
+                       try {
+                               qosEntryOperationalListener.close();
+                       } catch (Exception e) {
+                               LOG.debug(
+                                               "Failed to properly close qosEntryOperationalListener while waiting for qos entry creation {}",
+                                               qosInstanceIdentifier, e);
+                       }
+               }
+    }
+
+       private static QosEntries createQosEntry(String qosEntryId, Long queueNumber, InstanceIdentifier<Queues> queueId) {
+               QosEntriesBuilder qosEntriesBuilder = new QosEntriesBuilder();
+               qosEntriesBuilder.setQosId(new Uri(qosEntryId));
+               qosEntriesBuilder.setQosType(QosTypeLinuxHtb.class);
+
+               QueueListBuilder queueListBuilder = new QueueListBuilder();
+               queueListBuilder.setQueueNumber(queueNumber);
+               queueListBuilder.setQueueRef(new OvsdbQueueRef(queueId));
+               
+               List<QueueList> queueList = new LinkedList<>();
+               queueList.add(queueListBuilder.build());
+               
+               qosEntriesBuilder.setQueueList(queueList);
+               
+               return qosEntriesBuilder.build();
+       }
+
+       private static Queues createQueue(Long minRate, Long maxRate, String qosQueueId) {
+               QueuesBuilder queuesBuilder = new QueuesBuilder();
+               queuesBuilder.setQueueId(new Uri(qosQueueId));
+
+               LinkedList<QueuesOtherConfig> queuesOtherConfigList = new LinkedList<>();
+               QueuesOtherConfigBuilder queuesMaxRateOtherConfigBuilder = new QueuesOtherConfigBuilder();
+               queuesMaxRateOtherConfigBuilder.setQueueOtherConfigKey("max-rate");
+               queuesMaxRateOtherConfigBuilder.setQueueOtherConfigValue(maxRate.toString());
+               queuesOtherConfigList.add(queuesMaxRateOtherConfigBuilder.build());
+               
+               QueuesOtherConfigBuilder queuesMinRateOtherConfigBuilder = new QueuesOtherConfigBuilder();
+               queuesMinRateOtherConfigBuilder.setQueueOtherConfigKey("min-rate");
+               queuesMinRateOtherConfigBuilder.setQueueOtherConfigValue(minRate.toString());
+               queuesOtherConfigList.add(queuesMinRateOtherConfigBuilder.build());
+
+               queuesBuilder.setQueuesOtherConfig(queuesOtherConfigList);
+
+               return queuesBuilder.build();
+       }
+
+       
+       private static InstanceIdentifier<Queues> getQueueInstanceIdentifier(Queues queue) {
+               return InstanceIdentifier.create(NetworkTopology.class)
+                               .child(Topology.class, new TopologyKey(ovsdbTopoId))
+                               .child(Node.class, new NodeKey(odlNodeId))
+                               .augmentation(OvsdbNodeAugmentation.class)
+                               .child(Queues.class, queue.getKey());
+       }
+       
+       private static InstanceIdentifier<QosEntries> getQosEntryInstanceIdentifier(QosEntries qosEntries) {
+               return InstanceIdentifier.create(NetworkTopology.class)
+                               .child(Topology.class, new TopologyKey(ovsdbTopoId))
+                               .child(Node.class, new NodeKey(odlNodeId))
+                               .augmentation(OvsdbNodeAugmentation.class)
+                               .child(QosEntries.class, qosEntries.getKey());
+       }
+       
+    private static InstanceIdentifier<TerminationPoint> getTerminationPointInstanceIdentifier(Node bridgeNode, TerminationPoint tp){
+        return InstanceIdentifier
+                        .create(NetworkTopology.class)
+                        .child(Topology.class,
+                                new TopologyKey(ovsdbTopoId))
+                        .child(Node.class, bridgeNode.getKey())
+                        .child(TerminationPoint.class, tp.getKey());
+    }
+    
+    private static InstanceIdentifier<QosEntry> getTerminationPointQosEntryInstanceIdentifier(Node bridgeNode, TerminationPoint tp, QosEntry qosEntry){
+        return InstanceIdentifier
+                        .create(NetworkTopology.class)
+                        .child(Topology.class,
+                                new TopologyKey(ovsdbTopoId))
+                        .child(Node.class, bridgeNode.getKey())
+                        .child(TerminationPoint.class, tp.getKey())
+                        .augmentation(OvsdbTerminationPointAugmentation.class)
+                        .child(QosEntry.class, qosEntry.getKey());
+    }
+
+    private static TerminationPoint buildTerminationPoint(TerminationPoint tp, InstanceIdentifier<QosEntries> qosInstanceIdentifier){
+        TerminationPointBuilder terminationPointBuilder = new TerminationPointBuilder();
+                 terminationPointBuilder.setKey(tp.getKey());
+                terminationPointBuilder.setTpId(tp.getTpId());
+        terminationPointBuilder.addAugmentation(OvsdbTerminationPointAugmentation.class,addQosToPort(tp.getTpId().getValue(), qosInstanceIdentifier));
+        return terminationPointBuilder.build();
+    }
+
+    private static OvsdbTerminationPointAugmentation addQosToPort(String name, InstanceIdentifier<QosEntries> qosInstanceIdentifier){
+        OvsdbTerminationPointAugmentationBuilder ovsdbTerminationPointAugmentationBuilder = new OvsdbTerminationPointAugmentationBuilder();        
+               List<QosEntry> qosList = new ArrayList<>();
+               OvsdbQosRef qosRef = new OvsdbQosRef(qosInstanceIdentifier);
+               qosList.add(new QosEntryBuilder().setKey(new QosEntryKey(new Long(SouthboundConstants.PORT_QOS_LIST_KEY)))
+                               .setQosRef(qosRef).build());
+               ovsdbTerminationPointAugmentationBuilder.setQosEntry(qosList);
+
+        ovsdbTerminationPointAugmentationBuilder.setName(name);
+
+
+        return ovsdbTerminationPointAugmentationBuilder.build();
+    }
+    
+    private static Optional<TerminationPoint> findTerminationPoint(DataBroker dataBroker, String port){
+        List<Node> ovsdbNodes = getOvsdbNodes(dataBroker);
+        Optional<TerminationPoint> terminationPoint = Optional.empty();
+        if (!ovsdbNodes.isEmpty()) {
+            terminationPoint = ovsdbNodes.stream()
+                    .flatMap(node -> {
+                        if(node.getTerminationPoint()!=null)
+                            return node.getTerminationPoint().stream();
+                        return Stream.empty();
+                    })
+                    .filter(tp -> tp.getTpId().getValue().equals(port))
+                    .findFirst();
+        }
+
+        return terminationPoint;
+    }
+
+    private static Optional<Node> findBridgeNode(DataBroker dataBroker, String tpId){
+        List<Node> ovsdbNodes = getOvsdbNodes(dataBroker);
+        return ovsdbNodes.stream()
+                .filter(node -> {
+                            if(node.getTerminationPoint()!=null){
+                                return node.getTerminationPoint().stream()
+                                        .anyMatch(tp -> tp.getTpId().getValue().equals(tpId));
+                            } else {
+                                return false;
+                            }
+                        }
+                ).findFirst();
+
+    }
+
+    /**
+     * Retrieve a list of Ovsdb Nodes from the Operational DataStore.
+     *
+     * @param dataBroker The dataBroker instance to create transactions
+     * @return The Ovsdb Node retrieved from the Operational DataStore
+     */
+    public static List<Node> getOvsdbNodes(DataBroker dataBroker) {
+        final InstanceIdentifier<Topology> ovsdbTopoIdentifier = InstanceIdentifier
+                .create(NetworkTopology.class)
+                .child(Topology.class,
+                        new TopologyKey(ovsdbTopoId));
+
+        Topology topology = MdsalUtils.read(dataBroker,
+                LogicalDatastoreType.OPERATIONAL,
+                ovsdbTopoIdentifier);
+
+        if ((topology != null) && (topology.getNode() != null)){
+            return topology.getNode();
+        }
+        return Collections.emptyList();
+    }
+    
+    /**
+     * Retrieve a Ovsdb Odl node from the Operational DataStore.
+     *
+     * @param dataBroker The dataBroker instance to create transactions
+     * @return The Ovsdb Odl Node retrieved from the Operational DataStore
+     */
+    public static Node getOdlNode(DataBroker dataBroker) {
+        final InstanceIdentifier<Node> ovsdbOdlNodeIdentifier = InstanceIdentifier
+                .create(NetworkTopology.class)
+                .child(Topology.class,
+                        new TopologyKey(ovsdbTopoId))
+                .child(Node.class, new NodeKey(odlNodeId));
+
+        return MdsalUtils.read(dataBroker,
+                LogicalDatastoreType.OPERATIONAL,
+                ovsdbOdlNodeIdentifier);
+    }
+}
index 7421422a7c49a943d38b62a79ff56af75ca05503..a1a8b4f40e48876631bde838fa7cad309246197f 100644 (file)
@@ -57,6 +57,7 @@ public class OvsdbTopologyTestUtils {
         nodeBuilder.setNodeId(new NodeId(nodeId));
         nodeBuilder.setTerminationPoint(tps);
         nodeBuilder.addAugmentation(OvsdbBridgeAugmentation.class,createOvsdbBridgeAugmentation(nodeId));
+        nodeBuilder.addAugmentation(OvsdbNodeAugmentation.class, new OvsdbNodeAugmentationBuilder().build());
         return nodeBuilder.build();
     }
 
index 3bc5fcd821b606ecfd173b618d892a7c9e77f6d6..69c15463c68ce4705fa7e05c6d13ea1008c30df3 100644 (file)
@@ -33,10 +33,12 @@ import org.opendaylight.unimgr.mef.nrp.ovs.FlowTopologyTestUtils;
 import org.opendaylight.unimgr.mef.nrp.ovs.OpenFlowTopologyTestUtils;
 import org.opendaylight.unimgr.mef.nrp.ovs.OvsdbTopologyTestUtils;
 import org.opendaylight.unimgr.mef.nrp.ovs.util.OpenFlowUtils;
+import org.opendaylight.unimgr.mef.nrp.ovs.util.OvsdbUtils;
 import org.opendaylight.unimgr.utils.MdsalUtils;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.mef.types.rev170712.NaturalNumber;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.mef.types.rev170712.PositiveInteger;
-import org.opendaylight.yang.gen.v1.urn.mef.yang.nrm.connectivity.rev170712.VlanIdListAndUntag;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.nrm.connectivity.rev170712.carrier.eth.connectivity.end.point.resource.CeVlanIdListAndUntag;
+import org.opendaylight.yang.gen.v1.urn.mef.yang.nrm.connectivity.rev170712.carrier.eth.connectivity.end.point.resource.IngressBwpFlow;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.nrm.connectivity.rev170712.vlan.id.list.and.untag.VlanId;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.nrp._interface.rev170712.NrpConnectivityServiceEndPointAttrs;
 import org.opendaylight.yang.gen.v1.urn.mef.yang.nrp._interface.rev170712.nrp.connectivity.service.end.point.attrs.NrpCarrierEthConnectivityEndPointResource;
@@ -128,7 +130,6 @@ public class OvsActivatorTest extends AbstractDataBrokerTest{
 
         List<Flow> vlanFlows = flows.stream()
                 .filter(flow -> flow.getId().getValue().contains(vlanName))
-                .filter(flow -> flow.getMatch().getVlanMatch().getVlanId().getVlanId().getValue().equals(expectedVlanId))
                 .collect(Collectors.toList());
         assertEquals(interswitchPortCount+1,vlanFlows.size());
 
@@ -192,10 +193,18 @@ public class OvsActivatorTest extends AbstractDataBrokerTest{
         when(ceVlanIdList.getVlanId())
                 .thenReturn(vlanIds);
 
+        IngressBwpFlow ingressBwpFlow = mock(IngressBwpFlow.class);
+        when(ingressBwpFlow.getCir()).thenReturn(new NaturalNumber(4000000L));
+        when(ingressBwpFlow.getEir()).thenReturn(new NaturalNumber(4000000L));
+
         NrpCarrierEthConnectivityEndPointResource nrpCgEthFrameFlowCpaAspec =
                 mock(NrpCarrierEthConnectivityEndPointResource.class);
+
+               when(nrpCgEthFrameFlowCpaAspec.getIngressBwpFlow())
+                .thenReturn(ingressBwpFlow);
+
         when(nrpCgEthFrameFlowCpaAspec.getCeVlanIdListAndUntag())
-                .thenReturn(ceVlanIdList);
+        .thenReturn(ceVlanIdList);
 
         when(attrs.getNrpCarrierEthConnectivityEndPointResource())
                 .thenReturn(nrpCgEthFrameFlowCpaAspec);
@@ -214,6 +223,7 @@ public class OvsActivatorTest extends AbstractDataBrokerTest{
         bridges.add(createBridge("s3",4));
         bridges.add(createBridge("s4",3));
         bridges.add(createBridge("s5",4));
+        bridges.add(createBridge("odl", 0));
 
         bridges.forEach(node -> {
             OvsdbTopologyTestUtils.writeBridge(node,dataBroker);