55dc19fd1ee872d7faeeacd04da3a21f842d117e
[ovsdb.git] / southbound / southbound-it / src / test / java / org / opendaylight / ovsdb / southbound / it / SouthboundIT.java
1 /*
2  * Copyright (c) 2015 Red Hat, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.ovsdb.southbound.it;
9
10 import static org.junit.Assert.assertNotNull;
11 import static org.junit.Assert.fail;
12 import static org.ops4j.pax.exam.CoreOptions.maven;
13 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
14 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
15
16 import com.google.common.collect.ImmutableBiMap;
17 import com.google.common.collect.Lists;
18 import com.google.common.collect.ObjectArrays;
19
20 import java.net.InetAddress;
21 import java.net.UnknownHostException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Properties;
25
26 import javax.inject.Inject;
27
28 import org.junit.Assert;
29 import org.junit.Assume;
30 import org.junit.Before;
31 import org.junit.Test;
32 import org.junit.runner.RunWith;
33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
34 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
35 import org.opendaylight.ovsdb.southbound.SouthboundConstants;
36 import org.opendaylight.ovsdb.southbound.SouthboundMapper;
37 import org.opendaylight.ovsdb.southbound.SouthboundProvider;
38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
39 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.PortNumber;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbBridgeAugmentation;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbBridgeAugmentationBuilder;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbBridgeName;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbBridgeProtocolBase;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbNodeAugmentation;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbNodeRef;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentationBuilder;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.bridge.attributes.ProtocolEntry;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.bridge.attributes.ProtocolEntryBuilder;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.ConnectionInfo;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.ConnectionInfoBuilder;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.InterfaceTypeEntryBuilder;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.OpenvswitchOtherConfigs;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.InterfaceExternalIds;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.InterfaceExternalIdsBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.InterfaceOtherConfigs;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.InterfaceOtherConfigsBuilder;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.Options;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.OptionsBuilder;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.PortExternalIds;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.PortExternalIdsBuilder;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.PortOtherConfigs;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.port._interface.attributes.PortOtherConfigsBuilder;
64 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology;
65 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
66 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.TpId;
67 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology;
68 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.TopologyKey;
69 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
70 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.NodeBuilder;
71 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
72 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointBuilder;
73 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
74 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
75 import org.ops4j.pax.exam.Configuration;
76 import org.ops4j.pax.exam.Option;
77 import org.ops4j.pax.exam.junit.PaxExam;
78 import org.ops4j.pax.exam.karaf.options.LogLevelOption;
79 import org.ops4j.pax.exam.options.MavenUrlReference;
80 import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
81 import org.ops4j.pax.exam.spi.reactors.PerClass;
82 import org.osgi.framework.Bundle;
83 import org.osgi.framework.BundleContext;
84 import org.slf4j.Logger;
85 import org.slf4j.LoggerFactory;
86
87 /**
88  * Integration tests for southbound-impl
89  *
90  * @author Sam Hague (shague@redhat.com)
91  */
92 @RunWith(PaxExam.class)
93 @ExamReactorStrategy(PerClass.class)
94 public class SouthboundIT extends AbstractMdsalTestBase {
95     private static final Logger LOG = LoggerFactory.getLogger(SouthboundIT.class);
96     private static final int OVSDB_UPDATE_TIMEOUT = 1000;
97     private static DataBroker dataBroker = null;
98     private static String addressStr;
99     private static String portStr;
100     private static String connectionType;
101     private static Boolean setup = false;
102     private static MdsalUtils mdsalUtils = null;
103     private static String extras = "false";
104     private static final String NETVIRT = "org.opendaylight.ovsdb.openstack.net-virt";
105     private static final String NETVIRTPROVIDERS = "org.opendaylight.ovsdb.openstack.net-virt-providers";
106
107     @Inject
108     private BundleContext bundleContext;
109
110     @Configuration
111     public Option[] config() {
112         return super.config();
113     }
114
115     @Override
116     public String getModuleName() {
117         return "southbound-impl";
118     }
119
120     @Override
121     public String getInstanceName() {
122         return "southbound-default";
123     }
124
125     @Override
126     public MavenUrlReference getFeatureRepo() {
127         return maven()
128                 .groupId("org.opendaylight.ovsdb")
129                 .artifactId("southbound-features")
130                 .classifier("features")
131                 .type("xml")
132                 .versionAsInProject();
133     }
134
135     @Override
136     public String getFeatureName() {
137         setExtras();
138         return "odl-ovsdb-southbound-impl-ui";
139     }
140
141     protected String usage() {
142         return "Integration Test needs a valid connection configuration as follows :\n"
143                 + "active connection : mvn -Dovsdbserver.ipaddress=x.x.x.x -Dovsdbserver.port=yyyy verify\n"
144                 + "passive connection : mvn -Dovsdbserver.connection=passive verify\n";
145     }
146
147     @Override
148     public Option[] getFeaturesOptions() {
149         if (extras.equals("true")) {
150             Option[] options = new Option[] {
151                     features("mvn:org.opendaylight.ovsdb/features-ovsdb/1.1.0-SNAPSHOT/xml/features",
152                             "odl-ovsdb-openstack-sb")};
153             return options;
154         } else {
155             return new Option[]{};
156         }
157     }
158
159     @Override
160     public Option[] getLoggingOptions() {
161         Option[] options = new Option[] {
162                 /*editConfigurationFilePut(SouthboundITConstants.ORG_OPS4J_PAX_LOGGING_CFG,
163                         "log4j.logger.org.opendaylight.ovsdb",
164                         LogLevelOption.LogLevel.DEBUG.name()),*/
165                 editConfigurationFilePut(SouthboundITConstants.ORG_OPS4J_PAX_LOGGING_CFG,
166                         "log4j.logger.org.opendaylight.ovsdb.southbound-impl",
167                         LogLevelOption.LogLevel.DEBUG.name())
168         };
169
170         if (extras.equals("true")) {
171             Option[] extraOptions = new Option[] {
172                 editConfigurationFilePut(SouthboundITConstants.ORG_OPS4J_PAX_LOGGING_CFG,
173                         "log4j.logger.org.opendaylight.ovsdb",
174                         LogLevelOption.LogLevel.DEBUG.name()),
175                 editConfigurationFilePut(SouthboundITConstants.ORG_OPS4J_PAX_LOGGING_CFG,
176                         "log4j.logger.org.opendaylight.ovsdb.openstack.net-virt",
177                         LogLevelOption.LogLevel.DEBUG.name())
178             };
179             options = ObjectArrays.concat(options, extraOptions, Option.class);
180         }
181
182         options = ObjectArrays.concat(options, super.getLoggingOptions(), Option.class);
183         return options;
184     }
185
186     @Override
187     public Option[] getPropertiesOptions() {
188         Properties props = new Properties(System.getProperties());
189         String addressStr = props.getProperty(SouthboundITConstants.SERVER_IPADDRESS,
190                 SouthboundITConstants.DEFAULT_SERVER_IPADDRESS);
191         String portStr = props.getProperty(SouthboundITConstants.SERVER_PORT,
192                 SouthboundITConstants.DEFAULT_SERVER_PORT);
193         String connectionType = props.getProperty(SouthboundITConstants.CONNECTION_TYPE,
194                 SouthboundITConstants.CONNECTION_TYPE_ACTIVE);
195
196         LOG.info("Using the following properties: mode= {}, ip:port= {}:{}",
197                 connectionType, addressStr, portStr);
198
199         Option[] options = new Option[] {
200                 editConfigurationFilePut(SouthboundITConstants.CUSTOM_PROPERTIES,
201                         SouthboundITConstants.SERVER_IPADDRESS, addressStr),
202                 editConfigurationFilePut(SouthboundITConstants.CUSTOM_PROPERTIES,
203                         SouthboundITConstants.SERVER_PORT, portStr),
204                 editConfigurationFilePut(SouthboundITConstants.CUSTOM_PROPERTIES,
205                         SouthboundITConstants.CONNECTION_TYPE, connectionType)
206         };
207         return options;
208     }
209
210     private void setExtras() {
211         Properties props = new Properties(System.getProperties());
212         extras = props.getProperty(SouthboundITConstants.SERVER_EXTRAS,
213                 SouthboundITConstants.DEFAULT_SERVER_EXTRAS);
214         LOG.info("extras: {}", extras);
215         System.out.println("extras: " + extras);
216     }
217
218     @Before
219     public void setUp() throws InterruptedException {
220         if (setup == true) {
221             LOG.info("Skipping setUp, already initialized");
222             return;
223         }
224
225         try {
226             super.setup();
227         } catch (Exception e) {
228             e.printStackTrace();
229         }
230         //dataBroker = getSession().getSALService(DataBroker.class);
231         Thread.sleep(3000);
232         dataBroker = SouthboundProvider.getDb();
233         Assert.assertNotNull("db should not be null", dataBroker);
234
235         addressStr = bundleContext.getProperty(SouthboundITConstants.SERVER_IPADDRESS);
236         portStr = bundleContext.getProperty(SouthboundITConstants.SERVER_PORT);
237         connectionType = bundleContext.getProperty(SouthboundITConstants.CONNECTION_TYPE);
238
239         LOG.info("Using the following properties: mode= {}, ip:port= {}:{}",
240                 connectionType, addressStr, portStr);
241         if (connectionType.equalsIgnoreCase(SouthboundITConstants.CONNECTION_TYPE_ACTIVE)) {
242             if (addressStr == null) {
243                 fail(usage());
244             }
245         }
246
247         mdsalUtils = new MdsalUtils(dataBroker);
248         setup = true;
249
250         if (extras.equals("true")) {
251             isBundleReady(bundleContext, NETVIRT);
252             isBundleReady(bundleContext, NETVIRTPROVIDERS);
253         }
254     }
255
256     /**
257      * Test passive connection mode. The southbound starts in a listening mode waiting for connections on port
258      * 6640. This test will wait for incoming connections for {@link SouthboundITConstants.CONNECTION_INIT_TIMEOUT} ms.
259      *
260      * @throws InterruptedException
261      */
262     @Test
263     public void testPassiveNode() throws InterruptedException {
264         if (connectionType.equalsIgnoreCase(SouthboundITConstants.CONNECTION_TYPE_PASSIVE)) {
265             //Wait for CONNECTION_INIT_TIMEOUT for the Passive connection to be initiated by the ovsdb-server.
266             Thread.sleep(SouthboundITConstants.CONNECTION_INIT_TIMEOUT);
267         }
268     }
269
270     private ConnectionInfo getConnectionInfo(String addressStr, String portStr) {
271         InetAddress inetAddress = null;
272         try {
273             inetAddress = InetAddress.getByName(addressStr);
274         } catch (UnknownHostException e) {
275             fail("Could not allocate InetAddress: " + e);
276         }
277
278         IpAddress address = SouthboundMapper.createIpAddress(inetAddress);
279         PortNumber port = new PortNumber(Integer.parseInt(portStr));
280
281         LOG.info("connectionInfo: {}", new ConnectionInfoBuilder()
282                 .setRemoteIp(address)
283                 .setRemotePort(port)
284                 .build());
285         return new ConnectionInfoBuilder()
286                        .setRemoteIp(address)
287                        .setRemotePort(port)
288                        .build();
289     }
290
291     private String connectionInfoToString(ConnectionInfo connectionInfo) {
292         return new String(connectionInfo.getRemoteIp().getValue()) + ":" + connectionInfo.getRemotePort().getValue();
293     }
294
295     @Test
296     public void testNetworkTopology() throws InterruptedException {
297         NetworkTopology networkTopology = mdsalUtils.read(LogicalDatastoreType.CONFIGURATION,
298                 InstanceIdentifier.create(NetworkTopology.class));
299         Assert.assertNotNull("NetworkTopology could not be found in " + LogicalDatastoreType.CONFIGURATION,
300                 networkTopology);
301
302         networkTopology = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL,
303                 InstanceIdentifier.create(NetworkTopology.class));
304         Assert.assertNotNull("NetworkTopology could not be found in " + LogicalDatastoreType.OPERATIONAL,
305                 networkTopology);
306     }
307
308     @Test
309     public void testOvsdbTopology() throws InterruptedException {
310         InstanceIdentifier<Topology> path = InstanceIdentifier
311                 .create(NetworkTopology.class)
312                 .child(Topology.class, new TopologyKey(SouthboundConstants.OVSDB_TOPOLOGY_ID));
313
314         Topology topology = mdsalUtils.read(LogicalDatastoreType.CONFIGURATION, path);
315         Assert.assertNotNull("Topology could not be found in " + LogicalDatastoreType.CONFIGURATION,
316                 topology);
317
318         topology = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, path);
319
320         Assert.assertNotNull("Topology could not be found in " + LogicalDatastoreType.OPERATIONAL,
321                 topology);
322     }
323
324     private boolean addOvsdbNode(ConnectionInfo connectionInfo) throws InterruptedException {
325         boolean result = mdsalUtils.put(LogicalDatastoreType.CONFIGURATION,
326                 SouthboundMapper.createInstanceIdentifier(connectionInfo),
327                 SouthboundMapper.createNode(connectionInfo));
328         Thread.sleep(OVSDB_UPDATE_TIMEOUT);
329         return result;
330     }
331
332     private Node getOvsdbNode(ConnectionInfo connectionInfo) {
333         Node node = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL,
334                 SouthboundMapper.createInstanceIdentifier(connectionInfo));
335         return node;
336     }
337
338     private boolean deleteOvsdbNode(ConnectionInfo connectionInfo) throws InterruptedException {
339         boolean result = mdsalUtils.delete(LogicalDatastoreType.CONFIGURATION,
340                 SouthboundMapper.createInstanceIdentifier(connectionInfo));
341         Thread.sleep(OVSDB_UPDATE_TIMEOUT);
342         return result;
343     }
344
345     private Node connectOvsdbNode(ConnectionInfo connectionInfo) throws InterruptedException {
346         Assert.assertTrue(addOvsdbNode(connectionInfo));
347         Node node = getOvsdbNode(connectionInfo);
348         Assert.assertNotNull(node);
349         LOG.info("Connected to {}", connectionInfoToString(connectionInfo));
350         return node;
351     }
352
353     private boolean disconnectOvsdbNode(ConnectionInfo connectionInfo) throws InterruptedException {
354         Assert.assertTrue(deleteOvsdbNode(connectionInfo));
355         Node node = getOvsdbNode(connectionInfo);
356         //Assert.assertNull(node);
357         Assume.assumeNotNull(node);
358         LOG.info("Disconnected from {}", connectionInfoToString(connectionInfo));
359         return true;
360     }
361
362     @Test
363     public void testAddDeleteOvsdbNode() throws InterruptedException {
364         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
365         Node ovsdbNode = connectOvsdbNode(connectionInfo);
366         //Assert.assertFalse(disconnectOvsdbNode(connectionInfo));
367         Assume.assumeTrue(disconnectOvsdbNode(connectionInfo));
368     }
369
370     @Test
371     public void testOpenVSwitchOtherConfig() throws InterruptedException {
372         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
373         Node ovsdbNode = connectOvsdbNode(connectionInfo);
374         OvsdbNodeAugmentation ovsdbNodeAugmentation = ovsdbNode.getAugmentation(OvsdbNodeAugmentation.class);
375         assertNotNull(ovsdbNodeAugmentation);
376         List<OpenvswitchOtherConfigs> otherConfigsList = ovsdbNodeAugmentation.getOpenvswitchOtherConfigs();
377         if (otherConfigsList != null) {
378             for (OpenvswitchOtherConfigs otherConfig : otherConfigsList) {
379                 if (otherConfig.getOtherConfigKey().equals("local_ip")) {
380                     LOG.info("local_ip: {}", otherConfig.getOtherConfigValue());
381                     break;
382                 } else {
383                     LOG.info("other_config {}:{}", otherConfig.getOtherConfigKey(), otherConfig.getOtherConfigValue());
384                 }
385             }
386         } else {
387             LOG.info("other_config is not present");
388         }
389         //Assert.assertFalse(disconnectOvsdbNode(connectionInfo));
390         Assume.assumeTrue(disconnectOvsdbNode(connectionInfo));
391     }
392
393     private void setManagedBy(OvsdbBridgeAugmentationBuilder ovsdbBridgeAugmentationBuilder,
394                               ConnectionInfo connectionInfo) {
395         InstanceIdentifier<Node> connectionNodePath = SouthboundMapper.createInstanceIdentifier(connectionInfo);
396         ovsdbBridgeAugmentationBuilder.setManagedBy(new OvsdbNodeRef(connectionNodePath));
397     }
398
399     private List<ProtocolEntry> createMdsalProtocols() {
400         List<ProtocolEntry> protocolList = new ArrayList<ProtocolEntry>();
401         ImmutableBiMap<String, Class<? extends OvsdbBridgeProtocolBase>> mapper =
402                 SouthboundConstants.OVSDB_PROTOCOL_MAP.inverse();
403         protocolList.add(new ProtocolEntryBuilder().
404                 setProtocol((Class<? extends OvsdbBridgeProtocolBase>) mapper.get("OpenFlow13")).build());
405         return protocolList;
406     }
407
408     private OvsdbTerminationPointAugmentationBuilder createGenericOvsdbTerminationPointAugmentationBuilder() {
409         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationPointAugmentationBuilder =
410                 new OvsdbTerminationPointAugmentationBuilder();
411         ovsdbTerminationPointAugmentationBuilder.setInterfaceType(
412                 new InterfaceTypeEntryBuilder()
413                 .setInterfaceType(
414                         SouthboundMapper.createInterfaceType("internal"))
415                         .build().getInterfaceType());
416         return ovsdbTerminationPointAugmentationBuilder;
417     }
418
419     private boolean addTerminationPoint(NodeId bridgeNodeId, String portName,
420             OvsdbTerminationPointAugmentationBuilder ovsdbTerminationPointAugmentationBuilder)
421         throws InterruptedException {
422
423         InstanceIdentifier<Node> portIid = SouthboundMapper.createInstanceIdentifier(bridgeNodeId);
424         NodeBuilder portNodeBuilder = new NodeBuilder();
425         NodeId portNodeId = SouthboundMapper.createManagedNodeId(portIid);
426         portNodeBuilder.setNodeId(portNodeId);
427         TerminationPointBuilder entry = new TerminationPointBuilder();
428         entry.setKey(new TerminationPointKey(new TpId(portName)));
429         entry.addAugmentation(
430                 OvsdbTerminationPointAugmentation.class,
431                 ovsdbTerminationPointAugmentationBuilder.build());
432         portNodeBuilder.setTerminationPoint(Lists.newArrayList(entry.build()));
433         boolean result = mdsalUtils.merge(LogicalDatastoreType.CONFIGURATION,
434                 portIid, portNodeBuilder.build());
435         Thread.sleep(OVSDB_UPDATE_TIMEOUT);
436         return result;
437     }
438
439     private boolean addBridge(ConnectionInfo connectionInfo, String bridgeName) throws InterruptedException {
440         //Node node = SouthboundMapper.createNode(connectionInfo);
441         NodeBuilder bridgeNodeBuilder = new NodeBuilder();
442         InstanceIdentifier<Node> bridgeIid =
443                 SouthboundMapper.createInstanceIdentifier(connectionInfo, new OvsdbBridgeName(bridgeName));
444         NodeId bridgeNodeId = SouthboundMapper.createManagedNodeId(bridgeIid);
445         bridgeNodeBuilder.setNodeId(bridgeNodeId);
446         OvsdbBridgeAugmentationBuilder ovsdbBridgeAugmentationBuilder = new OvsdbBridgeAugmentationBuilder();
447         ovsdbBridgeAugmentationBuilder.setBridgeName(new OvsdbBridgeName(bridgeName));
448         ovsdbBridgeAugmentationBuilder.setProtocolEntry(createMdsalProtocols());
449         ovsdbBridgeAugmentationBuilder.setFailMode(
450                 SouthboundConstants.OVSDB_FAIL_MODE_MAP.inverse().get("secure"));
451         setManagedBy(ovsdbBridgeAugmentationBuilder, connectionInfo);
452         bridgeNodeBuilder.addAugmentation(OvsdbBridgeAugmentation.class, ovsdbBridgeAugmentationBuilder.build());
453
454         LOG.debug("Built with the intent to store bridge data {}",
455                 ovsdbBridgeAugmentationBuilder.toString());
456
457         boolean result = mdsalUtils.merge(LogicalDatastoreType.CONFIGURATION,
458                 bridgeIid, bridgeNodeBuilder.build());
459         Thread.sleep(OVSDB_UPDATE_TIMEOUT);
460         return result;
461     }
462
463     private OvsdbBridgeAugmentation getBridge(ConnectionInfo connectionInfo) {
464         InstanceIdentifier<Node> bridgeIid =
465                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
466                         new OvsdbBridgeName(SouthboundITConstants.BRIDGE_NAME));
467         Node bridgeNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, bridgeIid);
468         Assert.assertNotNull(bridgeNode);
469         OvsdbBridgeAugmentation ovsdbBridgeAugmentation = bridgeNode.getAugmentation(OvsdbBridgeAugmentation.class);
470         Assert.assertNotNull(ovsdbBridgeAugmentation);
471         return ovsdbBridgeAugmentation;
472     }
473
474     private boolean deleteBridge(ConnectionInfo connectionInfo) throws InterruptedException {
475         boolean result = mdsalUtils.delete(LogicalDatastoreType.CONFIGURATION,
476                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
477                         new OvsdbBridgeName(SouthboundITConstants.BRIDGE_NAME)));
478         Thread.sleep(OVSDB_UPDATE_TIMEOUT);
479         return result;
480     }
481
482     @Test
483     public void testAddDeleteBridge() throws InterruptedException {
484         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
485         Node ovsdbNode = connectOvsdbNode(connectionInfo);
486
487         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
488         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
489         Assert.assertNotNull(bridge);
490         LOG.info("bridge: {}", bridge);
491
492         Assert.assertTrue(deleteBridge(connectionInfo));
493
494         //Assert.assertFalse(disconnectOvsdbNode(connectionInfo));
495         Assume.assumeTrue(disconnectOvsdbNode(connectionInfo));
496     }
497
498     @Test
499     public void testTerminationPointOfPort() throws InterruptedException {
500         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
501         connectOvsdbNode(connectionInfo);
502
503         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
504         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
505         Assert.assertNotNull(bridge);
506         LOG.info("bridge: {}", bridge);
507         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
508                 connectionInfo, bridge.getBridgeName()));
509         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
510                 createGenericOvsdbTerminationPointAugmentationBuilder();
511         String portName = "testOfPort";
512         ovsdbTerminationBuilder.setName(portName);
513         Long ofPortExpected = new Long(45002);
514         ovsdbTerminationBuilder.setOfport(ofPortExpected);
515         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
516         InstanceIdentifier<Node> terminationPointIid =
517                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
518                 bridge.getBridgeName());
519         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
520
521         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
522         for (TerminationPoint terminationPoint : terminationPoints) {
523             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
524                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
525             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
526                 Long ofPort = ovsdbTerminationPointAugmentation.getOfport();
527                 // if ephemeral port 45002 is in use, ofPort is set to 1
528                 Assert.assertTrue(ofPort.equals(ofPortExpected) || ofPort.equals(new Long(1)));
529                 LOG.info("ofPort: {}", ofPort);
530             }
531         }
532         Assert.assertTrue(deleteBridge(connectionInfo));
533     }
534
535     @Test
536     public void testTerminationPointOfPortRequest() throws InterruptedException {
537         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
538         connectOvsdbNode(connectionInfo);
539         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
540         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
541         Assert.assertNotNull(bridge);
542         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
543                 connectionInfo, bridge.getBridgeName()));
544         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
545                 createGenericOvsdbTerminationPointAugmentationBuilder();
546         String portName = "testOfPortRequest";
547         ovsdbTerminationBuilder.setName(portName);
548         Long ofPortExpected = new Long(45008);
549         Integer ofPortRequestExpected = ofPortExpected.intValue();
550         Long ofPortInput = new Long(45008);
551         ovsdbTerminationBuilder.setOfport(ofPortInput);
552         ovsdbTerminationBuilder.setOfportRequest(ofPortRequestExpected);
553         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
554         InstanceIdentifier<Node> terminationPointIid =
555                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
556                 new OvsdbBridgeName(SouthboundITConstants.BRIDGE_NAME));
557         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
558
559         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
560         for (TerminationPoint terminationPoint : terminationPoints) {
561             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
562                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
563             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
564                 Long ofPort = ovsdbTerminationPointAugmentation.getOfport();
565                 // if ephemeral port 45002 is in use, ofPort is set to 1
566                 Assert.assertTrue(ofPort.equals(ofPortExpected) || ofPort.equals(new Long(1)));
567                 LOG.info("ofPort: {}", ofPort);
568
569                 Integer ofPortRequest = ovsdbTerminationPointAugmentation.getOfportRequest();
570                 Assert.assertTrue(ofPortRequest.equals(ofPortRequestExpected));
571                 LOG.info("ofPortRequest: {}", ofPortRequest);
572             }
573         }
574         Assert.assertTrue(deleteBridge(connectionInfo));
575     }
576
577     @Test
578     public void testTerminationPointPortExternalIds() throws InterruptedException {
579         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
580         connectOvsdbNode(connectionInfo);
581         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
582         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
583         Assert.assertNotNull(bridge);
584         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
585                 connectionInfo, bridge.getBridgeName()));
586         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
587                 createGenericOvsdbTerminationPointAugmentationBuilder();
588         String portName = "testPortExternalIds";
589         ovsdbTerminationBuilder.setName(portName);
590         //setup
591         PortExternalIdsBuilder externalIdsBuilder1 = new PortExternalIdsBuilder();
592         externalIdsBuilder1.setExternalIdKey("portExternalIdKey1");
593         externalIdsBuilder1.setExternalIdValue("portExternalIdValue1");
594         PortExternalIdsBuilder externalIdsBuilder2 = new PortExternalIdsBuilder();
595         externalIdsBuilder2.setExternalIdKey("portExternalIdKey2");
596         externalIdsBuilder2.setExternalIdValue("portExternalIdValue2");
597         List<PortExternalIds> portExternalIds = Lists.newArrayList(externalIdsBuilder1.build(),
598                 externalIdsBuilder2.build());
599         ovsdbTerminationBuilder.setPortExternalIds(portExternalIds);
600
601         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
602         InstanceIdentifier<Node> terminationPointIid =
603                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
604                 bridge.getBridgeName());
605         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
606
607         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
608         for (TerminationPoint terminationPoint : terminationPoints) {
609             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
610                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
611             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
612                 List<PortExternalIds> actualPortExternalIds = ovsdbTerminationPointAugmentation.getPortExternalIds();
613                 Assert.assertTrue((portExternalIds.size() == actualPortExternalIds.size()));
614                 for (PortExternalIds portExternalId : portExternalIds) {
615                     Assert.assertTrue(actualPortExternalIds.contains(portExternalId));
616                 }
617             }
618         }
619         Assert.assertTrue(deleteBridge(connectionInfo));
620     }
621
622     @Test
623     public void testTerminationPointInterfaceExternalIds() throws InterruptedException {
624         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
625         connectOvsdbNode(connectionInfo);
626         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
627         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
628         Assert.assertNotNull(bridge);
629         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
630                 connectionInfo, bridge.getBridgeName()));
631         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
632                 createGenericOvsdbTerminationPointAugmentationBuilder();
633         String portName = "testInterfaceExternalIds";
634         ovsdbTerminationBuilder.setName(portName);
635         //setup
636         InterfaceExternalIdsBuilder externalIdsBuilder1 = new InterfaceExternalIdsBuilder();
637         externalIdsBuilder1.setExternalIdKey("interfaceExternalIdKey1");
638         externalIdsBuilder1.setExternalIdValue("interfaceExternalIdValue1");
639         InterfaceExternalIdsBuilder externalIdsBuilder2 = new InterfaceExternalIdsBuilder();
640         externalIdsBuilder2.setExternalIdKey("interfaceExternalIdKey2");
641         externalIdsBuilder2.setExternalIdValue("interfaceExternalIdValue2");
642         List<InterfaceExternalIds> interfaceExternalIds = Lists.newArrayList(externalIdsBuilder1.build(),
643                 externalIdsBuilder2.build());
644         ovsdbTerminationBuilder.setInterfaceExternalIds(interfaceExternalIds);
645
646         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
647         InstanceIdentifier<Node> terminationPointIid =
648                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
649                 bridge.getBridgeName());
650         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
651
652         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
653         for (TerminationPoint terminationPoint : terminationPoints) {
654             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
655                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
656             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
657                 List<InterfaceExternalIds> actualInterfaceExternalIds = ovsdbTerminationPointAugmentation.
658                         getInterfaceExternalIds();
659                 Assert.assertTrue((interfaceExternalIds.size() == actualInterfaceExternalIds.size()));
660                 for (InterfaceExternalIds interfaceExternalId : interfaceExternalIds) {
661                     Assert.assertTrue(actualInterfaceExternalIds.contains(interfaceExternalId));
662                 }
663             }
664         }
665         Assert.assertTrue(deleteBridge(connectionInfo));
666     }
667
668     @Test
669     public void testTerminationPointOptions() throws InterruptedException {
670         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
671         connectOvsdbNode(connectionInfo);
672         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
673         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
674         Assert.assertNotNull(bridge);
675         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
676                 connectionInfo, bridge.getBridgeName()));
677         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
678                 createGenericOvsdbTerminationPointAugmentationBuilder();
679         String portName = "testInterfaceOptions";
680         ovsdbTerminationBuilder.setName(portName);
681         //setup
682         OptionsBuilder optionsBuilder1 = new OptionsBuilder();
683         optionsBuilder1.setOption("option1");
684         optionsBuilder1.setValue("optionValue1");
685         OptionsBuilder optionsBuilder2 = new OptionsBuilder();
686         optionsBuilder2.setOption("option2");
687         optionsBuilder2.setValue("optionValue2");
688         List<Options> options = Lists.newArrayList(optionsBuilder1.build(),
689                 optionsBuilder2.build());
690         ovsdbTerminationBuilder.setOptions(options);
691
692         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
693         InstanceIdentifier<Node> terminationPointIid =
694                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
695                 bridge.getBridgeName());
696         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
697
698         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
699         for (TerminationPoint terminationPoint : terminationPoints) {
700             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
701                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
702             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
703                 List<Options> actualOptions = ovsdbTerminationPointAugmentation.
704                         getOptions();
705                 Assert.assertTrue((options.size() == actualOptions.size()));
706                 for (Options option : options) {
707                     Assert.assertTrue(actualOptions.contains(option));
708                 }
709             }
710         }
711         Assert.assertTrue(deleteBridge(connectionInfo));
712     }
713
714     @Test
715     public void testTerminationPointInterfaceOtherConfigs() throws InterruptedException {
716         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
717         connectOvsdbNode(connectionInfo);
718         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
719         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
720         Assert.assertNotNull(bridge);
721         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
722                 connectionInfo, bridge.getBridgeName()));
723         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
724                 createGenericOvsdbTerminationPointAugmentationBuilder();
725         String portName = "testInterfaceOtherConfigs";
726         ovsdbTerminationBuilder.setName(portName);
727         //setup
728         InterfaceOtherConfigsBuilder interfaceBuilder1 = new InterfaceOtherConfigsBuilder();
729         interfaceBuilder1.setOtherConfigKey("interfaceOtherConfigsKey1");
730         interfaceBuilder1.setOtherConfigValue("interfaceOtherConfigsValue1");
731         InterfaceOtherConfigsBuilder interfaceBuilder2 = new InterfaceOtherConfigsBuilder();
732         interfaceBuilder2.setOtherConfigKey("interfaceOtherConfigsKey2");
733         interfaceBuilder2.setOtherConfigValue("interfaceOtherConfigsValue2");
734         List<InterfaceOtherConfigs> interfaceOtherConfigs = Lists.newArrayList(interfaceBuilder1.build(),
735                 interfaceBuilder2.build());
736         ovsdbTerminationBuilder.setInterfaceOtherConfigs(interfaceOtherConfigs);
737
738         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
739         Thread.sleep(1000);
740         InstanceIdentifier<Node> terminationPointIid =
741                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
742                 bridge.getBridgeName());
743         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
744
745         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
746         for (TerminationPoint terminationPoint : terminationPoints) {
747             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
748                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
749             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
750                 List<InterfaceOtherConfigs> actualInterfaceOtherConfigs = ovsdbTerminationPointAugmentation.
751                         getInterfaceOtherConfigs();
752                 Assert.assertNotNull(actualInterfaceOtherConfigs);
753                 Assert.assertNotNull(interfaceOtherConfigs);
754                 Assert.assertTrue(interfaceOtherConfigs.size() == actualInterfaceOtherConfigs.size());
755                 for (InterfaceOtherConfigs interfaceOtherConfig : interfaceOtherConfigs) {
756                     Assert.assertTrue(actualInterfaceOtherConfigs.contains(interfaceOtherConfig));
757                 }
758             }
759         }
760         Assert.assertTrue(deleteBridge(connectionInfo));
761     }
762
763     @Test
764     public void testTerminationPointPortOtherConfigs() throws InterruptedException {
765         ConnectionInfo connectionInfo = getConnectionInfo(addressStr, portStr);
766         connectOvsdbNode(connectionInfo);
767         Assert.assertTrue(addBridge(connectionInfo, SouthboundITConstants.BRIDGE_NAME));
768         OvsdbBridgeAugmentation bridge = getBridge(connectionInfo);
769         Assert.assertNotNull(bridge);
770         NodeId nodeId = SouthboundMapper.createManagedNodeId(SouthboundMapper.createInstanceIdentifier(
771                 connectionInfo, bridge.getBridgeName()));
772         OvsdbTerminationPointAugmentationBuilder ovsdbTerminationBuilder =
773                 createGenericOvsdbTerminationPointAugmentationBuilder();
774         String portName = "testPortOtherConfigs";
775         ovsdbTerminationBuilder.setName(portName);
776         //setup
777         PortOtherConfigsBuilder portBuilder1 = new PortOtherConfigsBuilder();
778         portBuilder1.setOtherConfigKey("portOtherConfigsKey1");
779         portBuilder1.setOtherConfigValue("portOtherConfigsValue1");
780         PortOtherConfigsBuilder portBuilder2 = new PortOtherConfigsBuilder();
781         portBuilder2.setOtherConfigKey("portOtherConfigsKey2");
782         portBuilder2.setOtherConfigValue("portOtherConfigsValue2");
783         List<PortOtherConfigs> portOtherConfigs = Lists.newArrayList(portBuilder1.build(),
784                 portBuilder2.build());
785         ovsdbTerminationBuilder.setPortOtherConfigs(portOtherConfigs);
786
787         Assert.assertTrue(addTerminationPoint(nodeId, portName, ovsdbTerminationBuilder));
788         InstanceIdentifier<Node> terminationPointIid =
789                 SouthboundMapper.createInstanceIdentifier(connectionInfo,
790                 bridge.getBridgeName());
791         Node terminationPointNode = mdsalUtils.read(LogicalDatastoreType.OPERATIONAL, terminationPointIid);
792
793         List<TerminationPoint> terminationPoints = terminationPointNode.getTerminationPoint();
794         for (TerminationPoint terminationPoint : terminationPoints) {
795             OvsdbTerminationPointAugmentation ovsdbTerminationPointAugmentation =
796                     terminationPoint.getAugmentation(OvsdbTerminationPointAugmentation.class);
797             if (ovsdbTerminationPointAugmentation.getName().equals(portName)) {
798                 List<PortOtherConfigs> actualPortOtherConfigs = ovsdbTerminationPointAugmentation.
799                         getPortOtherConfigs();
800                 Assert.assertTrue((portOtherConfigs.size() == actualPortOtherConfigs.size()));
801                 for (PortOtherConfigs portOtherConfig : portOtherConfigs) {
802                     Assert.assertTrue(actualPortOtherConfigs.contains(portOtherConfig));
803                 }
804             }
805         }
806         Assert.assertTrue(deleteBridge(connectionInfo));
807     }
808
809     /**
810      * isBundleReady is used to check if the requested bundle is Active
811      */
812     public void isBundleReady(BundleContext bundleContext, String bundleName) throws InterruptedException {
813         boolean ready = false;
814
815         while (!ready) {
816             int state = Bundle.UNINSTALLED;
817             Bundle[] bundles = bundleContext.getBundles();
818             for (Bundle element : bundles) {
819                 if (element.getSymbolicName().equals(bundleName)) {
820                     state = element.getState();
821                     LOG.info(">>>>> bundle is ready {}", bundleName);
822                     break;
823                 }
824             }
825             if (state != Bundle.ACTIVE) {
826                 LOG.info(">>>>> bundle not ready {}", bundleName);
827                 Thread.sleep(5000);
828             } else {
829                 ready = true;
830             }
831         }
832     }
833 }