X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=vpnmanager%2Fvpnmanager-impl%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fvpnservice%2FVpnManager.java;h=7c9a403957c0ff2dd6df05d1b88d9c65109416ab;hb=f30638fcc162d85f21f33cef401b4fb5da05caaf;hp=3689cbbe21395c9f5b07089a026d0d716888482a;hpb=42c322817cb799810b82212370b23ee92d866294;p=vpnservice.git diff --git a/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnManager.java b/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnManager.java index 3689cbbe..7c9a4039 100644 --- a/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnManager.java +++ b/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others. All rights reserved. + * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -9,35 +9,41 @@ package org.opendaylight.vpnservice; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; +import java.util.concurrent.*; +import com.google.common.util.concurrent.CheckedFuture; import org.opendaylight.bgpmanager.api.IBgpManager; import org.opendaylight.controller.md.sal.binding.api.DataBroker; import org.opendaylight.controller.md.sal.binding.api.DataChangeListener; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope; +import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException; +import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstanceToVpnId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstanceToVpnIdBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnRouteList; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey; import org.opendaylight.yangtools.concepts.ListenerRegistration; import org.opendaylight.yangtools.yang.binding.DataObject; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; -import org.opendaylight.yangtools.yang.common.RpcResult; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; import org.opendaylight.controller.md.sal.binding.api.WriteTransaction; import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig; +import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces; import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance; +import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey; import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances; import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstance1; -import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstance1Builder; +import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstanceOpData; +import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstanceOpDataBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries; import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables; import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry; -import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdInput; -import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdInputBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,11 +54,15 @@ import com.google.common.util.concurrent.Futures; public class VpnManager extends AbstractDataChangeListener implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(VpnManager.class); - private ListenerRegistration listenerRegistration, fibListenerRegistration; + private ListenerRegistration listenerRegistration, fibListenerRegistration, opListenerRegistration; + private ConcurrentMap vpnOpMap = new ConcurrentHashMap(); + private ExecutorService executorService = Executors.newSingleThreadExecutor(); private final DataBroker broker; private final IBgpManager bgpManager; private IdManagerService idManager; + private VpnInterfaceManager vpnInterfaceManager; private final FibEntriesListener fibListener; + private final VpnInstanceOpListener vpnInstOpListener; private static final FutureCallback DEFAULT_CALLBACK = new FutureCallback() { @@ -68,7 +78,7 @@ public class VpnManager extends AbstractDataChangeListener implemen /** * Listens for data change related to VPN Instance * Informs the BGP about VRF information - * + * * @param db - dataBroker reference */ public VpnManager(final DataBroker db, final IBgpManager bgpManager) { @@ -76,6 +86,7 @@ public class VpnManager extends AbstractDataChangeListener implemen broker = db; this.bgpManager = bgpManager; this.fibListener = new FibEntriesListener(); + this.vpnInstOpListener = new VpnInstanceOpListener(); registerListener(db); } @@ -83,10 +94,13 @@ public class VpnManager extends AbstractDataChangeListener implemen try { listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION, getWildCardPath(), VpnManager.this, DataChangeScope.SUBTREE); - fibListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION, + fibListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL, getFibEntryListenerPath(), fibListener, DataChangeScope.BASE); + opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION, + getVpnInstanceOpListenerPath(), vpnInstOpListener, DataChangeScope.SUBTREE); + } catch (final Exception e) { - LOG.error("VPN Service DataChange listener registration fail!", e); + LOG.error("VPN Service DataChange listener registration fail !", e); throw new IllegalStateException("VPN Service registration Listener failed.", e); } } @@ -95,18 +109,63 @@ public class VpnManager extends AbstractDataChangeListener implemen this.idManager = idManager; } + public void setVpnInterfaceManager(VpnInterfaceManager vpnInterfaceManager) { + this.vpnInterfaceManager = vpnInterfaceManager; + } + + private void waitForOpDataRemoval(String id) { + //wait till DCN for update on VPN Instance Op Data signals that vpn interfaces linked to this vpn instance is zero + Runnable notifyTask = new VpnNotifyTask(); + synchronized (id.intern()) { + vpnOpMap.put(id, notifyTask); + synchronized (notifyTask) { + try { + notifyTask.wait(VpnConstants.WAIT_TIME_IN_MILLISECONDS); + } catch (InterruptedException e) { + } + } + } + + } + @Override protected void remove(InstanceIdentifier identifier, VpnInstance del) { - LOG.trace("Remove event - Key: {}, value: {}", identifier, del); + LOG.trace("Remove VPN event - Key: {}, value: {}", identifier, del); String vpnName = del.getVpnInstanceName(); - InstanceIdentifier vpnIdentifier = VpnUtil.getVpnInstanceIdentifier(vpnName); - delete(LogicalDatastoreType.OPERATIONAL, vpnIdentifier); + //Clean up vpn Interface + InstanceIdentifier vpnInterfacesId = InstanceIdentifier.builder(VpnInterfaces.class).build(); + Optional optionalVpnInterfaces = read(LogicalDatastoreType.OPERATIONAL, vpnInterfacesId); + + if(optionalVpnInterfaces.isPresent()) { + List vpnInterfaces = optionalVpnInterfaces.get().getVpnInterface(); + for(VpnInterface vpnInterface : vpnInterfaces) { + if(vpnInterface.getVpnInstanceName().equals(vpnName)) { + LOG.debug("VpnInterface {} will be removed from VPN {}", vpnInterface.getName(), vpnName); + vpnInterfaceManager.remove( + VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface); + } + } + } + InstanceIdentifier + vpnIdentifier = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName); + delete(LogicalDatastoreType.CONFIGURATION, vpnIdentifier); + + VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME, vpnName); String rd = del.getIpv4Family().getRouteDistinguisher(); - try { - bgpManager.deleteVrf(rd); - } catch(Exception e) { - LOG.error("Exception when removing VRF from BGP", e); + + if (rd !=null) { + + try { + bgpManager.deleteVrf(rd); + } catch (Exception e) { + LOG.error("Exception when removing VRF from BGP", e); + } + waitForOpDataRemoval(rd); + delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getVpnInstanceOpDataIdentifier(rd)); + } else { + waitForOpDataRemoval(vpnName); + delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getVpnInstanceOpDataIdentifier(vpnName)); } } @@ -119,24 +178,70 @@ public class VpnManager extends AbstractDataChangeListener implemen @Override protected void add(InstanceIdentifier identifier, VpnInstance value) { - LOG.trace("key: {}, value: {}", identifier, value); + LOG.trace("VPN Instance key: {}, value: {}", identifier, value); + VpnAfConfig config = value.getIpv4Family(); + String rd = config.getRouteDistinguisher(); - long vpnId = getUniqueId(value.getVpnInstanceName()); + long vpnId = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, value.getVpnInstanceName()); + LOG.trace("VPN instance to ID generated."); + org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance + vpnInstanceToVpnId = VpnUtil.getVpnInstanceToVpnId(value.getVpnInstanceName(), vpnId, + (rd != null) ? rd : value.getVpnInstanceName()); - VpnInstance opValue = new VpnInstanceBuilder(value). - addAugmentation(VpnInstance1.class, new VpnInstance1Builder().setVpnId(vpnId).build()).build(); + syncWrite(LogicalDatastoreType.CONFIGURATION, + VpnUtil.getVpnInstanceToVpnIdIdentifier(value.getVpnInstanceName()), + vpnInstanceToVpnId, DEFAULT_CALLBACK); - asyncWrite(LogicalDatastoreType.OPERATIONAL, identifier, opValue, DEFAULT_CALLBACK); - //public void addVrf(String rd, Collection importRts, Collection exportRts) - VpnAfConfig config = value.getIpv4Family(); - String rd = config.getRouteDistinguisher(); - List importRts = Arrays.asList(config.getImportRoutePolicy().split(",")); - List exportRts = Arrays.asList(config.getExportRoutePolicy().split(",")); - try { - bgpManager.addVrf(rd, importRts, exportRts); - } catch(Exception e) { - LOG.error("Exception when adding VRF to BGP", e); + if(rd == null) { + syncWrite(LogicalDatastoreType.OPERATIONAL, + VpnUtil.getVpnInstanceOpDataIdentifier(value.getVpnInstanceName()), + VpnUtil.getVpnInstanceOpDataBuilder(value.getVpnInstanceName(), vpnId), DEFAULT_CALLBACK); + + } else { + syncWrite(LogicalDatastoreType.OPERATIONAL, + VpnUtil.getVpnInstanceOpDataIdentifier(rd), + VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId), DEFAULT_CALLBACK); + + List vpnTargetList = config.getVpnTargets().getVpnTarget(); + + List ertList = new ArrayList(); + List irtList = new ArrayList(); + + for (VpnTarget vpnTarget : vpnTargetList) { + if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.ExportExtcommunity) { + ertList.add(vpnTarget.getVrfRTValue()); + } + if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.ImportExtcommunity) { + irtList.add(vpnTarget.getVrfRTValue()); + } + if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.Both) { + ertList.add(vpnTarget.getVrfRTValue()); + irtList.add(vpnTarget.getVrfRTValue()); + } + } + + try { + bgpManager.addVrf(rd, irtList, ertList); + } catch(Exception e) { + LOG.error("Exception when adding VRF to BGP", e); + } + } + //Try to add up vpn Interfaces if already in Operational Datastore + LOG.trace("Trying to add the vpn interfaces -1."); + InstanceIdentifier vpnInterfacesId = InstanceIdentifier.builder(VpnInterfaces.class).build(); + Optional optionalVpnInterfaces = read(LogicalDatastoreType.CONFIGURATION, vpnInterfacesId); + + if(optionalVpnInterfaces.isPresent()) { + List vpnInterfaces = optionalVpnInterfaces.get().getVpnInterface(); + for(VpnInterface vpnInterface : vpnInterfaces) { + if(vpnInterface.getVpnInstanceName().equals(value.getVpnInstanceName())) { + LOG.debug("VpnInterface {} will be added from VPN {}", vpnInterface.getName(), value.getVpnInstanceName()); + vpnInterfaceManager.add( + VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface); + + } + } } } @@ -149,6 +254,11 @@ public class VpnManager extends AbstractDataChangeListener implemen .child(VrfEntry.class); } + private InstanceIdentifier getVpnInstanceOpListenerPath() { + return InstanceIdentifier.create(VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class); + + } + @Override public void close() throws Exception { if (listenerRegistration != null) { @@ -167,6 +277,15 @@ public class VpnManager extends AbstractDataChangeListener implemen } fibListenerRegistration = null; } + if (opListenerRegistration != null) { + try { + opListenerRegistration.close(); + } catch (final Exception e) { + LOG.error("Error when cleaning up VPN Instance Operational entries DataChangeListener.", e); + } + opListenerRegistration = null; + } + LOG.trace("VPN Manager Closed"); } @@ -192,37 +311,26 @@ public class VpnManager extends AbstractDataChangeListener implemen Futures.addCallback(tx.submit(), callback); } - private VpnInstance getVpnForRD(String rd) { - InstanceIdentifier id = InstanceIdentifier.create(VpnInstances.class); - Optional vpnInstances = read(LogicalDatastoreType.OPERATIONAL, id); - if(vpnInstances.isPresent()) { - List vpns = vpnInstances.get().getVpnInstance(); - for(VpnInstance vpn : vpns) { - if(vpn.getIpv4Family().getRouteDistinguisher().equals(rd)) { - return vpn; - } - } + private void syncWrite(LogicalDatastoreType datastoreType, + InstanceIdentifier path, T data, FutureCallback callback) { + WriteTransaction tx = broker.newWriteOnlyTransaction(); + tx.put(datastoreType, path, data, true); + CheckedFuture futures = tx.submit(); + try { + futures.get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Error writing VPN instance to ID info to datastore (path, data) : ({}, {})", path, data); + throw new RuntimeException(e.getMessage()); } - return null; } - private Integer getUniqueId(String idKey) { - GetUniqueIdInput getIdInput = new GetUniqueIdInputBuilder() - .setPoolName(VpnConstants.VPN_IDPOOL_NAME) - .setIdKey(idKey).build(); - - try { - Future> result = idManager.getUniqueId(getIdInput); - RpcResult rpcResult = result.get(); - if(rpcResult.isSuccessful()) { - return rpcResult.getResult().getIdValue().intValue(); - } else { - LOG.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors()); - } - } catch (NullPointerException | InterruptedException | ExecutionException e) { - LOG.warn("Exception when getting Unique Id",e); + protected VpnInstanceOpDataEntry getVpnInstanceOpData(String rd) { + InstanceIdentifier id = VpnUtil.getVpnInstanceOpDataIdentifier(rd); + Optional vpnInstanceOpData = read(LogicalDatastoreType.OPERATIONAL, id); + if(vpnInstanceOpData.isPresent()) { + return vpnInstanceOpData.get(); } - return 0; + return null; } private void delete(LogicalDatastoreType datastoreType, InstanceIdentifier path) { @@ -244,25 +352,17 @@ public class VpnManager extends AbstractDataChangeListener implemen final VrfTablesKey key = identifier.firstKeyOf(VrfTables.class, VrfTablesKey.class); String rd = key.getRouteDistinguisher(); Long label = del.getLabel(); - VpnInstance vpn = getVpnForRD(rd); - if(vpn != null) { - InstanceIdentifier id = VpnUtil.getVpnInstanceIdentifier(vpn.getVpnInstanceName()); - InstanceIdentifier augId = id.augmentation(VpnInstance1.class); - Optional vpnAugmenation = read(LogicalDatastoreType.OPERATIONAL, augId); - if(vpnAugmenation.isPresent()) { - VpnInstance1 vpnAug = vpnAugmenation.get(); - List routeIds = vpnAug.getRouteEntryId(); - if(routeIds == null) { - LOG.debug("Fib Route entry is empty."); - return; - } - LOG.debug("Removing label from vpn info - {}", label); - routeIds.remove(label); - asyncWrite(LogicalDatastoreType.OPERATIONAL, augId, - new VpnInstance1Builder(vpnAug).setRouteEntryId(routeIds).build(), DEFAULT_CALLBACK); - } else { - LOG.warn("VPN Augmentation not found for vpn instance {}", vpn.getVpnInstanceName()); + VpnInstanceOpDataEntry vpnInstanceOpData = getVpnInstanceOpData(rd); + if(vpnInstanceOpData != null) { + List routeIds = vpnInstanceOpData.getRouteEntryId(); + if(routeIds == null) { + LOG.debug("Fib Route entry is empty."); + return; } + LOG.debug("Removing label from vpn info - {}", label); + routeIds.remove(label); + asyncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil.getVpnInstanceOpDataIdentifier(rd), + new VpnInstanceOpDataEntryBuilder(vpnInstanceOpData).setRouteEntryId(routeIds).build(), DEFAULT_CALLBACK); } else { LOG.warn("No VPN Instance found for RD: {}", rd); } @@ -277,32 +377,62 @@ public class VpnManager extends AbstractDataChangeListener implemen @Override protected void add(InstanceIdentifier identifier, - VrfEntry add) { + VrfEntry add) { LOG.trace("Add Vrf Entry event - Key : {}, value : {}", identifier, add); final VrfTablesKey key = identifier.firstKeyOf(VrfTables.class, VrfTablesKey.class); String rd = key.getRouteDistinguisher(); Long label = add.getLabel(); - VpnInstance vpn = getVpnForRD(rd); + VpnInstanceOpDataEntry vpn = getVpnInstanceOpData(rd); if(vpn != null) { - InstanceIdentifier id = VpnUtil.getVpnInstanceIdentifier(vpn.getVpnInstanceName()); - InstanceIdentifier augId = id.augmentation(VpnInstance1.class); - Optional vpnAugmenation = read(LogicalDatastoreType.OPERATIONAL, augId); - if(vpnAugmenation.isPresent()) { - VpnInstance1 vpnAug = vpnAugmenation.get(); - List routeIds = vpnAug.getRouteEntryId(); - if(routeIds == null) { - routeIds = new ArrayList<>(); - } - LOG.debug("Adding label to vpn info - {}", label); - routeIds.add(label); - asyncWrite(LogicalDatastoreType.OPERATIONAL, augId, - new VpnInstance1Builder(vpnAug).setRouteEntryId(routeIds).build(), DEFAULT_CALLBACK); - } else { - LOG.warn("VPN Augmentation not found for vpn instance {}", vpn.getVpnInstanceName()); + List routeIds = vpn.getRouteEntryId(); + if(routeIds == null) { + routeIds = new ArrayList<>(); } + LOG.debug("Adding label to vpn info - {}", label); + routeIds.add(label); + asyncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil.getVpnInstanceOpDataIdentifier(rd), + new VpnInstanceOpDataEntryBuilder(vpn).setRouteEntryId(routeIds).build(), DEFAULT_CALLBACK); } else { LOG.warn("No VPN Instance found for RD: {}", rd); } } } + + class VpnInstanceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener { + + public VpnInstanceOpListener() { + super(VpnInstanceOpDataEntry.class); + } + + @Override + protected void remove(InstanceIdentifier identifier, VpnInstanceOpDataEntry del) { + + } + + @Override + protected void update(InstanceIdentifier identifier, VpnInstanceOpDataEntry original, VpnInstanceOpDataEntry update) { + final VpnInstanceOpDataEntryKey key = identifier.firstKeyOf(VpnInstanceOpDataEntry.class, VpnInstanceOpDataEntryKey.class); + String vpnName = key.getVrfId(); + + LOG.trace("VpnInstanceOpListener update: vpn name {} interface count in Old VpnOp Instance {} in New VpnOp Instance {}" , + vpnName, original.getVpnInterfaceCount(), update.getVpnInterfaceCount() ); + + //if((original.getVpnToDpnList().size() != update.getVpnToDpnList().size()) && (update.getVpnToDpnList().size() == 0)) { + if((original.getVpnInterfaceCount() != update.getVpnInterfaceCount()) && (update.getVpnInterfaceCount() == 0)) { + notifyTaskIfRequired(vpnName); + } + } + + private void notifyTaskIfRequired(String vpnName) { + Runnable notifyTask = vpnOpMap.remove(vpnName); + if (notifyTask == null) { + return; + } + executorService.execute(notifyTask); + } + + @Override + protected void add(InstanceIdentifier identifier, VpnInstanceOpDataEntry add) { + } + } }