2 * Copyright (c) 2014 SDN Hub, LLC. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.services;
10 import java.math.BigInteger;
11 import java.util.List;
14 import org.opendaylight.ovsdb.openstack.netvirt.NetworkHandler;
15 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
16 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration;
17 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerConfiguration.LoadBalancerPoolMember;
18 import org.opendaylight.ovsdb.openstack.netvirt.api.LoadBalancerProvider;
19 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
20 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
21 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
22 import org.opendaylight.ovsdb.openstack.netvirt.providers.ConfigInterface;
23 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.AbstractServiceInstance;
24 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.Service;
25 import org.opendaylight.ovsdb.utils.mdsal.openflow.ActionUtils;
26 import org.opendaylight.ovsdb.utils.mdsal.openflow.MatchUtils;
27 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.PortNumber;
29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
30 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
31 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionBuilder;
32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionKey;
33 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.address.address.Ipv4Builder;
34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
44 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.l2.types.rev130827.VlanId;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.dst.choice.grouping.dst.choice.DstNxRegCaseBuilder;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.action.rev140421.OfjNxHashFields;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.action.rev140421.OfjNxMpAlgorithm;
55 import org.osgi.framework.BundleContext;
56 import org.osgi.framework.ServiceReference;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
60 import com.google.common.collect.Lists;
62 public class LoadBalancerService extends AbstractServiceInstance implements LoadBalancerProvider, ConfigInterface {
64 private static final Logger LOG = LoggerFactory.getLogger(LoadBalancerProvider.class);
65 private static final int DEFAULT_FLOW_PRIORITY = 32768;
66 private static final Long FIRST_PASS_REGA_MATCH_VALUE = 0L;
67 private static final Long SECOND_PASS_REGA_MATCH_VALUE = 1L;
69 private static final Class<? extends NxmNxReg> REG_FIELD_A = NxmNxReg1.class;
70 private static final Class<? extends NxmNxReg> REG_FIELD_B = NxmNxReg2.class;
72 private volatile Southbound southbound;
74 public LoadBalancerService() {
75 super(Service.LOAD_BALANCER);
78 public LoadBalancerService(Service service) {
82 private String getDpid(Node node) {
83 long dpid = southbound.getDataPathId(node);
85 LOG.warn("getDpid: DPID could not be found for node: {}", node.getNodeId().getValue());
87 return String.valueOf(dpid);
91 * When this method is called, we do the following for minimizing flow updates:
92 * 1. Overwrite the solo multipath rule that applies to all members
93 * 2. Append second pass rule for the header rewriting specific to this member
94 * 3. Append reverse rules specific to this member
97 public Status programLoadBalancerPoolMemberRules(Node node,
98 LoadBalancerConfiguration lbConfig, LoadBalancerPoolMember member,
99 org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
100 if (lbConfig == null || member == null) {
101 LOG.error("Null value for LB config {} or Member {}", lbConfig, member);
102 return new Status(StatusCode.BADREQUEST);
104 if (!lbConfig.isValid()) {
105 LOG.error("LB config is invalid: {}", lbConfig);
106 return new Status(StatusCode.BADREQUEST);
108 LOG.debug("Performing {} rules for member {} with index {} on LB with VIP {} and total members {}",
109 action, member.getIP(), member.getIndex(), lbConfig.getVip(), lbConfig.getMembers().size());
111 NodeBuilder nodeBuilder = new NodeBuilder();
112 nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + getDpid(node)));
113 nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
115 //Update the multipath rule
116 manageLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig, true);
118 if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
119 manageLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig, member, true);
120 manageLoadBalancerMemberReverseRules(nodeBuilder, lbConfig, member, true);
121 return new Status(StatusCode.SUCCESS);
123 /* TODO: Delete single member.
124 * For now, removing a member requires deleting the full LB instance and re-adding
126 return new Status(StatusCode.NOTIMPLEMENTED);
130 * When this method is called, we perform the following:
131 * 1. Write the solo multipath rule that applies to all members
132 * 2. Append second pass rules for the header rewriting for all members
133 * 3. Append reverse rules for all the members, specific to the protocol/port
136 public Status programLoadBalancerRules(Node node, LoadBalancerConfiguration lbConfig,
137 org.opendaylight.ovsdb.openstack.netvirt.api.Action action) {
138 if (lbConfig == null || !lbConfig.isValid()) {
139 LOG.error("LB config is invalid: {}", lbConfig);
140 return new Status(StatusCode.BADREQUEST);
142 LOG.debug("Performing {} rules for VIP {} and {} members", action, lbConfig.getVip(), lbConfig.getMembers().size());
144 NodeBuilder nodeBuilder = new NodeBuilder();
145 nodeBuilder.setId(new NodeId(Constants.OPENFLOW_NODE_PREFIX + getDpid(node)));
146 nodeBuilder.setKey(new NodeKey(nodeBuilder.getId()));
148 if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.ADD)) {
149 manageLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig, true);
150 manageLoadBalancerVIPRulesSecondPass(nodeBuilder, lbConfig, true);
151 manageLoadBalancerReverseRules(nodeBuilder, lbConfig, true);
152 return new Status(StatusCode.SUCCESS);
154 else if (action.equals(org.opendaylight.ovsdb.openstack.netvirt.api.Action.DELETE)) {
155 manageLoadBalancerVIPRulesFirstPass(nodeBuilder, lbConfig, false);
156 manageLoadBalancerVIPRulesSecondPass(nodeBuilder, lbConfig, false);
157 manageLoadBalancerReverseRules(nodeBuilder, lbConfig, false);
158 return new Status(StatusCode.SUCCESS);
161 return new Status(StatusCode.NOTIMPLEMENTED);
165 * Method to insert/remove default rule for traffic destined to the VIP and no
166 * server selection performed yet
167 * @param nodeBuilder NodeBuilder
168 * @param lbConfig LoadBalancerConfiguration
169 * @param write Boolean to indicate of the flow is to be inserted or removed
171 private void manageLoadBalancerVIPRulesFirstPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig, boolean write) {
172 MatchBuilder matchBuilder = new MatchBuilder();
173 FlowBuilder flowBuilder = new FlowBuilder();
175 // Match Tunnel-ID, VIP, and Reg0==0
176 if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VXLAN) ||
177 lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_GRE)) {
178 MatchUtils.createTunnelIDMatch(matchBuilder, new BigInteger(lbConfig.getProviderSegmentationId()));
179 } else if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VLAN)) {
180 MatchUtils.createVlanIdMatch(matchBuilder, new VlanId(Integer.valueOf(lbConfig.getProviderSegmentationId())), true);
182 return; //Should not get here. TODO: Other types
185 MatchUtils.createDstL3IPv4Match(matchBuilder, MatchUtils.iPv4PrefixFromIPv4Address(lbConfig.getVip()));
186 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, FIRST_PASS_REGA_MATCH_VALUE));
188 String flowId = "LOADBALANCER_FORWARD_FLOW1_" + lbConfig.getProviderSegmentationId() + "_" + lbConfig.getVip();
189 flowBuilder.setId(new FlowId(flowId));
190 FlowKey key = new FlowKey(new FlowId(flowId));
191 flowBuilder.setMatch(matchBuilder.build());
192 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
193 flowBuilder.setBarrier(true);
194 flowBuilder.setTableId(this.getTable());
195 flowBuilder.setKey(key);
196 flowBuilder.setFlowName(flowId);
197 flowBuilder.setHardTimeout(0);
198 flowBuilder.setIdleTimeout(0);
201 // Create the OF Actions and Instructions
202 InstructionsBuilder isb = new InstructionsBuilder();
204 // Instructions List Stores Individual Instructions
205 List<Instruction> instructions = Lists.newArrayList();
207 List<Action> actionList = Lists.newArrayList();
209 ActionBuilder ab = new ActionBuilder();
210 ab.setAction(ActionUtils.nxLoadRegAction(new DstNxRegCaseBuilder().setNxReg(REG_FIELD_A).build(),
211 BigInteger.valueOf(SECOND_PASS_REGA_MATCH_VALUE)));
213 ab.setKey(new ActionKey(0));
214 actionList.add(ab.build());
216 ab = new ActionBuilder();
217 ab.setAction(ActionUtils.nxMultipathAction(OfjNxHashFields.NXHASHFIELDSSYMMETRICL4,
218 0, OfjNxMpAlgorithm.NXMPALGMODULON,
219 lbConfig.getMembers().size()-1, //By Nicira-Ext spec, this field is max_link minus 1
220 0L, new DstNxRegCaseBuilder().setNxReg(REG_FIELD_B).build(),
223 ab.setKey(new ActionKey(1));
224 actionList.add(ab.build());
226 ab = new ActionBuilder();
227 ab.setAction(ActionUtils.nxResubmitAction(null, this.getTable()));
229 ab.setKey(new ActionKey(2));
230 actionList.add(ab.build());
232 // Create an Apply Action
233 ApplyActionsBuilder aab = new ApplyActionsBuilder();
234 aab.setAction(actionList);
235 InstructionBuilder ib = new InstructionBuilder();
236 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
238 // Call the InstructionBuilder Methods Containing Actions
240 ib.setKey(new InstructionKey(0));
241 instructions.add(ib.build());
243 // Add InstructionBuilder to the Instruction(s)Builder List
244 isb.setInstruction(instructions);
246 // Add InstructionsBuilder to FlowBuilder
247 flowBuilder.setInstructions(isb.build());
249 writeFlow(flowBuilder, nodeBuilder);
252 removeFlow(flowBuilder, nodeBuilder);
257 * Method to program each rule that matches on Reg0 and Reg1 to insert appropriate header rewriting
258 * rules for all members. This function calls manageLoadBalancerMemberVIPRulesSecondPass in turn.
259 * @param nodeBuilder Node to insert rule to
260 * @param lbConfig Configuration for this LoadBalancer instance
261 * @param write Boolean to indicate of the flow is to be inserted or removed
263 private void manageLoadBalancerVIPRulesSecondPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig, boolean write) {
264 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
265 manageLoadBalancerMemberVIPRulesSecondPass(nodeBuilder, lbConfig, entry.getValue(), write);
269 private void manageLoadBalancerMemberVIPRulesSecondPass(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig, LoadBalancerPoolMember member, boolean write) {
270 String vip = lbConfig.getVip();
272 MatchBuilder matchBuilder = new MatchBuilder();
273 FlowBuilder flowBuilder = new FlowBuilder();
275 // Match Tunnel-ID, VIP, Reg0==1 and Reg1==Index of member
276 if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VXLAN) ||
277 lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_GRE)) {
278 MatchUtils.createTunnelIDMatch(matchBuilder, new BigInteger(lbConfig.getProviderSegmentationId()));
279 } else if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VLAN)) {
280 MatchUtils.createVlanIdMatch(matchBuilder, new VlanId(Integer.valueOf(lbConfig.getProviderSegmentationId())), true);
282 return; //Should not get here. TODO: Other types
285 MatchUtils.createDstL3IPv4Match(matchBuilder, MatchUtils.iPv4PrefixFromIPv4Address(vip));
286 MatchUtils.addNxRegMatch(matchBuilder, new MatchUtils.RegMatch(REG_FIELD_A, SECOND_PASS_REGA_MATCH_VALUE),
287 new MatchUtils.RegMatch(REG_FIELD_B, (long)member.getIndex()));
289 String flowId = "LOADBALANCER_FORWARD_FLOW2_" + lbConfig.getProviderSegmentationId() + "_" +
290 vip + "_" + member.getIP();
291 flowBuilder.setId(new FlowId(flowId));
292 FlowKey key = new FlowKey(new FlowId(flowId));
293 flowBuilder.setMatch(matchBuilder.build());
294 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY+1);
295 flowBuilder.setBarrier(true);
296 flowBuilder.setTableId(this.getTable());
297 flowBuilder.setKey(key);
298 flowBuilder.setFlowName(flowId);
299 flowBuilder.setHardTimeout(0);
300 flowBuilder.setIdleTimeout(0);
303 // Create the OF Actions and Instructions
304 InstructionsBuilder isb = new InstructionsBuilder();
306 // Instructions List Stores Individual Instructions
307 List<Instruction> instructions = Lists.newArrayList();
309 List<Action> actionList = Lists.newArrayList();
310 ActionBuilder ab = new ActionBuilder();
311 ab.setAction(ActionUtils.setDlDstAction(new MacAddress(member.getMAC())));
313 ab.setKey(new ActionKey(0));
314 actionList.add(ab.build());
316 ab = new ActionBuilder();
317 Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(MatchUtils.iPv4PrefixFromIPv4Address(member.getIP()));
318 ab.setAction(ActionUtils.setNwDstAction(ipb.build()));
320 ab.setKey(new ActionKey(1));
321 actionList.add(ab.build());
323 // Create an Apply Action
324 ApplyActionsBuilder aab = new ApplyActionsBuilder();
325 aab.setAction(actionList);
327 // Call the InstructionBuilder Methods Containing Actions
328 InstructionBuilder ib = new InstructionBuilder();
329 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
331 ib.setKey(new InstructionKey(0));
332 instructions.add(ib.build());
334 // Call the InstructionBuilder Methods Containing Actions
335 ib = this.getMutablePipelineInstructionBuilder();
337 ib.setKey(new InstructionKey(1));
338 instructions.add(ib.build());
340 // Add InstructionBuilder to the Instruction(s)Builder List
341 isb.setInstruction(instructions);
343 // Add InstructionsBuilder to FlowBuilder
344 flowBuilder.setInstructions(isb.build());
346 writeFlow(flowBuilder, nodeBuilder);
349 removeFlow(flowBuilder, nodeBuilder);
354 * Method to program all reverse rules that matches member {IP, Protocol, Port} for all members.
355 * This function calls manageLoadBalancerMemberReverseRules in turn.
356 * @param nodeBuilder Node to insert rule to
357 * @param lbConfig Configuration for this LoadBalancer instance
359 private void manageLoadBalancerReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig, boolean write) {
360 for(Map.Entry<String, LoadBalancerPoolMember> entry : lbConfig.getMembers().entrySet()){
361 manageLoadBalancerMemberReverseRules(nodeBuilder, lbConfig, entry.getValue(), write);
365 private void manageLoadBalancerMemberReverseRules(NodeBuilder nodeBuilder, LoadBalancerConfiguration lbConfig,
366 LoadBalancerPoolMember member, boolean write) {
368 String vip = lbConfig.getVip();
369 String vmac = lbConfig.getVmac();
371 MatchBuilder matchBuilder = new MatchBuilder();
372 FlowBuilder flowBuilder = new FlowBuilder();
374 // Match Tunnel-ID, MemberIP, and Protocol/Port
375 if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VXLAN) ||
376 lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_GRE)) {
377 MatchUtils.createTunnelIDMatch(matchBuilder, new BigInteger(lbConfig.getProviderSegmentationId()));
378 } else if (lbConfig.getProviderNetworkType().equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VLAN)) {
379 MatchUtils.createVlanIdMatch(matchBuilder, new VlanId(Integer.valueOf(lbConfig.getProviderSegmentationId())), true);
381 return; //Should not get here. TODO: Other types
384 MatchUtils.createSrcL3IPv4Match(matchBuilder, MatchUtils.iPv4PrefixFromIPv4Address(member.getIP()));
385 MatchUtils.createSetSrcTcpMatch(matchBuilder, new PortNumber(member.getPort()));
387 String flowId = "LOADBALANCER_REVERSE_FLOW_" + lbConfig.getProviderSegmentationId() +
388 vip + "_" + member.getIP();
389 flowBuilder.setId(new FlowId(flowId));
390 FlowKey key = new FlowKey(new FlowId(flowId));
391 flowBuilder.setMatch(matchBuilder.build());
392 flowBuilder.setPriority(DEFAULT_FLOW_PRIORITY);
393 flowBuilder.setBarrier(true);
394 flowBuilder.setTableId(this.getTable());
395 flowBuilder.setKey(key);
396 flowBuilder.setFlowName(flowId);
397 flowBuilder.setHardTimeout(0);
398 flowBuilder.setIdleTimeout(0);
401 // Create the OF Actions and Instructions
402 InstructionsBuilder isb = new InstructionsBuilder();
404 // Instructions List Stores Individual Instructions
405 List<Instruction> instructions = Lists.newArrayList();
407 List<Action> actionList = Lists.newArrayList();
408 ActionBuilder ab = new ActionBuilder();
409 Ipv4Builder ipb = new Ipv4Builder().setIpv4Address(MatchUtils.iPv4PrefixFromIPv4Address(vip));
410 ab.setAction(ActionUtils.setNwSrcAction(ipb.build()));
412 ab.setKey(new ActionKey(0));
413 actionList.add(ab.build());
415 /* If a dummy MAC is assigned to the VIP, we use that as the
416 * source MAC for the reverse traffic.
419 ab = new ActionBuilder();
420 ab.setAction(ActionUtils.setDlDstAction(new MacAddress(vmac)));
422 ab.setKey(new ActionKey(1));
423 actionList.add(ab.build());
426 // Create an Apply Action
427 ApplyActionsBuilder aab = new ApplyActionsBuilder();
428 aab.setAction(actionList);
430 // Call the InstructionBuilder Methods Containing Actions
431 InstructionBuilder ib = new InstructionBuilder();
432 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
434 ib.setKey(new InstructionKey(0));
435 instructions.add(ib.build());
437 // Call the InstructionBuilder Methods Containing Actions
438 ib = this.getMutablePipelineInstructionBuilder();
440 ib.setKey(new InstructionKey(1));
441 instructions.add(ib.build());
443 // Add InstructionBuilder to the Instruction(s)Builder List
444 isb.setInstruction(instructions);
446 // Add InstructionsBuilder to FlowBuilder
447 flowBuilder.setInstructions(isb.build());
449 writeFlow(flowBuilder, nodeBuilder);
452 removeFlow(flowBuilder, nodeBuilder);
457 public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
458 super.setDependencies(bundleContext.getServiceReference(LoadBalancerProvider.class.getName()), this);
459 southbound =(Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
463 public void setDependencies(Object impl) {