csit/libraries/XmlComparator.py

   1 '''
   2 Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3
   4 This program and the accompanying materials are made available under the
   5 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6 and is available at http://www.eclipse.org/legal/epl-v10.html
   7
   8 Created on May 21, 2014
   9
  10 @author: <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
  11 '''
  12 from xml.dom.minidom import Element
  13 import ipaddr
  14 import xml.dom.minidom as md
  15 import copy
  16
  17 KEY_NOT_FOUND = '<KEY_NOT_FOUND>'  # KeyNotFound for dictDiff
  18
  19
  20 class XMLtoDictParserTools():
  21
  22     @staticmethod
  23     def parseTreeToDict(node, returnedDict=None, ignoreList=[]):
  24         """
  25         Return Dictionary representation of the xml Tree DOM Element.
  26         Repeated tags are put to the array sorted by key (id or order)
  27         otherwise is the value represented by tag key name.
  28         @param node: DOM Element
  29         @param returnedDict : dictionary (default value None)
  30         @param ignereList : list of ignored tags for the xml Tree DOM Element
  31                             (default value is empty list)
  32         @return: dict representation for the input DOM Element
  33         """
  34         returnedDict = {} if returnedDict is None else returnedDict
  35         if (node.nodeType == Element.ELEMENT_NODE):
  36             nodeKey = (node.localName).encode('utf-8', 'ignore')
  37             if nodeKey not in ignoreList:
  38                 if node.childNodes is not None:
  39                     childDict = {}
  40                     for child in node.childNodes:
  41                         if child.nodeType == Element.TEXT_NODE:
  42                             nodeValue = (child.nodeValue).encode('utf-8', 'ignore')
  43                             if (len(nodeValue.strip(' \t\n\r'))) > 0:
  44                                 XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, nodeValue)
  45                                 nodeKey = None
  46                                 break
  47                         elif child.nodeType == Element.ELEMENT_NODE:
  48                             childDict = XMLtoDictParserTools.parseTreeToDict(child, childDict, ignoreList)
  49
  50                     XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, childDict)
  51
  52         return returnedDict
  53
  54     @staticmethod
  55     def addDictValue(m_dict, key, value):
  56
  57         def _allign_address(value):
  58             """unifies output"""
  59             n = ipaddr.IPNetwork(value)
  60             return '{0}/{1}'.format(n.network.exploded, n.prefixlen)
  61
  62         def _convert_numbers(value):
  63             if value.startswith("0x"):
  64                 return str(long(value, 16))
  65             return str(long(value))
  66
  67         if key is not None:
  68             if (isinstance(value, str)):
  69                 # we need to predict possible differences
  70                 # for same value in upper or lower case
  71                 value = value.lower()
  72             if key not in m_dict:
  73                 # lets add mask for ips withot mask
  74                 if key in ['ipv4-destination', 'ipv4-source', 'ipv6-destination', 'ipv6-source', 'ipv6-nd-target']:
  75                     nvalue = _allign_address(value)
  76                     m_dict[key] = nvalue
  77                 elif key in ['tunnel-mask', 'type', 'metadata-mask', 'out_port', 'out_group']:
  78                     nvalue = _convert_numbers(value)
  79                     m_dict[key] = nvalue
  80                 else:
  81                     m_dict[key] = value
  82             else:
  83                 exist_value = m_dict.get(key)
  84                 if (type(exist_value) is dict):
  85                     list_values = [exist_value, value]
  86                     key_for_sort = XMLtoDictParserTools.searchKey(exist_value)
  87                     if key_for_sort is not None:
  88                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  89                     m_dict[key] = list_values
  90                 elif (isinstance(exist_value, list)):
  91                     exist_value.append(value)
  92                     list_values = exist_value
  93                     key_for_sort = XMLtoDictParserTools.searchKey(value)
  94                     if key_for_sort is not None:
  95                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  96                     m_dict[key] = list_values
  97                 else:
  98                     m_dict[key] += value
  99
 100     @staticmethod
 101     def searchKey(dictionary):
 102         """
 103         Return an order key for the array ordering. OF_13
 104         allows only two possible kind of the order keys
 105         'order' or '*-id'
 106         @param dictionary: dictionary with data
 107         @return: the array order key
 108         """
 109         subKeyStr = ['-id', 'order']
 110         for substr in subKeyStr:
 111             for key in dictionary:
 112                 if key == substr:
 113                     return key
 114                 elif key.endswith(substr):
 115                     return key
 116         return None
 117
 118     @staticmethod
 119     def getDifferenceDict(original_dict, responded_dict):
 120         """
 121         Return a dict of keys that differ with another config object.  If a value is
 122         not found in one fo the configs, it will be represented by KEY_NOT_FOUND.
 123         @param original_dict:   Fist dictionary to diff.
 124         @param responded_dict:  Second dictionary to diff.
 125         @return diff:   Dict of Key => (original_dict.val, responded_dict.val)
 126                         Dict of Key => (original_key, KEY_NOT_FOUND)
 127                         Dict of Key => (KEY_NOT_FOUNE, original_key)
 128         """
 129         diff = {}
 130         # Check all keys in original_dict dict
 131         for key in original_dict.keys():
 132             if key not in responded_dict:
 133                 # missing key in responded dict
 134                 diff[key] = (key, KEY_NOT_FOUND)
 135             # check values of the dictionaries
 136             elif (original_dict[key] != responded_dict[key]):
 137                 # values are not the same #
 138
 139                 orig_dict_val = original_dict[key]
 140                 resp_dict_val = responded_dict[key]
 141
 142                 # check value is instance of dictionary
 143                 if isinstance(orig_dict_val, dict) and isinstance(resp_dict_val, dict):
 144                     sub_dif = XMLtoDictParserTools.getDifferenceDict(orig_dict_val, resp_dict_val)
 145                     if sub_dif:
 146                         diff[key] = sub_dif
 147
 148                 # check value is instance of list
 149                 # TODO - > change a basic comparator to compare by id or order
 150                 elif isinstance(orig_dict_val, list) and isinstance(resp_dict_val, list):
 151                     sub_list_diff = {}
 152                     # the list lengths
 153                     orig_i, resp_i = len(orig_dict_val), len(resp_dict_val)
 154                     # define a max iteration length (less from both)
 155                     min_index = orig_i if orig_i < resp_i else resp_i
 156                     for index in range(0, min_index, 1):
 157                         if (orig_dict_val[index] != resp_dict_val[index]):
 158                             sub_list_diff[index] = (orig_dict_val[index], resp_dict_val[index])
 159                     if (orig_i > min_index):
 160                         # original is longer as responded dict
 161                         for index in range(min_index, orig_i, 1):
 162                             sub_list_diff[index] = (orig_dict_val[index], None)
 163                     elif (resp_i > min_index):
 164                         # responded dict is longer as original
 165                         for index in range(min_index, resp_i, 1):
 166                             sub_list_diff[index] = (None, resp_dict_val[index])
 167                     if sub_list_diff:
 168                         diff[key] = sub_list_diff
 169
 170                 else:
 171                     diff[key] = (original_dict[key], responded_dict[key])
 172
 173         # Check all keys in responded_dict dict to find missing
 174         for key in responded_dict.keys():
 175             if key not in original_dict:
 176                 diff[key] = (KEY_NOT_FOUND, key)
 177         return diff
 178
 179
 180 IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON = ['id', 'flow-name', 'barrier', 'cookie_mask', 'installHw', 'flags',
 181                                            'strict', 'byte-count', 'duration', 'packet-count', 'in-port',
 182                                            'vlan-id-present', 'out_group', 'out_port', 'hard-timeout', 'idle-timeout',
 183                                            'flow-statistics', 'cookie', 'clear-actions',
 184                                            'ipv4-source-address-no-mask', 'ipv4-source-arbitrary-bitmask',
 185                                            'ipv4-destination-address-no-mask', 'ipv4-destination-arbitrary-bitmask',
 186                                            'ipv6-source-address-no-mask', 'ipv6-source-arbitrary-bitmask',
 187                                            'ipv6-destination-address-no-mask', 'ipv6-destination-arbitrary-bitmask']  # noqa
 188
 189 IGNORED_PATHS_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'controller-action'], True),  # noqa
 190                         (['flow', 'instructions', 'instruction', 'clear-actions', 'action'], False),
 191                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'push-vlan-action', 'vlan-id'], False),  # noqa
 192                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'drop-action'], True),
 193                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'flood-action'], True),
 194                         ]
 195
 196 TAGS_TO_ADD_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'output-action'], 'max-length', '0'),  # noqa
 197                       ]
 198
 199
 200 TAGS_TO_MODIFY_FOR_OC = [(['flow', 'match', 'metadata'], 'metadata', 'metadata-mask'),
 201                          (['flow', 'match', 'tunnel'], 'tunnel-id', 'tunnel-mask'),
 202                          ]
 203
 204
 205 class XmlComparator:
 206
 207     def is_flow_configured(self, requested_flow, configured_flows):
 208
 209         orig_tree = md.parseString(requested_flow)
 210         xml_resp_stream = configured_flows.encode('utf-8', 'ignore')
 211         xml_resp_tree = md.parseString(xml_resp_stream)
 212         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 213         origDict = XMLtoDictParserTools.parseTreeToDict(orig_tree._get_documentElement())
 214
 215         reportDict = {}
 216         index = 0
 217         for node in nodeListOperFlows:
 218             nodeDict = XMLtoDictParserTools.parseTreeToDict(node)
 219             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 220             index += 1
 221             if nodeDict == origDict:
 222                 return True, ''
 223             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 224                 return False, 'Flow found with diferences {0}'.format(
 225                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 226         return False, ''
 227
 228     def is_flow_operational2(self, requested_flow, oper_resp, check_id=False):
 229         def _rem_unimplemented_tags(tagpath, recurs, tdict):
 230             if len(tagpath) > 1 and tagpath[0] in tdict:
 231                 _rem_unimplemented_tags(tagpath[1:], recurs, tdict[tagpath[0]])
 232
 233             # when not to delete anything
 234             if len(tagpath) == 1 and tagpath[0] not in tdict:
 235                 return
 236             if len(tagpath) == 0:
 237                 return
 238
 239             # when to delete
 240             if len(tagpath) == 1 and tagpath[0] in tdict:
 241                 del tdict[tagpath[0]]
 242             if len(tagpath) > 1 and recurs is True and tagpath[0] in tdict and tdict[tagpath[0]] == {}:
 243                 del tdict[tagpath[0]]
 244             if tdict.keys() == ['order']:
 245                 del tdict['order']
 246
 247         def _add_tags(tagpath, newtag, value, tdict):
 248             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 249             if len(tagpath) > 0 and tagpath[0] in tdict:
 250                 _add_tags(tagpath[1:], newtag, value, tdict[tagpath[0]])
 251             elif len(tagpath) == 0 and newtag not in tdict:
 252                 tdict[newtag] = value
 253
 254         def _to_be_modified_tags(tagpath, tag, related_tag, tdict):
 255             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 256             if len(tagpath) > 0 and tagpath[0] in tdict:
 257                 _to_be_modified_tags(tagpath[1:], tag, related_tag, tdict[tagpath[0]])
 258             elif len(tagpath) == 0 and tag in tdict and related_tag in tdict:
 259                 tdict[tag] = str(long(tdict[tag]) & long(tdict[related_tag]))
 260
 261         IGNORED_TAGS_LIST = list(IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 262         if check_id:
 263             IGNORED_TAGS_LIST.remove('id')
 264         orig_tree = md.parseString(requested_flow)
 265         xml_resp_stream = oper_resp.encode('utf-8', 'ignore')
 266         xml_resp_tree = md.parseString(xml_resp_stream)
 267         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 268         origDict = XMLtoDictParserTools.parseTreeToDict(
 269             orig_tree._get_documentElement(),
 270             ignoreList=IGNORED_TAGS_LIST)
 271
 272         # origDict['flow-statistics'] = origDict.pop( 'flow' )
 273         reportDict = {}
 274         index = 0
 275         for node in nodeListOperFlows:
 276             nodeDict = XMLtoDictParserTools.parseTreeToDict(
 277                 node,
 278                 ignoreList=IGNORED_TAGS_LIST)
 279             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 280             index += 1
 281             if nodeDict == origDict:
 282                 return True, ''
 283             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 284                 for p in IGNORED_PATHS_FOR_OC:
 285                     td = copy.copy(origDict)
 286                     _rem_unimplemented_tags(p[0], p[1], td)
 287                     for (p, t, v) in TAGS_TO_ADD_FOR_OC:
 288                         _add_tags(p, t, v, td)
 289                     for (p, t, rt) in TAGS_TO_MODIFY_FOR_OC:
 290                         _to_be_modified_tags(p, t, rt, td)
 291
 292                     if nodeDict == td:
 293                         return True, ''
 294                 if nodeDict == origDict:
 295                     return True, ''
 296                 return False, 'Flow found with diferences {0}'.format(
 297                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 298         return False, ''
 299
 300     def get_data_for_flow_put_update(self, xml):
 301         # action only for yet
 302         xml_dom_input = md.parseString(xml)
 303         actionList = xml_dom_input.getElementsByTagName('action')
 304         if actionList is not None and len(actionList) > 0:
 305             action = actionList[0]
 306             for child in action.childNodes:
 307                 if child.nodeType == Element.ELEMENT_NODE:
 308                     nodeKey = (child.localName).encode('utf-8', 'ignore')
 309                     if nodeKey != 'order':
 310                         if nodeKey != 'drop-action':
 311                             new_act = child.ownerDocument.createElement('drop-action')
 312                         else:
 313                             new_act = child.ownerDocument.createElement('output-action')
 314                             onc = child.ownerDocument.createElement('output-node-connector')
 315                             onc_content = child.ownerDocument.createTextNode('TABLE')
 316                             onc.appendChild(onc_content)
 317                             new_act.appendChild(onc)
 318                             ml = child.ownerDocument.createElement('max-length')
 319                             ml_content = child.ownerDocument.createTextNode('60')
 320                             ml.appendChild(ml_content)
 321                             new_act.appendChild(ml)
 322                         child.parentNode.replaceChild(new_act, child)
 323         return xml_dom_input.toxml(encoding='utf-8')
 324
 325     def get_flow_content(self, tid=1, fid=1, priority=1):
 326         """Returns an xml flow content identified by given details.
 327
 328         Args:
 329             :param tid: table id
 330             :param fid: flow id
 331             :param priority: flow priority
 332         """
 333
 334         flow_template = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 335 <flow xmlns="urn:opendaylight:flow:inventory">
 336     <strict>false</strict>
 337     <instructions>
 338         <instruction>
 339             <order>0</order>
 340             <apply-actions>
 341                 <action>
 342                     <order>0</order>
 343                     <drop-action/>
 344                 </action>
 345             </apply-actions>
 346         </instruction>
 347     </instructions>
 348     <table_id>%s</table_id>
 349     <id>%s</id>
 350     <cookie_mask>4294967295</cookie_mask>
 351     <installHw>false</installHw>
 352     <match>
 353         <ethernet-match>
 354             <ethernet-type>
 355                 <type>2048</type>
 356             </ethernet-type>
 357         </ethernet-match>
 358         <ipv4-source>10.0.0.1/32</ipv4-source>
 359     </match>
 360     <cookie>%s</cookie>
 361     <flow-name>%s</flow-name>
 362     <priority>%s</priority>
 363     <barrier>false</barrier>
 364 </flow>'''
 365
 366         flow_data = flow_template % (tid, fid, fid, 'TestFlow-{0}'.format(fid), priority)
 367         return flow_data