csit/libraries/XmlComparator.py

   1 '''
   2 Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3
   4 This program and the accompanying materials are made available under the
   5 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6 and is available at http://www.eclipse.org/legal/epl-v10.html
   7
   8 Created on May 21, 2014
   9
  10 @author: <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
  11 '''
  12 from xml.dom.minidom import Element
  13 import ipaddr
  14 import xml.dom.minidom as md
  15 import copy
  16
  17 KEY_NOT_FOUND = '<KEY_NOT_FOUND>'  # KeyNotFound for dictDiff
  18
  19
  20 class XMLtoDictParserTools():
  21
  22     @staticmethod
  23     def parseTreeToDict(node, returnedDict=None, ignoreList=[]):
  24         """
  25         Return Dictionary representation of the xml Tree DOM Element.
  26         Repeated tags are put to the array sorted by key (id or order)
  27         otherwise is the value represented by tag key name.
  28         @param node: DOM Element
  29         @param returnedDict : dictionary (default value None)
  30         @param ignereList : list of ignored tags for the xml Tree DOM Element
  31                             (default value is empty list)
  32         @return: dict representation for the input DOM Element
  33         """
  34         returnedDict = {} if returnedDict is None else returnedDict
  35         if (node.nodeType == Element.ELEMENT_NODE):
  36             nodeKey = (node.localName).encode('utf-8', 'ignore')
  37             if nodeKey not in ignoreList:
  38                 if node.childNodes is not None:
  39                     childDict = {}
  40                     for child in node.childNodes:
  41                         if child.nodeType == Element.TEXT_NODE:
  42                             nodeValue = (child.nodeValue).encode('utf-8', 'ignore')
  43                             if (len(nodeValue.strip(' \t\n\r'))) > 0:
  44                                 XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, nodeValue)
  45                                 nodeKey = None
  46                                 break
  47                         elif child.nodeType == Element.ELEMENT_NODE:
  48                             childDict = XMLtoDictParserTools.parseTreeToDict(child, childDict, ignoreList)
  49
  50                     XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, childDict)
  51
  52         return returnedDict
  53
  54     @staticmethod
  55     def addDictValue(m_dict, key, value):
  56
  57         def _allign_address(value):
  58             """unifies output"""
  59             n = ipaddr.IPNetwork(value)
  60             return '{0}/{1}'.format(n.network.exploded, n.prefixlen)
  61
  62         def _convert_numbers(value):
  63             if value.startswith("0x"):
  64                 return str(long(value, 16))
  65             return str(long(value))
  66
  67         if key is not None:
  68             if (isinstance(value, str)):
  69                 # we need to predict possible differences
  70                 # for same value in upper or lower case
  71                 value = value.lower()
  72             if key not in m_dict:
  73                 # lets add mask for ips withot mask
  74                 if key in ['ipv4-destination', 'ipv4-source', 'ipv6-destination', 'ipv6-source', 'ipv6-nd-target']:
  75                     nvalue = _allign_address(value)
  76                     m_dict[key] = nvalue
  77                 elif key in ['tunnel-mask', 'type', 'metadata-mask', 'out_port', 'out_group']:
  78                     nvalue = _convert_numbers(value)
  79                     m_dict[key] = nvalue
  80                 else:
  81                     m_dict[key] = value
  82             else:
  83                 exist_value = m_dict.get(key)
  84                 if (type(exist_value) is dict):
  85                     list_values = [exist_value, value]
  86                     key_for_sort = XMLtoDictParserTools.searchKey(exist_value)
  87                     if key_for_sort is not None:
  88                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  89                     m_dict[key] = list_values
  90                 elif (isinstance(exist_value, list)):
  91                     exist_value.append(value)
  92                     list_values = exist_value
  93                     key_for_sort = XMLtoDictParserTools.searchKey(value)
  94                     if key_for_sort is not None:
  95                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  96                     m_dict[key] = list_values
  97                 else:
  98                     m_dict[key] += value
  99
 100     @staticmethod
 101     def searchKey(dictionary):
 102         """
 103         Return an order key for the array ordering. OF_13
 104         allows only two possible kind of the order keys
 105         'order' or '*-id'
 106         @param dictionary: dictionary with data
 107         @return: the array order key
 108         """
 109         subKeyStr = ['-id', 'order']
 110         for substr in subKeyStr:
 111             for key in dictionary:
 112                 if key == substr:
 113                     return key
 114                 elif key.endswith(substr):
 115                     return key
 116         return None
 117
 118     @staticmethod
 119     def getDifferenceDict(original_dict, responded_dict):
 120         """
 121         Return a dict of keys that differ with another config object.  If a value is
 122         not found in one fo the configs, it will be represented by KEY_NOT_FOUND.
 123         @param original_dict:   Fist dictionary to diff.
 124         @param responded_dict:  Second dictionary to diff.
 125         @return diff:   Dict of Key => (original_dict.val, responded_dict.val)
 126                         Dict of Key => (original_key, KEY_NOT_FOUND)
 127                         Dict of Key => (KEY_NOT_FOUNE, original_key)
 128         """
 129         diff = {}
 130         # Check all keys in original_dict dict
 131         for key in original_dict.keys():
 132             if key not in responded_dict:
 133                 # missing key in responded dict
 134                 diff[key] = (key, KEY_NOT_FOUND)
 135             # check values of the dictionaries
 136             elif (original_dict[key] != responded_dict[key]):
 137                 # values are not the same #
 138
 139                 orig_dict_val = original_dict[key]
 140                 resp_dict_val = responded_dict[key]
 141
 142                 # check value is instance of dictionary
 143                 if isinstance(orig_dict_val, dict) and isinstance(resp_dict_val, dict):
 144                     sub_dif = XMLtoDictParserTools.getDifferenceDict(orig_dict_val, resp_dict_val)
 145                     if sub_dif:
 146                         diff[key] = sub_dif
 147
 148                 # check value is instance of list
 149                 # TODO - > change a basic comparator to compare by id or order
 150                 elif isinstance(orig_dict_val, list) and isinstance(resp_dict_val, list):
 151                     sub_list_diff = {}
 152                     # the list lengths
 153                     orig_i, resp_i = len(orig_dict_val), len(resp_dict_val)
 154                     # define a max iteration length (less from both)
 155                     min_index = orig_i if orig_i < resp_i else resp_i
 156                     for index in range(0, min_index, 1):
 157                         if (orig_dict_val[index] != resp_dict_val[index]):
 158                             sub_list_diff[index] = (orig_dict_val[index], resp_dict_val[index])
 159                     if (orig_i > min_index):
 160                         # original is longer as responded dict
 161                         for index in range(min_index, orig_i, 1):
 162                             sub_list_diff[index] = (orig_dict_val[index], None)
 163                     elif (resp_i > min_index):
 164                         # responded dict is longer as original
 165                         for index in range(min_index, resp_i, 1):
 166                             sub_list_diff[index] = (None, resp_dict_val[index])
 167                     if sub_list_diff:
 168                         diff[key] = sub_list_diff
 169
 170                 else:
 171                     diff[key] = (original_dict[key], responded_dict[key])
 172
 173         # Check all keys in responded_dict dict to find missing
 174         for key in responded_dict.keys():
 175             if key not in original_dict:
 176                 diff[key] = (KEY_NOT_FOUND, key)
 177         return diff
 178
 179 IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON = ['id', 'flow-name', 'barrier', 'cookie_mask', 'installHw', 'flags',
 180                                            'strict', 'byte-count', 'duration', 'packet-count', 'in-port',
 181                                            'vlan-id-present', 'out_group', 'out_port', 'hard-timeout', 'idle-timeout',
 182                                            'flow-statistics', 'cookie', 'clear-actions',
 183                                            'ipv4-source-address-no-mask', 'ipv4-source-arbitrary-bitmask',
 184                                            'ipv4-destination-address-no-mask', 'ipv4-destination-arbitrary-bitmask',
 185                                            'ipv6-source-address-no-mask', 'ipv6-source-arbitrary-bitmask',
 186                                            'ipv6-destination-address-no-mask', 'ipv6-destination-arbitrary-bitmask']  # noqa
 187
 188 IGNORED_PATHS_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'controller-action'], True),  # noqa
 189                         (['flow', 'instructions', 'instruction', 'clear-actions', 'action'], False),
 190                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'push-vlan-action', 'vlan-id'], False),  # noqa
 191                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'drop-action'], True),
 192                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'flood-action'], True),
 193                         ]
 194
 195 TAGS_TO_ADD_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'output-action'], 'max-length', '0'),  # noqa
 196                       ]
 197
 198
 199 TAGS_TO_MODIFY_FOR_OC = [(['flow', 'match', 'metadata'], 'metadata', 'metadata-mask'),
 200                          (['flow', 'match', 'tunnel'], 'tunnel-id', 'tunnel-mask'),
 201                          ]
 202
 203
 204 class XmlComparator:
 205
 206     def is_flow_configured(self, requested_flow, configured_flows):
 207
 208         orig_tree = md.parseString(requested_flow)
 209         xml_resp_stream = configured_flows.encode('utf-8', 'ignore')
 210         xml_resp_tree = md.parseString(xml_resp_stream)
 211         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 212         origDict = XMLtoDictParserTools.parseTreeToDict(orig_tree._get_documentElement())
 213
 214         reportDict = {}
 215         index = 0
 216         for node in nodeListOperFlows:
 217             nodeDict = XMLtoDictParserTools.parseTreeToDict(node)
 218             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 219             index += 1
 220             # print nodeDict
 221             # print origDict
 222             if nodeDict == origDict:
 223                 return True, ''
 224             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 225                 return False, 'Flow found with diferences {0}'.format(
 226                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 227         return False, ''
 228
 229     def is_flow_operational2(self, requested_flow, oper_resp, check_id=False):
 230         def _rem_unimplemented_tags(tagpath, recurs, tdict):
 231             # print "_rem_unimplemented_tags", tagpath, tdict
 232             if len(tagpath) > 1 and tagpath[0] in tdict:
 233                 _rem_unimplemented_tags(tagpath[1:], recurs, tdict[tagpath[0]])
 234
 235             # when not to delete anything
 236             if len(tagpath) == 1 and tagpath[0] not in tdict:
 237                 return
 238             if len(tagpath) == 0:
 239                 return
 240
 241             # when to delete
 242             if len(tagpath) == 1 and tagpath[0] in tdict:
 243                 del tdict[tagpath[0]]
 244             if len(tagpath) > 1 and recurs is True and tagpath[0] in tdict and tdict[tagpath[0]] == {}:
 245                 del tdict[tagpath[0]]
 246             if tdict.keys() == ['order']:
 247                 del tdict['order']
 248             # print "leaving", tdict
 249
 250         def _add_tags(tagpath, newtag, value, tdict):
 251             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 252             # print "_add_tags", tagpath, newtag, value, tdict
 253             if len(tagpath) > 0 and tagpath[0] in tdict:
 254                 _add_tags(tagpath[1:], newtag, value, tdict[tagpath[0]])
 255             elif len(tagpath) == 0 and newtag not in tdict:
 256                 tdict[newtag] = value
 257
 258         def _to_be_modified_tags(tagpath, tag, related_tag, tdict):
 259             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 260             # print "_to_be_modified_tags", tagpath, tag, related_tag, tdict
 261             if len(tagpath) > 0 and tagpath[0] in tdict:
 262                 _to_be_modified_tags(tagpath[1:], tag, related_tag, tdict[tagpath[0]])
 263             elif len(tagpath) == 0 and tag in tdict and related_tag in tdict:
 264                 tdict[tag] = str(long(tdict[tag]) & long(tdict[related_tag]))
 265
 266         IGNORED_TAGS_LIST = list(IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 267         if check_id:
 268             IGNORED_TAGS_LIST.remove('id')
 269         orig_tree = md.parseString(requested_flow)
 270         xml_resp_stream = oper_resp.encode('utf-8', 'ignore')
 271         xml_resp_tree = md.parseString(xml_resp_stream)
 272         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 273         origDict = XMLtoDictParserTools.parseTreeToDict(
 274             orig_tree._get_documentElement(),
 275             ignoreList=IGNORED_TAGS_LIST)
 276
 277         # origDict['flow-statistics'] = origDict.pop( 'flow' )
 278         reportDict = {}
 279         index = 0
 280         for node in nodeListOperFlows:
 281             nodeDict = XMLtoDictParserTools.parseTreeToDict(
 282                 node,
 283                 ignoreList=IGNORED_TAGS_LIST)
 284             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 285             index += 1
 286             # print nodeDict
 287             # print origDict
 288             # print reportDict
 289             if nodeDict == origDict:
 290                 return True, ''
 291             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 292                 for p in IGNORED_PATHS_FOR_OC:
 293                     td = copy.copy(origDict)
 294                     _rem_unimplemented_tags(p[0], p[1], td)
 295                     for (p, t, v) in TAGS_TO_ADD_FOR_OC:
 296                         _add_tags(p, t, v, td)
 297                     for (p, t, rt) in TAGS_TO_MODIFY_FOR_OC:
 298                         _to_be_modified_tags(p, t, rt, td)
 299
 300                     # print "comparing1", nodeDict
 301                     # print "comparing2", td
 302                     if nodeDict == td:
 303                         return True, ''
 304                 if nodeDict == origDict:
 305                     return True, ''
 306                 return False, 'Flow found with diferences {0}'.format(
 307                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 308         return False, ''
 309
 310     def get_data_for_flow_put_update(self, xml):
 311         # action only for yet
 312         xml_dom_input = md.parseString(xml)
 313         actionList = xml_dom_input.getElementsByTagName('action')
 314         if actionList is not None and len(actionList) > 0:
 315             action = actionList[0]
 316             for child in action.childNodes:
 317                 if child.nodeType == Element.ELEMENT_NODE:
 318                     nodeKey = (child.localName).encode('utf-8', 'ignore')
 319                     if nodeKey != 'order':
 320                         if nodeKey != 'drop-action':
 321                             new_act = child.ownerDocument.createElement('drop-action')
 322                         else:
 323                             new_act = child.ownerDocument.createElement('output-action')
 324                             onc = child.ownerDocument.createElement('output-node-connector')
 325                             onc_content = child.ownerDocument.createTextNode('TABLE')
 326                             onc.appendChild(onc_content)
 327                             new_act.appendChild(onc)
 328                             ml = child.ownerDocument.createElement('max-length')
 329                             ml_content = child.ownerDocument.createTextNode('60')
 330                             ml.appendChild(ml_content)
 331                             new_act.appendChild(ml)
 332                         child.parentNode.replaceChild(new_act, child)
 333         return xml_dom_input.toxml(encoding='utf-8')
 334
 335     def get_flow_content(self, tid=1, fid=1, priority=1):
 336         """Returns an xml flow content identified by given details.
 337
 338         Args:
 339             :param tid: table id
 340             :param fid: flow id
 341             :param priority: flow priority
 342         """
 343
 344         flow_template = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 345 <flow xmlns="urn:opendaylight:flow:inventory">
 346     <strict>false</strict>
 347     <instructions>
 348         <instruction>
 349             <order>0</order>
 350             <apply-actions>
 351                 <action>
 352                     <order>0</order>
 353                     <drop-action/>
 354                 </action>
 355             </apply-actions>
 356         </instruction>
 357     </instructions>
 358     <table_id>%s</table_id>
 359     <id>%s</id>
 360     <cookie_mask>4294967295</cookie_mask>
 361     <installHw>false</installHw>
 362     <match>
 363         <ethernet-match>
 364             <ethernet-type>
 365                 <type>2048</type>
 366             </ethernet-type>
 367         </ethernet-match>
 368         <ipv4-source>10.0.0.1/32</ipv4-source>
 369     </match>
 370     <cookie>%s</cookie>
 371     <flow-name>%s</flow-name>
 372     <priority>%s</priority>
 373     <barrier>false</barrier>
 374 </flow>'''
 375
 376         flow_data = flow_template % (tid, fid, fid, 'TestFlow-{0}'.format(fid), priority)
 377         return flow_data