csit/libraries/XmlComparator.py

   1 '''
   2 Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3
   4 This program and the accompanying materials are made available under the
   5 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6 and is available at http://www.eclipse.org/legal/epl-v10.html
   7
   8 Created on May 21, 2014
   9
  10 @author: <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
  11 '''
  12 from xml.dom.minidom import Element
  13 import ipaddr
  14 import xml.dom.minidom as md
  15 import copy
  16
  17 KEY_NOT_FOUND = '<KEY_NOT_FOUND>'  # KeyNotFound for dictDiff
  18
  19
  20 class XMLtoDictParserTools():
  21
  22     @staticmethod
  23     def parseTreeToDict(node, returnedDict=None, ignoreList=[]):
  24         """
  25         Return Dictionary representation of the xml Tree DOM Element.
  26         Repeated tags are put to the array sorted by key (id or order)
  27         otherwise is the value represented by tag key name.
  28         @param node: DOM Element
  29         @param returnedDict : dictionary (default value None)
  30         @param ignereList : list of ignored tags for the xml Tree DOM Element
  31                             (default value is empty list)
  32         @return: dict representation for the input DOM Element
  33         """
  34         returnedDict = {} if returnedDict is None else returnedDict
  35         if (node.nodeType == Element.ELEMENT_NODE):
  36             nodeKey = (node.localName).encode('utf-8', 'ignore')
  37             if nodeKey not in ignoreList:
  38                 if node.childNodes is not None:
  39                     childDict = {}
  40                     for child in node.childNodes:
  41                         if child.nodeType == Element.TEXT_NODE:
  42                             nodeValue = (child.nodeValue).encode('utf-8', 'ignore')
  43                             if (len(nodeValue.strip(' \t\n\r'))) > 0:
  44                                 XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, nodeValue)
  45                                 nodeKey = None
  46                                 break
  47                         elif child.nodeType == Element.ELEMENT_NODE:
  48                             childDict = XMLtoDictParserTools.parseTreeToDict(child, childDict, ignoreList)
  49
  50                     XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, childDict)
  51
  52         return returnedDict
  53
  54     @staticmethod
  55     def addDictValue(m_dict, key, value):
  56
  57         def _allign_address(value):
  58             """unifies output"""
  59             n = ipaddr.IPNetwork(value)
  60             return '{0}/{1}'.format(n.network.exploded, n.prefixlen)
  61
  62         def _convert_numbers(value):
  63             if value.startswith("0x"):
  64                 return str(long(value, 16))
  65             return str(long(value))
  66
  67         if key is not None:
  68             if (isinstance(value, str)):
  69                 # we need to predict possible differences
  70                 # for same value in upper or lower case
  71                 value = value.lower()
  72             if key not in m_dict:
  73                 # lets add mask for ips withot mask
  74                 if key in ['ipv4-destination', 'ipv4-source', 'ipv6-destination', 'ipv6-source', 'ipv6-nd-target']:
  75                     nvalue = _allign_address(value)
  76                     m_dict[key] = nvalue
  77                 elif key in ['tunnel-mask', 'type', 'metadata-mask', 'out_port', 'out_group']:
  78                     nvalue = _convert_numbers(value)
  79                     m_dict[key] = nvalue
  80                 else:
  81                     m_dict[key] = value
  82             else:
  83                 exist_value = m_dict.get(key)
  84                 if (type(exist_value) is dict):
  85                     list_values = [exist_value, value]
  86                     key_for_sort = XMLtoDictParserTools.searchKey(exist_value)
  87                     if key_for_sort is not None:
  88                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  89                     m_dict[key] = list_values
  90                 elif (isinstance(exist_value, list)):
  91                     exist_value.append(value)
  92                     list_values = exist_value
  93                     key_for_sort = XMLtoDictParserTools.searchKey(value)
  94                     if key_for_sort is not None:
  95                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  96                     m_dict[key] = list_values
  97                 else:
  98                     m_dict[key] += value
  99
 100     @staticmethod
 101     def searchKey(dictionary):
 102         """
 103         Return an order key for the array ordering. OF_13
 104         allows only two possible kind of the order keys
 105         'order' or '*-id'
 106         @param dictionary: dictionary with data
 107         @return: the array order key
 108         """
 109         subKeyStr = ['-id', 'order']
 110         for substr in subKeyStr:
 111             for key in dictionary:
 112                 if key == substr:
 113                     return key
 114                 elif key.endswith(substr):
 115                     return key
 116         return None
 117
 118     @staticmethod
 119     def getDifferenceDict(original_dict, responded_dict):
 120         """
 121         Return a dict of keys that differ with another config object.  If a value is
 122         not found in one fo the configs, it will be represented by KEY_NOT_FOUND.
 123         @param original_dict:   Fist dictionary to diff.
 124         @param responded_dict:  Second dictionary to diff.
 125         @return diff:   Dict of Key => (original_dict.val, responded_dict.val)
 126                         Dict of Key => (original_key, KEY_NOT_FOUND)
 127                         Dict of Key => (KEY_NOT_FOUNE, original_key)
 128         """
 129         diff = {}
 130         # Check all keys in original_dict dict
 131         for key in original_dict.keys():
 132             if key not in responded_dict:
 133                 # missing key in responded dict
 134                 diff[key] = (key, KEY_NOT_FOUND)
 135             # check values of the dictionaries
 136             elif (original_dict[key] != responded_dict[key]):
 137                 # values are not the same #
 138
 139                 orig_dict_val = original_dict[key]
 140                 resp_dict_val = responded_dict[key]
 141
 142                 # check value is instance of dictionary
 143                 if isinstance(orig_dict_val, dict) and isinstance(resp_dict_val, dict):
 144                     sub_dif = XMLtoDictParserTools.getDifferenceDict(orig_dict_val, resp_dict_val)
 145                     if sub_dif:
 146                         diff[key] = sub_dif
 147
 148                 # check value is instance of list
 149                 # TODO - > change a basic comparator to compare by id or order
 150                 elif isinstance(orig_dict_val, list) and isinstance(resp_dict_val, list):
 151                     sub_list_diff = {}
 152                     # the list lengths
 153                     orig_i, resp_i = len(orig_dict_val), len(resp_dict_val)
 154                     # define a max iteration length (less from both)
 155                     min_index = orig_i if orig_i < resp_i else resp_i
 156                     for index in range(0, min_index, 1):
 157                         if (orig_dict_val[index] != resp_dict_val[index]):
 158                             sub_list_diff[index] = (orig_dict_val[index], resp_dict_val[index])
 159                     if (orig_i > min_index):
 160                         # original is longer as responded dict
 161                         for index in range(min_index, orig_i, 1):
 162                             sub_list_diff[index] = (orig_dict_val[index], None)
 163                     elif (resp_i > min_index):
 164                         # responded dict is longer as original
 165                         for index in range(min_index, resp_i, 1):
 166                             sub_list_diff[index] = (None, resp_dict_val[index])
 167                     if sub_list_diff:
 168                         diff[key] = sub_list_diff
 169
 170                 else:
 171                     diff[key] = (original_dict[key], responded_dict[key])
 172
 173         # Check all keys in responded_dict dict to find missing
 174         for key in responded_dict.keys():
 175             if key not in original_dict:
 176                 diff[key] = (KEY_NOT_FOUND, key)
 177         return diff
 178
 179
 180 IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON = ['id', 'flow-name', 'barrier', 'cookie_mask', 'installHw', 'flags',
 181                                            'strict', 'byte-count', 'duration', 'packet-count', 'in-port',
 182                                            'vlan-id-present', 'out_group', 'out_port', 'hard-timeout', 'idle-timeout',
 183                                            'flow-statistics', 'cookie', 'clear-actions',
 184                                            'ipv4-source-address-no-mask', 'ipv4-source-arbitrary-bitmask',
 185                                            'ipv4-destination-address-no-mask', 'ipv4-destination-arbitrary-bitmask',
 186                                            'ipv6-source-address-no-mask', 'ipv6-source-arbitrary-bitmask',
 187                                            'ipv6-destination-address-no-mask', 'ipv6-destination-arbitrary-bitmask']  # noqa
 188
 189 IGNORED_PATHS_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'controller-action'], True),  # noqa
 190                         (['flow', 'instructions', 'instruction', 'clear-actions', 'action'], False),
 191                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'push-vlan-action', 'vlan-id'], False),  # noqa
 192                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'drop-action'], True),
 193                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'flood-action'], True),
 194                         ]
 195
 196 TAGS_TO_ADD_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'output-action'], 'max-length', '0'),  # noqa
 197                       ]
 198
 199
 200 TAGS_TO_MODIFY_FOR_OC = [(['flow', 'match', 'metadata'], 'metadata', 'metadata-mask'),
 201                          (['flow', 'match', 'tunnel'], 'tunnel-id', 'tunnel-mask'),
 202                          ]
 203
 204
 205 class XmlComparator:
 206
 207     def is_flow_configured(self, requested_flow, configured_flows):
 208
 209         orig_tree = md.parseString(requested_flow)
 210         xml_resp_stream = configured_flows.encode('utf-8', 'ignore')
 211         xml_resp_tree = md.parseString(xml_resp_stream)
 212         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 213         origDict = XMLtoDictParserTools.parseTreeToDict(orig_tree._get_documentElement())
 214
 215         reportDict = {}
 216         index = 0
 217         for node in nodeListOperFlows:
 218             nodeDict = XMLtoDictParserTools.parseTreeToDict(node)
 219             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 220             index += 1
 221             # print nodeDict
 222             # print origDict
 223             if nodeDict == origDict:
 224                 return True, ''
 225             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 226                 return False, 'Flow found with diferences {0}'.format(
 227                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 228         return False, ''
 229
 230     def is_flow_operational2(self, requested_flow, oper_resp, check_id=False):
 231         def _rem_unimplemented_tags(tagpath, recurs, tdict):
 232             # print "_rem_unimplemented_tags", tagpath, tdict
 233             if len(tagpath) > 1 and tagpath[0] in tdict:
 234                 _rem_unimplemented_tags(tagpath[1:], recurs, tdict[tagpath[0]])
 235
 236             # when not to delete anything
 237             if len(tagpath) == 1 and tagpath[0] not in tdict:
 238                 return
 239             if len(tagpath) == 0:
 240                 return
 241
 242             # when to delete
 243             if len(tagpath) == 1 and tagpath[0] in tdict:
 244                 del tdict[tagpath[0]]
 245             if len(tagpath) > 1 and recurs is True and tagpath[0] in tdict and tdict[tagpath[0]] == {}:
 246                 del tdict[tagpath[0]]
 247             if tdict.keys() == ['order']:
 248                 del tdict['order']
 249             # print "leaving", tdict
 250
 251         def _add_tags(tagpath, newtag, value, tdict):
 252             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 253             # print "_add_tags", tagpath, newtag, value, tdict
 254             if len(tagpath) > 0 and tagpath[0] in tdict:
 255                 _add_tags(tagpath[1:], newtag, value, tdict[tagpath[0]])
 256             elif len(tagpath) == 0 and newtag not in tdict:
 257                 tdict[newtag] = value
 258
 259         def _to_be_modified_tags(tagpath, tag, related_tag, tdict):
 260             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 261             # print "_to_be_modified_tags", tagpath, tag, related_tag, tdict
 262             if len(tagpath) > 0 and tagpath[0] in tdict:
 263                 _to_be_modified_tags(tagpath[1:], tag, related_tag, tdict[tagpath[0]])
 264             elif len(tagpath) == 0 and tag in tdict and related_tag in tdict:
 265                 tdict[tag] = str(long(tdict[tag]) & long(tdict[related_tag]))
 266
 267         IGNORED_TAGS_LIST = list(IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 268         if check_id:
 269             IGNORED_TAGS_LIST.remove('id')
 270         orig_tree = md.parseString(requested_flow)
 271         xml_resp_stream = oper_resp.encode('utf-8', 'ignore')
 272         xml_resp_tree = md.parseString(xml_resp_stream)
 273         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 274         origDict = XMLtoDictParserTools.parseTreeToDict(
 275             orig_tree._get_documentElement(),
 276             ignoreList=IGNORED_TAGS_LIST)
 277
 278         # origDict['flow-statistics'] = origDict.pop( 'flow' )
 279         reportDict = {}
 280         index = 0
 281         for node in nodeListOperFlows:
 282             nodeDict = XMLtoDictParserTools.parseTreeToDict(
 283                 node,
 284                 ignoreList=IGNORED_TAGS_LIST)
 285             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 286             index += 1
 287             # print nodeDict
 288             # print origDict
 289             # print reportDict
 290             if nodeDict == origDict:
 291                 return True, ''
 292             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 293                 for p in IGNORED_PATHS_FOR_OC:
 294                     td = copy.copy(origDict)
 295                     _rem_unimplemented_tags(p[0], p[1], td)
 296                     for (p, t, v) in TAGS_TO_ADD_FOR_OC:
 297                         _add_tags(p, t, v, td)
 298                     for (p, t, rt) in TAGS_TO_MODIFY_FOR_OC:
 299                         _to_be_modified_tags(p, t, rt, td)
 300
 301                     # print "comparing1", nodeDict
 302                     # print "comparing2", td
 303                     if nodeDict == td:
 304                         return True, ''
 305                 if nodeDict == origDict:
 306                     return True, ''
 307                 return False, 'Flow found with diferences {0}'.format(
 308                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 309         return False, ''
 310
 311     def get_data_for_flow_put_update(self, xml):
 312         # action only for yet
 313         xml_dom_input = md.parseString(xml)
 314         actionList = xml_dom_input.getElementsByTagName('action')
 315         if actionList is not None and len(actionList) > 0:
 316             action = actionList[0]
 317             for child in action.childNodes:
 318                 if child.nodeType == Element.ELEMENT_NODE:
 319                     nodeKey = (child.localName).encode('utf-8', 'ignore')
 320                     if nodeKey != 'order':
 321                         if nodeKey != 'drop-action':
 322                             new_act = child.ownerDocument.createElement('drop-action')
 323                         else:
 324                             new_act = child.ownerDocument.createElement('output-action')
 325                             onc = child.ownerDocument.createElement('output-node-connector')
 326                             onc_content = child.ownerDocument.createTextNode('TABLE')
 327                             onc.appendChild(onc_content)
 328                             new_act.appendChild(onc)
 329                             ml = child.ownerDocument.createElement('max-length')
 330                             ml_content = child.ownerDocument.createTextNode('60')
 331                             ml.appendChild(ml_content)
 332                             new_act.appendChild(ml)
 333                         child.parentNode.replaceChild(new_act, child)
 334         return xml_dom_input.toxml(encoding='utf-8')
 335
 336     def get_flow_content(self, tid=1, fid=1, priority=1):
 337         """Returns an xml flow content identified by given details.
 338
 339         Args:
 340             :param tid: table id
 341             :param fid: flow id
 342             :param priority: flow priority
 343         """
 344
 345         flow_template = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 346 <flow xmlns="urn:opendaylight:flow:inventory">
 347     <strict>false</strict>
 348     <instructions>
 349         <instruction>
 350             <order>0</order>
 351             <apply-actions>
 352                 <action>
 353                     <order>0</order>
 354                     <drop-action/>
 355                 </action>
 356             </apply-actions>
 357         </instruction>
 358     </instructions>
 359     <table_id>%s</table_id>
 360     <id>%s</id>
 361     <cookie_mask>4294967295</cookie_mask>
 362     <installHw>false</installHw>
 363     <match>
 364         <ethernet-match>
 365             <ethernet-type>
 366                 <type>2048</type>
 367             </ethernet-type>
 368         </ethernet-match>
 369         <ipv4-source>10.0.0.1/32</ipv4-source>
 370     </match>
 371     <cookie>%s</cookie>
 372     <flow-name>%s</flow-name>
 373     <priority>%s</priority>
 374     <barrier>false</barrier>
 375 </flow>'''
 376
 377         flow_data = flow_template % (tid, fid, fid, 'TestFlow-{0}'.format(fid), priority)
 378         return flow_data