test/csit/libraries/XmlComparator.py

   1 '''
   2 Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3
   4 This program and the accompanying materials are made available under the
   5 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6 and is available at http://www.eclipse.org/legal/epl-v10.html
   7
   8 Created on May 21, 2014
   9
  10 @author: <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
  11 '''
  12 from xml.dom.minidom import Element
  13 import ipaddr
  14 import xml.dom.minidom as md
  15 import copy
  16
  17 KEY_NOT_FOUND = '<KEY_NOT_FOUND>'  # KeyNotFound for dictDiff
  18
  19
  20 class XMLtoDictParserTools():
  21
  22     @staticmethod
  23     def parseTreeToDict(node, returnedDict=None, ignoreList=[]):
  24         """
  25         Return Dictionary representation of the xml Tree DOM Element.
  26         Repeated tags are put to the array sorted by key (id or order)
  27         otherwise is the value represented by tag key name.
  28         @param node: DOM Element
  29         @param returnedDict : dictionary (default value None)
  30         @param ignereList : list of ignored tags for the xml Tree DOM Element
  31                             (default value is empty list)
  32         @return: dict representation for the input DOM Element
  33         """
  34         returnedDict = {} if returnedDict is None else returnedDict
  35         if (node.nodeType == Element.ELEMENT_NODE):
  36             nodeKey = (node.localName).encode('utf-8', 'ignore')
  37             if nodeKey not in ignoreList:
  38                 if node.childNodes is not None:
  39                     childDict = {}
  40                     for child in node.childNodes:
  41                         if child.nodeType == Element.TEXT_NODE:
  42                             nodeValue = (child.nodeValue).encode('utf-8', 'ignore')
  43                             if (len(nodeValue.strip(' \t\n\r'))) > 0:
  44                                 XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, nodeValue)
  45                                 nodeKey = None
  46                                 break
  47                         elif child.nodeType == Element.ELEMENT_NODE:
  48                             childDict = XMLtoDictParserTools.parseTreeToDict(child, childDict, ignoreList)
  49
  50                     XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, childDict)
  51
  52         return returnedDict
  53
  54     @staticmethod
  55     def addDictValue(m_dict, key, value):
  56
  57         def _allign_address(value):
  58             """unifies output"""
  59             n = ipaddr.IPNetwork(value)
  60             return '{0}/{1}'.format(n.network.exploded, n.prefixlen)
  61
  62         def _convert_numbers(value):
  63             if value.startswith("0x"):
  64                 return str(long(value, 16))
  65             return str(long(value))
  66
  67         if key is not None:
  68             if (isinstance(value, str)):
  69                 # we need to predict possible differences
  70                 # for same value in upper or lower case
  71                 value = value.lower()
  72             if key not in m_dict:
  73                 # lets add mask for ips withot mask
  74                 if key in ['ipv4-destination', 'ipv4-source', 'ipv6-destination', 'ipv6-source', 'ipv6-nd-target']:
  75                     nvalue = _allign_address(value)
  76                     m_dict[key] = nvalue
  77                 elif key in ['tunnel-mask', 'type', 'metadata-mask', 'out_port', 'out_group']:
  78                     nvalue = _convert_numbers(value)
  79                     m_dict[key] = nvalue
  80                 else:
  81                     m_dict[key] = value
  82             else:
  83                 exist_value = m_dict.get(key)
  84                 if (type(exist_value) is dict):
  85                     list_values = [exist_value, value]
  86                     key_for_sort = XMLtoDictParserTools.searchKey(exist_value)
  87                     if key_for_sort is not None:
  88                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  89                     m_dict[key] = list_values
  90                 elif (isinstance(exist_value, list)):
  91                     exist_value.append(value)
  92                     list_values = exist_value
  93                     key_for_sort = XMLtoDictParserTools.searchKey(value)
  94                     if key_for_sort is not None:
  95                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  96                     m_dict[key] = list_values
  97                 else:
  98                     m_dict[key] += value
  99
 100     @staticmethod
 101     def searchKey(dictionary):
 102         """
 103         Return an order key for the array ordering. OF_13
 104         allows only two possible kind of the order keys
 105         'order' or '*-id'
 106         @param dictionary: dictionary with data
 107         @return: the array order key
 108         """
 109         subKeyStr = ['-id', 'order']
 110         for substr in subKeyStr:
 111             for key in dictionary:
 112                 if key == substr:
 113                     return key
 114                 elif key.endswith(substr):
 115                     return key
 116         return None
 117
 118     @staticmethod
 119     def getDifferenceDict(original_dict, responded_dict):
 120         """
 121         Return a dict of keys that differ with another config object.  If a value is
 122         not found in one fo the configs, it will be represented by KEY_NOT_FOUND.
 123         @param original_dict:   Fist dictionary to diff.
 124         @param responded_dict:  Second dictionary to diff.
 125         @return diff:   Dict of Key => (original_dict.val, responded_dict.val)
 126                         Dict of Key => (original_key, KEY_NOT_FOUND)
 127                         Dict of Key => (KEY_NOT_FOUNE, original_key)
 128         """
 129         diff = {}
 130         # Check all keys in original_dict dict
 131         for key in original_dict.keys():
 132             if key not in responded_dict:
 133                 # missing key in responded dict
 134                 diff[key] = (key, KEY_NOT_FOUND)
 135             # check values of the dictionaries
 136             elif (original_dict[key] != responded_dict[key]):
 137                 # values are not the same #
 138
 139                 orig_dict_val = original_dict[key]
 140                 resp_dict_val = responded_dict[key]
 141
 142                 # check value is instance of dictionary
 143                 if isinstance(orig_dict_val, dict) and isinstance(resp_dict_val, dict):
 144                     sub_dif = XMLtoDictParserTools.getDifferenceDict(orig_dict_val, resp_dict_val)
 145                     if sub_dif:
 146                         diff[key] = sub_dif
 147
 148                 # check value is instance of list
 149                 # TODO - > change a basic comparator to compare by id or order
 150                 elif isinstance(orig_dict_val, list) and isinstance(resp_dict_val, list):
 151                     sub_list_diff = {}
 152                     # the list lengths
 153                     orig_i, resp_i = len(orig_dict_val), len(resp_dict_val)
 154                     # define a max iteration length (less from both)
 155                     min_index = orig_i if orig_i < resp_i else resp_i
 156                     for index in range(0, min_index, 1):
 157                         if (orig_dict_val[index] != resp_dict_val[index]):
 158                             sub_list_diff[index] = (orig_dict_val[index], resp_dict_val[index])
 159                     if (orig_i > min_index):
 160                         # original is longer as responded dict
 161                         for index in range(min_index, orig_i, 1):
 162                             sub_list_diff[index] = (orig_dict_val[index], None)
 163                     elif (resp_i > min_index):
 164                         # responded dict is longer as original
 165                         for index in range(min_index, resp_i, 1):
 166                             sub_list_diff[index] = (None, resp_dict_val[index])
 167                     if sub_list_diff:
 168                         diff[key] = sub_list_diff
 169
 170                 else:
 171                     diff[key] = (original_dict[key], responded_dict[key])
 172
 173         # Check all keys in responded_dict dict to find missing
 174         for key in responded_dict.keys():
 175             if key not in original_dict:
 176                 diff[key] = (KEY_NOT_FOUND, key)
 177         return diff
 178
 179 IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON = ['id', 'flow-name', 'barrier', 'cookie_mask', 'installHw', 'flags', 'strict', 'byte-count', 'duration', 'packet-count', 'in-port', 'vlan-id-present', 'out_group', 'out_port', 'hard-timeout', 'idle-timeout', 'flow-statistics', 'cookie']  # noqa
 180
 181 IGNORED_PATHS_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'controller-action'], True),  # noqa
 182                         (['flow', 'instructions', 'instruction', 'clear-actions', 'action'], False),
 183                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'push-vlan-action', 'vlan-id'], False),  # noqa
 184                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'drop-action'], True),
 185                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'flood-action'], True),
 186                         ]
 187
 188 TAGS_TO_ADD_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'output-action'], 'max-length', '0'),  # noqa
 189                       ]
 190
 191
 192 TAGS_TO_MODIFY_FOR_OC = [(['flow', 'match', 'metadata'], 'metadata', 'metadata-mask'),
 193                          (['flow', 'match', 'tunnel'], 'tunnel-id', 'tunnel-mask'),
 194                          ]
 195
 196
 197 class XmlComparator:
 198
 199     def is_flow_configured(self, requested_flow, configured_flows):
 200
 201         orig_tree = md.parseString(requested_flow)
 202         xml_resp_stream = configured_flows.encode('utf-8', 'ignore')
 203         xml_resp_tree = md.parseString(xml_resp_stream)
 204         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 205         origDict = XMLtoDictParserTools.parseTreeToDict(orig_tree._get_documentElement())
 206
 207         reportDict = {}
 208         index = 0
 209         for node in nodeListOperFlows:
 210             nodeDict = XMLtoDictParserTools.parseTreeToDict(node)
 211             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 212             index += 1
 213             # print nodeDict
 214             # print origDict
 215             if nodeDict == origDict:
 216                 return True, ''
 217             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 218                 return False, 'Flow found with diferences {0}'.format(
 219                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 220         return False, ''
 221
 222     def is_flow_operational2(self, requested_flow, oper_resp):
 223         def _rem_unimplemented_tags(tagpath, recurs, tdict):
 224             # print "_rem_unimplemented_tags", tagpath, tdict
 225             if len(tagpath) > 1 and tagpath[0] in tdict:
 226                 _rem_unimplemented_tags(tagpath[1:], recurs, tdict[tagpath[0]])
 227
 228             # when not to delete anything
 229             if len(tagpath) == 1 and tagpath[0] not in tdict:
 230                 return
 231             if len(tagpath) == 0:
 232                 return
 233
 234             # when to delete
 235             if len(tagpath) == 1 and tagpath[0] in tdict:
 236                 del tdict[tagpath[0]]
 237             if len(tagpath) > 1 and recurs is True and tagpath[0] in tdict and tdict[tagpath[0]] == {}:
 238                 del tdict[tagpath[0]]
 239             if tdict.keys() == ['order']:
 240                 del tdict['order']
 241             # print "leaving", tdict
 242
 243         def _add_tags(tagpath, newtag, value, tdict):
 244             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 245             # print "_add_tags", tagpath, newtag, value, tdict
 246             if len(tagpath) > 0 and tagpath[0] in tdict:
 247                 _add_tags(tagpath[1:], newtag, value, tdict[tagpath[0]])
 248             elif len(tagpath) == 0 and newtag not in tdict:
 249                 tdict[newtag] = value
 250
 251         def _to_be_modified_tags(tagpath, tag, related_tag, tdict):
 252             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 253             # print "_to_be_modified_tags", tagpath, tag, related_tag, tdict
 254             if len(tagpath) > 0 and tagpath[0] in tdict:
 255                 _to_be_modified_tags(tagpath[1:], tag, related_tag, tdict[tagpath[0]])
 256             elif len(tagpath) == 0 and tag in tdict and related_tag in tdict:
 257                 tdict[tag] = str(long(tdict[tag]) & long(tdict[related_tag]))
 258
 259         orig_tree = md.parseString(requested_flow)
 260         xml_resp_stream = oper_resp.encode('utf-8', 'ignore')
 261         xml_resp_tree = md.parseString(xml_resp_stream)
 262         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 263         origDict = XMLtoDictParserTools.parseTreeToDict(
 264             orig_tree._get_documentElement(),
 265             ignoreList=IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 266
 267         # origDict['flow-statistics'] = origDict.pop( 'flow' )
 268         reportDict = {}
 269         index = 0
 270         for node in nodeListOperFlows:
 271             nodeDict = XMLtoDictParserTools.parseTreeToDict(
 272                 node,
 273                 ignoreList=IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 274             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 275             index += 1
 276             # print nodeDict
 277             # print origDict
 278             # print reportDict
 279             if nodeDict == origDict:
 280                 return True, ''
 281             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 282                 for p in IGNORED_PATHS_FOR_OC:
 283                     td = copy.copy(origDict)
 284                     _rem_unimplemented_tags(p[0], p[1],  td)
 285                     for (p, t, v) in TAGS_TO_ADD_FOR_OC:
 286                         _add_tags(p, t, v, td)
 287                     for (p, t, rt) in TAGS_TO_MODIFY_FOR_OC:
 288                         _to_be_modified_tags(p, t, rt, td)
 289
 290                     # print "comparing1", nodeDict
 291                     # print "comparing2", td
 292                     if nodeDict == td:
 293                         return True, ''
 294                 if nodeDict == origDict:
 295                     return True, ''
 296                 return False, 'Flow found with diferences {0}'.format(
 297                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 298         return False, ''
 299
 300     def get_data_for_flow_put_update(self, xml):
 301         # action only for yet
 302         xml_dom_input = md.parseString(xml)
 303         actionList = xml_dom_input.getElementsByTagName('action')
 304         if actionList is not None and len(actionList) > 0:
 305             action = actionList[0]
 306             for child in action.childNodes:
 307                 if child.nodeType == Element.ELEMENT_NODE:
 308                     nodeKey = (child.localName).encode('utf-8', 'ignore')
 309                     if nodeKey != 'order':
 310                         if nodeKey != 'drop-action':
 311                             new_act = child.ownerDocument.createElement('drop-action')
 312                         else:
 313                             new_act = child.ownerDocument.createElement('output-action')
 314                             onc = child.ownerDocument.createElement('output-node-connector')
 315                             onc_content = child.ownerDocument.createTextNode('TABLE')
 316                             onc.appendChild(onc_content)
 317                             new_act.appendChild(onc)
 318                             ml = child.ownerDocument.createElement('max-length')
 319                             ml_content = child.ownerDocument.createTextNode('60')
 320                             ml.appendChild(ml_content)
 321                             new_act.appendChild(ml)
 322                         child.parentNode.replaceChild(new_act, child)
 323         return xml_dom_input.toxml(encoding='utf-8')