csit/libraries/XmlComparator.py

   1 '''
   2 Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3
   4 This program and the accompanying materials are made available under the
   5 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6 and is available at http://www.eclipse.org/legal/epl-v10.html
   7
   8 Created on May 21, 2014
   9
  10 @author: <a href="mailto:vdemcak@cisco.com">Vaclav Demcak</a>
  11 '''
  12 from xml.dom.minidom import Element
  13 import ipaddr
  14 import xml.dom.minidom as md
  15 import copy
  16
  17 KEY_NOT_FOUND = '<KEY_NOT_FOUND>'  # KeyNotFound for dictDiff
  18
  19
  20 class XMLtoDictParserTools():
  21
  22     @staticmethod
  23     def parseTreeToDict(node, returnedDict=None, ignoreList=[]):
  24         """
  25         Return Dictionary representation of the xml Tree DOM Element.
  26         Repeated tags are put to the array sorted by key (id or order)
  27         otherwise is the value represented by tag key name.
  28         @param node: DOM Element
  29         @param returnedDict : dictionary (default value None)
  30         @param ignereList : list of ignored tags for the xml Tree DOM Element
  31                             (default value is empty list)
  32         @return: dict representation for the input DOM Element
  33         """
  34         returnedDict = {} if returnedDict is None else returnedDict
  35         if (node.nodeType == Element.ELEMENT_NODE):
  36             nodeKey = (node.localName).encode('utf-8', 'ignore')
  37             if nodeKey not in ignoreList:
  38                 if node.childNodes is not None:
  39                     childDict = {}
  40                     for child in node.childNodes:
  41                         if child.nodeType == Element.TEXT_NODE:
  42                             nodeValue = (child.nodeValue).encode('utf-8', 'ignore')
  43                             if (len(nodeValue.strip(' \t\n\r'))) > 0:
  44                                 XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, nodeValue)
  45                                 nodeKey = None
  46                                 break
  47                         elif child.nodeType == Element.ELEMENT_NODE:
  48                             childDict = XMLtoDictParserTools.parseTreeToDict(child, childDict, ignoreList)
  49
  50                     XMLtoDictParserTools.addDictValue(returnedDict, nodeKey, childDict)
  51
  52         return returnedDict
  53
  54     @staticmethod
  55     def addDictValue(m_dict, key, value):
  56
  57         def _allign_address(value):
  58             """unifies output"""
  59             n = ipaddr.IPNetwork(value)
  60             return '{0}/{1}'.format(n.network.exploded, n.prefixlen)
  61
  62         def _convert_numbers(value):
  63             if value.startswith("0x"):
  64                 return str(long(value, 16))
  65             return str(long(value))
  66
  67         if key is not None:
  68             if (isinstance(value, str)):
  69                 # we need to predict possible differences
  70                 # for same value in upper or lower case
  71                 value = value.lower()
  72             if key not in m_dict:
  73                 # lets add mask for ips withot mask
  74                 if key in ['ipv4-destination', 'ipv4-source', 'ipv6-destination', 'ipv6-source', 'ipv6-nd-target']:
  75                     nvalue = _allign_address(value)
  76                     m_dict[key] = nvalue
  77                 elif key in ['tunnel-mask', 'type', 'metadata-mask', 'out_port', 'out_group']:
  78                     nvalue = _convert_numbers(value)
  79                     m_dict[key] = nvalue
  80                 else:
  81                     m_dict[key] = value
  82             else:
  83                 exist_value = m_dict.get(key)
  84                 if (type(exist_value) is dict):
  85                     list_values = [exist_value, value]
  86                     key_for_sort = XMLtoDictParserTools.searchKey(exist_value)
  87                     if key_for_sort is not None:
  88                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  89                     m_dict[key] = list_values
  90                 elif (isinstance(exist_value, list)):
  91                     exist_value.append(value)
  92                     list_values = exist_value
  93                     key_for_sort = XMLtoDictParserTools.searchKey(value)
  94                     if key_for_sort is not None:
  95                         list_values = sorted(list_values, key=lambda k: k[key_for_sort])
  96                     m_dict[key] = list_values
  97                 else:
  98                     m_dict[key] += value
  99
 100     @staticmethod
 101     def searchKey(dictionary):
 102         """
 103         Return an order key for the array ordering. OF_13
 104         allows only two possible kind of the order keys
 105         'order' or '*-id'
 106         @param dictionary: dictionary with data
 107         @return: the array order key
 108         """
 109         subKeyStr = ['-id', 'order']
 110         for substr in subKeyStr:
 111             for key in dictionary:
 112                 if key == substr:
 113                     return key
 114                 elif key.endswith(substr):
 115                     return key
 116         return None
 117
 118     @staticmethod
 119     def getDifferenceDict(original_dict, responded_dict):
 120         """
 121         Return a dict of keys that differ with another config object.  If a value is
 122         not found in one fo the configs, it will be represented by KEY_NOT_FOUND.
 123         @param original_dict:   Fist dictionary to diff.
 124         @param responded_dict:  Second dictionary to diff.
 125         @return diff:   Dict of Key => (original_dict.val, responded_dict.val)
 126                         Dict of Key => (original_key, KEY_NOT_FOUND)
 127                         Dict of Key => (KEY_NOT_FOUNE, original_key)
 128         """
 129         diff = {}
 130         # Check all keys in original_dict dict
 131         for key in original_dict.keys():
 132             if key not in responded_dict:
 133                 # missing key in responded dict
 134                 diff[key] = (key, KEY_NOT_FOUND)
 135             # check values of the dictionaries
 136             elif (original_dict[key] != responded_dict[key]):
 137                 # values are not the same #
 138
 139                 orig_dict_val = original_dict[key]
 140                 resp_dict_val = responded_dict[key]
 141
 142                 # check value is instance of dictionary
 143                 if isinstance(orig_dict_val, dict) and isinstance(resp_dict_val, dict):
 144                     sub_dif = XMLtoDictParserTools.getDifferenceDict(orig_dict_val, resp_dict_val)
 145                     if sub_dif:
 146                         diff[key] = sub_dif
 147
 148                 # check value is instance of list
 149                 # TODO - > change a basic comparator to compare by id or order
 150                 elif isinstance(orig_dict_val, list) and isinstance(resp_dict_val, list):
 151                     sub_list_diff = {}
 152                     # the list lengths
 153                     orig_i, resp_i = len(orig_dict_val), len(resp_dict_val)
 154                     # define a max iteration length (less from both)
 155                     min_index = orig_i if orig_i < resp_i else resp_i
 156                     for index in range(0, min_index, 1):
 157                         if (orig_dict_val[index] != resp_dict_val[index]):
 158                             sub_list_diff[index] = (orig_dict_val[index], resp_dict_val[index])
 159                     if (orig_i > min_index):
 160                         # original is longer as responded dict
 161                         for index in range(min_index, orig_i, 1):
 162                             sub_list_diff[index] = (orig_dict_val[index], None)
 163                     elif (resp_i > min_index):
 164                         # responded dict is longer as original
 165                         for index in range(min_index, resp_i, 1):
 166                             sub_list_diff[index] = (None, resp_dict_val[index])
 167                     if sub_list_diff:
 168                         diff[key] = sub_list_diff
 169
 170                 else:
 171                     diff[key] = (original_dict[key], responded_dict[key])
 172
 173         # Check all keys in responded_dict dict to find missing
 174         for key in responded_dict.keys():
 175             if key not in original_dict:
 176                 diff[key] = (KEY_NOT_FOUND, key)
 177         return diff
 178
 179 IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON = ['id', 'flow-name', 'barrier', 'cookie_mask', 'installHw', 'flags',
 180                                            'strict', 'byte-count', 'duration', 'packet-count', 'in-port',
 181                                            'vlan-id-present', 'out_group', 'out_port', 'hard-timeout', 'idle-timeout',
 182                                            'flow-statistics', 'cookie', 'clear-actions']  # noqa
 183
 184 IGNORED_PATHS_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'controller-action'], True),  # noqa
 185                         (['flow', 'instructions', 'instruction', 'clear-actions', 'action'], False),
 186                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'push-vlan-action', 'vlan-id'], False),  # noqa
 187                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'drop-action'], True),
 188                         (['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'flood-action'], True),
 189                         ]
 190
 191 TAGS_TO_ADD_FOR_OC = [(['flow', 'instructions', 'instruction', 'apply-actions', 'action', 'output-action'], 'max-length', '0'),  # noqa
 192                       ]
 193
 194
 195 TAGS_TO_MODIFY_FOR_OC = [(['flow', 'match', 'metadata'], 'metadata', 'metadata-mask'),
 196                          (['flow', 'match', 'tunnel'], 'tunnel-id', 'tunnel-mask'),
 197                          ]
 198
 199
 200 class XmlComparator:
 201
 202     def is_flow_configured(self, requested_flow, configured_flows):
 203
 204         orig_tree = md.parseString(requested_flow)
 205         xml_resp_stream = configured_flows.encode('utf-8', 'ignore')
 206         xml_resp_tree = md.parseString(xml_resp_stream)
 207         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 208         origDict = XMLtoDictParserTools.parseTreeToDict(orig_tree._get_documentElement())
 209
 210         reportDict = {}
 211         index = 0
 212         for node in nodeListOperFlows:
 213             nodeDict = XMLtoDictParserTools.parseTreeToDict(node)
 214             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 215             index += 1
 216             # print nodeDict
 217             # print origDict
 218             if nodeDict == origDict:
 219                 return True, ''
 220             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 221                 return False, 'Flow found with diferences {0}'.format(
 222                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 223         return False, ''
 224
 225     def is_flow_operational2(self, requested_flow, oper_resp):
 226         def _rem_unimplemented_tags(tagpath, recurs, tdict):
 227             # print "_rem_unimplemented_tags", tagpath, tdict
 228             if len(tagpath) > 1 and tagpath[0] in tdict:
 229                 _rem_unimplemented_tags(tagpath[1:], recurs, tdict[tagpath[0]])
 230
 231             # when not to delete anything
 232             if len(tagpath) == 1 and tagpath[0] not in tdict:
 233                 return
 234             if len(tagpath) == 0:
 235                 return
 236
 237             # when to delete
 238             if len(tagpath) == 1 and tagpath[0] in tdict:
 239                 del tdict[tagpath[0]]
 240             if len(tagpath) > 1 and recurs is True and tagpath[0] in tdict and tdict[tagpath[0]] == {}:
 241                 del tdict[tagpath[0]]
 242             if tdict.keys() == ['order']:
 243                 del tdict['order']
 244             # print "leaving", tdict
 245
 246         def _add_tags(tagpath, newtag, value, tdict):
 247             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 248             # print "_add_tags", tagpath, newtag, value, tdict
 249             if len(tagpath) > 0 and tagpath[0] in tdict:
 250                 _add_tags(tagpath[1:], newtag, value, tdict[tagpath[0]])
 251             elif len(tagpath) == 0 and newtag not in tdict:
 252                 tdict[newtag] = value
 253
 254         def _to_be_modified_tags(tagpath, tag, related_tag, tdict):
 255             '''if whole tagpath exists and the tag is not present, it is added with given value'''
 256             # print "_to_be_modified_tags", tagpath, tag, related_tag, tdict
 257             if len(tagpath) > 0 and tagpath[0] in tdict:
 258                 _to_be_modified_tags(tagpath[1:], tag, related_tag, tdict[tagpath[0]])
 259             elif len(tagpath) == 0 and tag in tdict and related_tag in tdict:
 260                 tdict[tag] = str(long(tdict[tag]) & long(tdict[related_tag]))
 261
 262         orig_tree = md.parseString(requested_flow)
 263         xml_resp_stream = oper_resp.encode('utf-8', 'ignore')
 264         xml_resp_tree = md.parseString(xml_resp_stream)
 265         nodeListOperFlows = xml_resp_tree.getElementsByTagNameNS("*", 'flow')
 266         origDict = XMLtoDictParserTools.parseTreeToDict(
 267             orig_tree._get_documentElement(),
 268             ignoreList=IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 269
 270         # origDict['flow-statistics'] = origDict.pop( 'flow' )
 271         reportDict = {}
 272         index = 0
 273         for node in nodeListOperFlows:
 274             nodeDict = XMLtoDictParserTools.parseTreeToDict(
 275                 node,
 276                 ignoreList=IGNORED_TAGS_FOR_OPERATIONAL_COMPARISON)
 277             XMLtoDictParserTools.addDictValue(reportDict, index, nodeDict)
 278             index += 1
 279             # print nodeDict
 280             # print origDict
 281             # print reportDict
 282             if nodeDict == origDict:
 283                 return True, ''
 284             if nodeDict['flow']['priority'] == origDict['flow']['priority']:
 285                 for p in IGNORED_PATHS_FOR_OC:
 286                     td = copy.copy(origDict)
 287                     _rem_unimplemented_tags(p[0], p[1], td)
 288                     for (p, t, v) in TAGS_TO_ADD_FOR_OC:
 289                         _add_tags(p, t, v, td)
 290                     for (p, t, rt) in TAGS_TO_MODIFY_FOR_OC:
 291                         _to_be_modified_tags(p, t, rt, td)
 292
 293                     # print "comparing1", nodeDict
 294                     # print "comparing2", td
 295                     if nodeDict == td:
 296                         return True, ''
 297                 if nodeDict == origDict:
 298                     return True, ''
 299                 return False, 'Flow found with diferences {0}'.format(
 300                     XMLtoDictParserTools.getDifferenceDict(nodeDict, origDict))
 301         return False, ''
 302
 303     def get_data_for_flow_put_update(self, xml):
 304         # action only for yet
 305         xml_dom_input = md.parseString(xml)
 306         actionList = xml_dom_input.getElementsByTagName('action')
 307         if actionList is not None and len(actionList) > 0:
 308             action = actionList[0]
 309             for child in action.childNodes:
 310                 if child.nodeType == Element.ELEMENT_NODE:
 311                     nodeKey = (child.localName).encode('utf-8', 'ignore')
 312                     if nodeKey != 'order':
 313                         if nodeKey != 'drop-action':
 314                             new_act = child.ownerDocument.createElement('drop-action')
 315                         else:
 316                             new_act = child.ownerDocument.createElement('output-action')
 317                             onc = child.ownerDocument.createElement('output-node-connector')
 318                             onc_content = child.ownerDocument.createTextNode('TABLE')
 319                             onc.appendChild(onc_content)
 320                             new_act.appendChild(onc)
 321                             ml = child.ownerDocument.createElement('max-length')
 322                             ml_content = child.ownerDocument.createTextNode('60')
 323                             ml.appendChild(ml_content)
 324                             new_act.appendChild(ml)
 325                         child.parentNode.replaceChild(new_act, child)
 326         return xml_dom_input.toxml(encoding='utf-8')
 327
 328     def get_flow_content(self, tid=1, fid=1, priority=1):
 329         """Returns an xml flow content identified by given details.
 330
 331         Args:
 332             :param tid: table id
 333             :param fid: flow id
 334             :param priority: flow priority
 335         """
 336
 337         flow_template = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 338 <flow xmlns="urn:opendaylight:flow:inventory">
 339     <strict>false</strict>
 340     <instructions>
 341         <instruction>
 342             <order>0</order>
 343             <apply-actions>
 344                 <action>
 345                     <order>0</order>
 346                     <drop-action/>
 347                 </action>
 348             </apply-actions>
 349         </instruction>
 350     </instructions>
 351     <table_id>%s</table_id>
 352     <id>%s</id>
 353     <cookie_mask>4294967295</cookie_mask>
 354     <installHw>false</installHw>
 355     <match>
 356         <ethernet-match>
 357             <ethernet-type>
 358                 <type>2048</type>
 359             </ethernet-type>
 360         </ethernet-match>
 361         <ipv4-source>10.0.0.1/32</ipv4-source>
 362     </match>
 363     <cookie>%s</cookie>
 364     <flow-name>%s</flow-name>
 365     <priority>%s</priority>
 366     <barrier>false</barrier>
 367 </flow>'''
 368
 369         flow_data = flow_template % (tid, fid, fid, 'TestFlow-{0}'.format(fid), priority)
 370         return flow_data