csit/libraries/netvirt/excepts.py

   1 import collections
   2 import errno
   3 import logging
   4 import os
   5 import re
   6
   7 # Make sure to have unique matches in different lines
   8 # Order the list in alphabetical order based on the "issue" key
   9 _whitelist = [
  10     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  11      "id": "ConflictingModificationAppliedException",
  12      "context": [
  13          "Node was created by other transaction",
  14          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  15          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow",
  16          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  17          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3."
  18      ]},
  19     # oxygen
  20     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  21      "id": "ConflictingModificationAppliedException",
  22      "context": [
  23          "Node was created by other transaction",
  24          "OptimisticLockFailedException: Optimistic lock failed."
  25          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  26          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  27          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  28          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  29      ]},
  30     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  31      "id": "ConflictingModificationAppliedException",
  32      "context": [
  33          "Node was created by other transaction",
  34          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  35          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  36      ]},
  37     # oxygen
  38     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  39      "id": "ConflictingModificationAppliedException",
  40      "context": [
  41          "OptimisticLockFailedException: Optimistic lock failed."
  42          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  43          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  44          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  45          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  46      ]},
  47     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  48      "id": "ConflictingModificationAppliedException",
  49      "context": [
  50          "Node was deleted by other transaction",
  51          "Optimistic lock failed for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  52          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  53          "elan-instance-name=",
  54      ]},
  55     # oxygen version of NETVIRT-1136
  56     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  57      "id": "ConflictingModificationAppliedException",
  58      "context": [
  59          "Node was deleted by other transaction",
  60          "OptimisticLockFailedException: Optimistic lock failed.",
  61          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  62          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  63          "elan-instance-name="
  64      ]},
  65     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1260",
  66      "id": "ConflictingModificationAppliedException",
  67      "context": [
  68          "Optimistic lock failed for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  69          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name=",
  70      ]},
  71     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  72      "id": "ConflictingModificationAppliedException",
  73      "context": [
  74          "OptimisticLockFailedException",
  75          "/(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)" +
  76          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  77          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id=",
  78          "vrf-id=", "/vpn-to-dpn-list/vpn-to-dpn-list", "dpnId="
  79      ]},
  80     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  81      "id": "ExecutionException",
  82      "context": [
  83          "OptimisticLockFailedException: Optimistic lock failed",
  84          "removeOrUpdateVpnToDpnList: Error removing from dpnToVpnList for vpn "
  85      ]},
  86     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  87      "id": "OptimisticLockFailedException",
  88      "context": [
  89          "OptimisticLockFailedException",
  90          "VpnInterfaceOpListener",
  91          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-",
  92          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  93          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id=",
  94          "vrf-id=", "/vpn-to-dpn-list/vpn-to-dpn-list", "dpnId="
  95      ]},
  96     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1281",
  97      "id": "OptimisticLockFailedException",
  98      "context": [
  99          "OptimisticLockFailedException: Optimistic lock failed.",
 100          "ConflictingModificationAppliedException: Node children was modified by other transaction",
 101          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-"
 102      ]},
 103     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1304",
 104      "id": "ModifiedNodeDoesNotExistException",
 105      "context": [
 106          "ModifiedNodeDoesNotExistException",
 107          "/(urn:opendaylight:netvirt:fibmanager?revision=2015-03-30)fibEntries/" +
 108          "vrfTables/vrfTables[{(urn:opendaylight:netvirt:fibmanager?revision=2015-03-30)routeDistinguisher="
 109      ]},
 110     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1304",
 111      "id": "TransactionCommitFailedException",
 112      "context": [
 113          "TransactionCommitFailedException",
 114          "/(urn:opendaylight:netvirt:fibmanager?revision=2015-03-30)fibEntries/" +
 115          "vrfTables/vrfTables[{(urn:opendaylight:netvirt:fibmanager?revision=2015-03-30)routeDistinguisher="
 116      ]},
 117     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1427",
 118      "id": "ModifiedNodeDoesNotExistException",
 119      "context": [
 120          "/(urn:huawei:params:xml:ns:yang:l3vpn?revision=2014-08-15)vpn-interfaces/vpn-interface/vpn-interface" +
 121          "[{(urn:huawei:params:xml:ns:yang:l3vpn?revision=2014-08-15)name=",
 122          "AugmentationIdentifier{childNames=[(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)adjacency]}"
 123      ]},
 124     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1428",
 125      "id": "ModifiedNodeDoesNotExistException",
 126      "context": [
 127          "/(urn:huawei:params:xml:ns:yang:l3vpn?revision=2014-08-15)vpn-interfaces/vpn-interface/vpn-interface" +
 128          "[{(urn:huawei:params:xml:ns:yang:l3vpn?revision=2014-08-15)name=",
 129      ]},
 130     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 131      "id": "ConflictingModificationAppliedException",
 132      "context": [
 133          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 134          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 135          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 136          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid="
 137      ]},
 138     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 139      "id": "OptimisticLockFailedException",
 140      "context": [
 141          "Got OptimisticLockFailedException",
 142          "AbstractTranscriberInterface"
 143      ]},
 144     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 145      "id": "ConflictingModificationAppliedException",
 146      "context": [
 147          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)neutron"
 148      ]},
 149     # oxygen
 150     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 151      "id": "ConflictingModificationAppliedException",
 152      "context": [
 153          "OptimisticLockFailedException: Optimistic lock failed.",
 154          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 155          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 156      ]},
 157     {"issue": "https://jira.opendaylight.org/browse/OPNFLWPLUG-917",
 158      "id": "IllegalStateException",
 159      "context": [
 160          "java.lang.IllegalStateException: Deserializer for key: msgVersion: 4 objectClass: " +
 161          "org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.oxm.rev150225.match.entries.grouping.MatchEntry " +
 162          "msgType: 1 oxm_field: 33 experimenterID: null was not found " +
 163          "- please verify that all needed deserializers ale loaded correctly"
 164      ]},
 165     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1640",
 166      "id": "ElasticsearchAppender",
 167      "context": [
 168          "Can't append into Elasticsearch",
 169          "org.apache.karaf.decanter.appender.elasticsearch - 1.0.0"
 170      ]}
 171 ]
 172
 173 _re_ts = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}")
 174 _re_ts_we = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}( \| ERROR \| | \| WARN  \| )")
 175 _re_ex = re.compile(r"(?i)exception")
 176 _ex_map = collections.OrderedDict()
 177 _ts_list = []
 178 _fail = []
 179
 180
 181 def get_exceptions(lines):
 182     """
 183     Create a map of exceptions that also has a list of warnings and errors preceeding
 184     the exception to use as context.
 185
 186     The lines are parsed to create a list where all lines related to a timestamp
 187     are aggregated. Timestamped lines with exception (case insensitive) are copied
 188     to the exception map keyed to the index of the timestamp line. Each exception value
 189     also has a list containing WARN and ERROR lines proceeding the exception.
 190
 191     :param list lines:
 192     :return OrderedDict _ex_map: map of exceptions
 193     """
 194     global _ex_map
 195     _ex_map = collections.OrderedDict()
 196     global _ts_list
 197     _ts_list = []
 198     cur_list = []
 199     warnerr_deq = collections.deque(maxlen=5)
 200
 201     for line in lines:
 202         ts = _re_ts.search(line)
 203
 204         # Check if this is the start or continuation of a timestamp line
 205         if ts:
 206             cur_list = [line]
 207             _ts_list.append(cur_list)
 208             ts_we = _re_ts_we.search(line)
 209             # Track WARN and ERROR lines
 210             if ts_we:
 211                 warn_err_index = len(_ts_list) - 1
 212                 warnerr_deq.append(warn_err_index)
 213         # Append to current timestamp line since this is not a timestamp line
 214         else:
 215             cur_list.append(line)
 216
 217         # Add the timestamp line to the exception map if it has an exception
 218         ex = _re_ex.search(line)
 219         if ex:
 220             index = len(_ts_list) - 1
 221             if index not in _ex_map:
 222                 _ex_map[index] = {"warnerr_list": list(warnerr_deq), 'lines': cur_list}
 223                 warnerr_deq.clear()  # reset the deque to only track new ERROR and WARN lines
 224
 225     return _ex_map
 226
 227
 228 def check_exceptions():
 229     """
 230     Return a list of exceptions that were not in the whitelist.
 231
 232     Each exception found is compared against all the patterns
 233     in the whitelist.
 234
 235     :return list _fail: list of exceptions not in the whitelist
 236     """
 237     global _fail
 238     _fail = []
 239     _match = []
 240     for ex_idx, ex in _ex_map.items():
 241         ex_str = "__".join(ex.get("lines"))
 242         for whitelist in _whitelist:
 243             # skip the current whitelist exception if not in the current exception
 244             if whitelist.get("id") not in ex_str:
 245                 continue
 246             whitelist_contexts = whitelist.get("context")
 247             num_context_matches = 0
 248             for whitelist_context in whitelist_contexts:
 249                 for exwe_index in reversed(ex.get("warnerr_list")):
 250                     exwe_str = "__".join(_ts_list[exwe_index])
 251                     if whitelist_context in exwe_str:
 252                         num_context_matches += 1
 253             # Mark this exception as a known issue if all the context's matched
 254             if num_context_matches >= len(whitelist_contexts):
 255                 ex["issue"] = whitelist.get("issue")
 256                 _match.append(ex)
 257                 logging.info("known exception was seen: {}".format(ex["issue"]))
 258                 break
 259         # A new exception when it isn't marked with a known issue.
 260         if "issue" not in ex:
 261             _fail.append(ex)
 262     return _fail, _match
 263
 264
 265 def verify_exceptions(lines):
 266     """
 267     Return a list of exceptions not in the whitelist for the given lines.
 268
 269     :param list lines: list of lines from a log
 270     :return list, list: one list of exceptions not in the whitelist, and a second with matching issues
 271     """
 272     if not lines:
 273         return
 274     get_exceptions(lines)
 275     return check_exceptions()
 276
 277
 278 def write_exceptions_map_to_file(testname, filename, mode="a+"):
 279     """
 280     Write the exceptions map to a file under the testname header. The output
 281     will include all lines in the exception itself as well as any previous
 282     contextual warning or error lines. The output will be appended or overwritten
 283     depending on the mode parameter. It is assumed that the caller has called
 284     verify_exceptions() earlier to populate the exceptions map, otherwise only
 285     the testname and header will be printed to the file.
 286
 287     :param str testname: The name of the test
 288     :param str filename: The file to open for writing
 289     :param str mode: Append (a+) or overwrite (w+)
 290     """
 291     try:
 292         os.makedirs(os.path.dirname(filename))
 293     except OSError as exception:
 294         if exception.errno != errno.EEXIST:
 295             raise
 296
 297     with open(filename, mode) as fp:
 298         fp.write("{}\n".format("=" * 60))
 299         fp.write("Starting test: {}\n".format(testname))
 300         for ex_idx, ex in _ex_map.items():
 301             fp.write("{}\n".format("-" * 40))
 302             if "issue" in ex:
 303                 fp.write("Exception was matched to: {}\n".format(ex.get("issue")))
 304             else:
 305                 fp.write("Exception is new\n")
 306             for exwe_index in ex.get("warnerr_list")[:-1]:
 307                 for line in _ts_list[exwe_index]:
 308                     fp.write("{}\n".format(line))
 309             fp.writelines(ex.get("lines"))
 310             fp.write("\n")