csit/libraries/netvirt/excepts.py

   1 import collections
   2 import logging
   3 import re
   4
   5 # Make sure to have unique matches in different lines
   6 # Order the list in alphabetical order based on the "issue" key
   7 _whitelist = [
   8     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
   9      "id": "ConflictingModificationAppliedException",
  10      "context": [
  11          "Node was created by other transaction",
  12          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  13          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow",
  14          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  15          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3."
  16          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  17          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  18          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  19          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  20      ]},
  21     # oxygen
  22     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  23      "id": "ConflictingModificationAppliedException",
  24      "context": [
  25          "Node was created by other transaction",
  26          "OptimisticLockFailedException: Optimistic lock failed."
  27          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  28          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  29          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  30          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  31      ]},
  32     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  33      "id": "ConflictingModificationAppliedException",
  34      "context": [
  35          "Node was created by other transaction",
  36          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  37          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  38          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  39          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  40          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  41          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  42      ]},
  43     # oxygen
  44     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  45      "id": "ConflictingModificationAppliedException",
  46      "context": [
  47          "OptimisticLockFailedException: Optimistic lock failed."
  48          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  49          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  50          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  51          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  52      ]},
  53     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  54      "id": "ConflictingModificationAppliedException",
  55      "context": [
  56          "Node was deleted by other transaction",
  57          "Optimistic lock failed for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  58          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  59          "elan-instance-name=",
  60          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  61          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  62          "elan-instance-name="
  63      ]},
  64     # oxygen version of NETVIRT-1136
  65     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  66      "id": "ConflictingModificationAppliedException",
  67      "context": [
  68          "Node was deleted by other transaction",
  69          "OptimisticLockFailedException: Optimistic lock failed."
  70          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  71          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  72          "elan-instance-name="
  73      ]},
  74     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1260",
  75      "id": "ConflictingModificationAppliedException",
  76      "context": [
  77          "Node was deleted by other transaction",
  78          "Optimistic lock failed for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  79          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name=",
  80          "Conflicting modification for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  81          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name="
  82      ]},
  83     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  84      "id": "ConflictingModificationAppliedException",
  85      "context": [
  86          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  87          "OptimisticLockFailedException",
  88          "Conflicting modification for path /(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)" +
  89          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  90          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id="
  91      ]},
  92     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  93      "id": "ExecutionException",
  94      "context": [
  95          "OptimisticLockFailedException: Optimistic lock failed.",
  96          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  97          "removeOrUpdateVpnToDpnList: Error removing from dpnToVpnList for vpn "
  98      ]},
  99     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1281",
 100      "id": "OptimisticLockFailedException",
 101      "context": [
 102          "OptimisticLockFailedException: Optimistic lock failed.",
 103          "ConflictingModificationAppliedException: Node children was modified by other transaction",
 104          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-"
 105      ]},
 106     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 107      "id": "ConflictingModificationAppliedException",
 108      "context": [
 109          "Node was deleted by other transaction",
 110          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 111          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 112          "Got OptimisticLockFailedException"
 113      ]},
 114     # oxygen
 115     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 116      "id": "ConflictingModificationAppliedException",
 117      "context": [
 118          "Node was deleted by other transaction",
 119          "OptimisticLockFailedException: Optimistic lock failed.",
 120          "Got OptimisticLockFailedException",
 121          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 122          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 123      ]},
 124     {"issue": "https://jira.opendaylight.org/browse/OPNFLWPLUG-917",
 125      "id": "IllegalStateException",
 126      "context": [
 127          "java.lang.IllegalStateException: Deserializer for key: msgVersion: 4 objectClass: " +
 128          "org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.oxm.rev150225.match.entries.grouping.MatchEntry " +
 129          "msgType: 1 oxm_field: 33 experimenterID: null was not found " +
 130          "- please verify that all needed deserializers ale loaded correctly"
 131      ]}
 132 ]
 133
 134 _re_ts = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}")
 135 _re_ts_we = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}( \| ERROR \| | \| WARN  \| )")
 136 _re_ex = re.compile(r"(?i)exception")
 137 _ex_map = collections.OrderedDict()
 138 _ts_list = []
 139 _fail = []
 140
 141
 142 def get_exceptions(lines):
 143     """
 144     Create a map of exceptions that also has a list of warnings and errors preceeding
 145     the exception to use as context.
 146
 147     The lines are parsed to create a list where all lines related to a timestamp
 148     are aggregated. Timestamped lines with exception (case insensitive) are copied
 149     to the exception map keyed to the index of the timestamp line. Each exception value
 150     also has a 3 element list containing the last three WARN and ERROR lines.
 151
 152     :param list lines:
 153     :return OrderedDict _ex_map: map of exceptions
 154     """
 155     global _ex_map
 156     _ex_map = collections.OrderedDict()
 157     global _ts_list
 158     _ts_list = []
 159     cur_list = []
 160     warnerr_deq = collections.deque(maxlen=5)
 161
 162     for line in lines:
 163         ts = _re_ts.search(line)
 164
 165         # Check if this is the start or continuation of a timestamp line
 166         if ts:
 167             cur_list = [line]
 168             _ts_list.append(cur_list)
 169             ts_we = _re_ts_we.search(line)
 170             # Track WARN and ERROR lines
 171             if ts_we:
 172                 warn_err_index = len(_ts_list) - 1
 173                 warnerr_deq.append(warn_err_index)
 174         # Append to current timestamp line since this is not a timestamp line
 175         else:
 176             cur_list.append(line)
 177
 178         # Add the timestamp line to the exception map if it has an exception
 179         ex = _re_ex.search(line)
 180         if ex:
 181             index = len(_ts_list) - 1
 182             if index not in _ex_map:
 183                 _ex_map[index] = {"warnerr_list": list(warnerr_deq), 'lines': cur_list}
 184                 warnerr_deq.clear()  # reset the deque to only track new ERROR and WARN lines
 185
 186     return _ex_map
 187
 188
 189 def check_exceptions():
 190     """
 191     Return a list of exceptions that were not in the whitelist.
 192
 193     Each exception found is compared against all the patterns
 194     in the whitelist.
 195
 196     :return list _fail: list of exceptions not in the whitelist
 197     """
 198     global _fail
 199     _fail = []
 200     _match = []
 201     for ex_idx, ex in _ex_map.items():
 202         ex_str = "__".join(ex.get("lines"))
 203         for whitelist in _whitelist:
 204             # skip the current whitelist exception if not in the current exception
 205             if whitelist.get("id") not in ex_str:
 206                 continue
 207             whitelist_contexts = whitelist.get("context")
 208             num_context_matches = 0
 209             for whitelist_context in whitelist_contexts:
 210                 for exwe_index in reversed(ex.get("warnerr_list")):
 211                     exwe_str = "__".join(_ts_list[exwe_index])
 212                     if whitelist_context in exwe_str:
 213                         num_context_matches += 1
 214                         break
 215             # Mark this exception as a known issue if all the context's matched
 216             if num_context_matches == len(whitelist_contexts):
 217                 ex["issue"] = whitelist.get("issue")
 218                 _match.append(ex)
 219                 logging.info("known exception was seen: {}".format(ex["issue"]))
 220                 break
 221         # A new exception when it isn't marked with a known issue.
 222         if "issue" not in ex:
 223             _fail.append(ex)
 224     return _fail, _match
 225
 226
 227 def verify_exceptions(lines):
 228     """
 229     Return a list of exceptions not in the whitelist for the given lines.
 230
 231     :param list lines: list of lines from a log
 232     :return list, list: one list of exceptions not in the whitelist, and a second with matching issues
 233     """
 234     if not lines:
 235         return
 236     get_exceptions(lines)
 237     return check_exceptions()