csit/libraries/netvirt/excepts.py

   1 import collections
   2 import logging
   3 import re
   4
   5 # Make sure to have unique matches in different lines
   6 # Order the list in alphabetical order based on the "issue" key
   7 _whitelist = [
   8     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
   9      "id": "ConflictingModificationAppliedException",
  10      "context": [
  11          "Node was created by other transaction",
  12          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  13          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow",
  14          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  15          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.",
  16          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  17          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  18          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  19          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  20      ]},
  21     # oxygen
  22     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  23      "id": "ConflictingModificationAppliedException",
  24      "context": [
  25          "Node was created by other transaction",
  26          "OptimisticLockFailedException: Optimistic lock failed."
  27          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  28          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  29          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  30          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  31      ]},
  32     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  33      "id": "ConflictingModificationAppliedException",
  34      "context": [
  35          "Node was created by other transaction",
  36          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  37          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  38          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  39          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  40          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  41          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  42      ]},
  43     # oxygen
  44     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  45      "id": "ConflictingModificationAppliedException",
  46      "context": [
  47          "OptimisticLockFailedException: Optimistic lock failed."
  48          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  49          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  50          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  51          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  52      ]},
  53     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  54      "id": "ConflictingModificationAppliedException",
  55      "context": [
  56          "Node was deleted by other transaction",
  57          "Optimistic lock failed for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  58          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  59          "elan-instance-name=",
  60          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  61          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  62          "elan-instance-name="
  63      ]},
  64     # oxygen version of NETVIRT-1136
  65     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  66      "id": "ConflictingModificationAppliedException",
  67      "context": [
  68          "Node was deleted by other transaction",
  69          "OptimisticLockFailedException: Optimistic lock failed.",
  70          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  71          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  72          "elan-instance-name="
  73      ]},
  74     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1260",
  75      "id": "ConflictingModificationAppliedException",
  76      "context": [
  77          "Node was deleted by other transaction",
  78          "Optimistic lock failed for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  79          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name=",
  80          "Conflicting modification for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  81          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name="
  82      ]},
  83     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  84      "id": "ConflictingModificationAppliedException",
  85      "context": [
  86          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  87          "OptimisticLockFailedException",
  88          "Conflicting modification for path /(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)" +
  89          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  90          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id="
  91      ]},
  92     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  93      "id": "ExecutionException",
  94      "context": [
  95          "OptimisticLockFailedException: Optimistic lock failed",
  96          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  97          "removeOrUpdateVpnToDpnList: Error removing from dpnToVpnList for vpn "
  98      ]},
  99     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1281",
 100      "id": "OptimisticLockFailedException",
 101      "context": [
 102          "OptimisticLockFailedException: Optimistic lock failed.",
 103          "ConflictingModificationAppliedException: Node children was modified by other transaction",
 104          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-"
 105      ]},
 106     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 107      "id": "ConflictingModificationAppliedException",
 108      "context": [
 109          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 110          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 111          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 112          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid="
 113      ]},
 114     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 115      "id": "OptimisticLockFailedException",
 116      "context": [
 117          "Got OptimisticLockFailedException",
 118          "AbstractTranscriberInterface"
 119      ]},
 120     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 121      "id": "ConflictingModificationAppliedException",
 122      "context": [
 123          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)neutron"
 124      ]},
 125     # oxygen
 126     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 127      "id": "ConflictingModificationAppliedException",
 128      "context": [
 129          "OptimisticLockFailedException: Optimistic lock failed.",
 130          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 131          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 132      ]},
 133     {"issue": "https://jira.opendaylight.org/browse/OPNFLWPLUG-917",
 134      "id": "IllegalStateException",
 135      "context": [
 136          "java.lang.IllegalStateException: Deserializer for key: msgVersion: 4 objectClass: " +
 137          "org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.oxm.rev150225.match.entries.grouping.MatchEntry " +
 138          "msgType: 1 oxm_field: 33 experimenterID: null was not found " +
 139          "- please verify that all needed deserializers ale loaded correctly"
 140      ]}
 141 ]
 142
 143 _re_ts = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}")
 144 _re_ts_we = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}( \| ERROR \| | \| WARN  \| )")
 145 _re_ex = re.compile(r"(?i)exception")
 146 _ex_map = collections.OrderedDict()
 147 _ts_list = []
 148 _fail = []
 149
 150
 151 def get_exceptions(lines):
 152     """
 153     Create a map of exceptions that also has a list of warnings and errors preceeding
 154     the exception to use as context.
 155
 156     The lines are parsed to create a list where all lines related to a timestamp
 157     are aggregated. Timestamped lines with exception (case insensitive) are copied
 158     to the exception map keyed to the index of the timestamp line. Each exception value
 159     also has a 3 element list containing the last three WARN and ERROR lines.
 160
 161     :param list lines:
 162     :return OrderedDict _ex_map: map of exceptions
 163     """
 164     global _ex_map
 165     _ex_map = collections.OrderedDict()
 166     global _ts_list
 167     _ts_list = []
 168     cur_list = []
 169     warnerr_deq = collections.deque(maxlen=5)
 170
 171     for line in lines:
 172         ts = _re_ts.search(line)
 173
 174         # Check if this is the start or continuation of a timestamp line
 175         if ts:
 176             cur_list = [line]
 177             _ts_list.append(cur_list)
 178             ts_we = _re_ts_we.search(line)
 179             # Track WARN and ERROR lines
 180             if ts_we:
 181                 warn_err_index = len(_ts_list) - 1
 182                 warnerr_deq.append(warn_err_index)
 183         # Append to current timestamp line since this is not a timestamp line
 184         else:
 185             cur_list.append(line)
 186
 187         # Add the timestamp line to the exception map if it has an exception
 188         ex = _re_ex.search(line)
 189         if ex:
 190             index = len(_ts_list) - 1
 191             if index not in _ex_map:
 192                 _ex_map[index] = {"warnerr_list": list(warnerr_deq), 'lines': cur_list}
 193                 warnerr_deq.clear()  # reset the deque to only track new ERROR and WARN lines
 194
 195     return _ex_map
 196
 197
 198 def check_exceptions():
 199     """
 200     Return a list of exceptions that were not in the whitelist.
 201
 202     Each exception found is compared against all the patterns
 203     in the whitelist.
 204
 205     :return list _fail: list of exceptions not in the whitelist
 206     """
 207     global _fail
 208     _fail = []
 209     _match = []
 210     for ex_idx, ex in _ex_map.items():
 211         ex_str = "__".join(ex.get("lines"))
 212         for whitelist in _whitelist:
 213             # skip the current whitelist exception if not in the current exception
 214             if whitelist.get("id") not in ex_str:
 215                 continue
 216             whitelist_contexts = whitelist.get("context")
 217             num_context_matches = 0
 218             for whitelist_context in whitelist_contexts:
 219                 for exwe_index in reversed(ex.get("warnerr_list")):
 220                     exwe_str = "__".join(_ts_list[exwe_index])
 221                     if whitelist_context in exwe_str:
 222                         num_context_matches += 1
 223             # Mark this exception as a known issue if all the context's matched
 224             if num_context_matches >= len(whitelist_contexts):
 225                 ex["issue"] = whitelist.get("issue")
 226                 _match.append(ex)
 227                 logging.info("known exception was seen: {}".format(ex["issue"]))
 228                 break
 229         # A new exception when it isn't marked with a known issue.
 230         if "issue" not in ex:
 231             _fail.append(ex)
 232     return _fail, _match
 233
 234
 235 def verify_exceptions(lines):
 236     """
 237     Return a list of exceptions not in the whitelist for the given lines.
 238
 239     :param list lines: list of lines from a log
 240     :return list, list: one list of exceptions not in the whitelist, and a second with matching issues
 241     """
 242     if not lines:
 243         return
 244     get_exceptions(lines)
 245     return check_exceptions()