csit/libraries/netvirt/excepts.py

   1 import collections
   2 import logging
   3 import re
   4
   5 # Make sure to have unique matches in different lines
   6 # Order the list in alphabetical order based on the "issue" key
   7 _whitelist = [
   8     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
   9      "id": "ConflictingModificationAppliedException",
  10      "context": [
  11          "Node was created by other transaction",
  12          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  13          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow",
  14          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  15          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.",
  16          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  17          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  18          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  19          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  20      ]},
  21     # oxygen
  22     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  23      "id": "ConflictingModificationAppliedException",
  24      "context": [
  25          "Node was created by other transaction",
  26          "OptimisticLockFailedException: Optimistic lock failed."
  27          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  28          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  29          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  30          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  31      ]},
  32     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  33      "id": "ConflictingModificationAppliedException",
  34      "context": [
  35          "Node was created by other transaction",
  36          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  37          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  38          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  39          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  40          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  41          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  42      ]},
  43     # oxygen
  44     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  45      "id": "ConflictingModificationAppliedException",
  46      "context": [
  47          "OptimisticLockFailedException: Optimistic lock failed."
  48          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  49          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  50          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  51          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  52      ]},
  53     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  54      "id": "ConflictingModificationAppliedException",
  55      "context": [
  56          "Node was deleted by other transaction",
  57          "Optimistic lock failed for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  58          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  59          "elan-instance-name=",
  60          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  61          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  62          "elan-instance-name="
  63      ]},
  64     # oxygen version of NETVIRT-1136
  65     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  66      "id": "ConflictingModificationAppliedException",
  67      "context": [
  68          "Node was deleted by other transaction",
  69          "OptimisticLockFailedException: Optimistic lock failed.",
  70          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  71          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  72          "elan-instance-name="
  73      ]},
  74     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1260",
  75      "id": "ConflictingModificationAppliedException",
  76      "context": [
  77          "Node was deleted by other transaction",
  78          "Optimistic lock failed for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  79          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name=",
  80          "Conflicting modification for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  81          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name="
  82      ]},
  83     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  84      "id": "ConflictingModificationAppliedException",
  85      "context": [
  86          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  87          "OptimisticLockFailedException",
  88          "Conflicting modification for path /(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)" +
  89          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  90          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id="
  91      ]},
  92     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  93      "id": "ExecutionException",
  94      "context": [
  95          "OptimisticLockFailedException: Optimistic lock failed.",
  96          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  97          "removeOrUpdateVpnToDpnList: Error removing from dpnToVpnList for vpn "
  98      ]},
  99     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1281",
 100      "id": "OptimisticLockFailedException",
 101      "context": [
 102          "OptimisticLockFailedException: Optimistic lock failed.",
 103          "ConflictingModificationAppliedException: Node children was modified by other transaction",
 104          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-"
 105      ]},
 106     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 107      "id": "ConflictingModificationAppliedException",
 108      "context": [
 109          "Node was deleted by other transaction",
 110          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 111          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 112          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 113          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid="
 114      ]},
 115     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 116      "id": "ConflictingModificationAppliedException",
 117      "context": [
 118          "Got OptimisticLockFailedException", "NeutronNetwork [networkUUID=",
 119          "AbstractTranscriberInterface"
 120      ]},
 121     # oxygen
 122     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 123      "id": "ConflictingModificationAppliedException",
 124      "context": [
 125          "Node was deleted by other transaction",
 126          "OptimisticLockFailedException: Optimistic lock failed.",
 127          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 128          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 129      ]},
 130     {"issue": "https://jira.opendaylight.org/browse/OPNFLWPLUG-917",
 131      "id": "IllegalStateException",
 132      "context": [
 133          "java.lang.IllegalStateException: Deserializer for key: msgVersion: 4 objectClass: " +
 134          "org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.oxm.rev150225.match.entries.grouping.MatchEntry " +
 135          "msgType: 1 oxm_field: 33 experimenterID: null was not found " +
 136          "- please verify that all needed deserializers ale loaded correctly"
 137      ]}
 138 ]
 139
 140 _re_ts = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}")
 141 _re_ts_we = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}( \| ERROR \| | \| WARN  \| )")
 142 _re_ex = re.compile(r"(?i)exception")
 143 _ex_map = collections.OrderedDict()
 144 _ts_list = []
 145 _fail = []
 146
 147
 148 def get_exceptions(lines):
 149     """
 150     Create a map of exceptions that also has a list of warnings and errors preceeding
 151     the exception to use as context.
 152
 153     The lines are parsed to create a list where all lines related to a timestamp
 154     are aggregated. Timestamped lines with exception (case insensitive) are copied
 155     to the exception map keyed to the index of the timestamp line. Each exception value
 156     also has a 3 element list containing the last three WARN and ERROR lines.
 157
 158     :param list lines:
 159     :return OrderedDict _ex_map: map of exceptions
 160     """
 161     global _ex_map
 162     _ex_map = collections.OrderedDict()
 163     global _ts_list
 164     _ts_list = []
 165     cur_list = []
 166     warnerr_deq = collections.deque(maxlen=5)
 167
 168     for line in lines:
 169         ts = _re_ts.search(line)
 170
 171         # Check if this is the start or continuation of a timestamp line
 172         if ts:
 173             cur_list = [line]
 174             _ts_list.append(cur_list)
 175             ts_we = _re_ts_we.search(line)
 176             # Track WARN and ERROR lines
 177             if ts_we:
 178                 warn_err_index = len(_ts_list) - 1
 179                 warnerr_deq.append(warn_err_index)
 180         # Append to current timestamp line since this is not a timestamp line
 181         else:
 182             cur_list.append(line)
 183
 184         # Add the timestamp line to the exception map if it has an exception
 185         ex = _re_ex.search(line)
 186         if ex:
 187             index = len(_ts_list) - 1
 188             if index not in _ex_map:
 189                 _ex_map[index] = {"warnerr_list": list(warnerr_deq), 'lines': cur_list}
 190                 warnerr_deq.clear()  # reset the deque to only track new ERROR and WARN lines
 191
 192     return _ex_map
 193
 194
 195 def check_exceptions():
 196     """
 197     Return a list of exceptions that were not in the whitelist.
 198
 199     Each exception found is compared against all the patterns
 200     in the whitelist.
 201
 202     :return list _fail: list of exceptions not in the whitelist
 203     """
 204     global _fail
 205     _fail = []
 206     _match = []
 207     for ex_idx, ex in _ex_map.items():
 208         ex_str = "__".join(ex.get("lines"))
 209         for whitelist in _whitelist:
 210             # skip the current whitelist exception if not in the current exception
 211             if whitelist.get("id") not in ex_str:
 212                 continue
 213             whitelist_contexts = whitelist.get("context")
 214             num_context_matches = 0
 215             for whitelist_context in whitelist_contexts:
 216                 for exwe_index in reversed(ex.get("warnerr_list")):
 217                     exwe_str = "__".join(_ts_list[exwe_index])
 218                     if whitelist_context in exwe_str:
 219                         num_context_matches += 1
 220                         break
 221             # Mark this exception as a known issue if all the context's matched
 222             if num_context_matches == len(whitelist_contexts):
 223                 ex["issue"] = whitelist.get("issue")
 224                 _match.append(ex)
 225                 logging.info("known exception was seen: {}".format(ex["issue"]))
 226                 break
 227         # A new exception when it isn't marked with a known issue.
 228         if "issue" not in ex:
 229             _fail.append(ex)
 230     return _fail, _match
 231
 232
 233 def verify_exceptions(lines):
 234     """
 235     Return a list of exceptions not in the whitelist for the given lines.
 236
 237     :param list lines: list of lines from a log
 238     :return list, list: one list of exceptions not in the whitelist, and a second with matching issues
 239     """
 240     if not lines:
 241         return
 242     get_exceptions(lines)
 243     return check_exceptions()