csit/libraries/netvirt/excepts.py

   1 import collections
   2 import logging
   3 import re
   4
   5 # Make sure to have unique matches in different lines
   6 # Order the list in alphabetical order based on the "issue" key
   7 _whitelist = [
   8     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
   9      "id": "ConflictingModificationAppliedException",
  10      "context": [
  11          "Node was created by other transaction",
  12          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  13          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow",
  14          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  15          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.",
  16          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  17          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  18          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  19          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  20      ]},
  21     # oxygen
  22     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-972",
  23      "id": "ConflictingModificationAppliedException",
  24      "context": [
  25          "Node was created by other transaction",
  26          "OptimisticLockFailedException: Optimistic lock failed."
  27          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  28          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=",
  29          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=21}]/flow/flow" +
  30          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=L3.", ".21.", ".42."
  31      ]},
  32     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  33      "id": "ConflictingModificationAppliedException",
  34      "context": [
  35          "Node was created by other transaction",
  36          "Optimistic lock failed for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  37          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  38          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  39          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  40          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  41          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  42      ]},
  43     # oxygen
  44     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1135",
  45      "id": "ConflictingModificationAppliedException",
  46      "context": [
  47          "OptimisticLockFailedException: Optimistic lock failed."
  48          "Conflicting modification for path /(urn:opendaylight:inventory?revision=2013-08-19)nodes/node/node" +
  49          "[{(urn:opendaylight:inventory?revision=2013-08-19)id=openflow:",
  50          "table/table[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=47}]/flow/flow" +
  51          "[{(urn:opendaylight:flow:inventory?revision=2013-08-19)id=SNAT.", ".47."
  52      ]},
  53     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  54      "id": "ConflictingModificationAppliedException",
  55      "context": [
  56          "Node was deleted by other transaction",
  57          "Optimistic lock failed for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  58          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  59          "elan-instance-name=",
  60          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  61          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  62          "elan-instance-name="
  63      ]},
  64     # oxygen version of NETVIRT-1136
  65     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1136",
  66      "id": "ConflictingModificationAppliedException",
  67      "context": [
  68          "Node was deleted by other transaction",
  69          "OptimisticLockFailedException: Optimistic lock failed.",
  70          "Conflicting modification for path /(urn:opendaylight:netvirt:elan?revision=2015-06-02)elan-" +
  71          "forwarding-tables/mac-table/mac-table[{(urn:opendaylight:netvirt:elan?revision=2015-06-02)" +
  72          "elan-instance-name="
  73      ]},
  74     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1260",
  75      "id": "ConflictingModificationAppliedException",
  76      "context": [
  77          "Node was deleted by other transaction",
  78          "Optimistic lock failed for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  79          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name=",
  80          "Conflicting modification for path /(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)" +
  81          "interfaces/interface/interface[{(urn:ietf:params:xml:ns:yang:ietf-interfaces?revision=2014-05-08)name="
  82      ]},
  83     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  84      "id": "ConflictingModificationAppliedException",
  85      "context": [
  86          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  87          "OptimisticLockFailedException",
  88          "Conflicting modification for path /(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)" +
  89          "vpn-instance-op-data/vpn-instance-op-data-entry/vpn-instance-op-data-entry" +
  90          "[{(urn:opendaylight:netvirt:l3vpn?revision=2013-09-11)vrf-id="
  91      ]},
  92     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1270",
  93      "id": "ExecutionException",
  94      "context": [
  95          "OptimisticLockFailedException: Optimistic lock failed",
  96          "ConflictingModificationAppliedException: Node children was modified by other transaction",
  97          "removeOrUpdateVpnToDpnList: Error removing from dpnToVpnList for vpn "
  98      ]},
  99     {"issue": "https://jira.opendaylight.org/browse/NETVIRT-1281",
 100      "id": "OptimisticLockFailedException",
 101      "context": [
 102          "OptimisticLockFailedException: Optimistic lock failed.",
 103          "ConflictingModificationAppliedException: Node children was modified by other transaction",
 104          "Direct Exception (not failed Future) when executing job, won't even retry: JobEntry{key='VPNINTERFACE-"
 105      ]},
 106     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 107      "id": "ConflictingModificationAppliedException",
 108      "context": [
 109          "Node was deleted by other transaction",
 110          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 111          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 112          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 113          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid="
 114      ]},
 115     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 116      "id": "ConflictingModificationAppliedException",
 117      "context": [
 118          "Got OptimisticLockFailedException", "NeutronNetwork [networkUUID=",
 119          "AbstractTranscriberInterface"
 120      ]},
 121     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 122      "id": "ConflictingModificationAppliedException",
 123      "context": [
 124          "Got OptimisticLockFailedException",
 125          "AbstractTranscriberInterface"
 126      ]},
 127     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 128      "id": "ConflictingModificationAppliedException",
 129      "context": [
 130          "Node was created by other transaction",
 131          "Optimistic lock failed for path /(urn:opendaylight:neutron?revision=2015-07-12)neutron"
 132      ]},
 133     # oxygen
 134     {"issue": "https://jira.opendaylight.org/browse/NEUTRON-157",
 135      "id": "ConflictingModificationAppliedException",
 136      "context": [
 137          "Node was deleted by other transaction",
 138          "OptimisticLockFailedException: Optimistic lock failed.",
 139          "Conflicting modification for path /(urn:opendaylight:neutron?revision=2015-07-12)" +
 140          "neutron/networks/network/network[{(urn:opendaylight:neutron?revision=2015-07-12)uuid=",
 141      ]},
 142     {"issue": "https://jira.opendaylight.org/browse/OPNFLWPLUG-917",
 143      "id": "IllegalStateException",
 144      "context": [
 145          "java.lang.IllegalStateException: Deserializer for key: msgVersion: 4 objectClass: " +
 146          "org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.oxm.rev150225.match.entries.grouping.MatchEntry " +
 147          "msgType: 1 oxm_field: 33 experimenterID: null was not found " +
 148          "- please verify that all needed deserializers ale loaded correctly"
 149      ]}
 150 ]
 151
 152 _re_ts = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}")
 153 _re_ts_we = re.compile(r"^[0-9]{4}(-[0-9]{2}){2}T([0-9]{2}:){2}[0-9]{2},[0-9]{3}( \| ERROR \| | \| WARN  \| )")
 154 _re_ex = re.compile(r"(?i)exception")
 155 _ex_map = collections.OrderedDict()
 156 _ts_list = []
 157 _fail = []
 158
 159
 160 def get_exceptions(lines):
 161     """
 162     Create a map of exceptions that also has a list of warnings and errors preceeding
 163     the exception to use as context.
 164
 165     The lines are parsed to create a list where all lines related to a timestamp
 166     are aggregated. Timestamped lines with exception (case insensitive) are copied
 167     to the exception map keyed to the index of the timestamp line. Each exception value
 168     also has a 3 element list containing the last three WARN and ERROR lines.
 169
 170     :param list lines:
 171     :return OrderedDict _ex_map: map of exceptions
 172     """
 173     global _ex_map
 174     _ex_map = collections.OrderedDict()
 175     global _ts_list
 176     _ts_list = []
 177     cur_list = []
 178     warnerr_deq = collections.deque(maxlen=5)
 179
 180     for line in lines:
 181         ts = _re_ts.search(line)
 182
 183         # Check if this is the start or continuation of a timestamp line
 184         if ts:
 185             cur_list = [line]
 186             _ts_list.append(cur_list)
 187             ts_we = _re_ts_we.search(line)
 188             # Track WARN and ERROR lines
 189             if ts_we:
 190                 warn_err_index = len(_ts_list) - 1
 191                 warnerr_deq.append(warn_err_index)
 192         # Append to current timestamp line since this is not a timestamp line
 193         else:
 194             cur_list.append(line)
 195
 196         # Add the timestamp line to the exception map if it has an exception
 197         ex = _re_ex.search(line)
 198         if ex:
 199             index = len(_ts_list) - 1
 200             if index not in _ex_map:
 201                 _ex_map[index] = {"warnerr_list": list(warnerr_deq), 'lines': cur_list}
 202                 warnerr_deq.clear()  # reset the deque to only track new ERROR and WARN lines
 203
 204     return _ex_map
 205
 206
 207 def check_exceptions():
 208     """
 209     Return a list of exceptions that were not in the whitelist.
 210
 211     Each exception found is compared against all the patterns
 212     in the whitelist.
 213
 214     :return list _fail: list of exceptions not in the whitelist
 215     """
 216     global _fail
 217     _fail = []
 218     _match = []
 219     for ex_idx, ex in _ex_map.items():
 220         ex_str = "__".join(ex.get("lines"))
 221         for whitelist in _whitelist:
 222             # skip the current whitelist exception if not in the current exception
 223             if whitelist.get("id") not in ex_str:
 224                 continue
 225             whitelist_contexts = whitelist.get("context")
 226             num_context_matches = 0
 227             for whitelist_context in whitelist_contexts:
 228                 for exwe_index in reversed(ex.get("warnerr_list")):
 229                     exwe_str = "__".join(_ts_list[exwe_index])
 230                     if whitelist_context in exwe_str:
 231                         num_context_matches += 1
 232             # Mark this exception as a known issue if all the context's matched
 233             if num_context_matches >= len(whitelist_contexts):
 234                 ex["issue"] = whitelist.get("issue")
 235                 _match.append(ex)
 236                 logging.info("known exception was seen: {}".format(ex["issue"]))
 237                 break
 238         # A new exception when it isn't marked with a known issue.
 239         if "issue" not in ex:
 240             _fail.append(ex)
 241     return _fail, _match
 242
 243
 244 def verify_exceptions(lines):
 245     """
 246     Return a list of exceptions not in the whitelist for the given lines.
 247
 248     :param list lines: list of lines from a log
 249     :return list, list: one list of exceptions not in the whitelist, and a second with matching issues
 250     """
 251     if not lines:
 252         return
 253     get_exceptions(lines)
 254     return check_exceptions()