resources/tools/odltools/odltools/csit/robotfiles.py

   1 import logging
   2 import xml.etree.cElementTree as ET
   3 from subprocess import Popen
   4
   5 import os
   6 import re
   7
   8 from odltools.ovs import flows
   9
  10 logger = logging.getLogger("csit.robotfiles")
  11
  12
  13 class RobotFiles:
  14     OUTDIR = "/tmp/robotjob"
  15     CHUNK_SIZE = 65536
  16     DUMP_FLOWS = "sudo ovs-ofctl dump-flows br-int -OOpenFlow13"
  17     TMP = "/tmp"
  18
  19     def __init__(self, infile, outdir):
  20         if outdir is None:
  21             self.outdir = RobotFiles.TMP
  22         else:
  23             self.outdir = outdir
  24         self.datafilepath = infile
  25         self.pdata = {}
  26         self.re_normalize_text = re.compile(r"( \n)|(\[A\[C.*)")
  27         # uri=restconf/config/interface-service-bindings:service-bindings, headers=None json=None</msg>
  28         self.re_uri = re.compile(r"uri=(?P<uri>.*),")
  29         logger.info("RobotFiles created")
  30
  31     def gunzip(self):
  32         infile = self.datafilepath
  33         basename = os.path.splitext(os.path.basename(self.datafilepath))[0]
  34         self.datafilepath = "{}/{}".format(self.outdir, basename)
  35         Popen("gunzip -cfk {} > {}".format(infile, self.datafilepath), shell=True).wait()
  36         logger.info("gunzip -cfk %s > %s", infile, self.datafilepath)
  37
  38     def mkdir(self, path):
  39         try:
  40             os.makedirs(path)
  41         except OSError:
  42             if not os.path.isdir(path):
  43                 raise
  44
  45     def mk_outdir(self):
  46         self.mkdir(self.outdir)
  47         logger.info("mk_outdir: %s created", self.outdir)
  48
  49     def read_chunks(self, fp):
  50         while True:
  51             data = fp.read(RobotFiles.CHUNK_SIZE)
  52             if not data:
  53                 break
  54             yield data
  55
  56     def parse_data_file(self):
  57         re_st = re.compile(r"dump-flows")
  58         cnt = 0
  59         with open(self.datafilepath, 'rb') as fp:
  60             for chunk in self.read_chunks(fp):
  61                 for m in re_st.finditer(chunk):
  62                     logger.info("%02d-%02d: %s", m.start(), m.end(), m.group(0))
  63                     cnt += 1
  64         logger.info("total matches: %d", cnt)
  65
  66     class State:
  67         def __init__(self):
  68             self.state = "init"
  69             self.pdata = {}
  70             self.test_id = None
  71             self.node = None
  72             self.command = None
  73             self.nodes = {}
  74             self.models = {}
  75
  76         def reset(self):
  77             self.state = "init"
  78             self.pdata = {}
  79             self.test_id = None
  80             self.node = None
  81             self.command = None
  82             self.nodes = {}
  83             self.models = {}
  84
  85     def normalize(self, intext):
  86         outtext = self.re_normalize_text.sub("", intext)
  87         return outtext
  88
  89     def print_config(self):
  90         logger.info("datafilepath: %s, outdir: %s", self.datafilepath, self.outdir)
  91
  92     # scan until test id= is seen. This indicates the start of a new test -> state=test
  93     # - scan until Get Test Teardown Debugs -> state=debugs
  94     #   - scan until Get DumpFlows And Ovsconfig -> state=nodes
  95     #   - scan until Get Model Dump -> state=models
  96     def process_element(self, state, event, element):
  97         tag = element.tag
  98         text = element.text
  99         attribs = element.attrib
 100         if logger.isEnabledFor(logging.DEBUG) and text is not None and attribs:
 101             logger.debug("process_element: %s - %s - %s - %s - %s - %s", state.state, state.command, event, tag, (text is not None), attribs)
 102         if event == "end":
 103             if element.tag == "test":
 104                 state.pdata['nodes'] = state.nodes
 105                 state.pdata['models'] = state.models
 106                 self.pdata[state.test_id] = state.pdata
 107                 state.reset()
 108                 return
 109             elif element.tag != "msg":
 110                 return
 111
 112         if event == "start" and state.state == "init":
 113             # <test id="s1-s1-s1-t1" name="Create VLAN Network (l2_network_1)">
 114             # <test id="s1-t1" name="Create VLAN Network net_1">
 115             if element.tag == "test":
 116                 state.test_id = element.get("id")
 117                 state.pdata["name"] = element.get("name")
 118                 state.state = "test"
 119                 state.command = ""
 120         elif event == "start" and state.state == "test":
 121             # <kw type="teardown" name="Get Test Teardown Debugs" library="OpenStackOperations">
 122             if element.tag == "kw" and element.get("name") == "Get Test Teardown Debugs":
 123                 state.state = "debugs"
 124                 state.command = ""
 125         elif event == "start" and state.state == "debugs":
 126             # <arg>Get DumpFlows And Ovsconfig</arg>
 127             if element.tag == "kw" and element.get("name") == "Get DumpFlows And Ovsconfig":
 128                 state.state = "nodes"
 129                 state.command = ""
 130             # <arg>Get Model Dump</arg>
 131             if element.tag == "kw" and element.get("name") == "Get Model Dump":
 132                 state.state = "models"
 133                 state.command = ""
 134         elif event == "start" and state.state == "nodes":
 135             # <arg>${OS_CONTROL_NODE_IP}</arg>
 136             if element.tag == "arg" and element.text is not None and "${OS_" in element.text:
 137                 state.node = element.text[element.text.find("{") + 1:element.text.find("}")]
 138                 state.nodes[state.node] = {}
 139                 state.state = "nodes2"
 140                 state.command = ""
 141         elif event == "start" and state.state == "nodes2":
 142             # <kw name="Write Commands Until Expected Prompt" library="Utils">
 143             if element.tag == "kw" and element.get("name") == "Write Commands Until Expected Prompt":
 144                 state.state = "kw"
 145                 state.command = ""
 146         elif event == "start" and state.state == "kw":
 147             # <arg>ip -o link</arg>
 148             if element.tag == "arg" and element.text is not None:
 149                 state.command = element.text
 150                 state.state = "command"
 151                 # only use the string before the ${...} since we don't know the ...
 152                 # <arg>sudo ip netns exec ${line} ip -o link</arg>
 153                 command_split = state.command.split("$")
 154                 if len(command_split) > 1:
 155                     state.command = command_split[0]
 156         elif event == "start" and state.state == "command":
 157             # <msg timestamp="20170414 07:31:21.769" level="INFO">ip -o link</msg>
 158             if element.tag == "msg" and element.text is not None:
 159                 text = self.normalize(element.text)
 160                 if text.find(state.command) != -1:
 161                     # <msg timestamp="20170414 07:31:34.425" level="INFO">sudo ip netns exec
 162                     # [jenkins@rele ^Mng-36618-350-devstack-newton-0 ~]&gt; ip -o link</msg>
 163                     if text.find("jenkins") != -1:
 164                         state.state = "nodes2"
 165                         state.command = ""
 166                     else:
 167                         state.state = "msg"
 168                         state.command = text
 169         elif state.state == "msg":
 170             # <msg timestamp="20170414 07:31:21.786" level="INFO">
 171             if element.tag == "msg" and element.text is not None:
 172                 state.nodes[state.node][state.command] = element.text
 173                 # are we at the end of the debugs for the node?
 174                 # this command is the last one
 175                 if state.command == "sudo ovs-ofctl dump-group-stats br-int -OOpenFlow13":
 176                     state.state = "debugs"
 177                     state.command = ""
 178                 else:
 179                     # still more debugs for this node
 180                     state.state = "nodes2"
 181         elif state.state == "models2":
 182             # <msg timestamp="20170813 08:20:11.806" level="INFO">Get Request using : alias=model_dump_session,
 183             # uri=restconf/config/interface-service-bindings:service-bindings, headers=None json=None</msg>
 184             if element.tag == "msg" and element.text is not None and element.text.find("uri") != -1:
 185                 uri = self.re_uri.search(element.text)
 186                 if uri is not None and "uri" in uri.group():
 187                     state.state = "uri"
 188                     state.command = uri.group("uri")
 189         elif event == "start" and state.state == "models":
 190             # <kw type="foritem" name="${model} = config/neutronvpn:router-interfaces-map">
 191             if element.tag == "kw" and "name" in element.attrib:
 192                 name_split = element.attrib["name"].split("${model} = ", 1)
 193                 model = None
 194                 if len(name_split) == 2:
 195                     model = name_split[1]
 196                 if model is not None:
 197                     state.state = "uri"
 198                     state.command = model
 199         elif state.state == "uri":
 200             if element.tag == "msg" and element.text is not None and element.text.find("pretty_output") != -1:
 201                 state.state = "dump"
 202                 # do not clear the state.command
 203         elif state.state == "dump":
 204             if element.tag == "msg" and element.text is not None:
 205                 state.models[state.command] = element.text
 206                 # if state.command == "restconf/operational/rendered-service-path:rendered-service-path":
 207                 if state.command == "restconf/operational/opendaylight-inventory:nodes":
 208                     state.state = "done"
 209                     state.command = ""
 210                 else:
 211                     state.state = "models"
 212                     state.command = ""
 213
 214     def parse_xml_data_file(self):
 215         state = self.State()
 216         with open(self.datafilepath, 'rb') as fp:
 217             iterparser = ET.iterparse(fp, events=("start", "end"))
 218             _, root = iterparser.next()
 219             for event, element in iterparser:
 220                 self.process_element(state, event, element)
 221                 element.clear()
 222                 # debugging code to stop after the named test case is processed
 223                 # if "s1-t1" in self.pdata:
 224                 #    break
 225             root.clear()
 226
 227     def write_pdata(self):
 228         for tindex, (testid, test) in enumerate(self.pdata.items()):
 229             tdir = self.outdir + "/" + testid + "_" + test["name"].replace(" ", "_")
 230             self.mkdir(tdir)
 231             for nindex, (nodeid, node) in enumerate(test['nodes'].items()):
 232                 ndir = tdir + "/" + nodeid
 233                 self.mkdir(ndir)
 234                 for cindex, (cmdid, cmd) in enumerate(node.items()):
 235                     filename = ndir + "/" + self.fix_command_names(cmdid) + ".txt"
 236                     with open(filename, 'w') as fp:
 237                         if cmd is not None:
 238                             fp.writelines(cmd)
 239             mdir = tdir + "/models"
 240             self.mkdir(mdir)
 241             for mindex, (model, mdata) in enumerate(test['models'].items()):
 242                 filename = mdir + "/" + self.fix_model_name(model) + ".json"
 243                 with open(filename, 'w') as fp:
 244                     if mdata is not None:
 245                         fp.writelines(mdata)
 246
 247     def write_debug_pdata(self):
 248         for tindex, (testid, test) in enumerate(self.pdata.items()):
 249             tdir = self.outdir + "/" + testid + "_" + test["name"].replace(" ", "_")
 250             for nindex, (nodeid, node) in enumerate(test['nodes'].items()):
 251                 ndir = tdir + "/" + nodeid
 252                 if RobotFiles.DUMP_FLOWS not in node:
 253                     continue
 254                 filename = ndir + "/" + self.fix_command_names(RobotFiles.DUMP_FLOWS)
 255                 logger.info("Processing: %s", filename)
 256                 filename = filename + ".f.txt"
 257                 dump_flows = node[RobotFiles.DUMP_FLOWS]
 258                 fls = flows.Flows(dump_flows)
 259                 fls.write_fdata(filename)
 260
 261     def fix_command_names(self, cmd):
 262         return cmd.replace(" ", "_")
 263
 264     def fix_model_name(self, model):
 265         name = model.replace("/", "___")
 266         name = name.replace(":", "__")
 267         return name
 268
 269
 270 def run(args):
 271     robotfile = RobotFiles(args.infile, args.outdir)
 272     robotfile.print_config()
 273     robotfile.mk_outdir()
 274     if args.gunzip:
 275         robotfile.gunzip()
 276     robotfile.print_config()
 277     robotfile.parse_xml_data_file()
 278     robotfile.write_pdata()
 279     if args.dump:
 280         robotfile.write_debug_pdata()