resources/tools/odltools/csit/robotfiles.py

   1 import logging
   2 import os
   3 import re
   4 import xml.etree.cElementTree as ET
   5 from subprocess import Popen
   6 from ovs import flows
   7
   8
   9 logger = logging.getLogger("csit.robotfiles")
  10
  11
  12 class RobotFiles:
  13     OUTDIR = "/tmp/robotjob"
  14     CHUNK_SIZE = 65536
  15     DUMP_FLOWS = "sudo ovs-ofctl dump-flows br-int -OOpenFlow13"
  16     TMP = "/tmp"
  17
  18     def __init__(self, infile, outdir):
  19         if outdir is None:
  20             self.outdir = RobotFiles.TMP
  21         else:
  22             self.outdir = outdir
  23         self.datafilepath = infile
  24         self.pdata = {}
  25         self.re_normalize_text = re.compile(r"( \n)|(\[A\[C.*)")
  26         self.re_uri = re.compile(r"uri=(?P<uri>.*),")
  27         self.re_model = re.compile(r"uri=(?P<uri>.*),")
  28
  29     def gunzip_output_file(self):
  30         infile = self.datafilepath
  31         basename = os.path.splitext(os.path.basename(self.datafilepath))[0]
  32         self.datafilepath = "{}/{}".format(self.outdir, basename)
  33         Popen("gunzip -cfk {} > {}".format(infile, self.datafilepath), shell=True).wait()
  34
  35     def mkdir(self, path):
  36         try:
  37             os.makedirs(path)
  38         except OSError:
  39             if not os.path.isdir(path):
  40                 raise
  41
  42     def mk_outdir(self):
  43         self.mkdir(self.outdir)
  44
  45     def read_chunks(self, fp):
  46         while True:
  47             data = fp.read(RobotFiles.CHUNK_SIZE)
  48             if not data:
  49                 break
  50             yield data
  51
  52     def parse_data_file(self):
  53         re_st = re.compile(r"dump-flows")
  54         cnt = 0
  55         with open(self.datafilepath, 'rb') as fp:
  56             for chunk in self.read_chunks(fp):
  57                 for m in re_st.finditer(chunk):
  58                     logger.info("%02d-%02d: %s", m.start(), m.end(), m.group(0))
  59                     cnt += 1
  60         logger.info("total matches: %d", cnt)
  61
  62     class State:
  63         def __init__(self):
  64             self.state = "init"
  65             self.pdata = {}
  66             self.test_id = None
  67             self.node = None
  68             self.command = None
  69             self.nodes = {}
  70             self.models = {}
  71
  72         def reset(self):
  73             self.state = "init"
  74             self.pdata = {}
  75             self.test_id = None
  76             self.node = None
  77             self.command = None
  78             self.nodes = {}
  79             self.models = {}
  80
  81     def normalize(self, intext):
  82         outtext = self.re_normalize_text.sub("", intext)
  83         return outtext
  84
  85     def print_config(self):
  86         logger.info("datafilepath: %s, outdir: %s", self.datafilepath, self.outdir)
  87
  88     # scan until test id= is seen. This indicates the start of a new test -> state=test
  89     # - scan until Get Test Teardown Debugs -> state=debugs
  90     #   - scan until Get DumpFlows And Ovsconfig -> state=nodes
  91     #   - scan until Get Model Dump -> state=models
  92     def process_element(self, state, event, element):
  93         tag = element.tag
  94         text = element.text
  95         attribs = element.attrib
  96         if logger.isEnabledFor(logging.DEBUG):
  97             logger.debug("%s - %s - %s - %s - %s - %s", state.state, state.command, event, tag, (text is not None), attribs)
  98         if event == "end":
  99             if element.tag == "test":
 100                 state.pdata['nodes'] = state.nodes
 101                 state.pdata['models'] = state.models
 102                 self.pdata[state.test_id] = state.pdata
 103                 state.reset()
 104                 return
 105             elif element.tag != "msg":
 106                 return
 107
 108         if event == "start" and state.state == "init":
 109             # <test id="s1-s1-s1-t1" name="Create VLAN Network (l2_network_1)">
 110             # <test id="s1-t1" name="Create VLAN Network net_1">
 111             if element.tag == "test":
 112                 state.test_id = element.get("id")
 113                 state.pdata["name"] = element.get("name")
 114                 state.state = "test"
 115         elif event == "start" and state.state == "test":
 116             # <kw type="teardown" name="Get Test Teardown Debugs" library="OpenStackOperations">
 117             if element.tag == "kw" and element.get("name") == "Get Test Teardown Debugs":
 118                 state.state = "debugs"
 119         elif event == "start" and state.state == "debugs":
 120             # <arg>Get DumpFlows And Ovsconfig</arg>
 121             if element.tag == "kw" and element.get("name") == "Get DumpFlows And Ovsconfig":
 122                 state.state = "nodes"
 123             # <arg>Get Model Dump</arg>
 124             if element.tag == "kw" and element.get("name") == "Get Model Dump":
 125                 state.state = "models"
 126         elif event == "start" and state.state == "nodes":
 127             # <arg>${OS_CONTROL_NODE_IP}</arg>
 128             if element.tag == "arg" and element.text is not None and "${OS_" in element.text:
 129                 state.node = element.text[element.text.find("{") + 1:element.text.find("}")]
 130                 state.nodes[state.node] = {}
 131                 state.state = "nodes2"
 132         elif event == "start" and state.state == "nodes2":
 133             # <kw name="Write Commands Until Expected Prompt" library="Utils">
 134             if element.tag == "kw" and element.get("name") == "Write Commands Until Expected Prompt":
 135                 state.state = "kw"
 136         elif event == "start" and state.state == "kw":
 137             # <arg>ip -o link</arg>
 138             if element.tag == "arg" and element.text is not None:
 139                 state.command = element.text
 140                 state.state = "command"
 141                 # only use the string before the ${...} since we don't know the ...
 142                 # <arg>sudo ip netns exec ${line} ip -o link</arg>
 143                 command_split = state.command.split("$")
 144                 if len(command_split) > 1:
 145                     state.command = command_split[0]
 146         elif event == "start" and state.state == "command":
 147             # <msg timestamp="20170414 07:31:21.769" level="INFO">ip -o link</msg>
 148             if element.tag == "msg" and element.text is not None:
 149                 text = self.normalize(element.text)
 150                 if text.find(state.command) != -1:
 151                     # <msg timestamp="20170414 07:31:34.425" level="INFO">sudo ip netns exec
 152                     # [jenkins@rele ^Mng-36618-350-devstack-newton-0 ~]&gt; ip -o link</msg>
 153                     if text.find("jenkins") != -1:
 154                         state.state = "nodes2"
 155                     else:
 156                         state.command = text
 157                         state.state = "msg"
 158         elif state.state == "msg":
 159             # <msg timestamp="20170414 07:31:21.786" level="INFO">
 160             if element.tag == "msg" and element.text is not None:
 161                 state.nodes[state.node][state.command] = element.text
 162                 # are we at the end of the debugs for the node?
 163                 # this command is the last one
 164                 if state.command == "sudo ovs-ofctl dump-group-stats br-int -OOpenFlow13":
 165                     state.state = "debugs"
 166                 else:
 167                     # still more debugs for this node
 168                     state.state = "nodes2"
 169         elif state.state == "models2":
 170             # <msg timestamp="20170813 08:20:11.806" level="INFO">Get Request using : alias=model_dump_session,
 171             # uri=restconf/config/interface-service-bindings:service-bindings, headers=None json=None</msg>
 172             if element.tag == "msg" and element.text is not None and element.text.find("uri") != -1:
 173                 uri = self.re_uri.search(element.text)
 174                 if uri is not None and "uri" in uri.group():
 175                     state.command = uri.group("uri")
 176                     state.state = "uri"
 177         elif event == "start" and state.state == "models":
 178             # <kw type="foritem" name="${model} = config/neutronvpn:router-interfaces-map">
 179             if element.tag == "kw" and "name" in element.attrib:
 180                 name_split = element.attrib["name"].split("${model} = ", 1)
 181                 model = None
 182                 if len(name_split) == 2:
 183                     model = name_split[1]
 184                 if model is not None:
 185                     state.command = model
 186                     state.state = "uri"
 187         elif state.state == "uri":
 188             if element.tag == "msg" and element.text is not None and element.text.find("pretty_output") != -1:
 189                 state.state = "dump"
 190         elif state.state == "dump":
 191             if element.tag == "msg" and element.text is not None:
 192                 state.models[state.command] = element.text
 193                 # if state.command == "restconf/operational/rendered-service-path:rendered-service-path":
 194                 if state.command == "restconf/operational/opendaylight-inventory:nodes":
 195                     state.state = "done"
 196                 else:
 197                     state.state = "models"
 198
 199     def parse_xml_data_file(self):
 200         state = self.State()
 201         with open(self.datafilepath, 'rb') as fp:
 202             iterparser = ET.iterparse(fp, events=("start", "end"))
 203             _, root = iterparser.next()
 204             for event, element in iterparser:
 205                 self.process_element(state, event, element)
 206                 element.clear()
 207                 # debugging code to stop after the named test case is processed
 208                 # if "s1-t2" in self.pdata:
 209                 #     break
 210             root.clear()
 211
 212     def write_pdata(self):
 213         for tindex, (testid, test) in enumerate(self.pdata.items()):
 214             tdir = self.outdir + "/" + testid + "_" + test["name"].replace(" ", "_")
 215             self.mkdir(tdir)
 216             for nindex, (nodeid, node) in enumerate(test['nodes'].items()):
 217                 ndir = tdir + "/" + nodeid
 218                 self.mkdir(ndir)
 219                 for cindex, (cmdid, cmd) in enumerate(node.items()):
 220                     filename = ndir + "/" + self.fix_command_names(cmdid) + ".txt"
 221                     with open(filename, 'w') as fp:
 222                         if cmd is not None:
 223                             fp.writelines(cmd)
 224             mdir = tdir + "/models"
 225             self.mkdir(mdir)
 226             for mindex, (model, mdata) in enumerate(test['models'].items()):
 227                 filename = mdir + "/" + self.fix_model_name(model) + ".txt"
 228                 with open(filename, 'w') as fp:
 229                     if mdata is not None:
 230                         fp.writelines(mdata)
 231
 232     def write_debug_pdata(self):
 233         for tindex, (testid, test) in enumerate(self.pdata.items()):
 234             tdir = self.outdir + "/" + testid + "_" + test["name"].replace(" ", "_")
 235             for nindex, (nodeid, node) in enumerate(test['nodes'].items()):
 236                 ndir = tdir + "/" + nodeid
 237                 if RobotFiles.DUMP_FLOWS not in node:
 238                     continue
 239                 filename = ndir + "/" + self.fix_command_names(RobotFiles.DUMP_FLOWS)
 240                 logger.info("Processing: %s", filename)
 241                 filename = filename + ".f.txt"
 242                 dump_flows = node[RobotFiles.DUMP_FLOWS]
 243                 fls = flows.Flows(dump_flows)
 244                 fls.write_fdata(filename)
 245
 246     def fix_command_names(self, cmd):
 247         return cmd.replace(" ", "_")
 248
 249     def fix_model_name(self, model):
 250         return model.replace("/", "_")
 251
 252
 253 def run(args):
 254     robotfile = RobotFiles(args.infile, args.outdir)
 255     robotfile.print_config()
 256     robotfile.mk_outdir()
 257     if args.gunzip:
 258         robotfile.gunzip_output_file()
 259     robotfile.print_config()
 260     robotfile.parse_xml_data_file()
 261     robotfile.write_pdata()
 262     if args.dump:
 263         robotfile.write_debug_pdata()
 264
 265
 266 def add_parser(subparsers):
 267     parser = subparsers.add_parser("csit")
 268     parser.add_argument("infile",
 269                         help="XML output from a Robot test, e.g. output_01_l2.xml.gz")
 270     parser.add_argument("outdir",
 271                         help="the directory that the parsed data is written into")
 272     parser.add_argument("-g", "--gunzip", action="store_true",
 273                         help="unzip the infile")
 274     parser.add_argument("-d", "--dump", action="store_true",
 275                         help="dump extra debugging, e.g. ovs metadata")
 276     parser.set_defaults(func=run)
 277