5 from jsonpathl import jsonpath
11 Library for checking differences between json files,
12 allowing pre-filtering those files using a number
13 of jsonpath expressions
14 This library supports the automated verification
15 of backup & restore use cases for applications
19 __author__ = "Diego Granados"
20 __copyright__ = "Copyright(c) 2017, Ericsson."
21 __license__ = "New-style BSD"
22 __email__ = "diego.jesus.granados.lopez@ericsson.com"
25 def from_path_to_jsonpatch(matchedpath):
26 """ Given a json path (using jsonpath notation), to a json patch (RFC 6902)
27 which can be used to remove the document fragment pointed by the input path
28 Note that such conversion is not formally specified anywhere, so the conversion
29 rules are experimentation-based
31 :param matchedpath: the input path (using jsonpath notation, see http://goessner.net/articles/JsonPath)
32 :return: the corresponding json patch for removing the fragment
35 logging.info('starting. filter path: %s', matchedpath)
37 # First step: path format change
38 # typical input: $['ietf-yang-library:modules-state']['module'][57]
39 # desired output: /ietf-yang-library:modules-state/module/57
41 matchedpath = matchedpath.replace('$.', '/')
42 matchedpath = matchedpath.replace('$[\'', '/')
43 matchedpath = matchedpath.replace('\'][\'', '/')
44 matchedpath = matchedpath.replace('\']', '/')
46 # this one is for the $[2] pattern
47 if '$[' in matchedpath and ']' in matchedpath:
48 matchedpath = matchedpath.replace('$[', '/')
49 matchedpath = matchedpath.replace(']', '')
51 matchedpath = matchedpath.replace('[', '')
52 matchedpath = matchedpath.replace(']', '')
53 matchedpath = matchedpath.rstrip('/')
55 # Now, for input: /ietf-yang-library:modules-state/module/57
56 # desired output: [{"op":"remove","path":"/ietf-yang-library:modules-state/module/57"}]
58 logging.info('final filter path: %s', matchedpath)
59 as_patch = '[{{"op\":\"remove\",\"path\":\"{0}\"}}]'.format(matchedpath)
60 logging.info('generated patch line: %s', as_patch)
64 def apply_filter(json_arg, filtering_line):
65 """ Filters a json document by removing the elements identified by a filtering pattern
67 :param json_arg: the document to filter
68 :param filtering_line: The filtering pattern. This is specified using jsonpath notation grammar
69 (see http://goessner.net/articles/JsonPath/)
70 :return: the filtered document
73 logging.info('apply_filter:starting. jsonPath filter=[%s]', filtering_line)
75 res = jsonpath(json_arg, filtering_line, result_type='PATH')
76 if isinstance(res, types.BooleanType) or len(res) == 0:
77 logging.info('apply_filter: The prefilter [%s] matched nothing', filtering_line)
80 raise AssertionError('Bad pre-filter [%s] (returned [%d] entries, should return one at most',
81 filtering_line, len(res))
82 as_json_patch = from_path_to_jsonpatch(res[0])
83 logging.info('apply_filter: applying patch! resolved patch =%s', as_json_patch)
84 patched_json = jsonpatch.apply_patch(json_arg, as_json_patch)
86 logging.info('apply_filter: json after patching: %s', patched_json)
90 def prefilter(json_arg, initial_prefilter):
91 """ Performs the prefiltering of a json file
92 :param json_arg: the json document to filter (as string)
94 :param initial_prefilter: a file containing a number of filtering patterns (using jsonpath notation)
95 :return: the original document, python-deserialized and having the fragments
96 matched by the filtering patterns removed
99 if not initial_prefilter:
100 logging.info('prefilter not found!')
101 # whether it is filtered or not, return as json so it can be handled uniformly from now on
102 return json.loads(json_arg)
104 with open(initial_prefilter) as f:
105 lines = f.read().splitlines()
106 logging.info('prefilter:lines in prefilter file: %d ', len(lines))
107 lines = filter(lambda k: not k.startswith('#'), lines)
108 logging.info('prefilter:lines after removing comments: %d ', len(lines))
109 json_args_as_json = json.loads(json_arg)
110 for filtering_line in lines:
111 json_args_as_json = apply_filter(json_args_as_json, filtering_line)
113 return json_args_as_json
116 def prefilter_json_files_then_compare(args):
117 """ Main function. Prefilters the input files using provided prefiltering patterns,
118 then returns number of differences (and the differences themselves, when requested)
120 :param args: Input arguments, already parsed
121 :return: the number of differences (from a jsonpatch standpoint) between the input
122 json files (those input files can be prefiltered using a number of patterns when
126 logging.info('prefilter_json_files_then_compare: starting!')
127 with open(args.initialFile) as f:
128 json_initial = file.read(f)
129 with open(args.finalFile) as f2:
130 json_final = file.read(f2)
132 patch = jsonpatch.JsonPatch.from_diff(json_initial, json_final)
133 logging.info('prefilter_json_files_then_compare:differences before patching: %d', len(list(patch)))
135 json_initial_filtered = prefilter(json_initial, args.initial_prefilter)
136 json_final_filtered = prefilter(json_final, args.finalPreFilter)
138 patch_after_filtering = jsonpatch.JsonPatch.from_diff(json_initial_filtered, json_final_filtered)
139 differences_after_patching = list(patch_after_filtering)
140 logging.info('prefilter_json_files_then_compare: differences after patching: %d', len(differences_after_patching))
142 if args.printDifferences:
143 for patchline in differences_after_patching:
144 print json.dumps(patchline)
146 print len(differences_after_patching)
147 return len(differences_after_patching)
150 def Json_Diff_Check_Keyword(json_before, json_after, filter_before, filter_after):
151 input_argv = ['-i', json_before, '-f', json_after, '-ipf', filter_before, '-fpf', filter_after, '-pd']
152 sys.argv[1:] = input_argv
153 logging.info('starting. constructed command line: %s', sys.argv)
154 return Json_Diff_Check()
157 def parse_args(args):
158 parser = argparse.ArgumentParser(description='both initial and final json files are compared for differences. '
159 'The program returns 0 when the json contents are the same, or the '
161 ' differences otherwise. Both json files can be prefiltered for '
163 ' before checking the differences')
165 parser.add_argument('-i', '--initialFile', required='true', dest='initialFile', action='store',
166 help='initial json file')
167 parser.add_argument('-f', '--finalFile', required='true', dest='finalFile', action='store', help='final json file')
168 parser.add_argument('-ipf', '--initial_prefilter', dest='initial_prefilter',
169 help='File with pre-filtering patterns to apply to the initial json file before comparing')
170 parser.add_argument('-fpf', '--finalPreFilter', dest='finalPreFilter',
171 help='File with pre-filtering patterns to apply to the final json file before comparing')
172 parser.add_argument('-pd', '--printDifferences', action='store_true',
173 help='on differences found, prints the list of paths for the found differences before exitting')
174 parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='generate log information')
175 return parser.parse_args(args)
178 def Json_Diff_Check():
179 args = parse_args(sys.argv[1:])
181 if hasattr(args, 'verbose'):
183 logging.basicConfig(level=logging.DEBUG)
185 if args.printDifferences:
186 logging.info('(will print differences)')
188 result = prefilter_json_files_then_compare(args)
192 if __name__ == '__main__':