5 from jsonpathl import jsonpath
11 Library for checking differences between json files,
12 allowing pre-filtering those files using a number
13 of jsonpath expressions
14 This library supports the automated verification
15 of backup & restore use cases for applications
19 __author__ = "Diego Granados"
20 __copyright__ = "Copyright(c) 2017, Ericsson."
21 __license__ = "New-style BSD"
22 __email__ = "diego.jesus.granados.lopez@ericsson.com"
25 def from_path_to_jsonpatch(matchedpath):
26 """Given a json path (using jsonpath notation), to a json patch (RFC 6902)
27 which can be used to remove the document fragment pointed by the input path
28 Note that such conversion is not formally specified anywhere, so the conversion
29 rules are experimentation-based
31 :param matchedpath: the input path (using jsonpath notation, see http://goessner.net/articles/JsonPath)
32 :return: the corresponding json patch for removing the fragment
35 logging.info("starting. filter path: %s", matchedpath)
37 # First step: path format change
38 # typical input: $['ietf-yang-library:modules-state']['module'][57]
39 # desired output: /ietf-yang-library:modules-state/module/57
41 matchedpath = matchedpath.replace("$.", "/")
42 matchedpath = matchedpath.replace("$['", "/")
43 matchedpath = matchedpath.replace("']['", "/")
44 matchedpath = matchedpath.replace("']", "/")
46 # this one is for the $[2] pattern
47 if "$[" in matchedpath and "]" in matchedpath:
48 matchedpath = matchedpath.replace("$[", "/")
49 matchedpath = matchedpath.replace("]", "")
51 matchedpath = matchedpath.replace("[", "")
52 matchedpath = matchedpath.replace("]", "")
53 matchedpath = matchedpath.rstrip("/")
55 # Now, for input: /ietf-yang-library:modules-state/module/57
56 # desired output: [{"op":"remove","path":"/ietf-yang-library:modules-state/module/57"}]
58 logging.info("final filter path: %s", matchedpath)
59 as_patch = '[{{"op":"remove","path":"{0}"}}]'.format(matchedpath)
60 logging.info("generated patch line: %s", as_patch)
64 def apply_filter(json_arg, filtering_line):
65 """Filters a json document by removing the elements identified by a filtering pattern
67 :param json_arg: the document to filter
68 :param filtering_line: The filtering pattern. This is specified using jsonpath notation grammar
69 (see http://goessner.net/articles/JsonPath/)
70 :return: the filtered document
73 logging.info("apply_filter:starting. jsonPath filter=[%s]", filtering_line)
75 res = jsonpath(json_arg, filtering_line, result_type="PATH")
76 if isinstance(res, types.BooleanType) or len(res) == 0:
77 logging.info("apply_filter: The prefilter [%s] matched nothing", filtering_line)
81 "Bad pre-filter [%s] (returned [%d] entries, should return one at most",
85 as_json_patch = from_path_to_jsonpatch(res[0])
86 logging.info("apply_filter: applying patch! resolved patch =%s", as_json_patch)
87 patched_json = jsonpatch.apply_patch(json_arg, as_json_patch)
89 logging.info("apply_filter: json after patching: %s", patched_json)
93 def prefilter(json_arg, initial_prefilter):
94 """Performs the prefiltering of a json file
95 :param json_arg: the json document to filter (as string)
97 :param initial_prefilter: a file containing a number of filtering patterns (using jsonpath notation)
98 :return: the original document, python-deserialized and having the fragments
99 matched by the filtering patterns removed
102 if not initial_prefilter:
103 logging.info("prefilter not found!")
104 # whether it is filtered or not, return as json so it can be handled uniformly from now on
105 return json.loads(json_arg)
107 with open(initial_prefilter) as f:
108 lines = f.read().splitlines()
109 logging.info("prefilter:lines in prefilter file: %d ", len(lines))
110 lines = filter(lambda k: not k.startswith("#"), lines)
111 logging.info("prefilter:lines after removing comments: %d ", len(lines))
112 json_args_as_json = json.loads(json_arg)
113 for filtering_line in lines:
114 json_args_as_json = apply_filter(json_args_as_json, filtering_line)
116 return json_args_as_json
119 def prefilter_json_files_then_compare(args):
120 """Main function. Prefilters the input files using provided prefiltering patterns,
121 then returns number of differences (and the differences themselves, when requested)
123 :param args: Input arguments, already parsed
124 :return: the number of differences (from a jsonpatch standpoint) between the input
125 json files (those input files can be prefiltered using a number of patterns when
129 logging.info("prefilter_json_files_then_compare: starting!")
130 with open(args.initialFile) as f:
131 json_initial = file.read(f)
132 with open(args.finalFile) as f2:
133 json_final = file.read(f2)
135 patch = jsonpatch.JsonPatch.from_diff(json_initial, json_final)
137 "prefilter_json_files_then_compare:differences before patching: %d",
141 json_initial_filtered = prefilter(json_initial, args.initial_prefilter)
142 json_final_filtered = prefilter(json_final, args.finalPreFilter)
144 patch_after_filtering = jsonpatch.JsonPatch.from_diff(
145 json_initial_filtered, json_final_filtered
147 differences_after_patching = list(patch_after_filtering)
149 "prefilter_json_files_then_compare: differences after patching: %d",
150 len(differences_after_patching),
153 if args.printDifferences:
154 for patchline in differences_after_patching:
155 print(json.dumps(patchline))
157 print(len(differences_after_patching))
158 return len(differences_after_patching)
161 def Json_Diff_Check_Keyword(json_before, json_after, filter_before, filter_after):
173 sys.argv[1:] = input_argv
174 logging.info("starting. constructed command line: %s", sys.argv)
175 return Json_Diff_Check()
178 def parse_args(args):
179 parser = argparse.ArgumentParser(
180 description="both initial and final json files are compared for differences. "
181 "The program returns 0 when the json contents are the same, or the "
183 " differences otherwise. Both json files can be prefiltered for "
185 " before checking the differences"
194 help="initial json file",
202 help="final json file",
206 "--initial_prefilter",
207 dest="initial_prefilter",
208 help="File with pre-filtering patterns to apply to the initial json file before comparing",
213 dest="finalPreFilter",
214 help="File with pre-filtering patterns to apply to the final json file before comparing",
218 "--printDifferences",
220 help="on differences found, prints the list of paths for the found differences before exitting",
227 help="generate log information",
229 return parser.parse_args(args)
232 def Json_Diff_Check():
233 args = parse_args(sys.argv[1:])
235 if hasattr(args, "verbose"):
237 logging.basicConfig(level=logging.DEBUG)
239 if args.printDifferences:
240 logging.info("(will print differences)")
242 result = prefilter_json_files_then_compare(args)
246 if __name__ == "__main__":