1#!/usr/bin/python
2#
3# Compares vmstate information stored in JSON format, obtained from
4# the -dump-vmstate QEMU command.
5#
6# Copyright 2014 Amit Shah <amit.shah@redhat.com>
7# Copyright 2014 Red Hat, Inc.
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation; either version 2 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, see <http://www.gnu.org/licenses/>.
21
22import argparse
23import json
24import sys
25
26# Count the number of errors found
27taint = 0
28
29def bump_taint():
30    global taint
31
32    # Ensure we don't wrap around or reset to 0 -- the shell only has
33    # an 8-bit return value.
34    if taint < 255:
35        taint = taint + 1
36
37
38def check_fields_match(name, s_field, d_field):
39    if s_field == d_field:
40        return True
41
42    # Some fields changed names between qemu versions.  This list
43    # is used to whitelist such changes in each section / description.
44    changed_names = {
45        'apic': ['timer', 'timer_expiry'],
46        'e1000': ['dev', 'parent_obj'],
47        'ehci': ['dev', 'pcidev'],
48        'I440FX': ['dev', 'parent_obj'],
49        'ich9_ahci': ['card', 'parent_obj'],
50        'ich9-ahci': ['ahci', 'ich9_ahci'],
51        'ioh3420': ['PCIDevice', 'PCIEDevice'],
52        'ioh-3240-express-root-port': ['port.br.dev',
53                                       'parent_obj.parent_obj.parent_obj',
54                                       'port.br.dev.exp.aer_log',
55                                'parent_obj.parent_obj.parent_obj.exp.aer_log'],
56        'cirrus_vga': ['hw_cursor_x', 'vga.hw_cursor_x',
57                       'hw_cursor_y', 'vga.hw_cursor_y'],
58        'lsiscsi': ['dev', 'parent_obj'],
59        'mch': ['d', 'parent_obj'],
60        'pci_bridge': ['bridge.dev', 'parent_obj', 'bridge.dev.shpc', 'shpc'],
61        'pcnet': ['pci_dev', 'parent_obj'],
62        'PIIX3': ['pci_irq_levels', 'pci_irq_levels_vmstate'],
63        'piix4_pm': ['dev', 'parent_obj', 'pci0_status',
64                     'acpi_pci_hotplug.acpi_pcihp_pci_status[0x0]',
65                     'pm1a.sts', 'ar.pm1.evt.sts', 'pm1a.en', 'ar.pm1.evt.en',
66                     'pm1_cnt.cnt', 'ar.pm1.cnt.cnt',
67                     'tmr.timer', 'ar.tmr.timer',
68                     'tmr.overflow_time', 'ar.tmr.overflow_time',
69                     'gpe', 'ar.gpe'],
70        'rtl8139': ['dev', 'parent_obj'],
71        'qxl': ['num_surfaces', 'ssd.num_surfaces'],
72        'usb-ccid': ['abProtocolDataStructure', 'abProtocolDataStructure.data'],
73        'usb-host': ['dev', 'parent_obj'],
74        'usb-mouse': ['usb-ptr-queue', 'HIDPointerEventQueue'],
75        'usb-tablet': ['usb-ptr-queue', 'HIDPointerEventQueue'],
76        'vmware_vga': ['card', 'parent_obj'],
77        'vmware_vga_internal': ['depth', 'new_depth'],
78        'xhci': ['pci_dev', 'parent_obj'],
79        'x3130-upstream': ['PCIDevice', 'PCIEDevice'],
80        'xio3130-express-downstream-port': ['port.br.dev',
81                                            'parent_obj.parent_obj.parent_obj',
82                                            'port.br.dev.exp.aer_log',
83                                'parent_obj.parent_obj.parent_obj.exp.aer_log'],
84        'xio3130-downstream': ['PCIDevice', 'PCIEDevice'],
85        'xio3130-express-upstream-port': ['br.dev', 'parent_obj.parent_obj',
86                                          'br.dev.exp.aer_log',
87                                          'parent_obj.parent_obj.exp.aer_log'],
88    }
89
90    if not name in changed_names:
91        return False
92
93    if s_field in changed_names[name] and d_field in changed_names[name]:
94        return True
95
96    return False
97
98def get_changed_sec_name(sec):
99    # Section names can change -- see commit 292b1634 for an example.
100    changes = {
101        "ICH9 LPC": "ICH9-LPC",
102        "e1000-82540em": "e1000",
103    }
104
105    for item in changes:
106        if item == sec:
107            return changes[item]
108        if changes[item] == sec:
109            return item
110    return ""
111
112def exists_in_substruct(fields, item):
113    # Some QEMU versions moved a few fields inside a substruct.  This
114    # kept the on-wire format the same.  This function checks if
115    # something got shifted inside a substruct.  For example, the
116    # change in commit 1f42d22233b4f3d1a2933ff30e8d6a6d9ee2d08f
117
118    if not "Description" in fields:
119        return False
120
121    if not "Fields" in fields["Description"]:
122        return False
123
124    substruct_fields = fields["Description"]["Fields"]
125
126    if substruct_fields == []:
127        return False
128
129    return check_fields_match(fields["Description"]["name"],
130                              substruct_fields[0]["field"], item)
131
132
133def check_fields(src_fields, dest_fields, desc, sec):
134    # This function checks for all the fields in a section.  If some
135    # fields got embedded into a substruct, this function will also
136    # attempt to check inside the substruct.
137
138    d_iter = iter(dest_fields)
139    s_iter = iter(src_fields)
140
141    # Using these lists as stacks to store previous value of s_iter
142    # and d_iter, so that when time comes to exit out of a substruct,
143    # we can go back one level up and continue from where we left off.
144
145    s_iter_list = []
146    d_iter_list = []
147
148    advance_src = True
149    advance_dest = True
150    unused_count = 0
151
152    while True:
153        if advance_src:
154            try:
155                s_item = s_iter.next()
156            except StopIteration:
157                if s_iter_list == []:
158                    break
159
160                s_iter = s_iter_list.pop()
161                continue
162        else:
163            if unused_count == 0:
164                # We want to avoid advancing just once -- when entering a
165                # dest substruct, or when exiting one.
166                advance_src = True
167
168        if advance_dest:
169            try:
170                d_item = d_iter.next()
171            except StopIteration:
172                if d_iter_list == []:
173                    # We were not in a substruct
174                    print "Section \"" + sec + "\",",
175                    print "Description " + "\"" + desc + "\":",
176                    print "expected field \"" + s_item["field"] + "\",",
177                    print "while dest has no further fields"
178                    bump_taint()
179                    break
180
181                d_iter = d_iter_list.pop()
182                advance_src = False
183                continue
184        else:
185            if unused_count == 0:
186                advance_dest = True
187
188        if unused_count != 0:
189            if advance_dest == False:
190                unused_count = unused_count - s_item["size"]
191                if unused_count == 0:
192                    advance_dest = True
193                    continue
194                if unused_count < 0:
195                    print "Section \"" + sec + "\",",
196                    print "Description \"" + desc + "\":",
197                    print "unused size mismatch near \"",
198                    print s_item["field"] + "\""
199                    bump_taint()
200                    break
201                continue
202
203            if advance_src == False:
204                unused_count = unused_count - d_item["size"]
205                if unused_count == 0:
206                    advance_src = True
207                    continue
208                if unused_count < 0:
209                    print "Section \"" + sec + "\",",
210                    print "Description \"" + desc + "\":",
211                    print "unused size mismatch near \"",
212                    print d_item["field"] + "\""
213                    bump_taint()
214                    break
215                continue
216
217        if not check_fields_match(desc, s_item["field"], d_item["field"]):
218            # Some fields were put in substructs, keeping the
219            # on-wire format the same, but breaking static tools
220            # like this one.
221
222            # First, check if dest has a new substruct.
223            if exists_in_substruct(d_item, s_item["field"]):
224                # listiterators don't have a prev() function, so we
225                # have to store our current location, descend into the
226                # substruct, and ensure we come out as if nothing
227                # happened when the substruct is over.
228                #
229                # Essentially we're opening the substructs that got
230                # added which didn't change the wire format.
231                d_iter_list.append(d_iter)
232                substruct_fields = d_item["Description"]["Fields"]
233                d_iter = iter(substruct_fields)
234                advance_src = False
235                continue
236
237            # Next, check if src has substruct that dest removed
238            # (can happen in backward migration: 2.0 -> 1.5)
239            if exists_in_substruct(s_item, d_item["field"]):
240                s_iter_list.append(s_iter)
241                substruct_fields = s_item["Description"]["Fields"]
242                s_iter = iter(substruct_fields)
243                advance_dest = False
244                continue
245
246            if s_item["field"] == "unused" or d_item["field"] == "unused":
247                if s_item["size"] == d_item["size"]:
248                    continue
249
250                if d_item["field"] == "unused":
251                    advance_dest = False
252                    unused_count = d_item["size"] - s_item["size"]
253                    continue
254
255                if s_item["field"] == "unused":
256                    advance_src = False
257                    unused_count = s_item["size"] - d_item["size"]
258                    continue
259
260            print "Section \"" + sec + "\",",
261            print "Description \"" + desc + "\":",
262            print "expected field \"" + s_item["field"] + "\",",
263            print "got \"" + d_item["field"] + "\"; skipping rest"
264            bump_taint()
265            break
266
267        check_version(s_item, d_item, sec, desc)
268
269        if not "Description" in s_item:
270            # Check size of this field only if it's not a VMSTRUCT entry
271            check_size(s_item, d_item, sec, desc, s_item["field"])
272
273        check_description_in_list(s_item, d_item, sec, desc)
274
275
276def check_subsections(src_sub, dest_sub, desc, sec):
277    for s_item in src_sub:
278        found = False
279        for d_item in dest_sub:
280            if s_item["name"] != d_item["name"]:
281                continue
282
283            found = True
284            check_descriptions(s_item, d_item, sec)
285
286        if not found:
287            print "Section \"" + sec + "\", Description \"" + desc + "\":",
288            print "Subsection \"" + s_item["name"] + "\" not found"
289            bump_taint()
290
291
292def check_description_in_list(s_item, d_item, sec, desc):
293    if not "Description" in s_item:
294        return
295
296    if not "Description" in d_item:
297        print "Section \"" + sec + "\", Description \"" + desc + "\",",
298        print "Field \"" + s_item["field"] + "\": missing description"
299        bump_taint()
300        return
301
302    check_descriptions(s_item["Description"], d_item["Description"], sec)
303
304
305def check_descriptions(src_desc, dest_desc, sec):
306    check_version(src_desc, dest_desc, sec, src_desc["name"])
307
308    if not check_fields_match(sec, src_desc["name"], dest_desc["name"]):
309        print "Section \"" + sec + "\":",
310        print "Description \"" + src_desc["name"] + "\"",
311        print "missing, got \"" + dest_desc["name"] + "\" instead; skipping"
312        bump_taint()
313        return
314
315    for f in src_desc:
316        if not f in dest_desc:
317            print "Section \"" + sec + "\"",
318            print "Description \"" + src_desc["name"] + "\":",
319            print "Entry \"" + f + "\" missing"
320            bump_taint()
321            continue
322
323        if f == 'Fields':
324            check_fields(src_desc[f], dest_desc[f], src_desc["name"], sec)
325
326        if f == 'Subsections':
327            check_subsections(src_desc[f], dest_desc[f], src_desc["name"], sec)
328
329
330def check_version(s, d, sec, desc=None):
331    if s["version_id"] > d["version_id"]:
332        print "Section \"" + sec + "\"",
333        if desc:
334            print "Description \"" + desc + "\":",
335        print "version error:", s["version_id"], ">", d["version_id"]
336        bump_taint()
337
338    if not "minimum_version_id" in d:
339        return
340
341    if s["version_id"] < d["minimum_version_id"]:
342        print "Section \"" + sec + "\"",
343        if desc:
344            print "Description \"" + desc + "\":",
345            print "minimum version error:", s["version_id"], "<",
346            print d["minimum_version_id"]
347            bump_taint()
348
349
350def check_size(s, d, sec, desc=None, field=None):
351    if s["size"] != d["size"]:
352        print "Section \"" + sec + "\"",
353        if desc:
354            print "Description \"" + desc + "\"",
355        if field:
356            print "Field \"" + field + "\"",
357        print "size mismatch:", s["size"], ",", d["size"]
358        bump_taint()
359
360
361def check_machine_type(s, d):
362    if s["Name"] != d["Name"]:
363        print "Warning: checking incompatible machine types:",
364        print "\"" + s["Name"] + "\", \"" + d["Name"] + "\""
365    return
366
367
368def main():
369    help_text = "Parse JSON-formatted vmstate dumps from QEMU in files SRC and DEST.  Checks whether migration from SRC to DEST QEMU versions would break based on the VMSTATE information contained within the JSON outputs.  The JSON output is created from a QEMU invocation with the -dump-vmstate parameter and a filename argument to it.  Other parameters to QEMU do not matter, except the -M (machine type) parameter."
370
371    parser = argparse.ArgumentParser(description=help_text)
372    parser.add_argument('-s', '--src', type=file, required=True,
373                        help='json dump from src qemu')
374    parser.add_argument('-d', '--dest', type=file, required=True,
375                        help='json dump from dest qemu')
376    parser.add_argument('--reverse', required=False, default=False,
377                        action='store_true',
378                        help='reverse the direction')
379    args = parser.parse_args()
380
381    src_data = json.load(args.src)
382    dest_data = json.load(args.dest)
383    args.src.close()
384    args.dest.close()
385
386    if args.reverse:
387        temp = src_data
388        src_data = dest_data
389        dest_data = temp
390
391    for sec in src_data:
392        dest_sec = sec
393        if not dest_sec in dest_data:
394            # Either the section name got changed, or the section
395            # doesn't exist in dest.
396            dest_sec = get_changed_sec_name(sec)
397            if not dest_sec in dest_data:
398                print "Section \"" + sec + "\" does not exist in dest"
399                bump_taint()
400                continue
401
402        s = src_data[sec]
403        d = dest_data[dest_sec]
404
405        if sec == "vmschkmachine":
406            check_machine_type(s, d)
407            continue
408
409        check_version(s, d, sec)
410
411        for entry in s:
412            if not entry in d:
413                print "Section \"" + sec + "\": Entry \"" + entry + "\"",
414                print "missing"
415                bump_taint()
416                continue
417
418            if entry == "Description":
419                check_descriptions(s[entry], d[entry], sec)
420
421    return taint
422
423
424if __name__ == '__main__':
425    sys.exit(main())
426