1#!/usr/bin/python
2#
3# Compares vmstate information stored in JSON format, obtained from
4# the -dump-vmstate QEMU command.
5#
6# Copyright 2014 Amit Shah <amit.shah@redhat.com>
7# Copyright 2014 Red Hat, Inc.
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation; either version 2 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, see <http://www.gnu.org/licenses/>.
21
22import argparse
23import json
24import sys
25
26# Count the number of errors found
27taint = 0
28
29def bump_taint():
30    global taint
31
32    # Ensure we don't wrap around or reset to 0 -- the shell only has
33    # an 8-bit return value.
34    if taint < 255:
35        taint = taint + 1
36
37
38def check_fields_match(name, s_field, d_field):
39    if s_field == d_field:
40        return True
41
42    # Some fields changed names between qemu versions.  This list
43    # is used to whitelist such changes in each section / description.
44    changed_names = {
45        'apic': ['timer', 'timer_expiry'],
46        'e1000': ['dev', 'parent_obj'],
47        'ehci': ['dev', 'pcidev'],
48        'I440FX': ['dev', 'parent_obj'],
49        'ich9_ahci': ['card', 'parent_obj'],
50        'ich9-ahci': ['ahci', 'ich9_ahci'],
51        'ioh3420': ['PCIDevice', 'PCIEDevice'],
52        'ioh-3240-express-root-port': ['port.br.dev',
53                                       'parent_obj.parent_obj.parent_obj',
54                                       'port.br.dev.exp.aer_log',
55                                'parent_obj.parent_obj.parent_obj.exp.aer_log'],
56        'cirrus_vga': ['hw_cursor_x', 'vga.hw_cursor_x',
57                       'hw_cursor_y', 'vga.hw_cursor_y'],
58        'lsiscsi': ['dev', 'parent_obj'],
59        'mch': ['d', 'parent_obj'],
60        'pci_bridge': ['bridge.dev', 'parent_obj', 'bridge.dev.shpc', 'shpc'],
61        'pcnet': ['pci_dev', 'parent_obj'],
62        'PIIX3': ['pci_irq_levels', 'pci_irq_levels_vmstate'],
63        'piix4_pm': ['dev', 'parent_obj', 'pci0_status',
64                     'acpi_pci_hotplug.acpi_pcihp_pci_status[0x0]',
65                     'pm1a.sts', 'ar.pm1.evt.sts', 'pm1a.en', 'ar.pm1.evt.en',
66                     'pm1_cnt.cnt', 'ar.pm1.cnt.cnt',
67                     'tmr.timer', 'ar.tmr.timer',
68                     'tmr.overflow_time', 'ar.tmr.overflow_time',
69                     'gpe', 'ar.gpe'],
70        'rtl8139': ['dev', 'parent_obj'],
71        'qxl': ['num_surfaces', 'ssd.num_surfaces'],
72        'usb-ccid': ['abProtocolDataStructure', 'abProtocolDataStructure.data'],
73        'usb-host': ['dev', 'parent_obj'],
74        'usb-mouse': ['usb-ptr-queue', 'HIDPointerEventQueue'],
75        'usb-tablet': ['usb-ptr-queue', 'HIDPointerEventQueue'],
76        'vmware_vga': ['card', 'parent_obj'],
77        'vmware_vga_internal': ['depth', 'new_depth'],
78        'xhci': ['pci_dev', 'parent_obj'],
79        'x3130-upstream': ['PCIDevice', 'PCIEDevice'],
80        'xio3130-express-downstream-port': ['port.br.dev',
81                                            'parent_obj.parent_obj.parent_obj',
82                                            'port.br.dev.exp.aer_log',
83                                'parent_obj.parent_obj.parent_obj.exp.aer_log'],
84        'xio3130-downstream': ['PCIDevice', 'PCIEDevice'],
85        'xio3130-express-upstream-port': ['br.dev', 'parent_obj.parent_obj',
86                                          'br.dev.exp.aer_log',
87                                          'parent_obj.parent_obj.exp.aer_log'],
88        'spapr_pci': ['dma_liobn[0]', 'mig_liobn',
89                      'mem_win_addr', 'mig_mem_win_addr',
90                      'mem_win_size', 'mig_mem_win_size',
91                      'io_win_addr', 'mig_io_win_addr',
92                      'io_win_size', 'mig_io_win_size'],
93    }
94
95    if not name in changed_names:
96        return False
97
98    if s_field in changed_names[name] and d_field in changed_names[name]:
99        return True
100
101    return False
102
103def get_changed_sec_name(sec):
104    # Section names can change -- see commit 292b1634 for an example.
105    changes = {
106        "ICH9 LPC": "ICH9-LPC",
107        "e1000-82540em": "e1000",
108    }
109
110    for item in changes:
111        if item == sec:
112            return changes[item]
113        if changes[item] == sec:
114            return item
115    return ""
116
117def exists_in_substruct(fields, item):
118    # Some QEMU versions moved a few fields inside a substruct.  This
119    # kept the on-wire format the same.  This function checks if
120    # something got shifted inside a substruct.  For example, the
121    # change in commit 1f42d22233b4f3d1a2933ff30e8d6a6d9ee2d08f
122
123    if not "Description" in fields:
124        return False
125
126    if not "Fields" in fields["Description"]:
127        return False
128
129    substruct_fields = fields["Description"]["Fields"]
130
131    if substruct_fields == []:
132        return False
133
134    return check_fields_match(fields["Description"]["name"],
135                              substruct_fields[0]["field"], item)
136
137
138def check_fields(src_fields, dest_fields, desc, sec):
139    # This function checks for all the fields in a section.  If some
140    # fields got embedded into a substruct, this function will also
141    # attempt to check inside the substruct.
142
143    d_iter = iter(dest_fields)
144    s_iter = iter(src_fields)
145
146    # Using these lists as stacks to store previous value of s_iter
147    # and d_iter, so that when time comes to exit out of a substruct,
148    # we can go back one level up and continue from where we left off.
149
150    s_iter_list = []
151    d_iter_list = []
152
153    advance_src = True
154    advance_dest = True
155    unused_count = 0
156
157    while True:
158        if advance_src:
159            try:
160                s_item = s_iter.next()
161            except StopIteration:
162                if s_iter_list == []:
163                    break
164
165                s_iter = s_iter_list.pop()
166                continue
167        else:
168            if unused_count == 0:
169                # We want to avoid advancing just once -- when entering a
170                # dest substruct, or when exiting one.
171                advance_src = True
172
173        if advance_dest:
174            try:
175                d_item = d_iter.next()
176            except StopIteration:
177                if d_iter_list == []:
178                    # We were not in a substruct
179                    print "Section \"" + sec + "\",",
180                    print "Description " + "\"" + desc + "\":",
181                    print "expected field \"" + s_item["field"] + "\",",
182                    print "while dest has no further fields"
183                    bump_taint()
184                    break
185
186                d_iter = d_iter_list.pop()
187                advance_src = False
188                continue
189        else:
190            if unused_count == 0:
191                advance_dest = True
192
193        if unused_count != 0:
194            if advance_dest == False:
195                unused_count = unused_count - s_item["size"]
196                if unused_count == 0:
197                    advance_dest = True
198                    continue
199                if unused_count < 0:
200                    print "Section \"" + sec + "\",",
201                    print "Description \"" + desc + "\":",
202                    print "unused size mismatch near \"",
203                    print s_item["field"] + "\""
204                    bump_taint()
205                    break
206                continue
207
208            if advance_src == False:
209                unused_count = unused_count - d_item["size"]
210                if unused_count == 0:
211                    advance_src = True
212                    continue
213                if unused_count < 0:
214                    print "Section \"" + sec + "\",",
215                    print "Description \"" + desc + "\":",
216                    print "unused size mismatch near \"",
217                    print d_item["field"] + "\""
218                    bump_taint()
219                    break
220                continue
221
222        if not check_fields_match(desc, s_item["field"], d_item["field"]):
223            # Some fields were put in substructs, keeping the
224            # on-wire format the same, but breaking static tools
225            # like this one.
226
227            # First, check if dest has a new substruct.
228            if exists_in_substruct(d_item, s_item["field"]):
229                # listiterators don't have a prev() function, so we
230                # have to store our current location, descend into the
231                # substruct, and ensure we come out as if nothing
232                # happened when the substruct is over.
233                #
234                # Essentially we're opening the substructs that got
235                # added which didn't change the wire format.
236                d_iter_list.append(d_iter)
237                substruct_fields = d_item["Description"]["Fields"]
238                d_iter = iter(substruct_fields)
239                advance_src = False
240                continue
241
242            # Next, check if src has substruct that dest removed
243            # (can happen in backward migration: 2.0 -> 1.5)
244            if exists_in_substruct(s_item, d_item["field"]):
245                s_iter_list.append(s_iter)
246                substruct_fields = s_item["Description"]["Fields"]
247                s_iter = iter(substruct_fields)
248                advance_dest = False
249                continue
250
251            if s_item["field"] == "unused" or d_item["field"] == "unused":
252                if s_item["size"] == d_item["size"]:
253                    continue
254
255                if d_item["field"] == "unused":
256                    advance_dest = False
257                    unused_count = d_item["size"] - s_item["size"]
258                    continue
259
260                if s_item["field"] == "unused":
261                    advance_src = False
262                    unused_count = s_item["size"] - d_item["size"]
263                    continue
264
265            print "Section \"" + sec + "\",",
266            print "Description \"" + desc + "\":",
267            print "expected field \"" + s_item["field"] + "\",",
268            print "got \"" + d_item["field"] + "\"; skipping rest"
269            bump_taint()
270            break
271
272        check_version(s_item, d_item, sec, desc)
273
274        if not "Description" in s_item:
275            # Check size of this field only if it's not a VMSTRUCT entry
276            check_size(s_item, d_item, sec, desc, s_item["field"])
277
278        check_description_in_list(s_item, d_item, sec, desc)
279
280
281def check_subsections(src_sub, dest_sub, desc, sec):
282    for s_item in src_sub:
283        found = False
284        for d_item in dest_sub:
285            if s_item["name"] != d_item["name"]:
286                continue
287
288            found = True
289            check_descriptions(s_item, d_item, sec)
290
291        if not found:
292            print "Section \"" + sec + "\", Description \"" + desc + "\":",
293            print "Subsection \"" + s_item["name"] + "\" not found"
294            bump_taint()
295
296
297def check_description_in_list(s_item, d_item, sec, desc):
298    if not "Description" in s_item:
299        return
300
301    if not "Description" in d_item:
302        print "Section \"" + sec + "\", Description \"" + desc + "\",",
303        print "Field \"" + s_item["field"] + "\": missing description"
304        bump_taint()
305        return
306
307    check_descriptions(s_item["Description"], d_item["Description"], sec)
308
309
310def check_descriptions(src_desc, dest_desc, sec):
311    check_version(src_desc, dest_desc, sec, src_desc["name"])
312
313    if not check_fields_match(sec, src_desc["name"], dest_desc["name"]):
314        print "Section \"" + sec + "\":",
315        print "Description \"" + src_desc["name"] + "\"",
316        print "missing, got \"" + dest_desc["name"] + "\" instead; skipping"
317        bump_taint()
318        return
319
320    for f in src_desc:
321        if not f in dest_desc:
322            print "Section \"" + sec + "\"",
323            print "Description \"" + src_desc["name"] + "\":",
324            print "Entry \"" + f + "\" missing"
325            bump_taint()
326            continue
327
328        if f == 'Fields':
329            check_fields(src_desc[f], dest_desc[f], src_desc["name"], sec)
330
331        if f == 'Subsections':
332            check_subsections(src_desc[f], dest_desc[f], src_desc["name"], sec)
333
334
335def check_version(s, d, sec, desc=None):
336    if s["version_id"] > d["version_id"]:
337        print "Section \"" + sec + "\"",
338        if desc:
339            print "Description \"" + desc + "\":",
340        print "version error:", s["version_id"], ">", d["version_id"]
341        bump_taint()
342
343    if not "minimum_version_id" in d:
344        return
345
346    if s["version_id"] < d["minimum_version_id"]:
347        print "Section \"" + sec + "\"",
348        if desc:
349            print "Description \"" + desc + "\":",
350            print "minimum version error:", s["version_id"], "<",
351            print d["minimum_version_id"]
352            bump_taint()
353
354
355def check_size(s, d, sec, desc=None, field=None):
356    if s["size"] != d["size"]:
357        print "Section \"" + sec + "\"",
358        if desc:
359            print "Description \"" + desc + "\"",
360        if field:
361            print "Field \"" + field + "\"",
362        print "size mismatch:", s["size"], ",", d["size"]
363        bump_taint()
364
365
366def check_machine_type(s, d):
367    if s["Name"] != d["Name"]:
368        print "Warning: checking incompatible machine types:",
369        print "\"" + s["Name"] + "\", \"" + d["Name"] + "\""
370    return
371
372
373def main():
374    help_text = "Parse JSON-formatted vmstate dumps from QEMU in files SRC and DEST.  Checks whether migration from SRC to DEST QEMU versions would break based on the VMSTATE information contained within the JSON outputs.  The JSON output is created from a QEMU invocation with the -dump-vmstate parameter and a filename argument to it.  Other parameters to QEMU do not matter, except the -M (machine type) parameter."
375
376    parser = argparse.ArgumentParser(description=help_text)
377    parser.add_argument('-s', '--src', type=file, required=True,
378                        help='json dump from src qemu')
379    parser.add_argument('-d', '--dest', type=file, required=True,
380                        help='json dump from dest qemu')
381    parser.add_argument('--reverse', required=False, default=False,
382                        action='store_true',
383                        help='reverse the direction')
384    args = parser.parse_args()
385
386    src_data = json.load(args.src)
387    dest_data = json.load(args.dest)
388    args.src.close()
389    args.dest.close()
390
391    if args.reverse:
392        temp = src_data
393        src_data = dest_data
394        dest_data = temp
395
396    for sec in src_data:
397        dest_sec = sec
398        if not dest_sec in dest_data:
399            # Either the section name got changed, or the section
400            # doesn't exist in dest.
401            dest_sec = get_changed_sec_name(sec)
402            if not dest_sec in dest_data:
403                print "Section \"" + sec + "\" does not exist in dest"
404                bump_taint()
405                continue
406
407        s = src_data[sec]
408        d = dest_data[dest_sec]
409
410        if sec == "vmschkmachine":
411            check_machine_type(s, d)
412            continue
413
414        check_version(s, d, sec)
415
416        for entry in s:
417            if not entry in d:
418                print "Section \"" + sec + "\": Entry \"" + entry + "\"",
419                print "missing"
420                bump_taint()
421                continue
422
423            if entry == "Description":
424                check_descriptions(s[entry], d[entry], sec)
425
426    return taint
427
428
429if __name__ == '__main__':
430    sys.exit(main())
431