1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import cPickle as pickle
13
14from robot.utils import DotDict
15from robot.libraries.BuiltIn import BuiltIn
16
17from boot_data import *
18import gen_print as gp
19import gen_robot_print as grp
20import gen_robot_plug_in as grpi
21import gen_robot_valid as grv
22import gen_misc as gm
23import gen_cmd as gc
24import state as st
25
26base_path = os.path.dirname(os.path.dirname(
27                            imp.find_module("gen_robot_print")[1])) +\
28    os.sep
29sys.path.append(base_path + "extended/")
30import run_keyword as rk
31
32# Program parameter processing.
33# Assign all program parms to python variables which are global to this module.
34parm_list = BuiltIn().get_variable_value("${parm_list}")
35int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'quiet', 'test_mode',
36            'debug']
37for parm in parm_list:
38    if parm in int_list:
39        sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
40                  "}\", \"0\"))"
41    else:
42        sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
43    cmd_buf = parm + " = " + sub_cmd
44    exec(cmd_buf)
45
46if ffdc_dir_path_style == "":
47    ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
48
49# Set up boot data structures.
50boot_table = create_boot_table()
51valid_boot_types = create_valid_boot_list(boot_table)
52
53# Setting master_pid correctly influences the behavior of plug-ins like
54# DB_Logging
55program_pid = os.getpid()
56master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
57
58boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
59                         str(master_pid) + ":boot_results"
60if os.path.isfile(boot_results_file_path):
61    # We've been called before in this run so we'll load the saved
62    # boot_results object.
63    boot_results = pickle.load(open(boot_results_file_path, 'rb'))
64else:
65    boot_results = boot_results(boot_table, boot_pass, boot_fail)
66
67boot_lists = read_boot_lists()
68last_ten = []
69# Convert these program parms to more useable lists.
70boot_list = filter(None, boot_list.split(":"))
71boot_stack = filter(None, boot_stack.split(":"))
72
73state = st.return_default_state()
74cp_setup_called = 0
75next_boot = ""
76base_tool_dir_path = os.path.normpath(os.environ.get(
77    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
78ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
79ffdc_list_file_path = base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST"
80boot_success = 0
81status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
82if status_dir_path != "":
83    status_dir_path = os.path.normpath(status_dir_path) + os.sep
84default_power_on = "REST Power On"
85default_power_off = "REST Power Off"
86boot_count = 0
87
88LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
89
90
91###############################################################################
92def initial_plug_in_setup():
93
94    r"""
95    Initialize all plug-in environment variables which do not change for the
96    duration of the program.
97
98    """
99
100    global LOG_LEVEL
101    BuiltIn().set_log_level("NONE")
102
103    BuiltIn().set_global_variable("${master_pid}", master_pid)
104    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
105    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
106    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
107    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
108                                  ffdc_list_file_path)
109
110    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
111                                  ffdc_dir_path_style)
112    BuiltIn().set_global_variable("${FFDC_CHECK}",
113                                  ffdc_check)
114
115    # For each program parameter, set the corresponding AUTOBOOT_ environment
116    # variable value.  Also, set an AUTOBOOT_ environment variable for every
117    # element in additional_values.
118    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
119                         "status_dir_path", "base_tool_dir_path",
120                         "ffdc_list_file_path"]
121
122    plug_in_vars = parm_list + additional_values
123
124    for var_name in plug_in_vars:
125        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
126        var_name = var_name.upper()
127        if var_value is None:
128            var_value = ""
129        os.environ["AUTOBOOT_" + var_name] = str(var_value)
130
131    BuiltIn().set_log_level(LOG_LEVEL)
132
133
134###############################################################################
135
136
137###############################################################################
138def plug_in_setup():
139
140    r"""
141    Initialize all changing plug-in environment variables for use by the
142    plug-in programs.
143    """
144
145    global LOG_LEVEL
146    global test_really_running
147
148    BuiltIn().set_log_level("NONE")
149
150    boot_pass, boot_fail = boot_results.return_total_pass_fail()
151    if boot_pass > 1:
152        test_really_running = 1
153    else:
154        test_really_running = 0
155
156    seconds = time.time()
157    loc_time = time.localtime(seconds)
158    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
159
160    ffdc_prefix = openbmc_nickname + "." + time_string
161
162    BuiltIn().set_global_variable("${test_really_running}",
163                                  test_really_running)
164    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
165    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
166    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
167    BuiltIn().set_global_variable("${boot_success}", boot_success)
168    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
169
170    # For each program parameter, set the corresponding AUTOBOOT_ environment
171    # variable value.  Also, set an AUTOBOOT_ environment variable for every
172    # element in additional_values.
173    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
174                         "boot_fail", "test_really_running", "ffdc_prefix"]
175
176    plug_in_vars = additional_values
177
178    for var_name in plug_in_vars:
179        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
180        var_name = var_name.upper()
181        if var_value is None:
182            var_value = ""
183        os.environ["AUTOBOOT_" + var_name] = str(var_value)
184
185    if debug:
186        shell_rc, out_buf = \
187            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
188
189    BuiltIn().set_log_level(LOG_LEVEL)
190
191###############################################################################
192
193
194###############################################################################
195def setup():
196
197    r"""
198    Do general program setup tasks.
199    """
200
201    global cp_setup_called
202
203    grp.rqprintn()
204
205    validate_parms()
206
207    grp.rqprint_pgm_header()
208
209    cmd_buf = ["Set BMC Power Policy", "RESTORE_LAST_STATE"]
210    grp.rpissuing_keyword(cmd_buf, test_mode)
211    if not test_mode:
212        BuiltIn().run_keyword(*cmd_buf)
213
214    initial_plug_in_setup()
215
216    plug_in_setup()
217    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
218        call_point='setup')
219    if rc != 0:
220        error_message = "Plug-in setup failed.\n"
221        grp.rprint_error_report(error_message)
222        BuiltIn().fail(error_message)
223    # Setting cp_setup_called lets our Teardown know that it needs to call
224    # the cleanup plug-in call point.
225    cp_setup_called = 1
226
227    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
228    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
229    # FFDC_LOG_PATH is used by "FFDC" keyword.
230    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
231
232    grp.rdprint_var(boot_table, 1)
233    grp.rdprint_var(boot_lists)
234
235###############################################################################
236
237
238###############################################################################
239def validate_parms():
240
241    r"""
242    Validate all program parameters.
243    """
244
245    grp.rqprintn()
246
247    grv.rvalid_value("openbmc_host")
248    grv.rvalid_value("openbmc_username")
249    grv.rvalid_value("openbmc_password")
250    if os_host != "":
251        grv.rvalid_value("os_username")
252        grv.rvalid_value("os_password")
253
254    if pdu_host != "":
255        grv.rvalid_value("pdu_username")
256        grv.rvalid_value("pdu_password")
257        grv.rvalid_integer("pdu_slot_no")
258    if openbmc_serial_host != "":
259        grv.rvalid_integer("openbmc_serial_port")
260    grv.rvalid_integer("max_num_tests")
261    grv.rvalid_value("openbmc_model")
262    grv.rvalid_integer("boot_pass")
263    grv.rvalid_integer("boot_fail")
264
265    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
266    BuiltIn().set_global_variable("${plug_in_packages_list}",
267                                  plug_in_packages_list)
268
269    if len(boot_list) == 0 and len(boot_stack) == 0:
270        error_message = "You must provide either a value for either the" +\
271            " boot_list or the boot_stack parm.\n"
272        BuiltIn().fail(gp.sprint_error(error_message))
273
274    valid_boot_list(boot_list, valid_boot_types)
275    valid_boot_list(boot_stack, valid_boot_types)
276
277    selected_PDU_boots = list(set(boot_list + boot_stack) &
278                              set(boot_lists['PDU_reboot']))
279
280    if len(selected_PDU_boots) > 0 and pdu_host == "":
281        error_message = "You have selected the following boots which" +\
282                        " require a PDU host but no value for pdu_host:\n"
283        error_message += gp.sprint_var(selected_PDU_boots)
284        error_message += gp.sprint_var(pdu_host, 2)
285        BuiltIn().fail(gp.sprint_error(error_message))
286
287    return
288
289###############################################################################
290
291
292###############################################################################
293def my_get_state():
294
295    r"""
296    Get the system state plus a little bit of wrapping.
297    """
298
299    global state
300
301    req_states = ['epoch_seconds'] + st.default_req_states
302
303    grp.rqprint_timen("Getting system state.")
304    if test_mode:
305        state['epoch_seconds'] = int(time.time())
306    else:
307        state = st.get_state(req_states=req_states, quiet=0)
308    grp.rprint_var(state)
309
310###############################################################################
311
312
313###############################################################################
314def select_boot():
315
316    r"""
317    Select a boot test to be run based on our current state and return the
318    chosen boot type.
319
320    Description of arguments:
321    state  The state of the machine.
322    """
323
324    global boot_stack
325
326    grp.rprint_timen("Selecting a boot test.")
327
328    my_get_state()
329
330    stack_popped = 0
331    if len(boot_stack) > 0:
332        stack_popped = 1
333        grp.rprint_dashes()
334        grp.rprint_var(boot_stack)
335        grp.rprint_dashes()
336        boot_candidate = boot_stack.pop()
337        if st.compare_states(state, boot_table[boot_candidate]['start']):
338            grp.rprint_timen("The machine state is valid for a '" +
339                             boot_candidate + "' boot test.")
340            grp.rprint_dashes()
341            grp.rprint_var(boot_stack)
342            grp.rprint_dashes()
343            return boot_candidate
344        else:
345            grp.rprint_timen("The machine state is not valid for a '" +
346                             boot_candidate + "' boot test.")
347            boot_stack.append(boot_candidate)
348            popped_boot = boot_candidate
349
350    # Loop through your list selecting a boot_candidates
351    boot_candidates = []
352    for boot_candidate in boot_list:
353        if st.compare_states(state, boot_table[boot_candidate]['start']):
354            if stack_popped:
355                if st.compare_states(boot_table[boot_candidate]['end'],
356                   boot_table[popped_boot]['start']):
357                    boot_candidates.append(boot_candidate)
358            else:
359                boot_candidates.append(boot_candidate)
360
361    if len(boot_candidates) == 0:
362        grp.rprint_timen("The user's boot list contained no boot tests" +
363                         " which are valid for the current machine state.")
364        boot_candidate = default_power_on
365        if not st.compare_states(state, boot_table[default_power_on]['start']):
366            boot_candidate = default_power_off
367        boot_candidates.append(boot_candidate)
368        grp.rprint_timen("Using default '" + boot_candidate +
369                         "' boot type to transtion to valid state.")
370
371    grp.rdprint_var(boot_candidates)
372
373    # Randomly select a boot from the candidate list.
374    boot = random.choice(boot_candidates)
375
376    return boot
377
378###############################################################################
379
380
381###############################################################################
382def print_last_boots():
383
384    r"""
385    Print the last ten boots done with their time stamps.
386    """
387
388    # indent 0, 90 chars wide, linefeed, char is "="
389    grp.rqprint_dashes(0, 90)
390    grp.rqprintn("Last 10 boots:\n")
391
392    for boot_entry in last_ten:
393        grp.rqprint(boot_entry)
394    grp.rqprint_dashes(0, 90)
395
396###############################################################################
397
398
399###############################################################################
400def print_defect_report():
401
402    r"""
403    Print a defect report.
404    """
405
406    grp.rqprintn()
407    # indent=0, width=90, linefeed=1, char="="
408    grp.rqprint_dashes(0, 90, 1, "=")
409    grp.rqprintn("Copy this data to the defect:\n")
410
411    grp.rqpvars(*parm_list)
412
413    grp.rqprintn()
414
415    print_last_boots()
416    grp.rqprintn()
417    grp.rqpvar(state)
418
419    # At some point I'd like to have the 'Call FFDC Methods' return a list
420    # of files it has collected.  In that case, the following "ls" command
421    # would no longer be needed.  For now, however, glob shows the files
422    # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
423    # out (so we don't see duplicates in the list).
424
425    LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
426
427    output = '\n'.join(glob.glob(LOG_PREFIX + '*'))
428    try:
429        ffdc_list = open(ffdc_list_file_path, 'r')
430    except IOError:
431        ffdc_list = ""
432
433    grp.rqprintn()
434    grp.rqprintn("FFDC data files:")
435    if status_file_path != "":
436        grp.rqprintn(status_file_path)
437
438    grp.rqprintn(output)
439    # grp.rqprintn(ffdc_list)
440    grp.rqprintn()
441
442    grp.rqprint_dashes(0, 90, 1, "=")
443
444###############################################################################
445
446
447###############################################################################
448def my_ffdc():
449
450    r"""
451    Collect FFDC data.
452    """
453
454    global state
455
456    plug_in_setup()
457    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
458        call_point='ffdc', stop_on_plug_in_failure=1)
459
460    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
461
462    cmd_buf = ["FFDC", "ffdc_prefix=" + AUTOBOOT_FFDC_PREFIX]
463    grp.rpissuing_keyword(cmd_buf)
464    try:
465        BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf)
466    except:
467        gp.print_error("Call to ffdc failed.\n")
468
469    my_get_state()
470
471    print_defect_report()
472
473###############################################################################
474
475
476###############################################################################
477def print_test_start_message(boot_keyword):
478
479    r"""
480    Print a message indicating what boot test is about to run.
481
482    Description of arguments:
483    boot_keyword  The name of the boot which is to be run
484                  (e.g. "BMC Power On").
485    """
486
487    global last_ten
488
489    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
490    grp.rqprint(doing_msg)
491
492    last_ten.append(doing_msg)
493
494    if len(last_ten) > 10:
495        del last_ten[0]
496
497###############################################################################
498
499
500###############################################################################
501def run_boot(boot):
502
503    r"""
504    Run the specified boot.
505
506    Description of arguments:
507    boot  The name of the boot test to be performed.
508    """
509
510    global state
511
512    print_test_start_message(boot)
513
514    plug_in_setup()
515    rc, shell_rc, failed_plug_in_name = \
516        grpi.rprocess_plug_in_packages(call_point="pre_boot")
517    if rc != 0:
518        error_message = "Plug-in failed with non-zero return code.\n" +\
519            gp.sprint_var(rc, 1)
520        BuiltIn().fail(gp.sprint_error(error_message))
521
522    if test_mode:
523        # In test mode, we'll pretend the boot worked by assigning its
524        # required end state to the default state value.
525        state = st.strip_anchor_state(boot_table[boot]['end'])
526    else:
527        # Assertion:  We trust that the state data was made fresh by the
528        # caller.
529
530        grp.rprintn()
531
532        if boot_table[boot]['method_type'] == "keyword":
533            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
534                               boot_table[boot]['method'])
535
536        if boot_table[boot]['bmc_reboot']:
537            st.wait_for_comm_cycle(int(state['epoch_seconds']))
538            plug_in_setup()
539            rc, shell_rc, failed_plug_in_name = \
540                grpi.rprocess_plug_in_packages(call_point="post_reboot")
541            if rc != 0:
542                error_message = "Plug-in failed with non-zero return code.\n"
543                error_message += gp.sprint_var(rc, 1)
544                BuiltIn().fail(gp.sprint_error(error_message))
545        else:
546            match_state = st.anchor_state(state)
547            del match_state['epoch_seconds']
548            # Wait for the state to change in any way.
549            st.wait_state(match_state, wait_time=state_change_timeout,
550                          interval="3 seconds", invert=1)
551
552        grp.rprintn()
553        if boot_table[boot]['end']['chassis'] == "Off":
554            boot_timeout = power_off_timeout
555        else:
556            boot_timeout = power_on_timeout
557        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
558                      interval="3 seconds")
559
560    plug_in_setup()
561    rc, shell_rc, failed_plug_in_name = \
562        grpi.rprocess_plug_in_packages(call_point="post_boot")
563    if rc != 0:
564        error_message = "Plug-in failed with non-zero return code.\n" +\
565            gp.sprint_var(rc, 1)
566        BuiltIn().fail(gp.sprint_error(error_message))
567
568###############################################################################
569
570
571###############################################################################
572def test_loop_body():
573
574    r"""
575    The main loop body for the loop in main_py.
576
577    Description of arguments:
578    boot_count  The iteration number (starts at 1).
579    """
580
581    global boot_count
582    global state
583    global next_boot
584    global boot_success
585
586    grp.rqprintn()
587
588    boot_count += 1
589
590    next_boot = select_boot()
591
592    grp.rqprint_timen("Starting boot " + str(boot_count) + ".")
593
594    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
595    try:
596        os.remove(ffdc_list_file_path)
597    except OSError:
598        pass
599
600    cmd_buf = ["run_boot", next_boot]
601    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
602    if boot_status == "FAIL":
603        grp.rprint(msg)
604
605    grp.rqprintn()
606    if boot_status == "PASS":
607        boot_success = 1
608        grp.rqprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
609    else:
610        boot_success = 0
611        grp.rqprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
612
613    boot_results.update(next_boot, boot_status)
614
615    plug_in_setup()
616    # NOTE: A post_test_case call point failure is NOT counted as a boot
617    # failure.
618    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
619        call_point='post_test_case', stop_on_plug_in_failure=1)
620
621    plug_in_setup()
622    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
623        call_point='ffdc_check', shell_rc=0x00000200,
624        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
625    if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
626        cmd_buf = ["my_ffdc"]
627        grp.rpissuing_keyword(cmd_buf)
628        try:
629            BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf)
630        except:
631            gp.print_error("Call to my_ffdc failed.\n")
632
633    boot_results.print_report()
634    grp.rqprint_timen("Finished boot " + str(boot_count) + ".")
635
636    plug_in_setup()
637    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
638        call_point='stop_check')
639    if rc != 0:
640        error_message = "Stopping as requested by user.\n"
641        grp.rprint_error_report(error_message)
642        BuiltIn().fail(error_message)
643
644    return True
645
646###############################################################################
647
648
649###############################################################################
650def main_keyword_teardown():
651
652    r"""
653    Clean up after the Main keyword.
654    """
655
656    if cp_setup_called:
657        plug_in_setup()
658        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
659            call_point='cleanup', stop_on_plug_in_failure=1)
660
661    # Save boot_results object to a file in case it is needed again.
662    grp.rprint_timen("Saving boot_results to the following path.")
663    grp.rprint_var(boot_results_file_path)
664    pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
665                pickle.HIGHEST_PROTOCOL)
666
667###############################################################################
668
669
670###############################################################################
671def test_teardown():
672
673    r"""
674    Clean up after this test case.
675    """
676
677    gp.qprintn()
678    cmd_buf = ["Print Error",
679               "A keyword timeout occurred ending this program.\n"]
680    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
681
682###############################################################################
683
684
685###############################################################################
686def main_py():
687
688    r"""
689    Do main program processing.
690    """
691
692    setup()
693
694    # Process caller's boot_stack.
695    while (len(boot_stack) > 0):
696        test_loop_body()
697
698    grp.rprint_timen("Finished processing stack.")
699
700    # Process caller's boot_list.
701    if len(boot_list) > 0:
702        for ix in range(1, max_num_tests + 1):
703            test_loop_body()
704
705    grp.rqprint_timen("Completed all requested boot tests.")
706
707###############################################################################
708