1 extern "C"
2 {
3 #include <libpdbg.h>
4 }
5 
6 #include "create_pel.hpp"
7 #include "dump_utils.hpp"
8 #include "extensions/phal/common_utils.hpp"
9 #include "phal_error.hpp"
10 
11 #include <attributes_info.H>
12 #include <fmt/format.h>
13 #include <libekb.H>
14 #include <libphal.H>
15 
16 #include <nlohmann/json.hpp>
17 #include <phosphor-logging/elog.hpp>
18 
19 #include <algorithm>
20 #include <cstdlib>
21 #include <cstring>
22 #include <iomanip>
23 #include <list>
24 #include <map>
25 #include <sstream>
26 #include <string>
27 
28 namespace openpower
29 {
30 namespace phal
31 {
32 using namespace phosphor::logging;
33 using namespace openpower::phal::exception;
34 
35 /**
36  * Used to pass buffer to pdbg callback api to get required target
37  * data (attributes) based on given data (attribute).
38  */
39 struct TargetInfo
40 {
41     ATTR_PHYS_BIN_PATH_Type physBinPath;
42     ATTR_LOCATION_CODE_Type locationCode;
43     ATTR_PHYS_DEV_PATH_Type physDevPath;
44     ATTR_MRU_ID_Type mruId;
45 
46     bool deconfigure;
47 
48     TargetInfo()
49     {
50         memset(&physBinPath, '\0', sizeof(physBinPath));
51         memset(&locationCode, '\0', sizeof(locationCode));
52         memset(&physDevPath, '\0', sizeof(physDevPath));
53         mruId = 0;
54         deconfigure = false;
55     }
56 };
57 
58 /**
59  * Used to return in callback function which are used to get
60  * physical path value and it binary format value.
61  *
62  * The value for constexpr defined based on pdbg_target_traverse function usage.
63  */
64 constexpr int continueTgtTraversal = 0;
65 constexpr int requireAttrFound = 1;
66 constexpr int requireAttrNotFound = 2;
67 
68 /**
69  * @brief Used to get target location code from phal device tree
70  *
71  * @param[in] target current device tree target
72  * @param[out] appPrivData used for accessing|storing from|to application
73  *
74  * @return 0 to continue traverse, non-zero to stop traverse
75  */
76 int pdbgCallbackToGetTgtReqAttrsVal(struct pdbg_target* target,
77                                     void* appPrivData)
78 {
79     using namespace openpower::phal::pdbg;
80 
81     TargetInfo* targetInfo = static_cast<TargetInfo*>(appPrivData);
82 
83     ATTR_PHYS_BIN_PATH_Type physBinPath;
84     /**
85      * TODO: Issue: phal/pdata#16
86      * Should not use direct pdbg api to read attribute. Need to use DT_GET_PROP
87      * macro for bmc app's and this will call libdt-api api but, it will print
88      * "pdbg_target_get_attribute failed" trace if attribute is not found and
89      * this callback will call recursively by using pdbg_target_traverse() until
90      * find expected attribute based on return code from this callback. Because,
91      * need to do target iteration to get actual attribute (ATTR_PHYS_BIN_PATH)
92      * value when device tree target info doesn't know to read attribute from
93      * device tree. So, Due to this error trace user will get confusion while
94      * looking traces. Hence using pdbg api to avoid trace until libdt-api
95      * provides log level setup.
96      */
97     if (!pdbg_target_get_attribute(
98             target, "ATTR_PHYS_BIN_PATH",
99             std::stoi(dtAttr::fapi2::ATTR_PHYS_BIN_PATH_Spec),
100             dtAttr::fapi2::ATTR_PHYS_BIN_PATH_ElementCount, physBinPath))
101     {
102         return continueTgtTraversal;
103     }
104 
105     if (std::memcmp(physBinPath, targetInfo->physBinPath,
106                     sizeof(physBinPath)) != 0)
107     {
108         return continueTgtTraversal;
109     }
110 
111     try
112     {
113         // Get location code information
114         openpower::phal::pdbg::getLocationCode(target,
115                                                targetInfo->locationCode);
116     }
117     catch (const std::exception& e)
118     {
119         // log message and continue with default data
120         log<level::ERR>(fmt::format("getLocationCode({}): Exception({})",
121                                     pdbg_target_path(target), e.what())
122                             .c_str());
123     }
124 
125     if (DT_GET_PROP(ATTR_PHYS_DEV_PATH, target, targetInfo->physDevPath))
126     {
127         log<level::ERR>("Could not read PHYS_DEV_PATH attribute");
128         return requireAttrNotFound;
129     }
130 
131     if (DT_GET_PROP(ATTR_MRU_ID, target, targetInfo->mruId))
132     {
133         log<level::ERR>("Could not read MRU_ID attribute");
134         return requireAttrNotFound;
135     }
136 
137     return requireAttrFound;
138 }
139 
140 /**
141  * @brief Used to get target info (attributes data)
142  *
143  * To get target required attributes value using another attribute value
144  * ("PHYS_BIN_PATH" which is present in same target attributes list) by using
145  * "ipdbg_target_traverse" api because, here we have attribute value only and
146  * doesn't have respective device tree target info to get required attributes
147  * values from it attributes list.
148  *
149  * @param[in] physBinPath to pass PHYS_BIN_PATH value
150  * @param[out] targetInfo to pas buufer to fill with required attributes
151  *
152  * @return true on success otherwise false
153  */
154 bool getTgtReqAttrsVal(const std::vector<uint8_t>& physBinPath,
155                        TargetInfo& targetInfo)
156 {
157     std::memcpy(&targetInfo.physBinPath, physBinPath.data(),
158                 sizeof(targetInfo.physBinPath));
159 
160     int ret = pdbg_target_traverse(NULL, pdbgCallbackToGetTgtReqAttrsVal,
161                                    &targetInfo);
162     if (ret == 0)
163     {
164         log<level::ERR>(fmt::format("Given ATTR_PHYS_BIN_PATH value({}) "
165                                     "not found in phal device tree",
166                                     targetInfo.physBinPath)
167                             .c_str());
168         return false;
169     }
170     else if (ret == requireAttrNotFound)
171     {
172         return false;
173     }
174 
175     return true;
176 }
177 } // namespace phal
178 
179 namespace pel
180 {
181 using namespace phosphor::logging;
182 
183 namespace detail
184 {
185 using json = nlohmann::json;
186 
187 // keys need to be unique so using counter value to generate unique key
188 static int counter = 0;
189 
190 // list of debug traces
191 static std::vector<std::pair<std::string, std::string>> traceLog;
192 
193 void processLogTraceCallback(void*, const char* fmt, va_list ap)
194 {
195     va_list vap;
196     va_copy(vap, ap);
197     std::vector<char> logData(1 + std::vsnprintf(nullptr, 0, fmt, ap));
198     std::vsnprintf(logData.data(), logData.size(), fmt, vap);
199     va_end(vap);
200     std::string logstr(logData.begin(), logData.end());
201 
202     log<level::INFO>(logstr.c_str());
203 
204     char timeBuf[80];
205     time_t t = time(0);
206     tm myTm{};
207     gmtime_r(&t, &myTm);
208     strftime(timeBuf, 80, "%Y-%m-%d %H:%M:%S", &myTm);
209 
210     // key values need to be unique for PEL
211     // TODO #openbmc/dev/issues/1563
212     // If written to Json no need to worry about unique KEY
213     std::stringstream str;
214     str << std::setfill('0');
215     str << "LOG" << std::setw(3) << counter;
216     str << " " << timeBuf;
217     traceLog.emplace_back(std::make_pair(str.str(), std::move(logstr)));
218     counter++;
219 }
220 
221 /**
222  * @brief GET PEL priority from pHAL priority
223  *
224  * The pHAL callout priority is in different format than PEL format
225  * so, this api is used to return current phal supported priority into
226  * PEL expected format.
227  *
228  * @param[in] phalPriority used to pass phal priority format string
229  *
230  * @return pel priority format string else empty if failure
231  *
232  * @note For "NONE" returning "L" (LOW)
233  */
234 static std::string getPelPriority(const std::string& phalPriority)
235 {
236     const std::map<std::string, std::string> priorityMap = {
237         {"HIGH", "H"}, {"MEDIUM", "M"}, {"LOW", "L"}, {"NONE", "L"}};
238 
239     auto it = priorityMap.find(phalPriority);
240     if (it == priorityMap.end())
241     {
242         log<level::ERR>(fmt::format("Unsupported phal priority({}) is given "
243                                     "to get pel priority format",
244                                     phalPriority)
245                             .c_str());
246         return "H";
247     }
248 
249     return it->second;
250 }
251 
252 void processIplErrorCallback(const ipl_error_info& errInfo)
253 {
254     log<level::INFO>(
255         fmt::format("processIplErrorCallback: Error type({})", errInfo.type)
256             .c_str());
257 
258     if (errInfo.type == IPL_ERR_OK)
259     {
260         // reset trace log and exit
261         reset();
262         return;
263     }
264 
265     if ((errInfo.type == IPL_ERR_SBE_BOOT) ||
266         (errInfo.type == IPL_ERR_SBE_CHIPOP))
267     {
268         // handle SBE related failures.
269         processSbeBootError();
270         return;
271     }
272 
273     if (errInfo.type == IPL_ERR_HWP)
274     {
275         // Handle hwp failure
276         processBootError(false);
277         return;
278     }
279 
280     // Log PEL for any other failures
281     if (errInfo.type != IPL_ERR_OK)
282     {
283         createPEL("org.open_power.PHAL.Error.Boot");
284         // reset trace log and exit
285         reset();
286         return;
287     }
288 }
289 
290 void processBootError(bool status)
291 {
292     log<level::INFO>("processBootError ", entry("STATUS=%d", status));
293     try
294     {
295         // return If no failure during hwp execution
296         if (status)
297             return;
298 
299         // Collecting ffdc details from phal
300         FFDC ffdc;
301         libekb_get_ffdc(ffdc);
302 
303         log<level::INFO>(
304             fmt::format("PHAL FFDC: Return Message[{}]", ffdc.message).c_str());
305 
306         // To store callouts details in json format as per pel expectation.
307         json jsonCalloutDataList;
308         jsonCalloutDataList = json::array();
309 
310         // To store phal trace and other additional data about ffdc.
311         FFDCData pelAdditionalData;
312 
313         if (ffdc.ffdc_type == FFDC_TYPE_HWP)
314         {
315             // Adding hardware procedures return code details
316             pelAdditionalData.emplace_back("HWP_RC", ffdc.hwp_errorinfo.rc);
317             pelAdditionalData.emplace_back("HWP_RC_DESC",
318                                            ffdc.hwp_errorinfo.rc_desc);
319 
320             // Adding hardware procedures required ffdc data for debug
321             for_each(ffdc.hwp_errorinfo.ffdcs_data.begin(),
322                      ffdc.hwp_errorinfo.ffdcs_data.end(),
323                      [&pelAdditionalData](
324                          std::pair<std::string, std::string>& ele) -> void {
325                          std::string keyWithPrefix("HWP_FFDC_");
326                          keyWithPrefix.append(ele.first);
327 
328                          pelAdditionalData.emplace_back(keyWithPrefix,
329                                                         ele.second);
330                      });
331 
332             // Adding hardware callout details
333             int calloutCount = 0;
334             for_each(ffdc.hwp_errorinfo.hwcallouts.begin(),
335                      ffdc.hwp_errorinfo.hwcallouts.end(),
336                      [&pelAdditionalData, &calloutCount, &jsonCalloutDataList](
337                          const HWCallout& hwCallout) -> void {
338                          calloutCount++;
339                          std::stringstream keyPrefix;
340                          keyPrefix << "HWP_HW_CO_" << std::setfill('0')
341                                    << std::setw(2) << calloutCount << "_";
342 
343                          pelAdditionalData.emplace_back(
344                              std::string(keyPrefix.str()).append("HW_ID"),
345                              hwCallout.hwid);
346 
347                          pelAdditionalData.emplace_back(
348                              std::string(keyPrefix.str()).append("PRIORITY"),
349                              hwCallout.callout_priority);
350 
351                          phal::TargetInfo targetInfo;
352                          phal::getTgtReqAttrsVal(hwCallout.target_entity_path,
353                                                  targetInfo);
354 
355                          std::string locationCode =
356                              std::string(targetInfo.locationCode);
357                          pelAdditionalData.emplace_back(
358                              std::string(keyPrefix.str()).append("LOC_CODE"),
359                              locationCode);
360 
361                          std::string physPath =
362                              std::string(targetInfo.physDevPath);
363                          pelAdditionalData.emplace_back(
364                              std::string(keyPrefix.str()).append("PHYS_PATH"),
365                              physPath);
366 
367                          pelAdditionalData.emplace_back(
368                              std::string(keyPrefix.str()).append("CLK_POS"),
369                              std::to_string(hwCallout.clkPos));
370 
371                          json jsonCalloutData;
372                          jsonCalloutData["LocationCode"] = locationCode;
373                          std::string pelPriority =
374                              getPelPriority(hwCallout.callout_priority);
375                          jsonCalloutData["Priority"] = pelPriority;
376 
377                          if (targetInfo.mruId != 0)
378                          {
379                              jsonCalloutData["MRUs"] = json::array({
380                                  {{"ID", targetInfo.mruId},
381                                   {"Priority", pelPriority}},
382                              });
383                          }
384 
385                          jsonCalloutDataList.emplace_back(jsonCalloutData);
386                      });
387 
388             // Adding CDG (callout, deconfigure and guard) targets details
389             calloutCount = 0;
390             for_each(ffdc.hwp_errorinfo.cdg_targets.begin(),
391                      ffdc.hwp_errorinfo.cdg_targets.end(),
392                      [&pelAdditionalData, &calloutCount,
393                       &jsonCalloutDataList](const CDG_Target& cdg_tgt) -> void {
394                          calloutCount++;
395                          std::stringstream keyPrefix;
396                          keyPrefix << "HWP_CDG_TGT_" << std::setfill('0')
397                                    << std::setw(2) << calloutCount << "_";
398 
399                          phal::TargetInfo targetInfo;
400                          targetInfo.deconfigure = cdg_tgt.deconfigure;
401 
402                          phal::getTgtReqAttrsVal(cdg_tgt.target_entity_path,
403                                                  targetInfo);
404 
405                          std::string locationCode =
406                              std::string(targetInfo.locationCode);
407                          pelAdditionalData.emplace_back(
408                              std::string(keyPrefix.str()).append("LOC_CODE"),
409                              locationCode);
410                          std::string physPath =
411                              std::string(targetInfo.physDevPath);
412                          pelAdditionalData.emplace_back(
413                              std::string(keyPrefix.str()).append("PHYS_PATH"),
414                              physPath);
415 
416                          pelAdditionalData.emplace_back(
417                              std::string(keyPrefix.str()).append("CO_REQ"),
418                              (cdg_tgt.callout == true ? "true" : "false"));
419 
420                          pelAdditionalData.emplace_back(
421                              std::string(keyPrefix.str()).append("CO_PRIORITY"),
422                              cdg_tgt.callout_priority);
423 
424                          pelAdditionalData.emplace_back(
425                              std::string(keyPrefix.str()).append("DECONF_REQ"),
426                              (cdg_tgt.deconfigure == true ? "true" : "false"));
427 
428                          pelAdditionalData.emplace_back(
429                              std::string(keyPrefix.str()).append("GUARD_REQ"),
430                              (cdg_tgt.guard == true ? "true" : "false"));
431 
432                          pelAdditionalData.emplace_back(
433                              std::string(keyPrefix.str()).append("GUARD_TYPE"),
434                              cdg_tgt.guard_type);
435 
436                          json jsonCalloutData;
437                          jsonCalloutData["LocationCode"] = locationCode;
438                          std::string pelPriority =
439                              getPelPriority(cdg_tgt.callout_priority);
440                          jsonCalloutData["Priority"] = pelPriority;
441 
442                          if (targetInfo.mruId != 0)
443                          {
444                              jsonCalloutData["MRUs"] = json::array({
445                                  {{"ID", targetInfo.mruId},
446                                   {"Priority", pelPriority}},
447                              });
448                          }
449                          jsonCalloutData["Deconfigured"] = cdg_tgt.deconfigure;
450                          jsonCalloutData["Guarded"] = cdg_tgt.guard;
451                          jsonCalloutData["GuardType"] = cdg_tgt.guard_type;
452                          jsonCalloutData["EntityPath"] =
453                              cdg_tgt.target_entity_path;
454 
455                          jsonCalloutDataList.emplace_back(jsonCalloutData);
456                      });
457         }
458         else if ((ffdc.ffdc_type != FFDC_TYPE_NONE) &&
459                  (ffdc.ffdc_type != FFDC_TYPE_UNSUPPORTED))
460         {
461             log<level::ERR>(
462                 fmt::format("Unsupported phal FFDC type to create PEL. "
463                             "MSG: {}",
464                             ffdc.message)
465                     .c_str());
466         }
467 
468         // Adding collected phal logs into PEL additional data
469         for_each(traceLog.begin(), traceLog.end(),
470                  [&pelAdditionalData](
471                      std::pair<std::string, std::string>& ele) -> void {
472                      pelAdditionalData.emplace_back(ele.first, ele.second);
473                  });
474 
475         // TODO: #ibm-openbmc/dev/issues/2595 : Once enabled this support,
476         // callout details is not required to sort in H,M and L orders which
477         // are expected by pel because, pel will take care for sorting callouts
478         // based on priority so, now adding support to send callout in order
479         // i.e High -> Medium -> Low.
480         std::sort(
481             jsonCalloutDataList.begin(), jsonCalloutDataList.end(),
482             [](const json& aEle, const json& bEle) -> bool {
483                 // Considering b element having higher priority than a element
484                 // or Both element will be same priorty (to keep same order
485                 // which are given by phal when two callouts are having same
486                 // priority)
487                 if (((aEle["Priority"] == "M") && (bEle["Priority"] == "H")) ||
488                     ((aEle["Priority"] == "L") &&
489                      ((bEle["Priority"] == "H") ||
490                       (bEle["Priority"] == "M"))) ||
491                     (aEle["Priority"] == bEle["Priority"]))
492                 {
493                     return false;
494                 }
495 
496                 // Considering a element having higher priority than b element
497                 return true;
498             });
499 
500         openpower::pel::createBootErrorPEL(pelAdditionalData,
501                                            jsonCalloutDataList);
502     }
503     catch (const std::exception& ex)
504     {
505         reset();
506         throw ex;
507     }
508     reset();
509 }
510 
511 void processSbeBootError()
512 {
513     log<level::INFO>("processSbeBootError : Entered ");
514 
515     using namespace openpower::phal::sbe;
516 
517     // To store phal trace and other additional data about ffdc.
518     FFDCData pelAdditionalData;
519 
520     // Adding collected phal logs into PEL additional data
521     for_each(
522         traceLog.begin(), traceLog.end(),
523         [&pelAdditionalData](std::pair<std::string, std::string>& ele) -> void {
524             pelAdditionalData.emplace_back(ele.first, ele.second);
525         });
526 
527     // reset the trace log and counter
528     reset();
529 
530     // get primary processor to collect FFDC/Dump information.
531     struct pdbg_target* procTarget;
532     pdbg_for_each_class_target("proc", procTarget)
533     {
534         if (openpower::phal::isPrimaryProc(procTarget))
535             break;
536         procTarget = nullptr;
537     }
538     // check valid primary processor is available
539     if (procTarget == nullptr)
540     {
541         log<level::ERR>("processSbeBootError: fail to get primary processor");
542         // Initialise the SRC6 with default data, not used in this use case.
543         pelAdditionalData.emplace_back("SRC6", "00000000");
544         openpower::pel::createPEL(
545             "org.open_power.Processor.Error.SbeBootFailure", pelAdditionalData);
546         return;
547     }
548     // SBE error object.
549     sbeError_t sbeError;
550     bool dumpIsRequired = false;
551 
552     try
553     {
554         // Capture FFDC information on primary processor
555         sbeError = captureFFDC(procTarget);
556     }
557     catch (const phalError_t& phalError)
558     {
559         // Fail to collect FFDC information , trigger Dump
560         log<level::ERR>(
561             fmt::format("captureFFDC: Exception({})", phalError.what())
562                 .c_str());
563         dumpIsRequired = true;
564     }
565 
566     std::string event;
567 
568     if ((sbeError.errType() == SBE_FFDC_NO_DATA) ||
569         (sbeError.errType() == SBE_CMD_TIMEOUT) || (dumpIsRequired))
570     {
571         event = "org.open_power.Processor.Error.SbeBootTimeout";
572         dumpIsRequired = true;
573     }
574     else
575     {
576         event = "org.open_power.Processor.Error.SbeBootFailure";
577     }
578     // SRC6 : [0:15] chip position
579     uint32_t index = pdbg_target_index(procTarget);
580     pelAdditionalData.emplace_back("SRC6", std::to_string(index << 16));
581     // Create SBE Error with FFDC data.
582     auto logId = createSbeErrorPEL(event, sbeError, pelAdditionalData);
583 
584     if (dumpIsRequired)
585     {
586         using namespace openpower::phal::dump;
587         DumpParameters dumpParameters = {logId, index, SBE_DUMP_TIMEOUT,
588                                          DumpType::SBE};
589         try
590         {
591             requestDump(dumpParameters);
592         }
593         catch (const std::runtime_error& e)
594         {
595             // Allowing call back to handle the error gracefully.
596             log<level::ERR>("Dump collection failed");
597             // TODO revist error handling.
598         }
599     }
600 }
601 
602 void reset()
603 {
604     // reset the trace log and counter
605     traceLog.clear();
606     counter = 0;
607 }
608 
609 void pDBGLogTraceCallbackHelper(int, const char* fmt, va_list ap)
610 {
611     processLogTraceCallback(NULL, fmt, ap);
612 }
613 } // namespace detail
614 
615 static inline uint8_t getLogLevelFromEnv(const char* env, const uint8_t dValue)
616 {
617     auto logLevel = dValue;
618     try
619     {
620         if (const char* env_p = std::getenv(env))
621         {
622             logLevel = std::stoi(env_p);
623         }
624     }
625     catch (const std::exception& e)
626     {
627         log<level::ERR>(("Conversion Failure"), entry("ENVIRONMENT=%s", env),
628                         entry("EXCEPTION=%s", e.what()));
629     }
630     return logLevel;
631 }
632 
633 void addBootErrorCallbacks()
634 {
635     // Get individual phal repos log level from environment variable
636     // and update the  log level.
637     pdbg_set_loglevel(getLogLevelFromEnv("PDBG_LOG", PDBG_INFO));
638     libekb_set_loglevel(getLogLevelFromEnv("LIBEKB_LOG", LIBEKB_LOG_IMP));
639     ipl_set_loglevel(getLogLevelFromEnv("IPL_LOG", IPL_INFO));
640 
641     // add callback for debug traces
642     pdbg_set_logfunc(detail::pDBGLogTraceCallbackHelper);
643     libekb_set_logfunc(detail::processLogTraceCallback, NULL);
644     ipl_set_logfunc(detail::processLogTraceCallback, NULL);
645 
646     // add callback for ipl failures
647     ipl_set_error_callback_func(detail::processIplErrorCallback);
648 }
649 
650 } // namespace pel
651 } // namespace openpower
652