1 /** 2 * Copyright © 2020 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "user_data_json.hpp" 18 19 #include "json_utils.hpp" 20 #include "pel_types.hpp" 21 #include "pel_values.hpp" 22 #include "stream.hpp" 23 #include "user_data_formats.hpp" 24 25 #include <Python.h> 26 27 #include <nlohmann/json.hpp> 28 #include <phosphor-logging/log.hpp> 29 30 #include <iomanip> 31 #include <sstream> 32 33 namespace openpower::pels::user_data 34 { 35 namespace pv = openpower::pels::pel_values; 36 using namespace phosphor::logging; 37 using orderedJSON = nlohmann::ordered_json; 38 39 void pyDecRef(PyObject* pyObj) 40 { 41 Py_XDECREF(pyObj); 42 } 43 44 /** 45 * @brief Returns a JSON string for use by PEL::printSectionInJSON(). 46 * 47 * The returning string will contain a JSON object, but without 48 * the outer {}. If the input JSON isn't a JSON object (dict), then 49 * one will be created with the input added to a 'Data' key. 50 * 51 * @param[in] creatorID - The creator ID for the PEL 52 * 53 * @param[in] json - The JSON to convert to a string 54 * 55 * @return std::string - The JSON string 56 */ 57 std::string prettyJSON(uint16_t componentID, uint8_t subType, uint8_t version, 58 uint8_t creatorID, const orderedJSON& json) 59 { 60 orderedJSON output; 61 output[pv::sectionVer] = std::to_string(version); 62 output[pv::subSection] = std::to_string(subType); 63 output[pv::createdBy] = getComponentName(componentID, creatorID); 64 65 if (!json.is_object()) 66 { 67 output["Data"] = json; 68 } 69 else 70 { 71 for (const auto& [key, value] : json.items()) 72 { 73 output[key] = value; 74 } 75 } 76 77 // Let nlohmann do the pretty printing. 78 std::stringstream stream; 79 stream << std::setw(4) << output; 80 81 auto jsonString = stream.str(); 82 83 // Now it looks like: 84 // { 85 // "Section Version": ... 86 // ... 87 // } 88 89 // Since PEL::printSectionInJSON() will supply the outer { }s, 90 // remove the existing ones. 91 92 // Replace the { and the following newline, and the } and its 93 // preceeding newline. 94 jsonString.erase(0, 2); 95 96 auto pos = jsonString.find_last_of('}'); 97 jsonString.erase(pos - 1); 98 99 return jsonString; 100 } 101 102 /** 103 * @brief Return a JSON string from the passed in CBOR data. 104 * 105 * @param[in] componentID - The comp ID from the UserData section header 106 * @param[in] subType - The subtype from the UserData section header 107 * @param[in] version - The version from the UserData section header 108 * @param[in] creatorID - The creator ID for the PEL 109 * @param[in] data - The CBOR data 110 * 111 * @return std::string - The JSON string 112 */ 113 std::string getCBORJSON(uint16_t componentID, uint8_t subType, uint8_t version, 114 uint8_t creatorID, const std::vector<uint8_t>& data) 115 { 116 // The CBOR parser needs the pad bytes added to 4 byte align 117 // removed. The number of bytes added to the pad is on the 118 // very end, so will remove both fields before parsing. 119 120 // If the data vector is too short, an exception will get 121 // thrown which will be handled up the call stack. 122 123 auto cborData = data; 124 uint32_t pad{}; 125 126 Stream stream{cborData}; 127 stream.offset(cborData.size() - 4); 128 stream >> pad; 129 130 if (cborData.size() > (pad + sizeof(pad))) 131 { 132 cborData.resize(data.size() - sizeof(pad) - pad); 133 } 134 135 orderedJSON json = orderedJSON::from_cbor(cborData); 136 137 return prettyJSON(componentID, subType, version, creatorID, json); 138 } 139 140 /** 141 * @brief Return a JSON string from the passed in text data. 142 * 143 * The function breaks up the input text into a vector of strings with 144 * newline as separator and converts that into JSON. It will convert any 145 * unprintable characters to periods. 146 * 147 * @param[in] componentID - The comp ID from the UserData section header 148 * @param[in] subType - The subtype from the UserData section header 149 * @param[in] version - The version from the UserData section header 150 * @param[in] creatorID - The creator ID for the PEL 151 * @param[in] data - The CBOR data 152 * 153 * @return std::string - The JSON string 154 */ 155 std::string getTextJSON(uint16_t componentID, uint8_t subType, uint8_t version, 156 uint8_t creatorID, const std::vector<uint8_t>& data) 157 { 158 std::vector<std::string> text; 159 size_t startPos = 0; 160 161 // Converts any unprintable characters to periods 162 auto validate = [](char& ch) { 163 if ((ch < ' ') || (ch > '~')) 164 { 165 ch = '.'; 166 } 167 }; 168 169 // Break up the data into an array of strings with newline as separator 170 for (size_t pos = 0; pos < data.size(); ++pos) 171 { 172 if (data[pos] == '\n') 173 { 174 std::string line{reinterpret_cast<const char*>(&data[startPos]), 175 pos - startPos}; 176 std::for_each(line.begin(), line.end(), validate); 177 text.push_back(std::move(line)); 178 startPos = pos + 1; 179 } 180 } 181 if (startPos < data.size()) 182 { 183 std::string line{reinterpret_cast<const char*>(&data[startPos]), 184 data.size() - startPos}; 185 std::for_each(line.begin(), line.end(), validate); 186 text.push_back(std::move(line)); 187 } 188 189 orderedJSON json = text; 190 return prettyJSON(componentID, subType, version, creatorID, json); 191 } 192 193 /** 194 * @brief Convert to an appropriate JSON string as the data is one of 195 * the formats that we natively support. 196 * 197 * @param[in] componentID - The comp ID from the UserData section header 198 * @param[in] subType - The subtype from the UserData section header 199 * @param[in] version - The version from the UserData section header 200 * @param[in] data - The data itself 201 * 202 * @return std::optional<std::string> - The JSON string if it could be created, 203 * else std::nullopt. 204 */ 205 std::optional<std::string> 206 getBuiltinFormatJSON(uint16_t componentID, uint8_t subType, uint8_t version, 207 const std::vector<uint8_t>& data, uint8_t creatorID) 208 { 209 switch (subType) 210 { 211 case static_cast<uint8_t>(UserDataFormat::json): 212 { 213 std::string jsonString{data.begin(), data.begin() + data.size()}; 214 215 orderedJSON json = orderedJSON::parse(jsonString); 216 217 return prettyJSON(componentID, subType, version, creatorID, json); 218 } 219 case static_cast<uint8_t>(UserDataFormat::cbor): 220 { 221 return getCBORJSON(componentID, subType, version, creatorID, data); 222 } 223 case static_cast<uint8_t>(UserDataFormat::text): 224 { 225 return getTextJSON(componentID, subType, version, creatorID, data); 226 } 227 default: 228 break; 229 } 230 return std::nullopt; 231 } 232 233 /** 234 * @brief Call Python modules to parse the data into a JSON string 235 * 236 * The module to call is based on the Creator Subsystem ID and the Component 237 * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy" 238 * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID. 239 * 240 * All modules must provide the following: 241 * Function: parseUDToJson 242 * Argument list: 243 * 1. (int) Sub-section type 244 * 2. (int) Section version 245 * 3. (memoryview): Data 246 *-Return data: 247 * 1. (str) JSON string 248 * 249 * @param[in] componentID - The comp ID from the UserData section header 250 * @param[in] subType - The subtype from the UserData section header 251 * @param[in] version - The version from the UserData section header 252 * @param[in] data - The data itself 253 * @param[in] creatorID - The creatorID from the PrivateHeader section 254 * @return std::optional<std::string> - The JSON string if it could be created, 255 * else std::nullopt 256 */ 257 std::optional<std::string> getPythonJSON(uint16_t componentID, uint8_t subType, 258 uint8_t version, 259 const std::vector<uint8_t>& data, 260 uint8_t creatorID) 261 { 262 PyObject *pName, *pModule, *eType, *eValue, *eTraceback, *pKey; 263 std::string pErrStr; 264 std::string module = getNumberString("%c", tolower(creatorID)) + 265 getNumberString("%04x", componentID); 266 pName = PyUnicode_FromString( 267 std::string("udparsers." + module + "." + module).c_str()); 268 std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef); 269 pModule = PyImport_Import(pName); 270 if (pModule == NULL) 271 { 272 pErrStr = "No error string found"; 273 PyErr_Fetch(&eType, &eValue, &eTraceback); 274 if (eType) 275 { 276 Py_XDECREF(eType); 277 } 278 if (eTraceback) 279 { 280 Py_XDECREF(eTraceback); 281 } 282 if (eValue) 283 { 284 PyObject* pStr = PyObject_Str(eValue); 285 Py_XDECREF(eValue); 286 if (pStr) 287 { 288 pErrStr = PyUnicode_AsUTF8(pStr); 289 Py_XDECREF(pStr); 290 } 291 } 292 } 293 else 294 { 295 std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr(pModule, 296 &pyDecRef); 297 std::string funcToCall = "parseUDToJson"; 298 pKey = PyUnicode_FromString(funcToCall.c_str()); 299 std::unique_ptr<PyObject, decltype(&pyDecRef)> keyPtr(pKey, &pyDecRef); 300 PyObject* pDict = PyModule_GetDict(pModule); 301 Py_INCREF(pDict); 302 if (!PyDict_Contains(pDict, pKey)) 303 { 304 Py_DECREF(pDict); 305 log<level::ERR>( 306 "Python module error", 307 entry("ERROR=%s", 308 std::string(funcToCall + " function missing").c_str()), 309 entry("PARSER_MODULE=%s", module.c_str()), 310 entry("SUBTYPE=0x%X", subType), entry("VERSION=%d", version), 311 entry("DATA_LENGTH=%lu\n", data.size())); 312 return std::nullopt; 313 } 314 PyObject* pFunc = PyDict_GetItemString(pDict, funcToCall.c_str()); 315 Py_DECREF(pDict); 316 Py_INCREF(pFunc); 317 if (PyCallable_Check(pFunc)) 318 { 319 auto ud = data.data(); 320 PyObject* pArgs = PyTuple_New(3); 321 std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr(pArgs, 322 &pyDecRef); 323 PyTuple_SetItem(pArgs, 0, 324 PyLong_FromUnsignedLong((unsigned long)subType)); 325 PyTuple_SetItem(pArgs, 1, 326 PyLong_FromUnsignedLong((unsigned long)version)); 327 PyObject* pData = PyMemoryView_FromMemory( 328 reinterpret_cast<char*>(const_cast<unsigned char*>(ud)), 329 data.size(), PyBUF_READ); 330 PyTuple_SetItem(pArgs, 2, pData); 331 PyObject* pResult = PyObject_CallObject(pFunc, pArgs); 332 Py_DECREF(pFunc); 333 if (pResult) 334 { 335 std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr( 336 pResult, &pyDecRef); 337 PyObject* pBytes = PyUnicode_AsEncodedString(pResult, "utf-8", 338 "~E~"); 339 std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr( 340 pBytes, &pyDecRef); 341 const char* output = PyBytes_AS_STRING(pBytes); 342 try 343 { 344 orderedJSON json = orderedJSON::parse(output); 345 if ((json.is_object() && !json.empty()) || 346 (json.is_array() && json.size() > 0) || 347 (json.is_string() && json != "")) 348 { 349 return prettyJSON(componentID, subType, version, 350 creatorID, json); 351 } 352 } 353 catch (const std::exception& e) 354 { 355 log<level::ERR>("Bad JSON from parser", 356 entry("ERROR=%s", e.what()), 357 entry("PARSER_MODULE=%s", module.c_str()), 358 entry("SUBTYPE=0x%X", subType), 359 entry("VERSION=%d", version), 360 entry("DATA_LENGTH=%lu\n", data.size())); 361 return std::nullopt; 362 } 363 } 364 else 365 { 366 pErrStr = "No error string found"; 367 PyErr_Fetch(&eType, &eValue, &eTraceback); 368 if (eType) 369 { 370 Py_XDECREF(eType); 371 } 372 if (eTraceback) 373 { 374 Py_XDECREF(eTraceback); 375 } 376 if (eValue) 377 { 378 PyObject* pStr = PyObject_Str(eValue); 379 Py_XDECREF(eValue); 380 if (pStr) 381 { 382 pErrStr = PyUnicode_AsUTF8(pStr); 383 Py_XDECREF(pStr); 384 } 385 } 386 } 387 } 388 } 389 if (!pErrStr.empty()) 390 { 391 log<level::DEBUG>("Python exception thrown by parser", 392 entry("ERROR=%s", pErrStr.c_str()), 393 entry("PARSER_MODULE=%s", module.c_str()), 394 entry("SUBTYPE=0x%X", subType), 395 entry("VERSION=%d", version), 396 entry("DATA_LENGTH=%lu\n", data.size())); 397 } 398 return std::nullopt; 399 } 400 401 std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType, 402 uint8_t version, 403 const std::vector<uint8_t>& data, 404 uint8_t creatorID, 405 const std::vector<std::string>& plugins) 406 { 407 std::string subsystem = getNumberString("%c", tolower(creatorID)); 408 std::string component = getNumberString("%04x", componentID); 409 try 410 { 411 if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" && 412 componentID == static_cast<uint16_t>(ComponentID::phosphorLogging)) 413 { 414 return getBuiltinFormatJSON(componentID, subType, version, data, 415 creatorID); 416 } 417 else if (std::find(plugins.begin(), plugins.end(), 418 subsystem + component) != plugins.end()) 419 { 420 return getPythonJSON(componentID, subType, version, data, 421 creatorID); 422 } 423 } 424 catch (const std::exception& e) 425 { 426 log<level::ERR>("Failed parsing UserData", entry("ERROR=%s", e.what()), 427 entry("COMP_ID=0x%X", componentID), 428 entry("SUBTYPE=0x%X", subType), 429 entry("VERSION=%d", version), 430 entry("DATA_LENGTH=%lu\n", data.size())); 431 } 432 433 return std::nullopt; 434 } 435 436 } // namespace openpower::pels::user_data 437