1 /** 2 * Copyright © 2020 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "user_data_json.hpp" 18 19 #include "json_utils.hpp" 20 #include "pel_types.hpp" 21 #include "pel_values.hpp" 22 #include "stream.hpp" 23 #include "user_data_formats.hpp" 24 25 #include <Python.h> 26 27 #include <nlohmann/json.hpp> 28 #include <phosphor-logging/lg2.hpp> 29 30 #include <iomanip> 31 #include <sstream> 32 33 namespace openpower::pels::user_data 34 { 35 namespace pv = openpower::pels::pel_values; 36 using orderedJSON = nlohmann::ordered_json; 37 38 void pyDecRef(PyObject* pyObj) 39 { 40 Py_XDECREF(pyObj); 41 } 42 43 /** 44 * @brief Returns a JSON string for use by PEL::printSectionInJSON(). 45 * 46 * The returning string will contain a JSON object, but without 47 * the outer {}. If the input JSON isn't a JSON object (dict), then 48 * one will be created with the input added to a 'Data' key. 49 * 50 * @param[in] creatorID - The creator ID for the PEL 51 * 52 * @param[in] json - The JSON to convert to a string 53 * 54 * @return std::string - The JSON string 55 */ 56 std::string prettyJSON(uint16_t componentID, uint8_t subType, uint8_t version, 57 uint8_t creatorID, const orderedJSON& json) 58 { 59 orderedJSON output; 60 output[pv::sectionVer] = std::to_string(version); 61 output[pv::subSection] = std::to_string(subType); 62 output[pv::createdBy] = getComponentName(componentID, creatorID); 63 64 if (!json.is_object()) 65 { 66 output["Data"] = json; 67 } 68 else 69 { 70 for (const auto& [key, value] : json.items()) 71 { 72 output[key] = value; 73 } 74 } 75 76 // Let nlohmann do the pretty printing. 77 std::stringstream stream; 78 stream << std::setw(4) << output; 79 80 auto jsonString = stream.str(); 81 82 // Now it looks like: 83 // { 84 // "Section Version": ... 85 // ... 86 // } 87 88 // Since PEL::printSectionInJSON() will supply the outer { }s, 89 // remove the existing ones. 90 91 // Replace the { and the following newline, and the } and its 92 // preceeding newline. 93 jsonString.erase(0, 2); 94 95 auto pos = jsonString.find_last_of('}'); 96 jsonString.erase(pos - 1); 97 98 return jsonString; 99 } 100 101 /** 102 * @brief Return a JSON string from the passed in CBOR data. 103 * 104 * @param[in] componentID - The comp ID from the UserData section header 105 * @param[in] subType - The subtype from the UserData section header 106 * @param[in] version - The version from the UserData section header 107 * @param[in] creatorID - The creator ID for the PEL 108 * @param[in] data - The CBOR data 109 * 110 * @return std::string - The JSON string 111 */ 112 std::string getCBORJSON(uint16_t componentID, uint8_t subType, uint8_t version, 113 uint8_t creatorID, const std::vector<uint8_t>& data) 114 { 115 // The CBOR parser needs the pad bytes added to 4 byte align 116 // removed. The number of bytes added to the pad is on the 117 // very end, so will remove both fields before parsing. 118 119 // If the data vector is too short, an exception will get 120 // thrown which will be handled up the call stack. 121 122 auto cborData = data; 123 uint32_t pad{}; 124 125 Stream stream{cborData}; 126 stream.offset(cborData.size() - 4); 127 stream >> pad; 128 129 if (cborData.size() > (pad + sizeof(pad))) 130 { 131 cborData.resize(data.size() - sizeof(pad) - pad); 132 } 133 134 orderedJSON json = orderedJSON::from_cbor(cborData); 135 136 return prettyJSON(componentID, subType, version, creatorID, json); 137 } 138 139 /** 140 * @brief Return a JSON string from the passed in text data. 141 * 142 * The function breaks up the input text into a vector of strings with 143 * newline as separator and converts that into JSON. It will convert any 144 * unprintable characters to periods. 145 * 146 * @param[in] componentID - The comp ID from the UserData section header 147 * @param[in] subType - The subtype from the UserData section header 148 * @param[in] version - The version from the UserData section header 149 * @param[in] creatorID - The creator ID for the PEL 150 * @param[in] data - The CBOR data 151 * 152 * @return std::string - The JSON string 153 */ 154 std::string getTextJSON(uint16_t componentID, uint8_t subType, uint8_t version, 155 uint8_t creatorID, const std::vector<uint8_t>& data) 156 { 157 std::vector<std::string> text; 158 size_t startPos = 0; 159 160 // Converts any unprintable characters to periods 161 auto validate = [](char& ch) { 162 if ((ch < ' ') || (ch > '~')) 163 { 164 ch = '.'; 165 } 166 }; 167 168 // Break up the data into an array of strings with newline as separator 169 for (size_t pos = 0; pos < data.size(); ++pos) 170 { 171 if (data[pos] == '\n') 172 { 173 std::string line{reinterpret_cast<const char*>(&data[startPos]), 174 pos - startPos}; 175 std::for_each(line.begin(), line.end(), validate); 176 text.push_back(std::move(line)); 177 startPos = pos + 1; 178 } 179 } 180 if (startPos < data.size()) 181 { 182 std::string line{reinterpret_cast<const char*>(&data[startPos]), 183 data.size() - startPos}; 184 std::for_each(line.begin(), line.end(), validate); 185 text.push_back(std::move(line)); 186 } 187 188 orderedJSON json = text; 189 return prettyJSON(componentID, subType, version, creatorID, json); 190 } 191 192 /** 193 * @brief Convert to an appropriate JSON string as the data is one of 194 * the formats that we natively support. 195 * 196 * @param[in] componentID - The comp ID from the UserData section header 197 * @param[in] subType - The subtype from the UserData section header 198 * @param[in] version - The version from the UserData section header 199 * @param[in] data - The data itself 200 * 201 * @return std::optional<std::string> - The JSON string if it could be created, 202 * else std::nullopt. 203 */ 204 std::optional<std::string> 205 getBuiltinFormatJSON(uint16_t componentID, uint8_t subType, uint8_t version, 206 const std::vector<uint8_t>& data, uint8_t creatorID) 207 { 208 switch (subType) 209 { 210 case static_cast<uint8_t>(UserDataFormat::json): 211 { 212 std::string jsonString{data.begin(), data.begin() + data.size()}; 213 214 orderedJSON json = orderedJSON::parse(jsonString); 215 216 return prettyJSON(componentID, subType, version, creatorID, json); 217 } 218 case static_cast<uint8_t>(UserDataFormat::cbor): 219 { 220 return getCBORJSON(componentID, subType, version, creatorID, data); 221 } 222 case static_cast<uint8_t>(UserDataFormat::text): 223 { 224 return getTextJSON(componentID, subType, version, creatorID, data); 225 } 226 default: 227 break; 228 } 229 return std::nullopt; 230 } 231 232 /** 233 * @brief Call Python modules to parse the data into a JSON string 234 * 235 * The module to call is based on the Creator Subsystem ID and the Component 236 * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy" 237 * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID. 238 * 239 * All modules must provide the following: 240 * Function: parseUDToJson 241 * Argument list: 242 * 1. (int) Sub-section type 243 * 2. (int) Section version 244 * 3. (memoryview): Data 245 *-Return data: 246 * 1. (str) JSON string 247 * 248 * @param[in] componentID - The comp ID from the UserData section header 249 * @param[in] subType - The subtype from the UserData section header 250 * @param[in] version - The version from the UserData section header 251 * @param[in] data - The data itself 252 * @param[in] creatorID - The creatorID from the PrivateHeader section 253 * @return std::optional<std::string> - The JSON string if it could be created, 254 * else std::nullopt 255 */ 256 std::optional<std::string> getPythonJSON(uint16_t componentID, uint8_t subType, 257 uint8_t version, 258 const std::vector<uint8_t>& data, 259 uint8_t creatorID) 260 { 261 PyObject *pName, *pModule, *eType, *eValue, *eTraceback, *pKey; 262 std::string pErrStr; 263 std::string module = getNumberString("%c", tolower(creatorID)) + 264 getNumberString("%04x", componentID); 265 pName = PyUnicode_FromString( 266 std::string("udparsers." + module + "." + module).c_str()); 267 std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef); 268 pModule = PyImport_Import(pName); 269 if (pModule == NULL) 270 { 271 pErrStr = "No error string found"; 272 PyErr_Fetch(&eType, &eValue, &eTraceback); 273 if (eType) 274 { 275 Py_XDECREF(eType); 276 } 277 if (eTraceback) 278 { 279 Py_XDECREF(eTraceback); 280 } 281 if (eValue) 282 { 283 PyObject* pStr = PyObject_Str(eValue); 284 Py_XDECREF(eValue); 285 if (pStr) 286 { 287 pErrStr = PyUnicode_AsUTF8(pStr); 288 Py_XDECREF(pStr); 289 } 290 } 291 } 292 else 293 { 294 std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr(pModule, 295 &pyDecRef); 296 std::string funcToCall = "parseUDToJson"; 297 pKey = PyUnicode_FromString(funcToCall.c_str()); 298 std::unique_ptr<PyObject, decltype(&pyDecRef)> keyPtr(pKey, &pyDecRef); 299 PyObject* pDict = PyModule_GetDict(pModule); 300 Py_INCREF(pDict); 301 if (!PyDict_Contains(pDict, pKey)) 302 { 303 Py_DECREF(pDict); 304 lg2::error("Python module error. Function missing: {FUNC}, " 305 "module = {MODULE}, subtype = {SUBTYPE}, " 306 "version = {VERSION}, data length = {LEN}", 307 "FUNC", funcToCall, "MODULE", module, "SUBTYPE", subType, 308 "VERSION", version, "LEN", data.size()); 309 return std::nullopt; 310 } 311 PyObject* pFunc = PyDict_GetItemString(pDict, funcToCall.c_str()); 312 Py_DECREF(pDict); 313 Py_INCREF(pFunc); 314 if (PyCallable_Check(pFunc)) 315 { 316 auto ud = data.data(); 317 PyObject* pArgs = PyTuple_New(3); 318 std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr(pArgs, 319 &pyDecRef); 320 PyTuple_SetItem(pArgs, 0, 321 PyLong_FromUnsignedLong((unsigned long)subType)); 322 PyTuple_SetItem(pArgs, 1, 323 PyLong_FromUnsignedLong((unsigned long)version)); 324 PyObject* pData = PyMemoryView_FromMemory( 325 reinterpret_cast<char*>(const_cast<unsigned char*>(ud)), 326 data.size(), PyBUF_READ); 327 PyTuple_SetItem(pArgs, 2, pData); 328 PyObject* pResult = PyObject_CallObject(pFunc, pArgs); 329 Py_DECREF(pFunc); 330 if (pResult) 331 { 332 std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr( 333 pResult, &pyDecRef); 334 PyObject* pBytes = PyUnicode_AsEncodedString(pResult, "utf-8", 335 "~E~"); 336 std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr( 337 pBytes, &pyDecRef); 338 const char* output = PyBytes_AS_STRING(pBytes); 339 try 340 { 341 orderedJSON json = orderedJSON::parse(output); 342 if ((json.is_object() && !json.empty()) || 343 (json.is_array() && json.size() > 0) || 344 (json.is_string() && json != "")) 345 { 346 return prettyJSON(componentID, subType, version, 347 creatorID, json); 348 } 349 } 350 catch (const std::exception& e) 351 { 352 lg2::error("Bad JSON from parser. Error = {ERROR}, " 353 "module = {MODULE}, subtype = {SUBTYPE}, " 354 "version = {VERSION}, data length = {LEN}", 355 "ERROR", e, "MODULE", module, "SUBTYPE", subType, 356 "VERSION", version, "LEN", data.size()); 357 return std::nullopt; 358 } 359 } 360 else 361 { 362 pErrStr = "No error string found"; 363 PyErr_Fetch(&eType, &eValue, &eTraceback); 364 if (eType) 365 { 366 Py_XDECREF(eType); 367 } 368 if (eTraceback) 369 { 370 Py_XDECREF(eTraceback); 371 } 372 if (eValue) 373 { 374 PyObject* pStr = PyObject_Str(eValue); 375 Py_XDECREF(eValue); 376 if (pStr) 377 { 378 pErrStr = PyUnicode_AsUTF8(pStr); 379 Py_XDECREF(pStr); 380 } 381 } 382 } 383 } 384 } 385 if (!pErrStr.empty()) 386 { 387 lg2::debug("Python exception thrown by parser. Error = {ERROR}, " 388 "module = {MODULE}, subtype = {SUBTYPE}, " 389 "version = {VERSION}, data length = {LEN}", 390 "ERROR", pErrStr, "MODULE", module, "SUBTYPE", subType, 391 "VERSION", version, "LEN", data.size()); 392 } 393 return std::nullopt; 394 } 395 396 std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType, 397 uint8_t version, 398 const std::vector<uint8_t>& data, 399 uint8_t creatorID, 400 const std::vector<std::string>& plugins) 401 { 402 std::string subsystem = getNumberString("%c", tolower(creatorID)); 403 std::string component = getNumberString("%04x", componentID); 404 try 405 { 406 if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" && 407 componentID == static_cast<uint16_t>(ComponentID::phosphorLogging)) 408 { 409 return getBuiltinFormatJSON(componentID, subType, version, data, 410 creatorID); 411 } 412 else if (std::find(plugins.begin(), plugins.end(), 413 subsystem + component) != plugins.end()) 414 { 415 return getPythonJSON(componentID, subType, version, data, 416 creatorID); 417 } 418 } 419 catch (const std::exception& e) 420 { 421 lg2::error("Failed parsing UserData. Error = {ERROR}, " 422 "component ID = {COMP_ID}, subtype = {SUBTYPE}, " 423 "version = {VERSION}, data length = {LEN}", 424 "ERROR", e, "COMP_ID", componentID, "SUBTYPE", subType, 425 "VERSION", version, "LEN", data.size()); 426 } 427 428 return std::nullopt; 429 } 430 431 } // namespace openpower::pels::user_data 432