1 /** 2 * Copyright © 2020 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "user_data_json.hpp" 18 19 #include "json_utils.hpp" 20 #include "pel_types.hpp" 21 #include "pel_values.hpp" 22 #include "stream.hpp" 23 #include "user_data_formats.hpp" 24 25 #include <Python.h> 26 27 #include <nlohmann/json.hpp> 28 #include <phosphor-logging/lg2.hpp> 29 30 #include <iomanip> 31 #include <sstream> 32 33 namespace openpower::pels::user_data 34 { 35 namespace pv = openpower::pels::pel_values; 36 using orderedJSON = nlohmann::ordered_json; 37 38 void pyDecRef(PyObject* pyObj) 39 { 40 Py_XDECREF(pyObj); 41 } 42 43 /** 44 * @brief Returns a JSON string for use by PEL::printSectionInJSON(). 45 * 46 * The returning string will contain a JSON object, but without 47 * the outer {}. If the input JSON isn't a JSON object (dict), then 48 * one will be created with the input added to a 'Data' key. 49 * 50 * @param[in] creatorID - The creator ID for the PEL 51 * 52 * @param[in] json - The JSON to convert to a string 53 * 54 * @return std::string - The JSON string 55 */ 56 std::string prettyJSON(uint16_t componentID, uint8_t subType, uint8_t version, 57 uint8_t creatorID, const orderedJSON& json) 58 { 59 orderedJSON output; 60 output[pv::sectionVer] = std::to_string(version); 61 output[pv::subSection] = std::to_string(subType); 62 output[pv::createdBy] = getComponentName(componentID, creatorID); 63 64 if (!json.is_object()) 65 { 66 output["Data"] = json; 67 } 68 else 69 { 70 for (const auto& [key, value] : json.items()) 71 { 72 output[key] = value; 73 } 74 } 75 76 // Let nlohmann do the pretty printing. 77 std::stringstream stream; 78 stream << std::setw(4) << output; 79 80 auto jsonString = stream.str(); 81 82 // Now it looks like: 83 // { 84 // "Section Version": ... 85 // ... 86 // } 87 88 // Since PEL::printSectionInJSON() will supply the outer { }s, 89 // remove the existing ones. 90 91 // Replace the { and the following newline, and the } and its 92 // preceeding newline. 93 jsonString.erase(0, 2); 94 95 auto pos = jsonString.find_last_of('}'); 96 jsonString.erase(pos - 1); 97 98 return jsonString; 99 } 100 101 /** 102 * @brief Return a JSON string from the passed in CBOR data. 103 * 104 * @param[in] componentID - The comp ID from the UserData section header 105 * @param[in] subType - The subtype from the UserData section header 106 * @param[in] version - The version from the UserData section header 107 * @param[in] creatorID - The creator ID for the PEL 108 * @param[in] data - The CBOR data 109 * 110 * @return std::string - The JSON string 111 */ 112 std::string getCBORJSON(uint16_t componentID, uint8_t subType, uint8_t version, 113 uint8_t creatorID, const std::vector<uint8_t>& data) 114 { 115 // The CBOR parser needs the pad bytes added to 4 byte align 116 // removed. The number of bytes added to the pad is on the 117 // very end, so will remove both fields before parsing. 118 119 // If the data vector is too short, an exception will get 120 // thrown which will be handled up the call stack. 121 122 auto cborData = data; 123 uint32_t pad{}; 124 125 Stream stream{cborData}; 126 stream.offset(cborData.size() - 4); 127 stream >> pad; 128 129 if (cborData.size() > (pad + sizeof(pad))) 130 { 131 cborData.resize(data.size() - sizeof(pad) - pad); 132 } 133 134 orderedJSON json = orderedJSON::from_cbor(cborData); 135 136 return prettyJSON(componentID, subType, version, creatorID, json); 137 } 138 139 /** 140 * @brief Return a JSON string from the passed in text data. 141 * 142 * The function breaks up the input text into a vector of strings with 143 * newline as separator and converts that into JSON. It will convert any 144 * unprintable characters to periods. 145 * 146 * @param[in] componentID - The comp ID from the UserData section header 147 * @param[in] subType - The subtype from the UserData section header 148 * @param[in] version - The version from the UserData section header 149 * @param[in] creatorID - The creator ID for the PEL 150 * @param[in] data - The CBOR data 151 * 152 * @return std::string - The JSON string 153 */ 154 std::string getTextJSON(uint16_t componentID, uint8_t subType, uint8_t version, 155 uint8_t creatorID, const std::vector<uint8_t>& data) 156 { 157 std::vector<std::string> text; 158 size_t startPos = 0; 159 160 // Converts any unprintable characters to periods 161 auto validate = [](char& ch) { 162 if ((ch < ' ') || (ch > '~')) 163 { 164 ch = '.'; 165 } 166 }; 167 168 // Break up the data into an array of strings with newline as separator 169 for (size_t pos = 0; pos < data.size(); ++pos) 170 { 171 if (data[pos] == '\n') 172 { 173 std::string line{reinterpret_cast<const char*>(&data[startPos]), 174 pos - startPos}; 175 std::for_each(line.begin(), line.end(), validate); 176 text.push_back(std::move(line)); 177 startPos = pos + 1; 178 } 179 } 180 if (startPos < data.size()) 181 { 182 std::string line{reinterpret_cast<const char*>(&data[startPos]), 183 data.size() - startPos}; 184 std::for_each(line.begin(), line.end(), validate); 185 text.push_back(std::move(line)); 186 } 187 188 orderedJSON json = text; 189 return prettyJSON(componentID, subType, version, creatorID, json); 190 } 191 192 /** 193 * @brief Convert to an appropriate JSON string as the data is one of 194 * the formats that we natively support. 195 * 196 * @param[in] componentID - The comp ID from the UserData section header 197 * @param[in] subType - The subtype from the UserData section header 198 * @param[in] version - The version from the UserData section header 199 * @param[in] data - The data itself 200 * 201 * @return std::optional<std::string> - The JSON string if it could be created, 202 * else std::nullopt. 203 */ 204 std::optional<std::string> 205 getBuiltinFormatJSON(uint16_t componentID, uint8_t subType, uint8_t version, 206 const std::vector<uint8_t>& data, uint8_t creatorID) 207 { 208 switch (subType) 209 { 210 case static_cast<uint8_t>(UserDataFormat::json): 211 { 212 std::string jsonString{data.begin(), data.begin() + data.size()}; 213 214 orderedJSON json = orderedJSON::parse(jsonString); 215 216 return prettyJSON(componentID, subType, version, creatorID, json); 217 } 218 case static_cast<uint8_t>(UserDataFormat::cbor): 219 { 220 return getCBORJSON(componentID, subType, version, creatorID, data); 221 } 222 case static_cast<uint8_t>(UserDataFormat::text): 223 { 224 return getTextJSON(componentID, subType, version, creatorID, data); 225 } 226 default: 227 break; 228 } 229 return std::nullopt; 230 } 231 232 /** 233 * @brief Call Python modules to parse the data into a JSON string 234 * 235 * The module to call is based on the Creator Subsystem ID and the Component 236 * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy" 237 * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID. 238 * 239 * All modules must provide the following: 240 * Function: parseUDToJson 241 * Argument list: 242 * 1. (int) Sub-section type 243 * 2. (int) Section version 244 * 3. (memoryview): Data 245 *-Return data: 246 * 1. (str) JSON string 247 * 248 * @param[in] componentID - The comp ID from the UserData section header 249 * @param[in] subType - The subtype from the UserData section header 250 * @param[in] version - The version from the UserData section header 251 * @param[in] data - The data itself 252 * @param[in] creatorID - The creatorID from the PrivateHeader section 253 * @return std::optional<std::string> - The JSON string if it could be created, 254 * else std::nullopt 255 */ 256 std::optional<std::string> 257 getPythonJSON(uint16_t componentID, uint8_t subType, uint8_t version, 258 const std::vector<uint8_t>& data, uint8_t creatorID) 259 { 260 PyObject *pName, *pModule, *eType, *eValue, *eTraceback, *pKey; 261 std::string pErrStr; 262 std::string module = getNumberString("%c", tolower(creatorID)) + 263 getNumberString("%04x", componentID); 264 pName = PyUnicode_FromString( 265 std::string("udparsers." + module + "." + module).c_str()); 266 std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef); 267 pModule = PyImport_Import(pName); 268 if (pModule == NULL) 269 { 270 pErrStr = "No error string found"; 271 PyErr_Fetch(&eType, &eValue, &eTraceback); 272 if (eType) 273 { 274 Py_XDECREF(eType); 275 } 276 if (eTraceback) 277 { 278 Py_XDECREF(eTraceback); 279 } 280 if (eValue) 281 { 282 PyObject* pStr = PyObject_Str(eValue); 283 Py_XDECREF(eValue); 284 if (pStr) 285 { 286 pErrStr = PyUnicode_AsUTF8(pStr); 287 Py_XDECREF(pStr); 288 } 289 } 290 } 291 else 292 { 293 std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr( 294 pModule, &pyDecRef); 295 std::string funcToCall = "parseUDToJson"; 296 pKey = PyUnicode_FromString(funcToCall.c_str()); 297 std::unique_ptr<PyObject, decltype(&pyDecRef)> keyPtr(pKey, &pyDecRef); 298 PyObject* pDict = PyModule_GetDict(pModule); 299 Py_INCREF(pDict); 300 if (!PyDict_Contains(pDict, pKey)) 301 { 302 Py_DECREF(pDict); 303 lg2::error("Python module error. Function missing: {FUNC}, " 304 "module = {MODULE}, subtype = {SUBTYPE}, " 305 "version = {VERSION}, data length = {LEN}", 306 "FUNC", funcToCall, "MODULE", module, "SUBTYPE", subType, 307 "VERSION", version, "LEN", data.size()); 308 return std::nullopt; 309 } 310 PyObject* pFunc = PyDict_GetItemString(pDict, funcToCall.c_str()); 311 Py_DECREF(pDict); 312 Py_INCREF(pFunc); 313 if (PyCallable_Check(pFunc)) 314 { 315 auto ud = data.data(); 316 PyObject* pArgs = PyTuple_New(3); 317 std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr( 318 pArgs, &pyDecRef); 319 PyTuple_SetItem(pArgs, 0, 320 PyLong_FromUnsignedLong((unsigned long)subType)); 321 PyTuple_SetItem(pArgs, 1, 322 PyLong_FromUnsignedLong((unsigned long)version)); 323 PyObject* pData = PyMemoryView_FromMemory( 324 reinterpret_cast<char*>(const_cast<unsigned char*>(ud)), 325 data.size(), PyBUF_READ); 326 PyTuple_SetItem(pArgs, 2, pData); 327 PyObject* pResult = PyObject_CallObject(pFunc, pArgs); 328 Py_DECREF(pFunc); 329 if (pResult) 330 { 331 std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr( 332 pResult, &pyDecRef); 333 PyObject* pBytes = 334 PyUnicode_AsEncodedString(pResult, "utf-8", "~E~"); 335 std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr( 336 pBytes, &pyDecRef); 337 const char* output = PyBytes_AS_STRING(pBytes); 338 try 339 { 340 orderedJSON json = orderedJSON::parse(output); 341 if ((json.is_object() && !json.empty()) || 342 (json.is_array() && json.size() > 0) || 343 (json.is_string() && json != "")) 344 { 345 return prettyJSON(componentID, subType, version, 346 creatorID, json); 347 } 348 } 349 catch (const std::exception& e) 350 { 351 lg2::error("Bad JSON from parser. Error = {ERROR}, " 352 "module = {MODULE}, subtype = {SUBTYPE}, " 353 "version = {VERSION}, data length = {LEN}", 354 "ERROR", e, "MODULE", module, "SUBTYPE", subType, 355 "VERSION", version, "LEN", data.size()); 356 return std::nullopt; 357 } 358 } 359 else 360 { 361 pErrStr = "No error string found"; 362 PyErr_Fetch(&eType, &eValue, &eTraceback); 363 if (eType) 364 { 365 Py_XDECREF(eType); 366 } 367 if (eTraceback) 368 { 369 Py_XDECREF(eTraceback); 370 } 371 if (eValue) 372 { 373 PyObject* pStr = PyObject_Str(eValue); 374 Py_XDECREF(eValue); 375 if (pStr) 376 { 377 pErrStr = PyUnicode_AsUTF8(pStr); 378 Py_XDECREF(pStr); 379 } 380 } 381 } 382 } 383 } 384 if (!pErrStr.empty()) 385 { 386 lg2::debug("Python exception thrown by parser. Error = {ERROR}, " 387 "module = {MODULE}, subtype = {SUBTYPE}, " 388 "version = {VERSION}, data length = {LEN}", 389 "ERROR", pErrStr, "MODULE", module, "SUBTYPE", subType, 390 "VERSION", version, "LEN", data.size()); 391 } 392 return std::nullopt; 393 } 394 395 std::optional<std::string> 396 getJSON(uint16_t componentID, uint8_t subType, uint8_t version, 397 const std::vector<uint8_t>& data, uint8_t creatorID, 398 const std::vector<std::string>& plugins) 399 { 400 std::string subsystem = getNumberString("%c", tolower(creatorID)); 401 std::string component = getNumberString("%04x", componentID); 402 try 403 { 404 if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" && 405 componentID == static_cast<uint16_t>(ComponentID::phosphorLogging)) 406 { 407 return getBuiltinFormatJSON(componentID, subType, version, data, 408 creatorID); 409 } 410 else if (std::find(plugins.begin(), plugins.end(), 411 subsystem + component) != plugins.end()) 412 { 413 return getPythonJSON(componentID, subType, version, data, 414 creatorID); 415 } 416 } 417 catch (const std::exception& e) 418 { 419 lg2::error("Failed parsing UserData. Error = {ERROR}, " 420 "component ID = {COMP_ID}, subtype = {SUBTYPE}, " 421 "version = {VERSION}, data length = {LEN}", 422 "ERROR", e, "COMP_ID", componentID, "SUBTYPE", subType, 423 "VERSION", version, "LEN", data.size()); 424 } 425 426 return std::nullopt; 427 } 428 429 } // namespace openpower::pels::user_data 430