1 /** 2 * Copyright © 2020 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "user_data_json.hpp" 18 19 #include "json_utils.hpp" 20 #include "pel_types.hpp" 21 #include "pel_values.hpp" 22 #include "stream.hpp" 23 #include "user_data_formats.hpp" 24 25 #include <Python.h> 26 27 #include <iomanip> 28 #include <nlohmann/json.hpp> 29 #include <phosphor-logging/log.hpp> 30 #include <sstream> 31 32 namespace openpower::pels::user_data 33 { 34 namespace pv = openpower::pels::pel_values; 35 using namespace phosphor::logging; 36 using orderedJSON = nlohmann::ordered_json; 37 38 void pyDecRef(PyObject* pyObj) 39 { 40 Py_XDECREF(pyObj); 41 } 42 43 /** 44 * @brief Returns a JSON string for use by PEL::printSectionInJSON(). 45 * 46 * The returning string will contain a JSON object, but without 47 * the outer {}. If the input JSON isn't a JSON object (dict), then 48 * one will be created with the input added to a 'Data' key. 49 * 50 * @param[in] json - The JSON to convert to a string 51 * 52 * @return std::string - The JSON string 53 */ 54 std::string prettyJSON(uint16_t componentID, uint8_t subType, uint8_t version, 55 const orderedJSON& json) 56 { 57 orderedJSON output; 58 output[pv::sectionVer] = std::to_string(version); 59 output[pv::subSection] = std::to_string(subType); 60 output[pv::createdBy] = getNumberString("0x%04X", componentID); 61 62 if (!json.is_object()) 63 { 64 output["Data"] = json; 65 } 66 else 67 { 68 for (const auto& [key, value] : json.items()) 69 { 70 output[key] = value; 71 } 72 } 73 74 // Let nlohmann do the pretty printing. 75 std::stringstream stream; 76 stream << std::setw(4) << output; 77 78 auto jsonString = stream.str(); 79 80 // Now it looks like: 81 // { 82 // "Section Version": ... 83 // ... 84 // } 85 86 // Since PEL::printSectionInJSON() will supply the outer { }s, 87 // remove the existing ones. 88 89 // Replace the { and the following newline, and the } and its 90 // preceeding newline. 91 jsonString.erase(0, 2); 92 93 auto pos = jsonString.find_last_of('}'); 94 jsonString.erase(pos - 1); 95 96 return jsonString; 97 } 98 99 /** 100 * @brief Return a JSON string from the passed in CBOR data. 101 * 102 * @param[in] componentID - The comp ID from the UserData section header 103 * @param[in] subType - The subtype from the UserData section header 104 * @param[in] version - The version from the UserData section header 105 * @param[in] data - The CBOR data 106 * 107 * @return std::string - The JSON string 108 */ 109 std::string getCBORJSON(uint16_t componentID, uint8_t subType, uint8_t version, 110 const std::vector<uint8_t>& data) 111 { 112 // The CBOR parser needs the pad bytes added to 4 byte align 113 // removed. The number of bytes added to the pad is on the 114 // very end, so will remove both fields before parsing. 115 116 // If the data vector is too short, an exception will get 117 // thrown which will be handled up the call stack. 118 119 auto cborData = data; 120 uint32_t pad{}; 121 122 Stream stream{cborData}; 123 stream.offset(cborData.size() - 4); 124 stream >> pad; 125 126 if (cborData.size() > (pad + sizeof(pad))) 127 { 128 cborData.resize(data.size() - sizeof(pad) - pad); 129 } 130 131 orderedJSON json = orderedJSON::from_cbor(cborData); 132 133 return prettyJSON(componentID, subType, version, json); 134 } 135 136 /** 137 * @brief Return a JSON string from the passed in text data. 138 * 139 * The function breaks up the input text into a vector of strings with 140 * newline as separator and converts that into JSON. It will convert any 141 * unprintable characters to periods. 142 * 143 * @param[in] componentID - The comp ID from the UserData section header 144 * @param[in] subType - The subtype from the UserData section header 145 * @param[in] version - The version from the UserData section header 146 * @param[in] data - The CBOR data 147 * 148 * @return std::string - The JSON string 149 */ 150 std::string getTextJSON(uint16_t componentID, uint8_t subType, uint8_t version, 151 const std::vector<uint8_t>& data) 152 { 153 std::vector<std::string> text; 154 size_t startPos = 0; 155 156 // Converts any unprintable characters to periods 157 auto validate = [](char& ch) { 158 if ((ch < ' ') || (ch > '~')) 159 { 160 ch = '.'; 161 } 162 }; 163 164 // Break up the data into an array of strings with newline as separator 165 for (size_t pos = 0; pos < data.size(); ++pos) 166 { 167 if (data[pos] == '\n') 168 { 169 std::string line{reinterpret_cast<const char*>(&data[startPos]), 170 pos - startPos}; 171 std::for_each(line.begin(), line.end(), validate); 172 text.push_back(std::move(line)); 173 startPos = pos + 1; 174 } 175 } 176 if (startPos < data.size()) 177 { 178 std::string line{reinterpret_cast<const char*>(&data[startPos]), 179 data.size() - startPos}; 180 std::for_each(line.begin(), line.end(), validate); 181 text.push_back(std::move(line)); 182 } 183 184 orderedJSON json = text; 185 return prettyJSON(componentID, subType, version, json); 186 } 187 188 /** 189 * @brief Convert to an appropriate JSON string as the data is one of 190 * the formats that we natively support. 191 * 192 * @param[in] componentID - The comp ID from the UserData section header 193 * @param[in] subType - The subtype from the UserData section header 194 * @param[in] version - The version from the UserData section header 195 * @param[in] data - The data itself 196 * 197 * @return std::optional<std::string> - The JSON string if it could be created, 198 * else std::nullopt. 199 */ 200 std::optional<std::string> 201 getBuiltinFormatJSON(uint16_t componentID, uint8_t subType, uint8_t version, 202 const std::vector<uint8_t>& data) 203 { 204 switch (subType) 205 { 206 case static_cast<uint8_t>(UserDataFormat::json): 207 { 208 std::string jsonString{data.begin(), data.begin() + data.size()}; 209 210 orderedJSON json = orderedJSON::parse(jsonString); 211 212 return prettyJSON(componentID, subType, version, json); 213 } 214 case static_cast<uint8_t>(UserDataFormat::cbor): 215 { 216 return getCBORJSON(componentID, subType, version, data); 217 } 218 case static_cast<uint8_t>(UserDataFormat::text): 219 { 220 return getTextJSON(componentID, subType, version, data); 221 } 222 default: 223 break; 224 } 225 return std::nullopt; 226 } 227 228 /** 229 * @brief Call Python modules to parse the data into a JSON string 230 * 231 * The module to call is based on the Creator Subsystem ID and the Component 232 * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy" 233 * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID. 234 * 235 * All modules must provide the following: 236 * Function: parseUDToJson 237 * Argument list: 238 * 1. (int) Sub-section type 239 * 2. (int) Section version 240 * 3. (memoryview): Data 241 *-Return data: 242 * 1. (str) JSON string 243 * 244 * @param[in] componentID - The comp ID from the UserData section header 245 * @param[in] subType - The subtype from the UserData section header 246 * @param[in] version - The version from the UserData section header 247 * @param[in] data - The data itself 248 * @param[in] creatorID - The creatorID from the PrivateHeader section 249 * @return std::optional<std::string> - The JSON string if it could be created, 250 * else std::nullopt 251 */ 252 std::optional<std::string> getPythonJSON(uint16_t componentID, uint8_t subType, 253 uint8_t version, 254 const std::vector<uint8_t>& data, 255 uint8_t creatorID) 256 { 257 PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pData, *pResult, 258 *pBytes, *eType, *eValue, *eTraceback, *pKey; 259 std::string pErrStr; 260 std::string module = getNumberString("%c", tolower(creatorID)) + 261 getNumberString("%04x", componentID); 262 pName = PyUnicode_FromString( 263 std::string("udparsers." + module + "." + module).c_str()); 264 std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef); 265 pModule = PyImport_Import(pName); 266 if (pModule == NULL) 267 { 268 pErrStr = "No error string found"; 269 PyErr_Fetch(&eType, &eValue, &eTraceback); 270 if (eType) 271 { 272 Py_XDECREF(eType); 273 } 274 if (eTraceback) 275 { 276 Py_XDECREF(eTraceback); 277 } 278 if (eValue) 279 { 280 PyObject* pStr = PyObject_Str(eValue); 281 Py_XDECREF(eValue); 282 if (pStr) 283 { 284 pErrStr = PyUnicode_AsUTF8(pStr); 285 Py_XDECREF(pStr); 286 } 287 } 288 } 289 else 290 { 291 std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr(pModule, 292 &pyDecRef); 293 std::string funcToCall = "parseUDToJson"; 294 pKey = PyUnicode_FromString(funcToCall.c_str()); 295 std::unique_ptr<PyObject, decltype(&pyDecRef)> keyPtr(pKey, &pyDecRef); 296 pDict = PyModule_GetDict(pModule); 297 Py_INCREF(pDict); 298 if (!PyDict_Contains(pDict, pKey)) 299 { 300 Py_DECREF(pDict); 301 log<level::ERR>( 302 "Python module error", 303 entry("ERROR=%s", 304 std::string(funcToCall + " function missing").c_str()), 305 entry("PARSER_MODULE=%s", module.c_str()), 306 entry("SUBTYPE=0x%X", subType), entry("VERSION=%d", version), 307 entry("DATA_LENGTH=%lu\n", data.size())); 308 return std::nullopt; 309 } 310 pFunc = PyDict_GetItemString(pDict, funcToCall.c_str()); 311 Py_DECREF(pDict); 312 Py_INCREF(pFunc); 313 if (PyCallable_Check(pFunc)) 314 { 315 auto ud = data.data(); 316 pArgs = PyTuple_New(3); 317 std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr(pArgs, 318 &pyDecRef); 319 PyTuple_SetItem(pArgs, 0, 320 PyLong_FromUnsignedLong((unsigned long)subType)); 321 PyTuple_SetItem(pArgs, 1, 322 PyLong_FromUnsignedLong((unsigned long)version)); 323 pData = PyMemoryView_FromMemory( 324 reinterpret_cast<char*>(const_cast<unsigned char*>(ud)), 325 data.size(), PyBUF_READ); 326 PyTuple_SetItem(pArgs, 2, pData); 327 pResult = PyObject_CallObject(pFunc, pArgs); 328 Py_DECREF(pFunc); 329 if (pResult) 330 { 331 std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr( 332 pResult, &pyDecRef); 333 pBytes = PyUnicode_AsEncodedString(pResult, "utf-8", "~E~"); 334 std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr( 335 pBytes, &pyDecRef); 336 const char* output = PyBytes_AS_STRING(pBytes); 337 try 338 { 339 orderedJSON json = orderedJSON::parse(output); 340 if ((json.is_object() && !json.empty()) || 341 (json.is_array() && json.size() > 0) || 342 (json.is_string() && json != "")) 343 { 344 return prettyJSON(componentID, subType, version, json); 345 } 346 } 347 catch (const std::exception& e) 348 { 349 log<level::ERR>("Bad JSON from parser", 350 entry("ERROR=%s", e.what()), 351 entry("PARSER_MODULE=%s", module.c_str()), 352 entry("SUBTYPE=0x%X", subType), 353 entry("VERSION=%d", version), 354 entry("DATA_LENGTH=%lu\n", data.size())); 355 return std::nullopt; 356 } 357 } 358 else 359 { 360 pErrStr = "No error string found"; 361 PyErr_Fetch(&eType, &eValue, &eTraceback); 362 if (eType) 363 { 364 Py_XDECREF(eType); 365 } 366 if (eTraceback) 367 { 368 Py_XDECREF(eTraceback); 369 } 370 if (eValue) 371 { 372 PyObject* pStr = PyObject_Str(eValue); 373 Py_XDECREF(eValue); 374 if (pStr) 375 { 376 pErrStr = PyUnicode_AsUTF8(pStr); 377 Py_XDECREF(pStr); 378 } 379 } 380 } 381 } 382 } 383 if (!pErrStr.empty()) 384 { 385 log<level::DEBUG>("Python exception thrown by parser", 386 entry("ERROR=%s", pErrStr.c_str()), 387 entry("PARSER_MODULE=%s", module.c_str()), 388 entry("SUBTYPE=0x%X", subType), 389 entry("VERSION=%d", version), 390 entry("DATA_LENGTH=%lu\n", data.size())); 391 } 392 return std::nullopt; 393 } 394 395 std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType, 396 uint8_t version, 397 const std::vector<uint8_t>& data, 398 uint8_t creatorID, 399 const std::vector<std::string>& plugins) 400 { 401 std::string subsystem = getNumberString("%c", tolower(creatorID)); 402 std::string component = getNumberString("%04x", componentID); 403 try 404 { 405 if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" && 406 componentID == static_cast<uint16_t>(ComponentID::phosphorLogging)) 407 { 408 return getBuiltinFormatJSON(componentID, subType, version, data); 409 } 410 else if (std::find(plugins.begin(), plugins.end(), 411 subsystem + component) != plugins.end()) 412 { 413 return getPythonJSON(componentID, subType, version, data, 414 creatorID); 415 } 416 } 417 catch (const std::exception& e) 418 { 419 log<level::ERR>("Failed parsing UserData", entry("ERROR=%s", e.what()), 420 entry("COMP_ID=0x%X", componentID), 421 entry("SUBTYPE=0x%X", subType), 422 entry("VERSION=%d", version), 423 entry("DATA_LENGTH=%lu\n", data.size())); 424 } 425 426 return std::nullopt; 427 } 428 429 } // namespace openpower::pels::user_data 430