1 /** 2 * Copyright © 2020 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "user_data_json.hpp" 18 19 #include "json_utils.hpp" 20 #include "pel_types.hpp" 21 #include "pel_values.hpp" 22 #include "stream.hpp" 23 #include "user_data_formats.hpp" 24 25 #include <Python.h> 26 27 #include <fifo_map.hpp> 28 #include <iomanip> 29 #include <nlohmann/json.hpp> 30 #include <phosphor-logging/log.hpp> 31 #include <sstream> 32 33 namespace openpower::pels::user_data 34 { 35 namespace pv = openpower::pels::pel_values; 36 using namespace phosphor::logging; 37 38 // Use fifo_map as nlohmann::json's map. We are just ignoring the 'less' 39 // compare. With this map the keys are kept in FIFO order. 40 template <class K, class V, class dummy_compare, class A> 41 using fifoMap = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>; 42 using fifoJSON = nlohmann::basic_json<fifoMap>; 43 44 void pyDecRef(PyObject* pyObj) 45 { 46 Py_XDECREF(pyObj); 47 } 48 49 /** 50 * @brief Returns a JSON string for use by PEL::printSectionInJSON(). 51 * 52 * The returning string will contain a JSON object, but without 53 * the outer {}. If the input JSON isn't a JSON object (dict), then 54 * one will be created with the input added to a 'Data' key. 55 * 56 * @param[in] json - The JSON to convert to a string 57 * 58 * @return std::string - The JSON string 59 */ 60 std::string prettyJSON(uint16_t componentID, uint8_t subType, uint8_t version, 61 const fifoJSON& json) 62 { 63 fifoJSON output; 64 output[pv::sectionVer] = std::to_string(version); 65 output[pv::subSection] = std::to_string(subType); 66 output[pv::createdBy] = getNumberString("0x%04X", componentID); 67 68 if (!json.is_object()) 69 { 70 output["Data"] = json; 71 } 72 else 73 { 74 for (const auto& [key, value] : json.items()) 75 { 76 output[key] = value; 77 } 78 } 79 80 // Let nlohmann do the pretty printing. 81 std::stringstream stream; 82 stream << std::setw(4) << output; 83 84 auto jsonString = stream.str(); 85 86 // Now it looks like: 87 // { 88 // "Section Version": ... 89 // ... 90 // } 91 92 // Since PEL::printSectionInJSON() will supply the outer { }s, 93 // remove the existing ones. 94 95 // Replace the { and the following newline, and the } and its 96 // preceeding newline. 97 jsonString.erase(0, 2); 98 99 auto pos = jsonString.find_last_of('}'); 100 jsonString.erase(pos - 1); 101 102 return jsonString; 103 } 104 105 /** 106 * @brief Return a JSON string from the passed in CBOR data. 107 * 108 * @param[in] componentID - The comp ID from the UserData section header 109 * @param[in] subType - The subtype from the UserData section header 110 * @param[in] version - The version from the UserData section header 111 * @param[in] data - The CBOR data 112 * 113 * @return std::string - The JSON string 114 */ 115 std::string getCBORJSON(uint16_t componentID, uint8_t subType, uint8_t version, 116 const std::vector<uint8_t>& data) 117 { 118 // The CBOR parser needs the pad bytes added to 4 byte align 119 // removed. The number of bytes added to the pad is on the 120 // very end, so will remove both fields before parsing. 121 122 // If the data vector is too short, an exception will get 123 // thrown which will be handled up the call stack. 124 125 auto cborData = data; 126 uint32_t pad{}; 127 128 Stream stream{cborData}; 129 stream.offset(cborData.size() - 4); 130 stream >> pad; 131 132 if (cborData.size() > (pad + sizeof(pad))) 133 { 134 cborData.resize(data.size() - sizeof(pad) - pad); 135 } 136 137 fifoJSON json = nlohmann::json::from_cbor(cborData); 138 139 return prettyJSON(componentID, subType, version, json); 140 } 141 142 /** 143 * @brief Return a JSON string from the passed in text data. 144 * 145 * The function breaks up the input text into a vector of strings with 146 * newline as separator and converts that into JSON. It will convert any 147 * unprintable characters to periods. 148 * 149 * @param[in] componentID - The comp ID from the UserData section header 150 * @param[in] subType - The subtype from the UserData section header 151 * @param[in] version - The version from the UserData section header 152 * @param[in] data - The CBOR data 153 * 154 * @return std::string - The JSON string 155 */ 156 std::string getTextJSON(uint16_t componentID, uint8_t subType, uint8_t version, 157 const std::vector<uint8_t>& data) 158 { 159 std::vector<std::string> text; 160 size_t startPos = 0; 161 162 // Converts any unprintable characters to periods 163 auto validate = [](char& ch) { 164 if ((ch < ' ') || (ch > '~')) 165 { 166 ch = '.'; 167 } 168 }; 169 170 // Break up the data into an array of strings with newline as separator 171 for (size_t pos = 0; pos < data.size(); ++pos) 172 { 173 if (data[pos] == '\n') 174 { 175 std::string line{reinterpret_cast<const char*>(&data[startPos]), 176 pos - startPos}; 177 std::for_each(line.begin(), line.end(), validate); 178 text.push_back(std::move(line)); 179 startPos = pos + 1; 180 } 181 } 182 if (startPos < data.size()) 183 { 184 std::string line{reinterpret_cast<const char*>(&data[startPos]), 185 data.size() - startPos}; 186 std::for_each(line.begin(), line.end(), validate); 187 text.push_back(std::move(line)); 188 } 189 190 fifoJSON json = text; 191 return prettyJSON(componentID, subType, version, json); 192 } 193 194 /** 195 * @brief Convert to an appropriate JSON string as the data is one of 196 * the formats that we natively support. 197 * 198 * @param[in] componentID - The comp ID from the UserData section header 199 * @param[in] subType - The subtype from the UserData section header 200 * @param[in] version - The version from the UserData section header 201 * @param[in] data - The data itself 202 * 203 * @return std::optional<std::string> - The JSON string if it could be created, 204 * else std::nullopt. 205 */ 206 std::optional<std::string> 207 getBuiltinFormatJSON(uint16_t componentID, uint8_t subType, uint8_t version, 208 const std::vector<uint8_t>& data) 209 { 210 switch (subType) 211 { 212 case static_cast<uint8_t>(UserDataFormat::json): 213 { 214 std::string jsonString{data.begin(), data.begin() + data.size()}; 215 216 fifoJSON json = nlohmann::json::parse(jsonString); 217 218 return prettyJSON(componentID, subType, version, json); 219 } 220 case static_cast<uint8_t>(UserDataFormat::cbor): 221 { 222 return getCBORJSON(componentID, subType, version, data); 223 } 224 case static_cast<uint8_t>(UserDataFormat::text): 225 { 226 return getTextJSON(componentID, subType, version, data); 227 } 228 default: 229 break; 230 } 231 return std::nullopt; 232 } 233 234 /** 235 * @brief Call Python modules to parse the data into a JSON string 236 * 237 * The module to call is based on the Creator Subsystem ID and the Component 238 * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy" 239 * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID. 240 * 241 * All modules must provide the following: 242 * Function: parseUDToJson 243 * Argument list: 244 * 1. (int) Sub-section type 245 * 2. (int) Section version 246 * 3. (memoryview): Data 247 *-Return data: 248 * 1. (str) JSON string 249 * 250 * @param[in] componentID - The comp ID from the UserData section header 251 * @param[in] subType - The subtype from the UserData section header 252 * @param[in] version - The version from the UserData section header 253 * @param[in] data - The data itself 254 * @param[in] creatorID - The creatorID from the PrivateHeader section 255 * @return std::optional<std::string> - The JSON string if it could be created, 256 * else std::nullopt 257 */ 258 std::optional<std::string> getPythonJSON(uint16_t componentID, uint8_t subType, 259 uint8_t version, 260 const std::vector<uint8_t>& data, 261 uint8_t creatorID) 262 { 263 PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pData, *pResult, 264 *pBytes, *eType, *eValue, *eTraceback; 265 std::string pErrStr; 266 std::string module = getNumberString("%c", tolower(creatorID)) + 267 getNumberString("%04x", componentID); 268 pName = PyUnicode_FromString( 269 std::string("udparsers." + module + "." + module).c_str()); 270 std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef); 271 pModule = PyImport_Import(pName); 272 std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr(pModule, &pyDecRef); 273 if (pModule == NULL) 274 { 275 pErrStr = "No error string found"; 276 PyErr_Fetch(&eType, &eValue, &eTraceback); 277 if (eValue) 278 { 279 PyObject* pStr = PyObject_Str(eValue); 280 if (pStr) 281 { 282 pErrStr = PyUnicode_AsUTF8(pStr); 283 } 284 Py_XDECREF(pStr); 285 } 286 } 287 else 288 { 289 pDict = PyModule_GetDict(pModule); 290 pFunc = PyDict_GetItemString(pDict, "parseUDToJson"); 291 if (PyCallable_Check(pFunc)) 292 { 293 auto ud = data.data(); 294 pArgs = PyTuple_New(3); 295 std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr(pArgs, 296 &pyDecRef); 297 PyTuple_SetItem(pArgs, 0, 298 PyLong_FromUnsignedLong((unsigned long)subType)); 299 PyTuple_SetItem(pArgs, 1, 300 PyLong_FromUnsignedLong((unsigned long)version)); 301 pData = PyMemoryView_FromMemory( 302 reinterpret_cast<char*>(const_cast<unsigned char*>(ud)), 303 data.size(), PyBUF_READ); 304 std::unique_ptr<PyObject, decltype(&pyDecRef)> dataPtr(pData, 305 &pyDecRef); 306 PyTuple_SetItem(pArgs, 2, pData); 307 pResult = PyObject_CallObject(pFunc, pArgs); 308 std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr(pResult, 309 &pyDecRef); 310 if (pResult) 311 { 312 pBytes = PyUnicode_AsEncodedString(pResult, "utf-8", "~E~"); 313 std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr( 314 pBytes, &pyDecRef); 315 const char* output = PyBytes_AS_STRING(pBytes); 316 try 317 { 318 fifoJSON json = nlohmann::json::parse(output); 319 return prettyJSON(componentID, subType, version, json); 320 } 321 catch (std::exception& e) 322 { 323 log<level::ERR>("Bad JSON from parser", 324 entry("ERROR=%s", e.what()), 325 entry("PARSER_MODULE=%s", module.c_str()), 326 entry("SUBTYPE=0x%X", subType), 327 entry("VERSION=%d", version), 328 entry("DATA_LENGTH=%lu\n", data.size())); 329 return std::nullopt; 330 } 331 } 332 else 333 { 334 pErrStr = "No error string found"; 335 PyErr_Fetch(&eType, &eValue, &eTraceback); 336 if (eValue) 337 { 338 PyObject* pStr = PyObject_Str(eValue); 339 if (pStr) 340 { 341 pErrStr = PyUnicode_AsUTF8(pStr); 342 } 343 Py_XDECREF(pStr); 344 } 345 } 346 } 347 } 348 if (!pErrStr.empty()) 349 { 350 log<level::ERR>("Python exception thrown by parser", 351 entry("ERROR=%s", pErrStr.c_str()), 352 entry("PARSER_MODULE=%s", module.c_str()), 353 entry("SUBTYPE=0x%X", subType), 354 entry("VERSION=%d", version), 355 entry("DATA_LENGTH=%lu\n", data.size())); 356 } 357 Py_XDECREF(eType); 358 Py_XDECREF(eValue); 359 Py_XDECREF(eTraceback); 360 return std::nullopt; 361 } 362 363 std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType, 364 uint8_t version, 365 const std::vector<uint8_t>& data, 366 uint8_t creatorID, 367 const std::vector<std::string>& plugins) 368 { 369 std::string subsystem = getNumberString("%c", tolower(creatorID)); 370 std::string component = getNumberString("%04x", componentID); 371 try 372 { 373 if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" && 374 componentID == static_cast<uint16_t>(ComponentID::phosphorLogging)) 375 { 376 return getBuiltinFormatJSON(componentID, subType, version, data); 377 } 378 else if (std::find(plugins.begin(), plugins.end(), 379 subsystem + component) != plugins.end()) 380 { 381 return getPythonJSON(componentID, subType, version, data, 382 creatorID); 383 } 384 } 385 catch (std::exception& e) 386 { 387 log<level::ERR>("Failed parsing UserData", entry("ERROR=%s", e.what()), 388 entry("COMP_ID=0x%X", componentID), 389 entry("SUBTYPE=0x%X", subType), 390 entry("VERSION=%d", version), 391 entry("DATA_LENGTH=%lu\n", data.size())); 392 } 393 394 return std::nullopt; 395 } 396 397 } // namespace openpower::pels::user_data 398