xref: /openbmc/bmcweb/scripts/update_schemas.py (revision f78fc98d) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2023.3"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "AggregationService",
31    "AggregationSource",
32    "AggregationSourceCollection",
33    "Assembly",
34    "AttributeRegistry",
35    "Bios",
36    "Cable",
37    "CableCollection",
38    "Certificate",
39    "CertificateCollection",
40    "CertificateLocations",
41    "CertificateService",
42    "Chassis",
43    "ChassisCollection",
44    "ComputerSystem",
45    "ComputerSystemCollection",
46    "ComponentIntegrity",
47    "ComponentIntegrityCollection",
48    "Drive",
49    "DriveCollection",
50    "EnvironmentMetrics",
51    "EthernetInterface",
52    "EthernetInterfaceCollection",
53    "Event",
54    "EventDestination",
55    "EventDestinationCollection",
56    "EventService",
57    "FabricAdapter",
58    "FabricAdapterCollection",
59    "Fan",
60    "FanCollection",
61    "IPAddresses",
62    "JsonSchemaFile",
63    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
64    "LogEntry",
65    "LogEntryCollection",
66    "LogService",
67    "LogServiceCollection",
68    "Manager",
69    "ManagerAccount",
70    "ManagerAccountCollection",
71    "ManagerCollection",
72    "ManagerDiagnosticData",
73    "ManagerNetworkProtocol",
74    "Memory",
75    "MemoryCollection",
76    "Message",
77    "MessageRegistry",
78    "MessageRegistryCollection",
79    "MessageRegistryFile",
80    "MessageRegistryFileCollection",
81    "MetricDefinition",
82    "MetricDefinitionCollection",
83    "MetricReport",
84    "MetricReportCollection",
85    "MetricReportDefinition",
86    "MetricReportDefinitionCollection",
87    "OperatingConfig",
88    "OperatingConfigCollection",
89    "PCIeDevice",
90    "PCIeDeviceCollection",
91    "PCIeFunction",
92    "PCIeFunctionCollection",
93    "PhysicalContext",
94    "PCIeSlots",
95    "Port",
96    "PortCollection",
97    "Power",
98    "PowerSubsystem",
99    "PowerSupply",
100    "PowerSupplyCollection",
101    "Privileges",  # Used in Role
102    "Processor",
103    "ProcessorCollection",
104    "RedfishError",
105    "RedfishExtensions",
106    "Redundancy",
107    "Resource",
108    "Role",
109    "RoleCollection",
110    "Sensor",
111    "SensorCollection",
112    "ServiceRoot",
113    "Session",
114    "SessionCollection",
115    "SessionService",
116    "Settings",
117    "SoftwareInventory",
118    "SoftwareInventoryCollection",
119    "Storage",
120    "StorageCollection",
121    "StorageController",
122    "StorageControllerCollection",
123    "Task",
124    "TaskCollection",
125    "TaskService",
126    "TelemetryService",
127    "Thermal",
128    "ThermalMetrics",
129    "ThermalSubsystem",
130    "Triggers",
131    "TriggersCollection",
132    "UpdateService",
133    "VirtualMedia",
134    "VirtualMediaCollection",
135    "odata",
136    "odata-v4",
137    "redfish-error",
138    "redfish-payload-annotations",
139    "redfish-schema",
140    "redfish-schema-v1",
141]
142
143# OEM schemas
144oem_schema_names = [
145    "OemManager",
146    "OemComputerSystem",
147    "OemVirtualMedia",
148    "OpenBMCAccountService",
149]
150
151SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
152
153proxies = {"https": os.environ.get("https_proxy", None)}
154
155r = requests.get(
156    "https://www.dmtf.org/sites/default/files/standards/documents/"
157    + VERSION
158    + ".zip",
159    proxies=proxies,
160)
161
162r.raise_for_status()
163
164
165static_path = os.path.realpath(
166    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
167)
168
169
170cpp_path = os.path.realpath(
171    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
172)
173
174
175schema_path = os.path.join(static_path, "schema")
176json_schema_path = os.path.join(static_path, "JsonSchemas")
177metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
178
179zipBytesIO = BytesIO(r.content)
180zip_ref = zipfile.ZipFile(zipBytesIO)
181
182
183class SchemaVersion:
184    """
185    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
186    versions in the way humans expect, by comparing version strings as lists
187    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
188    insensitive schema name comparisons
189    """
190
191    def __init__(self, key):
192        key = str.casefold(key)
193
194        split_tup = key.split(".")
195        self.version_pieces = [split_tup[0]]
196        if len(split_tup) < 2:
197            return
198        version = split_tup[1]
199
200        if version.startswith("v"):
201            version = version[1:]
202        if any(char.isdigit() for char in version):
203            self.version_pieces.extend([int(x) for x in version.split("_")])
204
205    def __lt__(self, other):
206        return self.version_pieces < other.version_pieces
207
208
209# Remove the old files
210skip_prefixes = ["Oem", "OpenBMC"]
211if os.path.exists(schema_path):
212    files = [
213        os.path.join(schema_path, f)
214        for f in os.listdir(schema_path)
215        if not any([f.startswith(prefix) for prefix in skip_prefixes])
216    ]
217    for f in files:
218        os.remove(f)
219if os.path.exists(json_schema_path):
220    files = [
221        os.path.join(json_schema_path, f)
222        for f in os.listdir(json_schema_path)
223        if not any([f.startswith(prefix) for prefix in skip_prefixes])
224    ]
225    for f in files:
226        if os.path.isfile(f):
227            os.remove(f)
228        else:
229            shutil.rmtree(f)
230try:
231    os.remove(metadata_index_path)
232except FileNotFoundError:
233    pass
234
235if not os.path.exists(schema_path):
236    os.makedirs(schema_path)
237if not os.path.exists(json_schema_path):
238    os.makedirs(json_schema_path)
239
240csdl_filenames = []
241json_schema_files = defaultdict(list)
242
243for zip_file in zip_ref.infolist():
244    if zip_file.is_dir():
245        continue
246    if zip_file.filename.startswith("csdl/"):
247        csdl_filenames.append(os.path.basename(zip_file.filename))
248    elif zip_file.filename.startswith("json-schema/"):
249        filename = os.path.basename(zip_file.filename)
250        filenamesplit = filename.split(".")
251        # exclude schemas again to save flash space
252        if filenamesplit[0] not in include_list:
253            continue
254        json_schema_files[filenamesplit[0]].append(filename)
255    elif zip_file.filename.startswith("openapi/"):
256        pass
257    elif zip_file.filename.startswith("dictionaries/"):
258        pass
259
260# sort the json files by version
261for key, value in json_schema_files.items():
262    value.sort(key=SchemaVersion, reverse=True)
263
264# Create a dictionary ordered by schema name
265json_schema_files = OrderedDict(
266    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
267)
268
269csdl_filenames.sort(key=SchemaVersion)
270
271# Create oem filenames - from oem json names
272oem_csdl_filenames = []
273for filename in oem_schema_names:
274    oem_csdl_filenames.append(filename + "_v1.xml")
275
276# Append Oem csdl files
277csdl_filenames += oem_csdl_filenames
278
279with open(metadata_index_path, "w") as metadata_index:
280    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
281    metadata_index.write(
282        "<edmx:Edmx xmlns:edmx="
283        '"http://docs.oasis-open.org/odata/ns/edmx"'
284        ' Version="4.0">\n'
285    )
286
287    for filename in csdl_filenames:
288        # filename looks like Zone_v1.xml
289        if filename in oem_csdl_filenames:
290            with open(
291                os.path.join(schema_path, filename), "rb"
292            ) as oem_csdl_in:
293                content = oem_csdl_in.read()
294                content = content.replace(b"\r\n", b"\n")
295        else:
296            with open(os.path.join(schema_path, filename), "wb") as schema_out:
297                content = zip_ref.read(os.path.join("csdl", filename))
298                content = content.replace(b"\r\n", b"\n")
299                schema_out.write(content)
300                filenamesplit = filename.split("_")
301                if filenamesplit[0] not in include_list:
302                    continue
303
304        metadata_index.write(
305            '    <edmx:Reference Uri="/redfish/v1/schema/' + filename + '">\n'
306        )
307
308        xml_root = ET.fromstring(content)
309        edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
310        edm = "{http://docs.oasis-open.org/odata/ns/edm}"
311        for edmx_child in xml_root:
312            if edmx_child.tag == edmx + "DataServices":
313                for data_child in edmx_child:
314                    if data_child.tag == edm + "Schema":
315                        namespace = data_child.attrib["Namespace"]
316                        if namespace.startswith("RedfishExtensions"):
317                            metadata_index.write(
318                                '        <edmx:Include Namespace="'
319                                + namespace
320                                + '"  Alias="Redfish"/>\n'
321                            )
322
323                        else:
324                            metadata_index.write(
325                                '        <edmx:Include Namespace="'
326                                + namespace
327                                + '"/>\n'
328                            )
329        metadata_index.write("    </edmx:Reference>\n")
330
331    metadata_index.write(
332        "    <edmx:DataServices>\n"
333        "        <Schema "
334        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
335        'Namespace="Service">\n'
336        '            <EntityContainer Name="Service" '
337        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
338        "        </Schema>\n"
339        "    </edmx:DataServices>\n"
340    )
341    metadata_index.write("</edmx:Edmx>\n")
342
343
344for schema, version in json_schema_files.items():
345    zip_filepath = os.path.join("json-schema", version[0])
346    schemadir = os.path.join(json_schema_path, schema)
347    os.makedirs(schemadir)
348
349    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
350        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
351
352with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
353    hpp_file.write(
354        "#pragma once\n"
355        "{WARNING}\n"
356        "// clang-format off\n"
357        "#include <array>\n"
358        "\n"
359        "namespace redfish\n"
360        "{{\n"
361        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
362    )
363    for schema_file in json_schema_files:
364        hpp_file.write('        "{}",\n'.format(schema_file))
365
366    for schema_file in oem_schema_names:
367        hpp_file.write('        "{}",\n'.format(schema_file))
368
369    hpp_file.write("    };\n}\n")
370
371zip_ref.close()
372
373generate_schema_enums.main()
374generate_top_collections()
375
376# Now delete the xml schema files we aren't supporting
377if os.path.exists(schema_path):
378    files = [
379        os.path.join(schema_path, f)
380        for f in os.listdir(schema_path)
381        if not any([f.startswith(prefix) for prefix in skip_prefixes])
382    ]
383    for filename in files:
384        # filename will include the absolute path
385        filenamesplit = filename.split("/")
386        name = filenamesplit.pop()
387        namesplit = name.split("_")
388        if namesplit[0] not in include_list:
389            print("excluding schema: " + filename)
390            os.remove(filename)
391