xref: /openbmc/bmcweb/scripts/update_schemas.py (revision 5e7e2dc5)
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2022.3"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "Assembly",
31    "AttributeRegistry",
32    "Bios",
33    "Cable",
34    "CableCollection",
35    "Certificate",
36    "CertificateCollection",
37    "CertificateLocations",
38    "CertificateService",
39    "Chassis",
40    "ChassisCollection",
41    "ComputerSystem",
42    "ComputerSystemCollection",
43    "Drive",
44    "DriveCollection",
45    "EnvironmentMetrics",
46    "EthernetInterface",
47    "EthernetInterfaceCollection",
48    "Event",
49    "EventDestination",
50    "EventDestinationCollection",
51    "EventService",
52    "FabricAdapter",
53    "FabricAdapterCollection",
54    "Fan",
55    "FanCollection",
56    "IPAddresses",
57    "JsonSchemaFile",
58    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
59    "LogEntry",
60    "LogEntryCollection",
61    "LogService",
62    "LogServiceCollection",
63    "Manager",
64    "ManagerAccount",
65    "ManagerAccountCollection",
66    "ManagerCollection",
67    "ManagerDiagnosticData",
68    "ManagerNetworkProtocol",
69    "Memory",
70    "MemoryCollection",
71    "Message",
72    "MessageRegistry",
73    "MessageRegistryCollection",
74    "MessageRegistryFile",
75    "MessageRegistryFileCollection",
76    "MetricDefinition",
77    "MetricDefinitionCollection",
78    "MetricReport",
79    "MetricReportCollection",
80    "MetricReportDefinition",
81    "MetricReportDefinitionCollection",
82    "OperatingConfig",
83    "OperatingConfigCollection",
84    "PCIeDevice",
85    "PCIeDeviceCollection",
86    "PCIeFunction",
87    "PCIeFunctionCollection",
88    "PhysicalContext",
89    "PCIeSlots",
90    "Port",
91    "PortCollection",
92    "Power",
93    "PowerSubsystem",
94    "PowerSupply",
95    "PowerSupplyCollection",
96    "Privileges",  # Used in Role
97    "Processor",
98    "ProcessorCollection",
99    "RedfishError",
100    "RedfishExtensions",
101    "Redundancy",
102    "Resource",
103    "Role",
104    "RoleCollection",
105    "Sensor",
106    "SensorCollection",
107    "ServiceRoot",
108    "Session",
109    "SessionCollection",
110    "SessionService",
111    "Settings",
112    "SoftwareInventory",
113    "SoftwareInventoryCollection",
114    "Storage",
115    "StorageCollection",
116    "StorageController",
117    "StorageControllerCollection",
118    "Task",
119    "TaskCollection",
120    "TaskService",
121    "TelemetryService",
122    "Thermal",
123    "ThermalMetrics",
124    "ThermalSubsystem",
125    "Triggers",
126    "TriggersCollection",
127    "UpdateService",
128    "VLanNetworkInterfaceCollection",
129    "VLanNetworkInterface",
130    "VirtualMedia",
131    "VirtualMediaCollection",
132    "odata",
133    "odata-v4",
134    "redfish-error",
135    "redfish-payload-annotations",
136    "redfish-schema",
137    "redfish-schema-v1",
138]
139
140SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
141
142proxies = {"https": os.environ.get("https_proxy", None)}
143
144r = requests.get(
145    "https://www.dmtf.org/sites/default/files/standards/documents/"
146    + VERSION
147    + ".zip",
148    proxies=proxies,
149)
150
151r.raise_for_status()
152
153
154static_path = os.path.realpath(
155    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
156)
157
158
159cpp_path = os.path.realpath(
160    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
161)
162
163
164schema_path = os.path.join(static_path, "schema")
165json_schema_path = os.path.join(static_path, "JsonSchemas")
166metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
167
168zipBytesIO = BytesIO(r.content)
169zip_ref = zipfile.ZipFile(zipBytesIO)
170
171
172class SchemaVersion:
173    """
174    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
175    versions in the way humans expect, by comparing version strings as lists
176    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
177    insensitive schema name comparisons
178    """
179
180    def __init__(self, key):
181        key = str.casefold(key)
182
183        split_tup = key.split(".")
184        self.version_pieces = [split_tup[0]]
185        if len(split_tup) < 2:
186            return
187        version = split_tup[1]
188
189        if version.startswith("v"):
190            version = version[1:]
191        if any(char.isdigit() for char in version):
192            self.version_pieces.extend([int(x) for x in version.split("_")])
193
194    def __lt__(self, other):
195        return self.version_pieces < other.version_pieces
196
197
198# Remove the old files
199skip_prefixes = ["Oem", "OpenBMC"]
200if os.path.exists(schema_path):
201    files = [
202        os.path.join(schema_path, f)
203        for f in os.listdir(schema_path)
204        if not any([f.startswith(prefix) for prefix in skip_prefixes])
205    ]
206    for f in files:
207        os.remove(f)
208if os.path.exists(json_schema_path):
209    files = [
210        os.path.join(json_schema_path, f)
211        for f in os.listdir(json_schema_path)
212        if not any([f.startswith(prefix) for prefix in skip_prefixes])
213    ]
214    for f in files:
215        if os.path.isfile(f):
216            os.remove(f)
217        else:
218            shutil.rmtree(f)
219try:
220    os.remove(metadata_index_path)
221except FileNotFoundError:
222    pass
223
224if not os.path.exists(schema_path):
225    os.makedirs(schema_path)
226if not os.path.exists(json_schema_path):
227    os.makedirs(json_schema_path)
228
229csdl_filenames = []
230json_schema_files = defaultdict(list)
231
232for zip_file in zip_ref.infolist():
233    if zip_file.is_dir():
234        continue
235    if zip_file.filename.startswith("csdl/"):
236        csdl_filenames.append(os.path.basename(zip_file.filename))
237    elif zip_file.filename.startswith("json-schema/"):
238        filename = os.path.basename(zip_file.filename)
239        filenamesplit = filename.split(".")
240        # exclude schemas again to save flash space
241        if filenamesplit[0] not in include_list:
242            continue
243        json_schema_files[filenamesplit[0]].append(filename)
244    elif zip_file.filename.startswith("openapi/"):
245        pass
246    elif zip_file.filename.startswith("dictionaries/"):
247        pass
248
249# sort the json files by version
250for key, value in json_schema_files.items():
251    value.sort(key=SchemaVersion, reverse=True)
252
253# Create a dictionary ordered by schema name
254json_schema_files = OrderedDict(
255    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
256)
257
258csdl_filenames.sort(key=SchemaVersion)
259with open(metadata_index_path, "w") as metadata_index:
260    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
261    metadata_index.write(
262        "<edmx:Edmx xmlns:edmx="
263        '"http://docs.oasis-open.org/odata/ns/edmx"'
264        ' Version="4.0">\n'
265    )
266
267    for filename in csdl_filenames:
268        # filename looks like Zone_v1.xml
269        with open(os.path.join(schema_path, filename), "wb") as schema_out:
270            content = zip_ref.read(os.path.join("csdl", filename))
271            content = content.replace(b"\r\n", b"\n")
272
273            schema_out.write(content)
274
275            filenamesplit = filename.split("_")
276            if filenamesplit[0] not in include_list:
277                continue
278            metadata_index.write(
279                '    <edmx:Reference Uri="/redfish/v1/schema/'
280                + filename
281                + '">\n'
282            )
283
284            xml_root = ET.fromstring(content)
285            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
286            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
287            for edmx_child in xml_root:
288                if edmx_child.tag == edmx + "DataServices":
289                    for data_child in edmx_child:
290                        if data_child.tag == edm + "Schema":
291                            namespace = data_child.attrib["Namespace"]
292                            if namespace.startswith("RedfishExtensions"):
293                                metadata_index.write(
294                                    '        <edmx:Include Namespace="'
295                                    + namespace
296                                    + '"  Alias="Redfish"/>\n'
297                                )
298
299                            else:
300                                metadata_index.write(
301                                    '        <edmx:Include Namespace="'
302                                    + namespace
303                                    + '"/>\n'
304                                )
305            metadata_index.write("    </edmx:Reference>\n")
306
307    metadata_index.write(
308        "    <edmx:DataServices>\n"
309        "        <Schema "
310        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
311        'Namespace="Service">\n'
312        '            <EntityContainer Name="Service" '
313        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
314        "        </Schema>\n"
315        "    </edmx:DataServices>\n"
316    )
317    # TODO:Issue#32 There's a bug in the script that currently deletes this
318    # schema (because it's an OEM schema). Because it's the only six, and we
319    # don't update schemas very often, we just manually fix it. Need a
320    # permanent fix to the script.
321    metadata_index.write(
322        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
323    )
324    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
325    metadata_index.write("    </edmx:Reference>\n")
326
327    metadata_index.write(
328        '    <edmx:Reference Uri="'
329        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
330    )
331    metadata_index.write(
332        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
333    )
334    metadata_index.write("    </edmx:Reference>\n")
335
336    metadata_index.write(
337        '    <edmx:Reference Uri="'
338        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
339    )
340    metadata_index.write(
341        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
342    )
343    metadata_index.write(
344        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
345    )
346    metadata_index.write("    </edmx:Reference>\n")
347
348    metadata_index.write(
349        '    <edmx:Reference Uri="'
350        '/redfish/v1/schema/OpenBMCAccountService_v1.xml">\n'
351    )
352    metadata_index.write(
353        '        <edmx:Include Namespace="OpenBMCAccountService"/>\n'
354    )
355    metadata_index.write(
356        '        <edmx:Include Namespace="OpenBMCAccountService.v1_0_0"/>\n'
357    )
358    metadata_index.write("    </edmx:Reference>\n")
359
360    metadata_index.write("</edmx:Edmx>\n")
361
362
363for schema, version in json_schema_files.items():
364    zip_filepath = os.path.join("json-schema", version[0])
365    schemadir = os.path.join(json_schema_path, schema)
366    os.makedirs(schemadir)
367
368    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
369        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
370
371with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
372    hpp_file.write(
373        "#pragma once\n"
374        "{WARNING}\n"
375        "// clang-format off\n"
376        "#include <array>\n"
377        "\n"
378        "namespace redfish\n"
379        "{{\n"
380        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
381    )
382    for schema_file in json_schema_files:
383        hpp_file.write('        "{}",\n'.format(schema_file))
384
385    hpp_file.write("    };\n}\n")
386
387zip_ref.close()
388
389generate_schema_enums.main()
390generate_top_collections()
391
392# Now delete the xml schema files we aren't supporting
393if os.path.exists(schema_path):
394    files = [
395        os.path.join(schema_path, f)
396        for f in os.listdir(schema_path)
397        if not any([f.startswith(prefix) for prefix in skip_prefixes])
398    ]
399    for filename in files:
400        # filename will include the absolute path
401        filenamesplit = filename.split("/")
402        name = filenamesplit.pop()
403        namesplit = name.split("_")
404        if namesplit[0] not in include_list:
405            print("excluding schema: " + filename)
406            os.remove(filename)
407