xref: /openbmc/bmcweb/scripts/update_schemas.py (revision 1b8b02a4) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2023.1"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "AggregationService",
31    "AggregationSource",
32    "AggregationSourceCollection",
33    "Assembly",
34    "AttributeRegistry",
35    "Bios",
36    "Cable",
37    "CableCollection",
38    "Certificate",
39    "CertificateCollection",
40    "CertificateLocations",
41    "CertificateService",
42    "Chassis",
43    "ChassisCollection",
44    "ComputerSystem",
45    "ComputerSystemCollection",
46    "ComponentIntegrity",
47    "ComponentIntegrityCollection",
48    "Drive",
49    "DriveCollection",
50    "EnvironmentMetrics",
51    "EthernetInterface",
52    "EthernetInterfaceCollection",
53    "Event",
54    "EventDestination",
55    "EventDestinationCollection",
56    "EventService",
57    "FabricAdapter",
58    "FabricAdapterCollection",
59    "Fan",
60    "FanCollection",
61    "IPAddresses",
62    "JsonSchemaFile",
63    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
64    "LogEntry",
65    "LogEntryCollection",
66    "LogService",
67    "LogServiceCollection",
68    "Manager",
69    "ManagerAccount",
70    "ManagerAccountCollection",
71    "ManagerCollection",
72    "ManagerDiagnosticData",
73    "ManagerNetworkProtocol",
74    "Memory",
75    "MemoryCollection",
76    "Message",
77    "MessageRegistry",
78    "MessageRegistryCollection",
79    "MessageRegistryFile",
80    "MessageRegistryFileCollection",
81    "MetricDefinition",
82    "MetricDefinitionCollection",
83    "MetricReport",
84    "MetricReportCollection",
85    "MetricReportDefinition",
86    "MetricReportDefinitionCollection",
87    "OperatingConfig",
88    "OperatingConfigCollection",
89    "PCIeDevice",
90    "PCIeDeviceCollection",
91    "PCIeFunction",
92    "PCIeFunctionCollection",
93    "PhysicalContext",
94    "PCIeSlots",
95    "Port",
96    "PortCollection",
97    "Power",
98    "PowerSubsystem",
99    "PowerSupply",
100    "PowerSupplyCollection",
101    "Privileges",  # Used in Role
102    "Processor",
103    "ProcessorCollection",
104    "RedfishError",
105    "RedfishExtensions",
106    "Redundancy",
107    "Resource",
108    "Role",
109    "RoleCollection",
110    "Sensor",
111    "SensorCollection",
112    "ServiceRoot",
113    "Session",
114    "SessionCollection",
115    "SessionService",
116    "Settings",
117    "SoftwareInventory",
118    "SoftwareInventoryCollection",
119    "Storage",
120    "StorageCollection",
121    "StorageController",
122    "StorageControllerCollection",
123    "Task",
124    "TaskCollection",
125    "TaskService",
126    "TelemetryService",
127    "Thermal",
128    "ThermalMetrics",
129    "ThermalSubsystem",
130    "Triggers",
131    "TriggersCollection",
132    "UpdateService",
133    "VirtualMedia",
134    "VirtualMediaCollection",
135    "odata",
136    "odata-v4",
137    "redfish-error",
138    "redfish-payload-annotations",
139    "redfish-schema",
140    "redfish-schema-v1",
141]
142
143SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
144
145proxies = {"https": os.environ.get("https_proxy", None)}
146
147r = requests.get(
148    "https://www.dmtf.org/sites/default/files/standards/documents/"
149    + VERSION
150    + ".zip",
151    proxies=proxies,
152)
153
154r.raise_for_status()
155
156
157static_path = os.path.realpath(
158    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
159)
160
161
162cpp_path = os.path.realpath(
163    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
164)
165
166
167schema_path = os.path.join(static_path, "schema")
168json_schema_path = os.path.join(static_path, "JsonSchemas")
169metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
170
171zipBytesIO = BytesIO(r.content)
172zip_ref = zipfile.ZipFile(zipBytesIO)
173
174
175class SchemaVersion:
176    """
177    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
178    versions in the way humans expect, by comparing version strings as lists
179    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
180    insensitive schema name comparisons
181    """
182
183    def __init__(self, key):
184        key = str.casefold(key)
185
186        split_tup = key.split(".")
187        self.version_pieces = [split_tup[0]]
188        if len(split_tup) < 2:
189            return
190        version = split_tup[1]
191
192        if version.startswith("v"):
193            version = version[1:]
194        if any(char.isdigit() for char in version):
195            self.version_pieces.extend([int(x) for x in version.split("_")])
196
197    def __lt__(self, other):
198        return self.version_pieces < other.version_pieces
199
200
201# Remove the old files
202skip_prefixes = ["Oem", "OpenBMC"]
203if os.path.exists(schema_path):
204    files = [
205        os.path.join(schema_path, f)
206        for f in os.listdir(schema_path)
207        if not any([f.startswith(prefix) for prefix in skip_prefixes])
208    ]
209    for f in files:
210        os.remove(f)
211if os.path.exists(json_schema_path):
212    files = [
213        os.path.join(json_schema_path, f)
214        for f in os.listdir(json_schema_path)
215        if not any([f.startswith(prefix) for prefix in skip_prefixes])
216    ]
217    for f in files:
218        if os.path.isfile(f):
219            os.remove(f)
220        else:
221            shutil.rmtree(f)
222try:
223    os.remove(metadata_index_path)
224except FileNotFoundError:
225    pass
226
227if not os.path.exists(schema_path):
228    os.makedirs(schema_path)
229if not os.path.exists(json_schema_path):
230    os.makedirs(json_schema_path)
231
232csdl_filenames = []
233json_schema_files = defaultdict(list)
234
235for zip_file in zip_ref.infolist():
236    if zip_file.is_dir():
237        continue
238    if zip_file.filename.startswith("csdl/"):
239        csdl_filenames.append(os.path.basename(zip_file.filename))
240    elif zip_file.filename.startswith("json-schema/"):
241        filename = os.path.basename(zip_file.filename)
242        filenamesplit = filename.split(".")
243        # exclude schemas again to save flash space
244        if filenamesplit[0] not in include_list:
245            continue
246        json_schema_files[filenamesplit[0]].append(filename)
247    elif zip_file.filename.startswith("openapi/"):
248        pass
249    elif zip_file.filename.startswith("dictionaries/"):
250        pass
251
252# sort the json files by version
253for key, value in json_schema_files.items():
254    value.sort(key=SchemaVersion, reverse=True)
255
256# Create a dictionary ordered by schema name
257json_schema_files = OrderedDict(
258    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
259)
260
261csdl_filenames.sort(key=SchemaVersion)
262with open(metadata_index_path, "w") as metadata_index:
263    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
264    metadata_index.write(
265        "<edmx:Edmx xmlns:edmx="
266        '"http://docs.oasis-open.org/odata/ns/edmx"'
267        ' Version="4.0">\n'
268    )
269
270    for filename in csdl_filenames:
271        # filename looks like Zone_v1.xml
272        with open(os.path.join(schema_path, filename), "wb") as schema_out:
273            content = zip_ref.read(os.path.join("csdl", filename))
274            content = content.replace(b"\r\n", b"\n")
275
276            schema_out.write(content)
277
278            filenamesplit = filename.split("_")
279            if filenamesplit[0] not in include_list:
280                continue
281            metadata_index.write(
282                '    <edmx:Reference Uri="/redfish/v1/schema/'
283                + filename
284                + '">\n'
285            )
286
287            xml_root = ET.fromstring(content)
288            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
289            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
290            for edmx_child in xml_root:
291                if edmx_child.tag == edmx + "DataServices":
292                    for data_child in edmx_child:
293                        if data_child.tag == edm + "Schema":
294                            namespace = data_child.attrib["Namespace"]
295                            if namespace.startswith("RedfishExtensions"):
296                                metadata_index.write(
297                                    '        <edmx:Include Namespace="'
298                                    + namespace
299                                    + '"  Alias="Redfish"/>\n'
300                                )
301
302                            else:
303                                metadata_index.write(
304                                    '        <edmx:Include Namespace="'
305                                    + namespace
306                                    + '"/>\n'
307                                )
308            metadata_index.write("    </edmx:Reference>\n")
309
310    metadata_index.write(
311        "    <edmx:DataServices>\n"
312        "        <Schema "
313        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
314        'Namespace="Service">\n'
315        '            <EntityContainer Name="Service" '
316        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
317        "        </Schema>\n"
318        "    </edmx:DataServices>\n"
319    )
320    # TODO:Issue#32 There's a bug in the script that currently deletes this
321    # schema (because it's an OEM schema). Because it's the only six, and we
322    # don't update schemas very often, we just manually fix it. Need a
323    # permanent fix to the script.
324    metadata_index.write(
325        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
326    )
327    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
328    metadata_index.write("    </edmx:Reference>\n")
329
330    metadata_index.write(
331        '    <edmx:Reference Uri="'
332        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
333    )
334    metadata_index.write(
335        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
336    )
337    metadata_index.write("    </edmx:Reference>\n")
338
339    metadata_index.write(
340        '    <edmx:Reference Uri="'
341        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
342    )
343    metadata_index.write(
344        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
345    )
346    metadata_index.write(
347        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
348    )
349    metadata_index.write("    </edmx:Reference>\n")
350
351    metadata_index.write(
352        '    <edmx:Reference Uri="'
353        '/redfish/v1/schema/OpenBMCAccountService_v1.xml">\n'
354    )
355    metadata_index.write(
356        '        <edmx:Include Namespace="OpenBMCAccountService"/>\n'
357    )
358    metadata_index.write(
359        '        <edmx:Include Namespace="OpenBMCAccountService.v1_0_0"/>\n'
360    )
361    metadata_index.write("    </edmx:Reference>\n")
362
363    metadata_index.write("</edmx:Edmx>\n")
364
365
366for schema, version in json_schema_files.items():
367    zip_filepath = os.path.join("json-schema", version[0])
368    schemadir = os.path.join(json_schema_path, schema)
369    os.makedirs(schemadir)
370
371    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
372        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
373
374with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
375    hpp_file.write(
376        "#pragma once\n"
377        "{WARNING}\n"
378        "// clang-format off\n"
379        "#include <array>\n"
380        "\n"
381        "namespace redfish\n"
382        "{{\n"
383        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
384    )
385    for schema_file in json_schema_files:
386        hpp_file.write('        "{}",\n'.format(schema_file))
387
388    hpp_file.write("    };\n}\n")
389
390zip_ref.close()
391
392generate_schema_enums.main()
393generate_top_collections()
394
395# Now delete the xml schema files we aren't supporting
396if os.path.exists(schema_path):
397    files = [
398        os.path.join(schema_path, f)
399        for f in os.listdir(schema_path)
400        if not any([f.startswith(prefix) for prefix in skip_prefixes])
401    ]
402    for filename in files:
403        # filename will include the absolute path
404        filenamesplit = filename.split("/")
405        name = filenamesplit.pop()
406        namesplit = name.split("_")
407        if namesplit[0] not in include_list:
408            print("excluding schema: " + filename)
409            os.remove(filename)
410