xref: /openbmc/bmcweb/scripts/update_schemas.py (revision a8d8f9d8) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2022.3"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "Assembly",
31    "AttributeRegistry",
32    "Bios",
33    "Cable",
34    "CableCollection",
35    "Certificate",
36    "CertificateCollection",
37    "CertificateLocations",
38    "CertificateService",
39    "Chassis",
40    "ChassisCollection",
41    "ComputerSystem",
42    "ComputerSystemCollection",
43    "Drive",
44    "DriveCollection",
45    "EnvironmentMetrics",
46    "EthernetInterface",
47    "EthernetInterfaceCollection",
48    "Event",
49    "EventDestination",
50    "EventDestinationCollection",
51    "EventService",
52    "FabricAdapter",
53    "FabricAdapterCollection",
54    "Fan",
55    "FanCollection",
56    "IPAddresses",
57    "JsonSchemaFile",
58    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
59    "LogEntry",
60    "LogEntryCollection",
61    "LogService",
62    "LogServiceCollection",
63    "Manager",
64    "ManagerAccount",
65    "ManagerAccountCollection",
66    "ManagerCollection",
67    "ManagerDiagnosticData",
68    "ManagerNetworkProtocol",
69    "Memory",
70    "MemoryCollection",
71    "Message",
72    "MessageRegistry",
73    "MessageRegistryCollection",
74    "MessageRegistryFile",
75    "MessageRegistryFileCollection",
76    "MetricDefinition",
77    "MetricDefinitionCollection",
78    "MetricReport",
79    "MetricReportCollection",
80    "MetricReportDefinition",
81    "MetricReportDefinitionCollection",
82    "OperatingConfig",
83    "OperatingConfigCollection",
84    "PCIeDevice",
85    "PCIeDeviceCollection",
86    "PCIeFunction",
87    "PCIeFunctionCollection",
88    "PhysicalContext",
89    "PCIeSlots",
90    "Power",
91    "PowerSubsystem",
92    "PowerSupply",
93    "PowerSupplyCollection",
94    "Privileges",  # Used in Role
95    "Processor",
96    "ProcessorCollection",
97    "RedfishError",
98    "RedfishExtensions",
99    "Redundancy",
100    "Resource",
101    "Role",
102    "RoleCollection",
103    "Sensor",
104    "SensorCollection",
105    "ServiceRoot",
106    "Session",
107    "SessionCollection",
108    "SessionService",
109    "Settings",
110    "SoftwareInventory",
111    "SoftwareInventoryCollection",
112    "Storage",
113    "StorageCollection",
114    "StorageController",
115    "StorageControllerCollection",
116    "Task",
117    "TaskCollection",
118    "TaskService",
119    "TelemetryService",
120    "Thermal",
121    "ThermalMetrics",
122    "ThermalSubsystem",
123    "Triggers",
124    "TriggersCollection",
125    "UpdateService",
126    "VLanNetworkInterfaceCollection",
127    "VLanNetworkInterface",
128    "VirtualMedia",
129    "VirtualMediaCollection",
130    "odata",
131    "odata-v4",
132    "redfish-error",
133    "redfish-payload-annotations",
134    "redfish-schema",
135    "redfish-schema-v1",
136]
137
138SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
139
140proxies = {"https": os.environ.get("https_proxy", None)}
141
142r = requests.get(
143    "https://www.dmtf.org/sites/default/files/standards/documents/"
144    + VERSION
145    + ".zip",
146    proxies=proxies,
147)
148
149r.raise_for_status()
150
151
152static_path = os.path.realpath(
153    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
154)
155
156
157cpp_path = os.path.realpath(
158    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
159)
160
161
162schema_path = os.path.join(static_path, "schema")
163json_schema_path = os.path.join(static_path, "JsonSchemas")
164metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
165
166zipBytesIO = BytesIO(r.content)
167zip_ref = zipfile.ZipFile(zipBytesIO)
168
169
170class SchemaVersion:
171    """
172    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
173    versions in the way humans expect, by comparing version strings as lists
174    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
175    insensitive schema name comparisons
176    """
177
178    def __init__(self, key):
179        key = str.casefold(key)
180
181        split_tup = key.split(".")
182        self.version_pieces = [split_tup[0]]
183        if len(split_tup) < 2:
184            return
185        version = split_tup[1]
186
187        if version.startswith("v"):
188            version = version[1:]
189        if any(char.isdigit() for char in version):
190            self.version_pieces.extend([int(x) for x in version.split("_")])
191
192    def __lt__(self, other):
193        return self.version_pieces < other.version_pieces
194
195
196# Remove the old files
197skip_prefixes = "Oem"
198if os.path.exists(schema_path):
199    files = [
200        os.path.join(schema_path, f)
201        for f in os.listdir(schema_path)
202        if not f.startswith(skip_prefixes)
203    ]
204    for f in files:
205        os.remove(f)
206if os.path.exists(json_schema_path):
207    files = [
208        os.path.join(json_schema_path, f)
209        for f in os.listdir(json_schema_path)
210        if not f.startswith(skip_prefixes)
211    ]
212    for f in files:
213        if os.path.isfile(f):
214            os.remove(f)
215        else:
216            shutil.rmtree(f)
217try:
218    os.remove(metadata_index_path)
219except FileNotFoundError:
220    pass
221
222if not os.path.exists(schema_path):
223    os.makedirs(schema_path)
224if not os.path.exists(json_schema_path):
225    os.makedirs(json_schema_path)
226
227csdl_filenames = []
228json_schema_files = defaultdict(list)
229
230for zip_file in zip_ref.infolist():
231    if zip_file.is_dir():
232        continue
233    if zip_file.filename.startswith("csdl/"):
234        csdl_filenames.append(os.path.basename(zip_file.filename))
235    elif zip_file.filename.startswith("json-schema/"):
236        filename = os.path.basename(zip_file.filename)
237        filenamesplit = filename.split(".")
238        # exclude schemas again to save flash space
239        if filenamesplit[0] not in include_list:
240            continue
241        json_schema_files[filenamesplit[0]].append(filename)
242    elif zip_file.filename.startswith("openapi/"):
243        pass
244    elif zip_file.filename.startswith("dictionaries/"):
245        pass
246
247# sort the json files by version
248for key, value in json_schema_files.items():
249    value.sort(key=SchemaVersion, reverse=True)
250
251# Create a dictionary ordered by schema name
252json_schema_files = OrderedDict(
253    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
254)
255
256csdl_filenames.sort(key=SchemaVersion)
257with open(metadata_index_path, "w") as metadata_index:
258    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
259    metadata_index.write(
260        "<edmx:Edmx xmlns:edmx="
261        '"http://docs.oasis-open.org/odata/ns/edmx"'
262        ' Version="4.0">\n'
263    )
264
265    for filename in csdl_filenames:
266        # filename looks like Zone_v1.xml
267        with open(os.path.join(schema_path, filename), "wb") as schema_out:
268            content = zip_ref.read(os.path.join("csdl", filename))
269            content = content.replace(b"\r\n", b"\n")
270
271            schema_out.write(content)
272
273            filenamesplit = filename.split("_")
274            if filenamesplit[0] not in include_list:
275                continue
276            metadata_index.write(
277                '    <edmx:Reference Uri="/redfish/v1/schema/'
278                + filename
279                + '">\n'
280            )
281
282            xml_root = ET.fromstring(content)
283            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
284            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
285            for edmx_child in xml_root:
286                if edmx_child.tag == edmx + "DataServices":
287                    for data_child in edmx_child:
288                        if data_child.tag == edm + "Schema":
289                            namespace = data_child.attrib["Namespace"]
290                            if namespace.startswith("RedfishExtensions"):
291                                metadata_index.write(
292                                    '        <edmx:Include Namespace="'
293                                    + namespace
294                                    + '"  Alias="Redfish"/>\n'
295                                )
296
297                            else:
298                                metadata_index.write(
299                                    '        <edmx:Include Namespace="'
300                                    + namespace
301                                    + '"/>\n'
302                                )
303            metadata_index.write("    </edmx:Reference>\n")
304
305    metadata_index.write(
306        "    <edmx:DataServices>\n"
307        "        <Schema "
308        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
309        'Namespace="Service">\n'
310        '            <EntityContainer Name="Service" '
311        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
312        "        </Schema>\n"
313        "    </edmx:DataServices>\n"
314    )
315    # TODO:Issue#32 There's a bug in the script that currently deletes this
316    # schema (because it's an OEM schema). Because it's the only six, and we
317    # don't update schemas very often, we just manually fix it. Need a
318    # permanent fix to the script.
319    metadata_index.write(
320        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
321    )
322    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
323    metadata_index.write("    </edmx:Reference>\n")
324
325    metadata_index.write(
326        '    <edmx:Reference Uri="'
327        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
328    )
329    metadata_index.write(
330        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
331    )
332    metadata_index.write("    </edmx:Reference>\n")
333
334    metadata_index.write(
335        '    <edmx:Reference Uri="'
336        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
337    )
338    metadata_index.write(
339        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
340    )
341    metadata_index.write(
342        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
343    )
344    metadata_index.write("    </edmx:Reference>\n")
345
346    metadata_index.write(
347        '    <edmx:Reference Uri="'
348        '/redfish/v1/schema/OemAccountService_v1.xml">\n'
349    )
350    metadata_index.write(
351        '        <edmx:Include Namespace="OemAccountService"/>\n'
352    )
353    metadata_index.write(
354        '        <edmx:Include Namespace="OemAccountService.v1_0_0"/>\n'
355    )
356    metadata_index.write("    </edmx:Reference>\n")
357
358    metadata_index.write("</edmx:Edmx>\n")
359
360
361for schema, version in json_schema_files.items():
362    zip_filepath = os.path.join("json-schema", version[0])
363    schemadir = os.path.join(json_schema_path, schema)
364    os.makedirs(schemadir)
365
366    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
367        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
368
369with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
370    hpp_file.write(
371        "#pragma once\n"
372        "{WARNING}\n"
373        "// clang-format off\n"
374        "#include <array>\n"
375        "\n"
376        "namespace redfish\n"
377        "{{\n"
378        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
379    )
380    for schema_file in json_schema_files:
381        hpp_file.write('        "{}",\n'.format(schema_file))
382
383    hpp_file.write("    };\n}\n")
384
385zip_ref.close()
386
387generate_schema_enums.main()
388generate_top_collections()
389
390# Now delete the xml schema files we aren't supporting
391if os.path.exists(schema_path):
392    files = [
393        os.path.join(schema_path, f)
394        for f in os.listdir(schema_path)
395        if not f.startswith(skip_prefixes)
396    ]
397    for filename in files:
398        # filename will include the absolute path
399        filenamesplit = filename.split("/")
400        name = filenamesplit.pop()
401        namesplit = name.split("_")
402        if namesplit[0] not in include_list:
403            print("excluding schema: " + filename)
404            os.remove(filename)
405