xref: /openbmc/bmcweb/scripts/update_schemas.py (revision dfa3fdc3) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import requests
10from packaging.version import parse
11
12VERSION = "DSP8010_2022.2"
13
14WARNING = """/****************************************************************
15 *                 READ THIS WARNING FIRST
16 * This is an auto-generated header which contains definitions
17 * for Redfish DMTF defined schemas.
18 * DO NOT modify this registry outside of running the
19 * update_schemas.py script.  The definitions contained within
20 * this file are owned by DMTF.  Any modifications to these files
21 * should be first pushed to the relevant registry in the DMTF
22 * github organization.
23 ***************************************************************/"""
24
25# To use a new schema, add to list and rerun tool
26include_list = [
27    "AccountService",
28    "ActionInfo",
29    "Assembly",
30    "AttributeRegistry",
31    "Bios",
32    "Cable",
33    "CableCollection",
34    "Certificate",
35    "CertificateCollection",
36    "CertificateLocations",
37    "CertificateService",
38    "Chassis",
39    "ChassisCollection",
40    "ComputerSystem",
41    "ComputerSystemCollection",
42    "Drive",
43    "DriveCollection",
44    "EnvironmentMetrics",
45    "EthernetInterface",
46    "EthernetInterfaceCollection",
47    "Event",
48    "EventDestination",
49    "EventDestinationCollection",
50    "EventService",
51    "Fan",
52    "FanCollection",
53    "IPAddresses",
54    "JsonSchemaFile",
55    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
56    "LogEntry",
57    "LogEntryCollection",
58    "LogService",
59    "LogServiceCollection",
60    "Manager",
61    "ManagerAccount",
62    "ManagerAccountCollection",
63    "ManagerCollection",
64    "ManagerDiagnosticData",
65    "ManagerNetworkProtocol",
66    "Memory",
67    "MemoryCollection",
68    "Message",
69    "MessageRegistry",
70    "MessageRegistryCollection",
71    "MessageRegistryFile",
72    "MessageRegistryFileCollection",
73    "MetricDefinition",
74    "MetricDefinitionCollection",
75    "MetricReport",
76    "MetricReportCollection",
77    "MetricReportDefinition",
78    "MetricReportDefinitionCollection",
79    "OperatingConfig",
80    "OperatingConfigCollection",
81    "PCIeDevice",
82    "PCIeDeviceCollection",
83    "PCIeFunction",
84    "PCIeFunctionCollection",
85    "PhysicalContext",
86    "PCIeSlots",
87    "Power",
88    "PowerSubsystem",
89    "PowerSupply",
90    "PowerSupplyCollection",
91    "Privileges",  # Used in Role
92    "Processor",
93    "ProcessorCollection",
94    "RedfishError",
95    "RedfishExtensions",
96    "Redundancy",
97    "Resource",
98    "Role",
99    "RoleCollection",
100    "Sensor",
101    "SensorCollection",
102    "ServiceRoot",
103    "Session",
104    "SessionCollection",
105    "SessionService",
106    "Settings",
107    "SoftwareInventory",
108    "SoftwareInventoryCollection",
109    "Storage",
110    "StorageCollection",
111    "StorageController",
112    "StorageControllerCollection",
113    "Task",
114    "TaskCollection",
115    "TaskService",
116    "TelemetryService",
117    "Thermal",
118    "ThermalMetrics",
119    "ThermalSubsystem",
120    "Triggers",
121    "TriggersCollection",
122    "UpdateService",
123    "VLanNetworkInterfaceCollection",
124    "VLanNetworkInterface",
125    "VirtualMedia",
126    "VirtualMediaCollection",
127    "odata",
128    "odata-v4",
129    "redfish-error",
130    "redfish-payload-annotations",
131    "redfish-schema",
132    "redfish-schema-v1",
133]
134
135SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
136
137proxies = {"https": os.environ.get("https_proxy", None)}
138
139r = requests.get(
140    "https://www.dmtf.org/sites/default/files/standards/documents/"
141    + VERSION
142    + ".zip",
143    proxies=proxies,
144)
145
146r.raise_for_status()
147
148
149static_path = os.path.realpath(
150    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
151)
152
153
154cpp_path = os.path.realpath(
155    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
156)
157
158
159schema_path = os.path.join(static_path, "schema")
160json_schema_path = os.path.join(static_path, "JsonSchemas")
161metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
162
163zipBytesIO = BytesIO(r.content)
164zip_ref = zipfile.ZipFile(zipBytesIO)
165
166
167def version_sort_key(key):
168    """
169    Method that computes a sort key that zero pads all numbers, such that
170    version sorting like
171    0_2_0
172    0_10_0
173    sorts in the way humans expect.
174    it also does case insensitive comparisons.
175    """
176    key = str.casefold(key)
177
178    # Decription of this class calls it "Version numbering for anarchists and
179    # software realists.".  That seems like exactly what we need here.
180
181    if not any(char.isdigit() for char in key):
182        split_tup = os.path.splitext(key)
183        key = split_tup[0] + ".v0_0_0" + split_tup[1]
184
185    # special case some files that don't seem to follow the naming convention,
186    # and cause sort problems.  These need brought up with DMTF TODO(Ed)
187    if key == "odata.4.0.0.json":
188        key = "odata.v4_0_0.json"
189    if key == "redfish-schema.1.0.0.json":
190        key = "redfish-schema.v1_0_0.json"
191
192    return parse(key)
193
194
195# Remove the old files
196
197skip_prefixes = "Oem"
198if os.path.exists(schema_path):
199    files = [
200        os.path.join(schema_path, f)
201        for f in os.listdir(schema_path)
202        if not f.startswith(skip_prefixes)
203    ]
204    for f in files:
205        os.remove(f)
206if os.path.exists(json_schema_path):
207    files = [
208        os.path.join(json_schema_path, f)
209        for f in os.listdir(json_schema_path)
210        if not f.startswith(skip_prefixes)
211    ]
212    for f in files:
213        if os.path.isfile(f):
214            os.remove(f)
215        else:
216            shutil.rmtree(f)
217try:
218    os.remove(metadata_index_path)
219except FileNotFoundError:
220    pass
221
222if not os.path.exists(schema_path):
223    os.makedirs(schema_path)
224if not os.path.exists(json_schema_path):
225    os.makedirs(json_schema_path)
226
227csdl_filenames = []
228json_schema_files = defaultdict(list)
229
230for zip_filepath in zip_ref.namelist():
231    if zip_filepath.startswith("csdl/") and (zip_filepath != "csdl/"):
232        csdl_filenames.append(os.path.basename(zip_filepath))
233    elif zip_filepath.startswith("json-schema/"):
234        filename = os.path.basename(zip_filepath)
235        filenamesplit = filename.split(".")
236        # exclude schemas again to save flash space
237        if filenamesplit[0] not in include_list:
238            continue
239        json_schema_files[filenamesplit[0]].append(filename)
240    elif zip_filepath.startswith("openapi/"):
241        pass
242    elif zip_filepath.startswith("dictionaries/"):
243        pass
244
245# sort the json files by version
246for key, value in json_schema_files.items():
247    value.sort(key=version_sort_key, reverse=True)
248
249# Create a dictionary ordered by schema name
250json_schema_files = OrderedDict(
251    sorted(json_schema_files.items(), key=lambda x: version_sort_key(x[0]))
252)
253
254csdl_filenames.sort(key=version_sort_key)
255with open(metadata_index_path, "w") as metadata_index:
256    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
257    metadata_index.write(
258        "<edmx:Edmx xmlns:edmx="
259        '"http://docs.oasis-open.org/odata/ns/edmx"'
260        ' Version="4.0">\n'
261    )
262
263    for filename in csdl_filenames:
264        # filename looks like Zone_v1.xml
265        filenamesplit = filename.split("_")
266        if filenamesplit[0] not in include_list:
267            print("excluding schema: " + filename)
268            continue
269
270        with open(os.path.join(schema_path, filename), "wb") as schema_out:
271            metadata_index.write(
272                '    <edmx:Reference Uri="/redfish/v1/schema/'
273                + filename
274                + '">\n'
275            )
276
277            content = zip_ref.read(os.path.join("csdl", filename))
278            content = content.replace(b"\r\n", b"\n")
279            xml_root = ET.fromstring(content)
280            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
281            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
282            for edmx_child in xml_root:
283                if edmx_child.tag == edmx + "DataServices":
284                    for data_child in edmx_child:
285                        if data_child.tag == edm + "Schema":
286                            namespace = data_child.attrib["Namespace"]
287                            if namespace.startswith("RedfishExtensions"):
288                                metadata_index.write(
289                                    '        <edmx:Include Namespace="'
290                                    + namespace
291                                    + '"  Alias="Redfish"/>\n'
292                                )
293
294                            else:
295                                metadata_index.write(
296                                    '        <edmx:Include Namespace="'
297                                    + namespace
298                                    + '"/>\n'
299                                )
300            schema_out.write(content)
301            metadata_index.write("    </edmx:Reference>\n")
302
303    metadata_index.write(
304        "    <edmx:DataServices>\n"
305        "        <Schema "
306        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
307        'Namespace="Service">\n'
308        '            <EntityContainer Name="Service" '
309        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
310        "        </Schema>\n"
311        "    </edmx:DataServices>\n"
312    )
313    # TODO:Issue#32 There's a bug in the script that currently deletes this
314    # schema (because it's an OEM schema). Because it's the only six, and we
315    # don't update schemas very often, we just manually fix it. Need a
316    # permanent fix to the script.
317    metadata_index.write(
318        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
319    )
320    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
321    metadata_index.write("    </edmx:Reference>\n")
322
323    metadata_index.write(
324        '    <edmx:Reference Uri="'
325        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
326    )
327    metadata_index.write(
328        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
329    )
330    metadata_index.write("    </edmx:Reference>\n")
331
332    metadata_index.write(
333        '    <edmx:Reference Uri="'
334        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
335    )
336    metadata_index.write(
337        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
338    )
339    metadata_index.write(
340        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
341    )
342    metadata_index.write("    </edmx:Reference>\n")
343
344    metadata_index.write(
345        '    <edmx:Reference Uri="'
346        '/redfish/v1/schema/OemAccountService_v1.xml">\n'
347    )
348    metadata_index.write(
349        '        <edmx:Include Namespace="OemAccountService"/>\n'
350    )
351    metadata_index.write(
352        '        <edmx:Include Namespace="OemAccountService.v1_0_0"/>\n'
353    )
354    metadata_index.write("    </edmx:Reference>\n")
355
356    metadata_index.write(
357        '    <edmx:Reference Uri="/redfish/v1/schema/OemSession_v1.xml">\n'
358    )
359    metadata_index.write('        <edmx:Include Namespace="OemSession"/>\n')
360    metadata_index.write(
361        '        <edmx:Include Namespace="OemSession.v1_0_0"/>\n'
362    )
363    metadata_index.write("    </edmx:Reference>\n")
364
365    metadata_index.write("</edmx:Edmx>\n")
366
367
368for schema, version in json_schema_files.items():
369    zip_filepath = os.path.join("json-schema", version[0])
370    schemadir = os.path.join(json_schema_path, schema)
371    os.makedirs(schemadir)
372
373    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
374        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
375
376with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
377    hpp_file.write(
378        "#pragma once\n"
379        "{WARNING}\n"
380        "// clang-format off\n"
381        "#include <array>\n"
382        "\n"
383        "namespace redfish\n"
384        "{{\n"
385        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
386    )
387    for schema_file in json_schema_files:
388        hpp_file.write('        "{}",\n'.format(schema_file))
389
390    hpp_file.write("    };\n}\n")
391
392zip_ref.close()
393