xref: /openbmc/bmcweb/scripts/update_schemas.py (revision f165deef) 
1#!/usr/bin/env python3
2import requests
3import zipfile
4from io import BytesIO
5from packaging.version import Version, parse
6
7import os
8from collections import OrderedDict, defaultdict
9import shutil
10import json
11
12import xml.etree.ElementTree as ET
13
14VERSION = "DSP8010_2022.2"
15
16WARNING = """/****************************************************************
17 *                 READ THIS WARNING FIRST
18 * This is an auto-generated header which contains definitions
19 * for Redfish DMTF defined schemas.
20 * DO NOT modify this registry outside of running the
21 * update_schemas.py script.  The definitions contained within
22 * this file are owned by DMTF.  Any modifications to these files
23 * should be first pushed to the relevant registry in the DMTF
24 * github organization.
25 ***************************************************************/"""
26
27# To use a new schema, add to list and rerun tool
28include_list = [
29    "AccountService",
30    "ActionInfo",
31    "Assembly",
32    "AttributeRegistry",
33    "Bios",
34    "Cable",
35    "CableCollection",
36    "Certificate",
37    "CertificateCollection",
38    "CertificateLocations",
39    "CertificateService",
40    "Chassis",
41    "ChassisCollection",
42    "ComputerSystem",
43    "ComputerSystemCollection",
44    "Drive",
45    "DriveCollection",
46    'EnvironmentMetrics',
47    "EthernetInterface",
48    "EthernetInterfaceCollection",
49    "Event",
50    "EventDestination",
51    "EventDestinationCollection",
52    "EventService",
53    "Fan",
54    "FanCollection",
55    "IPAddresses",
56    "JsonSchemaFile",
57    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
58    "LogEntry",
59    "LogEntryCollection",
60    "LogService",
61    "LogServiceCollection",
62    "Manager",
63    "ManagerAccount",
64    "ManagerAccountCollection",
65    "ManagerCollection",
66    "ManagerDiagnosticData",
67    "ManagerNetworkProtocol",
68    "Memory",
69    "MemoryCollection",
70    "Message",
71    "MessageRegistry",
72    "MessageRegistryCollection",
73    "MessageRegistryFile",
74    "MessageRegistryFileCollection",
75    "MetricDefinition",
76    "MetricDefinitionCollection",
77    "MetricReport",
78    "MetricReportCollection",
79    "MetricReportDefinition",
80    "MetricReportDefinitionCollection",
81    "OperatingConfig",
82    "OperatingConfigCollection",
83    "PCIeDevice",
84    "PCIeDeviceCollection",
85    "PCIeFunction",
86    "PCIeFunctionCollection",
87    "PhysicalContext",
88    "PCIeSlots",
89    "Power",
90    "PowerSubsystem",
91    "PowerSupply",
92    "PowerSupplyCollection",
93    "Privileges",  # Used in Role
94    "Processor",
95    "ProcessorCollection",
96    "RedfishError",
97    "RedfishExtensions",
98    "Redundancy",
99    "Resource",
100    "Role",
101    "RoleCollection",
102    "Sensor",
103    "SensorCollection",
104    "ServiceRoot",
105    "Session",
106    "SessionCollection",
107    "SessionService",
108    "Settings",
109    "SoftwareInventory",
110    "SoftwareInventoryCollection",
111    "Storage",
112    "StorageCollection",
113    "StorageController",
114    "StorageControllerCollection",
115    "Task",
116    "TaskCollection",
117    "TaskService",
118    "TelemetryService",
119    "Thermal",
120    "ThermalSubsystem",
121    "Triggers",
122    "TriggersCollection",
123    "UpdateService",
124    "VLanNetworkInterfaceCollection",
125    "VLanNetworkInterface",
126    "VirtualMedia",
127    "VirtualMediaCollection",
128    "odata",
129    "odata-v4",
130    "redfish-error",
131    "redfish-payload-annotations",
132    "redfish-schema",
133    "redfish-schema-v1",
134]
135
136SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
137
138proxies = {"https": os.environ.get("https_proxy", None)}
139
140r = requests.get(
141    "https://www.dmtf.org/sites/default/files/standards/documents/"
142    + VERSION
143    + ".zip",
144    proxies=proxies,
145)
146
147r.raise_for_status()
148
149
150static_path = os.path.realpath(
151    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
152)
153
154
155cpp_path = os.path.realpath(
156    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
157)
158
159
160schema_path = os.path.join(static_path, "schema")
161json_schema_path = os.path.join(static_path, "JsonSchemas")
162metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
163
164zipBytesIO = BytesIO(r.content)
165zip_ref = zipfile.ZipFile(zipBytesIO)
166
167
168def version_sort_key(key):
169    """
170    Method that computes a sort key that zero pads all numbers, such that
171    version sorting like
172    0_2_0
173    0_10_0
174    sorts in the way humans expect.
175    it also does case insensitive comparisons.
176    """
177    key = str.casefold(key)
178
179    # Decription of this class calls it "Version numbering for anarchists and
180    # software realists.".  That seems like exactly what we need here.
181
182    if not any(char.isdigit() for char in key):
183        split_tup = os.path.splitext(key)
184        key = split_tup[0] + ".v0_0_0" + split_tup[1]
185
186    # special case some files that don't seem to follow the naming convention,
187    # and cause sort problems.  These need brought up with DMTF TODO(Ed)
188    if key == "odata.4.0.0.json":
189        key = "odata.v4_0_0.json"
190    if key == "redfish-schema.1.0.0.json":
191        key = "redfish-schema.v1_0_0.json"
192
193    return parse(key)
194
195
196# Remove the old files
197
198skip_prefixes = "Oem"
199if os.path.exists(schema_path):
200    files = [
201        os.path.join(schema_path, f)
202        for f in os.listdir(schema_path)
203        if not f.startswith(skip_prefixes)
204    ]
205    for f in files:
206        os.remove(f)
207if os.path.exists(json_schema_path):
208    files = [
209        os.path.join(json_schema_path, f)
210        for f in os.listdir(json_schema_path)
211        if not f.startswith(skip_prefixes)
212    ]
213    for f in files:
214        if os.path.isfile(f):
215            os.remove(f)
216        else:
217            shutil.rmtree(f)
218try:
219    os.remove(metadata_index_path)
220except FileNotFoundError:
221    pass
222
223if not os.path.exists(schema_path):
224    os.makedirs(schema_path)
225if not os.path.exists(json_schema_path):
226    os.makedirs(json_schema_path)
227
228csdl_filenames = []
229json_schema_files = defaultdict(list)
230
231for zip_filepath in zip_ref.namelist():
232    if zip_filepath.startswith("csdl/") and (zip_filepath != "csdl/"):
233        csdl_filenames.append(os.path.basename(zip_filepath))
234    elif zip_filepath.startswith("json-schema/"):
235        filename = os.path.basename(zip_filepath)
236        filenamesplit = filename.split(".")
237        # exclude schemas again to save flash space
238        if filenamesplit[0] not in include_list:
239            continue
240        json_schema_files[filenamesplit[0]].append(filename)
241    elif zip_filepath.startswith("openapi/"):
242        pass
243    elif zip_filepath.startswith("dictionaries/"):
244        pass
245
246# sort the json files by version
247for key, value in json_schema_files.items():
248    value.sort(key=version_sort_key, reverse=True)
249
250# Create a dictionary ordered by schema name
251json_schema_files = OrderedDict(
252    sorted(json_schema_files.items(), key=lambda x: version_sort_key(x[0]))
253)
254
255csdl_filenames.sort(key=version_sort_key)
256with open(metadata_index_path, "w") as metadata_index:
257
258    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
259    metadata_index.write(
260        "<edmx:Edmx xmlns:edmx="
261        '"http://docs.oasis-open.org/odata/ns/edmx"'
262        ' Version="4.0">\n'
263    )
264
265    for filename in csdl_filenames:
266        # filename looks like Zone_v1.xml
267        filenamesplit = filename.split("_")
268        if filenamesplit[0] not in include_list:
269            print("excluding schema: " + filename)
270            continue
271
272        with open(os.path.join(schema_path, filename), "wb") as schema_out:
273
274            metadata_index.write(
275                '    <edmx:Reference Uri="/redfish/v1/schema/'
276                + filename
277                + '">\n'
278            )
279
280            content = zip_ref.read(os.path.join("csdl", filename))
281            content = content.replace(b"\r\n", b"\n")
282            xml_root = ET.fromstring(content)
283            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
284            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
285            for edmx_child in xml_root:
286                if edmx_child.tag == edmx + "DataServices":
287                    for data_child in edmx_child:
288                        if data_child.tag == edm + "Schema":
289                            namespace = data_child.attrib["Namespace"]
290                            if namespace.startswith("RedfishExtensions"):
291                                metadata_index.write(
292                                    "        "
293                                    '<edmx:Include Namespace="'
294                                    + namespace
295                                    + '"  Alias="Redfish"/>\n'
296                                )
297
298                            else:
299                                metadata_index.write(
300                                    "        "
301                                    '<edmx:Include Namespace="'
302                                    + namespace
303                                    + '"/>\n'
304                                )
305            schema_out.write(content)
306            metadata_index.write("    </edmx:Reference>\n")
307
308    metadata_index.write(
309        "    <edmx:DataServices>\n"
310        "        <Schema "
311        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
312        'Namespace="Service">\n'
313        '            <EntityContainer Name="Service" '
314        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
315        "        </Schema>\n"
316        "    </edmx:DataServices>\n"
317    )
318    # TODO:Issue#32 There's a bug in the script that currently deletes this
319    # schema (because it's an OEM schema). Because it's the only six, and we
320    # don't update schemas very often, we just manually fix it. Need a
321    # permanent fix to the script.
322    metadata_index.write(
323        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
324    )
325    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
326    metadata_index.write("    </edmx:Reference>\n")
327
328    metadata_index.write(
329        '    <edmx:Reference Uri="'
330        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
331    )
332    metadata_index.write(
333        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
334    )
335    metadata_index.write("    </edmx:Reference>\n")
336
337    metadata_index.write(
338        '    <edmx:Reference Uri="'
339        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
340    )
341    metadata_index.write(
342        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
343    )
344    metadata_index.write(
345        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
346    )
347    metadata_index.write("    </edmx:Reference>\n")
348
349    metadata_index.write(
350        '    <edmx:Reference Uri="'
351        '/redfish/v1/schema/OemAccountService_v1.xml">\n'
352    )
353    metadata_index.write(
354        '        <edmx:Include Namespace="OemAccountService"/>\n'
355    )
356    metadata_index.write(
357        '        <edmx:Include Namespace="OemAccountService.v1_0_0"/>\n'
358    )
359    metadata_index.write("    </edmx:Reference>\n")
360
361    metadata_index.write(
362        '    <edmx:Reference Uri="/redfish/v1/schema/OemSession_v1.xml">\n'
363    )
364    metadata_index.write('        <edmx:Include Namespace="OemSession"/>\n')
365    metadata_index.write(
366        '        <edmx:Include Namespace="OemSession.v1_0_0"/>\n'
367    )
368    metadata_index.write("    </edmx:Reference>\n")
369
370    metadata_index.write("</edmx:Edmx>\n")
371
372
373for schema, version in json_schema_files.items():
374    zip_filepath = os.path.join("json-schema", version[0])
375    schemadir = os.path.join(json_schema_path, schema)
376    os.makedirs(schemadir)
377
378    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
379        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
380
381with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
382    hpp_file.write(
383        "#pragma once\n"
384        "{WARNING}\n"
385        "// clang-format off\n"
386        "#include <array>\n"
387        "\n"
388        "namespace redfish\n"
389        "{{\n"
390        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
391    )
392    for schema_file in json_schema_files:
393        hpp_file.write('        "{}",\n'.format(schema_file))
394
395    hpp_file.write("    };\n" "}\n")
396
397zip_ref.close()
398