xref: /openbmc/bmcweb/scripts/update_schemas.py (revision 7a1dbc48) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2022.2"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "Assembly",
31    "AttributeRegistry",
32    "Bios",
33    "Cable",
34    "CableCollection",
35    "Certificate",
36    "CertificateCollection",
37    "CertificateLocations",
38    "CertificateService",
39    "Chassis",
40    "ChassisCollection",
41    "ComputerSystem",
42    "ComputerSystemCollection",
43    "Drive",
44    "DriveCollection",
45    "EnvironmentMetrics",
46    "EthernetInterface",
47    "EthernetInterfaceCollection",
48    "Event",
49    "EventDestination",
50    "EventDestinationCollection",
51    "EventService",
52    "Fan",
53    "FanCollection",
54    "IPAddresses",
55    "JsonSchemaFile",
56    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
57    "LogEntry",
58    "LogEntryCollection",
59    "LogService",
60    "LogServiceCollection",
61    "Manager",
62    "ManagerAccount",
63    "ManagerAccountCollection",
64    "ManagerCollection",
65    "ManagerDiagnosticData",
66    "ManagerNetworkProtocol",
67    "Memory",
68    "MemoryCollection",
69    "Message",
70    "MessageRegistry",
71    "MessageRegistryCollection",
72    "MessageRegistryFile",
73    "MessageRegistryFileCollection",
74    "MetricDefinition",
75    "MetricDefinitionCollection",
76    "MetricReport",
77    "MetricReportCollection",
78    "MetricReportDefinition",
79    "MetricReportDefinitionCollection",
80    "OperatingConfig",
81    "OperatingConfigCollection",
82    "PCIeDevice",
83    "PCIeDeviceCollection",
84    "PCIeFunction",
85    "PCIeFunctionCollection",
86    "PhysicalContext",
87    "PCIeSlots",
88    "Power",
89    "PowerSubsystem",
90    "PowerSupply",
91    "PowerSupplyCollection",
92    "Privileges",  # Used in Role
93    "Processor",
94    "ProcessorCollection",
95    "RedfishError",
96    "RedfishExtensions",
97    "Redundancy",
98    "Resource",
99    "Role",
100    "RoleCollection",
101    "Sensor",
102    "SensorCollection",
103    "ServiceRoot",
104    "Session",
105    "SessionCollection",
106    "SessionService",
107    "Settings",
108    "SoftwareInventory",
109    "SoftwareInventoryCollection",
110    "Storage",
111    "StorageCollection",
112    "StorageController",
113    "StorageControllerCollection",
114    "Task",
115    "TaskCollection",
116    "TaskService",
117    "TelemetryService",
118    "Thermal",
119    "ThermalMetrics",
120    "ThermalSubsystem",
121    "Triggers",
122    "TriggersCollection",
123    "UpdateService",
124    "VLanNetworkInterfaceCollection",
125    "VLanNetworkInterface",
126    "VirtualMedia",
127    "VirtualMediaCollection",
128    "odata",
129    "odata-v4",
130    "redfish-error",
131    "redfish-payload-annotations",
132    "redfish-schema",
133    "redfish-schema-v1",
134]
135
136SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
137
138proxies = {"https": os.environ.get("https_proxy", None)}
139
140r = requests.get(
141    "https://www.dmtf.org/sites/default/files/standards/documents/"
142    + VERSION
143    + ".zip",
144    proxies=proxies,
145)
146
147r.raise_for_status()
148
149
150static_path = os.path.realpath(
151    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
152)
153
154
155cpp_path = os.path.realpath(
156    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
157)
158
159
160schema_path = os.path.join(static_path, "schema")
161json_schema_path = os.path.join(static_path, "JsonSchemas")
162metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
163
164zipBytesIO = BytesIO(r.content)
165zip_ref = zipfile.ZipFile(zipBytesIO)
166
167
168class SchemaVersion:
169    """
170    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
171    versions in the way humans expect, by comparing version strings as lists
172    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
173    insensitive schema name comparisons
174    """
175
176    def __init__(self, key):
177        key = str.casefold(key)
178
179        split_tup = key.split(".")
180        self.version_pieces = [split_tup[0]]
181        if len(split_tup) < 2:
182            return
183        version = split_tup[1]
184
185        if version.startswith("v"):
186            version = version[1:]
187        if any(char.isdigit() for char in version):
188            self.version_pieces.extend([int(x) for x in version.split("_")])
189
190    def __lt__(self, other):
191        return self.version_pieces < other.version_pieces
192
193
194# Remove the old files
195skip_prefixes = "Oem"
196if os.path.exists(schema_path):
197    files = [
198        os.path.join(schema_path, f)
199        for f in os.listdir(schema_path)
200        if not f.startswith(skip_prefixes)
201    ]
202    for f in files:
203        os.remove(f)
204if os.path.exists(json_schema_path):
205    files = [
206        os.path.join(json_schema_path, f)
207        for f in os.listdir(json_schema_path)
208        if not f.startswith(skip_prefixes)
209    ]
210    for f in files:
211        if os.path.isfile(f):
212            os.remove(f)
213        else:
214            shutil.rmtree(f)
215try:
216    os.remove(metadata_index_path)
217except FileNotFoundError:
218    pass
219
220if not os.path.exists(schema_path):
221    os.makedirs(schema_path)
222if not os.path.exists(json_schema_path):
223    os.makedirs(json_schema_path)
224
225csdl_filenames = []
226json_schema_files = defaultdict(list)
227
228for zip_file in zip_ref.infolist():
229    if zip_file.is_dir():
230        continue
231    if zip_file.filename.startswith("csdl/"):
232        csdl_filenames.append(os.path.basename(zip_file.filename))
233    elif zip_file.filename.startswith("json-schema/"):
234        filename = os.path.basename(zip_file.filename)
235        filenamesplit = filename.split(".")
236        # exclude schemas again to save flash space
237        if filenamesplit[0] not in include_list:
238            continue
239        json_schema_files[filenamesplit[0]].append(filename)
240    elif zip_file.filename.startswith("openapi/"):
241        pass
242    elif zip_file.filename.startswith("dictionaries/"):
243        pass
244
245# sort the json files by version
246for key, value in json_schema_files.items():
247    value.sort(key=SchemaVersion, reverse=True)
248
249# Create a dictionary ordered by schema name
250json_schema_files = OrderedDict(
251    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
252)
253
254csdl_filenames.sort(key=SchemaVersion)
255with open(metadata_index_path, "w") as metadata_index:
256    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
257    metadata_index.write(
258        "<edmx:Edmx xmlns:edmx="
259        '"http://docs.oasis-open.org/odata/ns/edmx"'
260        ' Version="4.0">\n'
261    )
262
263    for filename in csdl_filenames:
264        # filename looks like Zone_v1.xml
265        with open(os.path.join(schema_path, filename), "wb") as schema_out:
266            content = zip_ref.read(os.path.join("csdl", filename))
267            content = content.replace(b"\r\n", b"\n")
268
269            schema_out.write(content)
270
271            filenamesplit = filename.split("_")
272            if filenamesplit[0] not in include_list:
273                continue
274            metadata_index.write(
275                '    <edmx:Reference Uri="/redfish/v1/schema/'
276                + filename
277                + '">\n'
278            )
279
280            xml_root = ET.fromstring(content)
281            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
282            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
283            for edmx_child in xml_root:
284                if edmx_child.tag == edmx + "DataServices":
285                    for data_child in edmx_child:
286                        if data_child.tag == edm + "Schema":
287                            namespace = data_child.attrib["Namespace"]
288                            if namespace.startswith("RedfishExtensions"):
289                                metadata_index.write(
290                                    '        <edmx:Include Namespace="'
291                                    + namespace
292                                    + '"  Alias="Redfish"/>\n'
293                                )
294
295                            else:
296                                metadata_index.write(
297                                    '        <edmx:Include Namespace="'
298                                    + namespace
299                                    + '"/>\n'
300                                )
301            metadata_index.write("    </edmx:Reference>\n")
302
303    metadata_index.write(
304        "    <edmx:DataServices>\n"
305        "        <Schema "
306        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
307        'Namespace="Service">\n'
308        '            <EntityContainer Name="Service" '
309        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
310        "        </Schema>\n"
311        "    </edmx:DataServices>\n"
312    )
313    # TODO:Issue#32 There's a bug in the script that currently deletes this
314    # schema (because it's an OEM schema). Because it's the only six, and we
315    # don't update schemas very often, we just manually fix it. Need a
316    # permanent fix to the script.
317    metadata_index.write(
318        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
319    )
320    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
321    metadata_index.write("    </edmx:Reference>\n")
322
323    metadata_index.write(
324        '    <edmx:Reference Uri="'
325        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
326    )
327    metadata_index.write(
328        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
329    )
330    metadata_index.write("    </edmx:Reference>\n")
331
332    metadata_index.write(
333        '    <edmx:Reference Uri="'
334        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
335    )
336    metadata_index.write(
337        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
338    )
339    metadata_index.write(
340        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
341    )
342    metadata_index.write("    </edmx:Reference>\n")
343
344    metadata_index.write(
345        '    <edmx:Reference Uri="'
346        '/redfish/v1/schema/OemAccountService_v1.xml">\n'
347    )
348    metadata_index.write(
349        '        <edmx:Include Namespace="OemAccountService"/>\n'
350    )
351    metadata_index.write(
352        '        <edmx:Include Namespace="OemAccountService.v1_0_0"/>\n'
353    )
354    metadata_index.write("    </edmx:Reference>\n")
355
356    metadata_index.write(
357        '    <edmx:Reference Uri="/redfish/v1/schema/OemSession_v1.xml">\n'
358    )
359    metadata_index.write('        <edmx:Include Namespace="OemSession"/>\n')
360    metadata_index.write(
361        '        <edmx:Include Namespace="OemSession.v1_0_0"/>\n'
362    )
363    metadata_index.write("    </edmx:Reference>\n")
364
365    metadata_index.write("</edmx:Edmx>\n")
366
367
368for schema, version in json_schema_files.items():
369    zip_filepath = os.path.join("json-schema", version[0])
370    schemadir = os.path.join(json_schema_path, schema)
371    os.makedirs(schemadir)
372
373    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
374        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
375
376with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
377    hpp_file.write(
378        "#pragma once\n"
379        "{WARNING}\n"
380        "// clang-format off\n"
381        "#include <array>\n"
382        "\n"
383        "namespace redfish\n"
384        "{{\n"
385        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
386    )
387    for schema_file in json_schema_files:
388        hpp_file.write('        "{}",\n'.format(schema_file))
389
390    hpp_file.write("    };\n}\n")
391
392zip_ref.close()
393
394generate_schema_enums.main()
395generate_top_collections()
396
397# Now delete the xml schema files we aren't supporting
398if os.path.exists(schema_path):
399    files = [
400        os.path.join(schema_path, f)
401        for f in os.listdir(schema_path)
402        if not f.startswith(skip_prefixes)
403    ]
404    for filename in files:
405        # filename will include the absolute path
406        filenamesplit = filename.split("/")
407        name = filenamesplit.pop()
408        namesplit = name.split("_")
409        if namesplit[0] not in include_list:
410            print("excluding schema: " + filename)
411            os.remove(filename)
412