xref: /openbmc/bmcweb/scripts/update_schemas.py (revision ed76121b) 
1#!/usr/bin/env python3
2import requests
3import zipfile
4from io import BytesIO
5from packaging.version import Version, parse
6
7import os
8from collections import OrderedDict, defaultdict
9import shutil
10import json
11
12import xml.etree.ElementTree as ET
13
14VERSION = "DSP8010_2022.1"
15
16WARNING = """/****************************************************************
17 *                 READ THIS WARNING FIRST
18 * This is an auto-generated header which contains definitions
19 * for Redfish DMTF defined schemas.
20 * DO NOT modify this registry outside of running the
21 * update_schemas.py script.  The definitions contained within
22 * this file are owned by DMTF.  Any modifications to these files
23 * should be first pushed to the relevant registry in the DMTF
24 * github organization.
25 ***************************************************************/"""
26
27# To use a new schema, add to list and rerun tool
28include_list = [
29    "AccountService",
30    "ActionInfo",
31    "Assembly",
32    "AttributeRegistry",
33    "Bios",
34    "Cable",
35    "CableCollection",
36    "Certificate",
37    "CertificateCollection",
38    "CertificateLocations",
39    "CertificateService",
40    "Chassis",
41    "ChassisCollection",
42    "ComputerSystem",
43    "ComputerSystemCollection",
44    "Drive",
45    "DriveCollection",
46    "EthernetInterface",
47    "EthernetInterfaceCollection",
48    "Event",
49    "EventDestination",
50    "EventDestinationCollection",
51    "EventService",
52    "IPAddresses",
53    "JsonSchemaFile",
54    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
55    "LogEntry",
56    "LogEntryCollection",
57    "LogService",
58    "LogServiceCollection",
59    "Manager",
60    "ManagerAccount",
61    "ManagerAccountCollection",
62    "ManagerCollection",
63    "ManagerDiagnosticData",
64    "ManagerNetworkProtocol",
65    "Memory",
66    "MemoryCollection",
67    "Message",
68    "MessageRegistry",
69    "MessageRegistryCollection",
70    "MessageRegistryFile",
71    "MessageRegistryFileCollection",
72    "MetricDefinition",
73    "MetricDefinitionCollection",
74    "MetricReport",
75    "MetricReportCollection",
76    "MetricReportDefinition",
77    "MetricReportDefinitionCollection",
78    "OperatingConfig",
79    "OperatingConfigCollection",
80    "PCIeDevice",
81    "PCIeDeviceCollection",
82    "PCIeFunction",
83    "PCIeFunctionCollection",
84    "PhysicalContext",
85    "PCIeSlots",
86    "Power",
87    "Privileges",  # Used in Role
88    "Processor",
89    "ProcessorCollection",
90    "RedfishError",
91    "RedfishExtensions",
92    "Redundancy",
93    "Resource",
94    "Role",
95    "RoleCollection",
96    "Sensor",
97    "SensorCollection",
98    "ServiceRoot",
99    "Session",
100    "SessionCollection",
101    "SessionService",
102    "Settings",
103    "SoftwareInventory",
104    "SoftwareInventoryCollection",
105    "Storage",
106    "StorageCollection",
107    "StorageController",
108    "StorageControllerCollection",
109    "Task",
110    "TaskCollection",
111    "TaskService",
112    "TelemetryService",
113    "Thermal",
114    "ThermalSubsystem",
115    "Triggers",
116    "TriggersCollection",
117    "UpdateService",
118    "VLanNetworkInterfaceCollection",
119    "VLanNetworkInterface",
120    "VirtualMedia",
121    "VirtualMediaCollection",
122    "odata",
123    "odata-v4",
124    "redfish-error",
125    "redfish-payload-annotations",
126    "redfish-schema",
127    "redfish-schema-v1",
128]
129
130SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
131
132proxies = {"https": os.environ.get("https_proxy", None)}
133
134r = requests.get(
135    "https://www.dmtf.org/sites/default/files/standards/documents/"
136    + VERSION
137    + ".zip",
138    proxies=proxies,
139)
140
141r.raise_for_status()
142
143
144static_path = os.path.realpath(
145    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
146)
147
148
149cpp_path = os.path.realpath(
150    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
151)
152
153
154schema_path = os.path.join(static_path, "schema")
155json_schema_path = os.path.join(static_path, "JsonSchemas")
156metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
157
158zipBytesIO = BytesIO(r.content)
159zip_ref = zipfile.ZipFile(zipBytesIO)
160
161
162def version_sort_key(key):
163    """
164    Method that computes a sort key that zero pads all numbers, such that
165    version sorting like
166    0_2_0
167    0_10_0
168    sorts in the way humans expect.
169    it also does case insensitive comparisons.
170    """
171    key = str.casefold(key)
172
173    # Decription of this class calls it "Version numbering for anarchists and
174    # software realists.".  That seems like exactly what we need here.
175
176    if not any(char.isdigit() for char in key):
177        split_tup = os.path.splitext(key)
178        key = split_tup[0] + ".v0_0_0" + split_tup[1]
179
180    # special case some files that don't seem to follow the naming convention,
181    # and cause sort problems.  These need brought up with DMTF TODO(Ed)
182    if key == "odata.4.0.0.json":
183        key = "odata.v4_0_0.json"
184    if key == "redfish-schema.1.0.0.json":
185        key = "redfish-schema.v1_0_0.json"
186
187    return parse(key)
188
189
190# Remove the old files
191
192skip_prefixes = "Oem"
193if os.path.exists(schema_path):
194    files = [
195        os.path.join(schema_path, f)
196        for f in os.listdir(schema_path)
197        if not f.startswith(skip_prefixes)
198    ]
199    for f in files:
200        os.remove(f)
201if os.path.exists(json_schema_path):
202    files = [
203        os.path.join(json_schema_path, f)
204        for f in os.listdir(json_schema_path)
205        if not f.startswith(skip_prefixes)
206    ]
207    for f in files:
208        if os.path.isfile(f):
209            os.remove(f)
210        else:
211            shutil.rmtree(f)
212try:
213    os.remove(metadata_index_path)
214except FileNotFoundError:
215    pass
216
217if not os.path.exists(schema_path):
218    os.makedirs(schema_path)
219if not os.path.exists(json_schema_path):
220    os.makedirs(json_schema_path)
221
222csdl_filenames = []
223json_schema_files = defaultdict(list)
224
225for zip_filepath in zip_ref.namelist():
226    if zip_filepath.startswith("csdl/") and (zip_filepath != "csdl/"):
227        csdl_filenames.append(os.path.basename(zip_filepath))
228    elif zip_filepath.startswith("json-schema/"):
229        filename = os.path.basename(zip_filepath)
230        filenamesplit = filename.split(".")
231        # exclude schemas again to save flash space
232        if filenamesplit[0] not in include_list:
233            continue
234        json_schema_files[filenamesplit[0]].append(filename)
235    elif zip_filepath.startswith("openapi/"):
236        pass
237    elif zip_filepath.startswith("dictionaries/"):
238        pass
239
240# sort the json files by version
241for key, value in json_schema_files.items():
242    value.sort(key=version_sort_key, reverse=True)
243
244# Create a dictionary ordered by schema name
245json_schema_files = OrderedDict(
246    sorted(json_schema_files.items(), key=lambda x: version_sort_key(x[0]))
247)
248
249csdl_filenames.sort(key=version_sort_key)
250with open(metadata_index_path, "w") as metadata_index:
251
252    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
253    metadata_index.write(
254        "<edmx:Edmx xmlns:edmx="
255        '"http://docs.oasis-open.org/odata/ns/edmx"'
256        ' Version="4.0">\n'
257    )
258
259    for filename in csdl_filenames:
260        # filename looks like Zone_v1.xml
261        filenamesplit = filename.split("_")
262        if filenamesplit[0] not in include_list:
263            print("excluding schema: " + filename)
264            continue
265
266        with open(os.path.join(schema_path, filename), "wb") as schema_out:
267
268            metadata_index.write(
269                '    <edmx:Reference Uri="/redfish/v1/schema/'
270                + filename
271                + '">\n'
272            )
273
274            content = zip_ref.read(os.path.join("csdl", filename))
275            content = content.replace(b"\r\n", b"\n")
276            xml_root = ET.fromstring(content)
277            edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
278            edm = "{http://docs.oasis-open.org/odata/ns/edm}"
279            for edmx_child in xml_root:
280                if edmx_child.tag == edmx + "DataServices":
281                    for data_child in edmx_child:
282                        if data_child.tag == edm + "Schema":
283                            namespace = data_child.attrib["Namespace"]
284                            if namespace.startswith("RedfishExtensions"):
285                                metadata_index.write(
286                                    "        "
287                                    '<edmx:Include Namespace="'
288                                    + namespace
289                                    + '"  Alias="Redfish"/>\n'
290                                )
291
292                            else:
293                                metadata_index.write(
294                                    "        "
295                                    '<edmx:Include Namespace="'
296                                    + namespace
297                                    + '"/>\n'
298                                )
299            schema_out.write(content)
300            metadata_index.write("    </edmx:Reference>\n")
301
302    metadata_index.write(
303        "    <edmx:DataServices>\n"
304        "        <Schema "
305        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
306        'Namespace="Service">\n'
307        '            <EntityContainer Name="Service" '
308        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
309        "        </Schema>\n"
310        "    </edmx:DataServices>\n"
311    )
312    # TODO:Issue#32 There's a bug in the script that currently deletes this
313    # schema (because it's an OEM schema). Because it's the only six, and we
314    # don't update schemas very often, we just manually fix it. Need a
315    # permanent fix to the script.
316    metadata_index.write(
317        '    <edmx:Reference Uri="/redfish/v1/schema/OemManager_v1.xml">\n'
318    )
319    metadata_index.write('        <edmx:Include Namespace="OemManager"/>\n')
320    metadata_index.write("    </edmx:Reference>\n")
321
322    metadata_index.write(
323        '    <edmx:Reference Uri="'
324        '/redfish/v1/schema/OemComputerSystem_v1.xml">\n'
325    )
326    metadata_index.write(
327        '        <edmx:Include Namespace="OemComputerSystem"/>\n'
328    )
329    metadata_index.write("    </edmx:Reference>\n")
330
331    metadata_index.write(
332        '    <edmx:Reference Uri="'
333        '/redfish/v1/schema/OemVirtualMedia_v1.xml">\n'
334    )
335    metadata_index.write(
336        '        <edmx:Include Namespace="OemVirtualMedia"/>\n'
337    )
338    metadata_index.write(
339        '        <edmx:Include Namespace="OemVirtualMedia.v1_0_0"/>\n'
340    )
341    metadata_index.write("    </edmx:Reference>\n")
342
343    metadata_index.write(
344        '    <edmx:Reference Uri="'
345        '/redfish/v1/schema/OemAccountService_v1.xml">\n'
346    )
347    metadata_index.write(
348        '        <edmx:Include Namespace="OemAccountService"/>\n'
349    )
350    metadata_index.write(
351        '        <edmx:Include Namespace="OemAccountService.v1_0_0"/>\n'
352    )
353    metadata_index.write("    </edmx:Reference>\n")
354
355    metadata_index.write(
356        '    <edmx:Reference Uri="/redfish/v1/schema/OemSession_v1.xml">\n'
357    )
358    metadata_index.write('        <edmx:Include Namespace="OemSession"/>\n')
359    metadata_index.write(
360        '        <edmx:Include Namespace="OemSession.v1_0_0"/>\n'
361    )
362    metadata_index.write("    </edmx:Reference>\n")
363
364    metadata_index.write("</edmx:Edmx>\n")
365
366
367for schema, version in json_schema_files.items():
368    zip_filepath = os.path.join("json-schema", version[0])
369    schemadir = os.path.join(json_schema_path, schema)
370    os.makedirs(schemadir)
371
372    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
373        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
374
375with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
376    hpp_file.write(
377        "#pragma once\n"
378        "{WARNING}\n"
379        "// clang-format off\n"
380        "\n"
381        "namespace redfish\n"
382        "{{\n"
383        "    constexpr std::array schemas {{\n".format(WARNING=WARNING)
384    )
385    for schema_file in json_schema_files:
386        hpp_file.write('        "{}",\n'.format(schema_file))
387
388    hpp_file.write("    };\n" "}\n")
389
390zip_ref.close()
391