xref: /openbmc/bmcweb/scripts/update_schemas.py (revision 56d0bb068b69c5b8853e44d7af179030adcacd64) 
1#!/usr/bin/env python3
2import os
3import shutil
4import xml.etree.ElementTree as ET
5import zipfile
6from collections import OrderedDict, defaultdict
7from io import BytesIO
8
9import generate_schema_enums
10import requests
11from generate_schema_collections import generate_top_collections
12
13VERSION = "DSP8010_2023.3"
14
15WARNING = """/****************************************************************
16 *                 READ THIS WARNING FIRST
17 * This is an auto-generated header which contains definitions
18 * for Redfish DMTF defined schemas.
19 * DO NOT modify this registry outside of running the
20 * update_schemas.py script.  The definitions contained within
21 * this file are owned by DMTF.  Any modifications to these files
22 * should be first pushed to the relevant registry in the DMTF
23 * github organization.
24 ***************************************************************/"""
25
26# To use a new schema, add to list and rerun tool
27include_list = [
28    "AccountService",
29    "ActionInfo",
30    "AggregationService",
31    "AggregationSource",
32    "AggregationSourceCollection",
33    "Assembly",
34    "AttributeRegistry",
35    "Bios",
36    "Cable",
37    "CableCollection",
38    "Certificate",
39    "CertificateCollection",
40    "CertificateLocations",
41    "CertificateService",
42    "Chassis",
43    "ChassisCollection",
44    "ComputerSystem",
45    "ComputerSystemCollection",
46    "ComponentIntegrity",
47    "ComponentIntegrityCollection",
48    "Drive",
49    "DriveCollection",
50    "EnvironmentMetrics",
51    "EthernetInterface",
52    "EthernetInterfaceCollection",
53    "Event",
54    "EventDestination",
55    "EventDestinationCollection",
56    "EventService",
57    "FabricAdapter",
58    "FabricAdapterCollection",
59    "Fan",
60    "FanCollection",
61    "IPAddresses",
62    "JsonSchemaFile",
63    "JsonSchemaFileCollection",  # redfish/v1/JsonSchemas
64    "LogEntry",
65    "LogEntryCollection",
66    "LogService",
67    "LogServiceCollection",
68    "Manager",
69    "ManagerAccount",
70    "ManagerAccountCollection",
71    "ManagerCollection",
72    "ManagerDiagnosticData",
73    "ManagerNetworkProtocol",
74    "Memory",
75    "MemoryCollection",
76    "Message",
77    "MessageRegistry",
78    "MessageRegistryCollection",
79    "MessageRegistryFile",
80    "MessageRegistryFileCollection",
81    "MetricDefinition",
82    "MetricDefinitionCollection",
83    "MetricReport",
84    "MetricReportCollection",
85    "MetricReportDefinition",
86    "MetricReportDefinitionCollection",
87    "OperatingConfig",
88    "OperatingConfigCollection",
89    "PCIeDevice",
90    "PCIeDeviceCollection",
91    "PCIeFunction",
92    "PCIeFunctionCollection",
93    "PhysicalContext",
94    "PCIeSlots",
95    "Port",
96    "PortCollection",
97    "Power",
98    "PowerSubsystem",
99    "PowerSupply",
100    "PowerSupplyCollection",
101    "Privileges",  # Used in Role
102    "Processor",
103    "ProcessorCollection",
104    "Protocol",
105    "RedfishError",
106    "RedfishExtensions",
107    "Redundancy",
108    "Resource",
109    "Role",
110    "RoleCollection",
111    "Sensor",
112    "SensorCollection",
113    "ServiceRoot",
114    "Session",
115    "SessionCollection",
116    "SessionService",
117    "Settings",
118    "SoftwareInventory",
119    "SoftwareInventoryCollection",
120    "Storage",
121    "StorageCollection",
122    "StorageController",
123    "StorageControllerCollection",
124    "Task",
125    "TaskCollection",
126    "TaskService",
127    "TelemetryService",
128    "Thermal",
129    "ThermalMetrics",
130    "ThermalSubsystem",
131    "Triggers",
132    "TriggersCollection",
133    "UpdateService",
134    "VirtualMedia",
135    "VirtualMediaCollection",
136    "odata",
137    "odata-v4",
138    "redfish-error",
139    "redfish-payload-annotations",
140    "redfish-schema",
141    "redfish-schema-v1",
142]
143
144# OEM schemas
145oem_schema_names = [
146    "OemManager",
147    "OemComputerSystem",
148    "OemVirtualMedia",
149    "OpenBMCAccountService",
150]
151
152SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
153
154proxies = {"https": os.environ.get("https_proxy", None)}
155
156r = requests.get(
157    "https://www.dmtf.org/sites/default/files/standards/documents/"
158    + VERSION
159    + ".zip",
160    proxies=proxies,
161)
162
163r.raise_for_status()
164
165
166static_path = os.path.realpath(
167    os.path.join(SCRIPT_DIR, "..", "static", "redfish", "v1")
168)
169
170
171cpp_path = os.path.realpath(
172    os.path.join(SCRIPT_DIR, "..", "redfish-core", "include")
173)
174
175
176schema_path = os.path.join(static_path, "schema")
177json_schema_path = os.path.join(static_path, "JsonSchemas")
178metadata_index_path = os.path.join(static_path, "$metadata", "index.xml")
179
180zipBytesIO = BytesIO(r.content)
181zip_ref = zipfile.ZipFile(zipBytesIO)
182
183
184class SchemaVersion:
185    """
186    A Python class for sorting Redfish schema versions.  Allows sorting Redfish
187    versions in the way humans expect, by comparing version strings as lists
188    (ie 0_2_0 comes before 0_10_0) in the way humans expect.  It does case
189    insensitive schema name comparisons
190    """
191
192    def __init__(self, key):
193        key = str.casefold(key)
194
195        split_tup = key.split(".")
196        self.version_pieces = [split_tup[0]]
197        if len(split_tup) < 2:
198            return
199        version = split_tup[1]
200
201        if version.startswith("v"):
202            version = version[1:]
203        if any(char.isdigit() for char in version):
204            self.version_pieces.extend([int(x) for x in version.split("_")])
205
206    def __lt__(self, other):
207        return self.version_pieces < other.version_pieces
208
209
210# Remove the old files
211skip_prefixes = ["Oem", "OpenBMC"]
212if os.path.exists(schema_path):
213    files = [
214        os.path.join(schema_path, f)
215        for f in os.listdir(schema_path)
216        if not any([f.startswith(prefix) for prefix in skip_prefixes])
217    ]
218    for f in files:
219        os.remove(f)
220if os.path.exists(json_schema_path):
221    files = [
222        os.path.join(json_schema_path, f)
223        for f in os.listdir(json_schema_path)
224        if not any([f.startswith(prefix) for prefix in skip_prefixes])
225    ]
226    for f in files:
227        if os.path.isfile(f):
228            os.remove(f)
229        else:
230            shutil.rmtree(f)
231try:
232    os.remove(metadata_index_path)
233except FileNotFoundError:
234    pass
235
236if not os.path.exists(schema_path):
237    os.makedirs(schema_path)
238if not os.path.exists(json_schema_path):
239    os.makedirs(json_schema_path)
240
241csdl_filenames = []
242json_schema_files = defaultdict(list)
243
244for zip_file in zip_ref.infolist():
245    if zip_file.is_dir():
246        continue
247    if zip_file.filename.startswith("csdl/"):
248        csdl_filenames.append(os.path.basename(zip_file.filename))
249    elif zip_file.filename.startswith("json-schema/"):
250        filename = os.path.basename(zip_file.filename)
251        filenamesplit = filename.split(".")
252        # exclude schemas again to save flash space
253        if filenamesplit[0] not in include_list:
254            continue
255        json_schema_files[filenamesplit[0]].append(filename)
256    elif zip_file.filename.startswith("openapi/"):
257        pass
258    elif zip_file.filename.startswith("dictionaries/"):
259        pass
260
261# sort the json files by version
262for key, value in json_schema_files.items():
263    value.sort(key=SchemaVersion, reverse=True)
264
265# Create a dictionary ordered by schema name
266json_schema_files = OrderedDict(
267    sorted(json_schema_files.items(), key=lambda x: SchemaVersion(x[0]))
268)
269
270csdl_filenames.sort(key=SchemaVersion)
271
272# Create oem filenames - from oem json names
273oem_csdl_filenames = []
274for filename in oem_schema_names:
275    oem_csdl_filenames.append(filename + "_v1.xml")
276
277# Append Oem csdl files
278csdl_filenames += oem_csdl_filenames
279
280with open(metadata_index_path, "w") as metadata_index:
281    metadata_index.write('<?xml version="1.0" encoding="UTF-8"?>\n')
282    metadata_index.write(
283        "<edmx:Edmx xmlns:edmx="
284        '"http://docs.oasis-open.org/odata/ns/edmx"'
285        ' Version="4.0">\n'
286    )
287
288    for filename in csdl_filenames:
289        # filename looks like Zone_v1.xml
290        if filename in oem_csdl_filenames:
291            with open(
292                os.path.join(schema_path, filename), "rb"
293            ) as oem_csdl_in:
294                content = oem_csdl_in.read()
295                content = content.replace(b"\r\n", b"\n")
296        else:
297            with open(os.path.join(schema_path, filename), "wb") as schema_out:
298                content = zip_ref.read(os.path.join("csdl", filename))
299                content = content.replace(b"\r\n", b"\n")
300                schema_out.write(content)
301                filenamesplit = filename.split("_")
302                if filenamesplit[0] not in include_list:
303                    continue
304
305        metadata_index.write(
306            '    <edmx:Reference Uri="/redfish/v1/schema/' + filename + '">\n'
307        )
308
309        xml_root = ET.fromstring(content)
310        edmx = "{http://docs.oasis-open.org/odata/ns/edmx}"
311        edm = "{http://docs.oasis-open.org/odata/ns/edm}"
312        for edmx_child in xml_root:
313            if edmx_child.tag == edmx + "DataServices":
314                for data_child in edmx_child:
315                    if data_child.tag == edm + "Schema":
316                        namespace = data_child.attrib["Namespace"]
317                        if namespace.startswith("RedfishExtensions"):
318                            metadata_index.write(
319                                '        <edmx:Include Namespace="'
320                                + namespace
321                                + '"  Alias="Redfish"/>\n'
322                            )
323
324                        else:
325                            metadata_index.write(
326                                '        <edmx:Include Namespace="'
327                                + namespace
328                                + '"/>\n'
329                            )
330        metadata_index.write("    </edmx:Reference>\n")
331
332    metadata_index.write(
333        "    <edmx:DataServices>\n"
334        "        <Schema "
335        'xmlns="http://docs.oasis-open.org/odata/ns/edm" '
336        'Namespace="Service">\n'
337        '            <EntityContainer Name="Service" '
338        'Extends="ServiceRoot.v1_0_0.ServiceContainer"/>\n'
339        "        </Schema>\n"
340        "    </edmx:DataServices>\n"
341    )
342    metadata_index.write("</edmx:Edmx>\n")
343
344
345for schema, version in json_schema_files.items():
346    zip_filepath = os.path.join("json-schema", version[0])
347    schemadir = os.path.join(json_schema_path, schema)
348    os.makedirs(schemadir)
349
350    with open(os.path.join(schemadir, schema + ".json"), "wb") as schema_file:
351        schema_file.write(zip_ref.read(zip_filepath).replace(b"\r\n", b"\n"))
352
353with open(os.path.join(cpp_path, "schemas.hpp"), "w") as hpp_file:
354    hpp_file.write(
355        "#pragma once\n"
356        "{WARNING}\n"
357        "// clang-format off\n"
358        "#include <array>\n"
359        "#include <string_view>\n"
360        "\n"
361        "namespace redfish\n"
362        "{{\n"
363        "    constexpr std::array<std::string_view,{SIZE}> schemas {{\n".format(
364            WARNING=WARNING,
365            SIZE=len(json_schema_files) + len(oem_schema_names),
366        )
367    )
368    for schema_file in json_schema_files:
369        hpp_file.write('        "{}",\n'.format(schema_file))
370
371    for schema_file in oem_schema_names:
372        hpp_file.write('        "{}",\n'.format(schema_file))
373
374    hpp_file.write("    };\n}\n")
375
376zip_ref.close()
377
378generate_schema_enums.main()
379generate_top_collections()
380
381# Now delete the xml schema files we aren't supporting
382if os.path.exists(schema_path):
383    files = [
384        os.path.join(schema_path, f)
385        for f in os.listdir(schema_path)
386        if not any([f.startswith(prefix) for prefix in skip_prefixes])
387    ]
388    for filename in files:
389        # filename will include the absolute path
390        filenamesplit = filename.split("/")
391        name = filenamesplit.pop()
392        namesplit = name.split("_")
393        if namesplit[0] not in include_list:
394            print("excluding schema: " + filename)
395            os.remove(filename)
396