1#!/usr/bin/env python3
  2# SPDX-License-Identifier: GPL-2.0
  3# -*- coding: utf-8; mode: python -*-
  4
  5"""
  6    Script to auto generate the documentation for Netlink specifications.
  7
  8    :copyright:  Copyright (C) 2023  Breno Leitao <leitao@debian.org>
  9    :license:    GPL Version 2, June 1991 see linux/COPYING for details.
 10
 11    This script performs extensive parsing to the Linux kernel's netlink YAML
 12    spec files, in an effort to avoid needing to heavily mark up the original
 13    YAML file.
 14
 15    This code is split in three big parts:
 16        1) RST formatters: Use to convert a string to a RST output
 17        2) Parser helpers: Functions to parse the YAML data structure
 18        3) Main function and small helpers
 19"""
 20
 21from typing import Any, Dict, List
 22import os.path
 23import sys
 24import argparse
 25import logging
 26import yaml
 27
 28
 29SPACE_PER_LEVEL = 4
 30
 31
 32# RST Formatters
 33# ==============
 34def headroom(level: int) -> str:
 35    """Return space to format"""
 36    return " " * (level * SPACE_PER_LEVEL)
 37
 38
 39def bold(text: str) -> str:
 40    """Format bold text"""
 41    return f"**{text}**"
 42
 43
 44def inline(text: str) -> str:
 45    """Format inline text"""
 46    return f"``{text}``"
 47
 48
 49def sanitize(text: str) -> str:
 50    """Remove newlines and multiple spaces"""
 51    # This is useful for some fields that are spread across multiple lines
 52    return str(text).replace("\n", " ").strip()
 53
 54
 55def rst_fields(key: str, value: str, level: int = 0) -> str:
 56    """Return a RST formatted field"""
 57    return headroom(level) + f":{key}: {value}"
 58
 59
 60def rst_definition(key: str, value: Any, level: int = 0) -> str:
 61    """Format a single rst definition"""
 62    return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
 63
 64
 65def rst_paragraph(paragraph: str, level: int = 0) -> str:
 66    """Return a formatted paragraph"""
 67    return headroom(level) + paragraph
 68
 69
 70def rst_bullet(item: str, level: int = 0) -> str:
 71    """Return a formatted a bullet"""
 72    return headroom(level) + f"- {item}"
 73
 74
 75def rst_subsection(title: str) -> str:
 76    """Add a sub-section to the document"""
 77    return f"{title}\n" + "-" * len(title)
 78
 79
 80def rst_subsubsection(title: str) -> str:
 81    """Add a sub-sub-section to the document"""
 82    return f"{title}\n" + "~" * len(title)
 83
 84
 85def rst_section(namespace: str, prefix: str, title: str) -> str:
 86    """Add a section to the document"""
 87    return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title)
 88
 89
 90def rst_subtitle(title: str) -> str:
 91    """Add a subtitle to the document"""
 92    return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
 93
 94
 95def rst_title(title: str) -> str:
 96    """Add a title to the document"""
 97    return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
 98
 99
100def rst_list_inline(list_: List[str], level: int = 0) -> str:
101    """Format a list using inlines"""
102    return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
103
104
105def rst_ref(namespace: str, prefix: str, name: str) -> str:
106    """Add a hyperlink to the document"""
107    mappings = {'enum': 'definition',
108                'fixed-header': 'definition',
109                'nested-attributes': 'attribute-set',
110                'struct': 'definition'}
111    if prefix in mappings:
112        prefix = mappings[prefix]
113    return f":ref:`{namespace}-{prefix}-{name}`"
114
115
116def rst_header() -> str:
117    """The headers for all the auto generated RST files"""
118    lines = []
119
120    lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
121    lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
122
123    return "\n".join(lines)
124
125
126def rst_toctree(maxdepth: int = 2) -> str:
127    """Generate a toctree RST primitive"""
128    lines = []
129
130    lines.append(".. toctree::")
131    lines.append(f"   :maxdepth: {maxdepth}\n\n")
132
133    return "\n".join(lines)
134
135
136def rst_label(title: str) -> str:
137    """Return a formatted label"""
138    return f".. _{title}:\n\n"
139
140
141# Parsers
142# =======
143
144
145def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
146    """Parse 'multicast' group list and return a formatted string"""
147    lines = []
148    for group in mcast_group:
149        lines.append(rst_bullet(group["name"]))
150
151    return "\n".join(lines)
152
153
154def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
155    """Parse 'do' section and return a formatted string"""
156    lines = []
157    for key in do_dict.keys():
158        lines.append(rst_paragraph(bold(key), level + 1))
159        if key in ['request', 'reply']:
160            lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
161        else:
162            lines.append(headroom(level + 2) + do_dict[key] + "\n")
163
164    return "\n".join(lines)
165
166
167def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
168    """Parse 'attributes' section"""
169    if "attributes" not in attrs:
170        return ""
171    lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
172
173    return "\n".join(lines)
174
175
176def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str:
177    """Parse operations block"""
178    preprocessed = ["name", "doc", "title", "do", "dump", "flags"]
179    linkable = ["fixed-header", "attribute-set"]
180    lines = []
181
182    for operation in operations:
183        lines.append(rst_section(namespace, 'operation', operation["name"]))
184        lines.append(rst_paragraph(operation["doc"]) + "\n")
185
186        for key in operation.keys():
187            if key in preprocessed:
188                # Skip the special fields
189                continue
190            value = operation[key]
191            if key in linkable:
192                value = rst_ref(namespace, key, value)
193            lines.append(rst_fields(key, value, 0))
194        if 'flags' in operation:
195            lines.append(rst_fields('flags', rst_list_inline(operation['flags'])))
196
197        if "do" in operation:
198            lines.append(rst_paragraph(":do:", 0))
199            lines.append(parse_do(operation["do"], 0))
200        if "dump" in operation:
201            lines.append(rst_paragraph(":dump:", 0))
202            lines.append(parse_do(operation["dump"], 0))
203
204        # New line after fields
205        lines.append("\n")
206
207    return "\n".join(lines)
208
209
210def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
211    """Parse a list of entries"""
212    ignored = ["pad"]
213    lines = []
214    for entry in entries:
215        if isinstance(entry, dict):
216            # entries could be a list or a dictionary
217            field_name = entry.get("name", "")
218            if field_name in ignored:
219                continue
220            type_ = entry.get("type")
221            if type_:
222                field_name += f" ({inline(type_)})"
223            lines.append(
224                rst_fields(field_name, sanitize(entry.get("doc", "")), level)
225            )
226        elif isinstance(entry, list):
227            lines.append(rst_list_inline(entry, level))
228        else:
229            lines.append(rst_bullet(inline(sanitize(entry)), level))
230
231    lines.append("\n")
232    return "\n".join(lines)
233
234
235def parse_definitions(defs: Dict[str, Any], namespace: str) -> str:
236    """Parse definitions section"""
237    preprocessed = ["name", "entries", "members"]
238    ignored = ["render-max"]  # This is not printed
239    lines = []
240
241    for definition in defs:
242        lines.append(rst_section(namespace, 'definition', definition["name"]))
243        for k in definition.keys():
244            if k in preprocessed + ignored:
245                continue
246            lines.append(rst_fields(k, sanitize(definition[k]), 0))
247
248        # Field list needs to finish with a new line
249        lines.append("\n")
250        if "entries" in definition:
251            lines.append(rst_paragraph(":entries:", 0))
252            lines.append(parse_entries(definition["entries"], 1))
253        if "members" in definition:
254            lines.append(rst_paragraph(":members:", 0))
255            lines.append(parse_entries(definition["members"], 1))
256
257    return "\n".join(lines)
258
259
260def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str:
261    """Parse attribute from attribute-set"""
262    preprocessed = ["name", "type"]
263    linkable = ["enum", "nested-attributes", "struct", "sub-message"]
264    ignored = ["checks"]
265    lines = []
266
267    for entry in entries:
268        lines.append(rst_section(namespace, 'attribute-set', entry["name"]))
269        for attr in entry["attributes"]:
270            type_ = attr.get("type")
271            attr_line = attr["name"]
272            if type_:
273                # Add the attribute type in the same line
274                attr_line += f" ({inline(type_)})"
275
276            lines.append(rst_subsubsection(attr_line))
277
278            for k in attr.keys():
279                if k in preprocessed + ignored:
280                    continue
281                if k in linkable:
282                    value = rst_ref(namespace, k, attr[k])
283                else:
284                    value = sanitize(attr[k])
285                lines.append(rst_fields(k, value, 0))
286            lines.append("\n")
287
288    return "\n".join(lines)
289
290
291def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str:
292    """Parse sub-message definitions"""
293    lines = []
294
295    for entry in entries:
296        lines.append(rst_section(namespace, 'sub-message', entry["name"]))
297        for fmt in entry["formats"]:
298            value = fmt["value"]
299
300            lines.append(rst_bullet(bold(value)))
301            for attr in ['fixed-header', 'attribute-set']:
302                if attr in fmt:
303                    lines.append(rst_fields(attr,
304                                            rst_ref(namespace, attr, fmt[attr]),
305                                            1))
306            lines.append("\n")
307
308    return "\n".join(lines)
309
310
311def parse_yaml(obj: Dict[str, Any]) -> str:
312    """Format the whole YAML into a RST string"""
313    lines = []
314
315    # Main header
316
317    lines.append(rst_header())
318
319    family = obj['name']
320
321    title = f"Family ``{family}`` netlink specification"
322    lines.append(rst_title(title))
323    lines.append(rst_paragraph(".. contents:: :depth: 3\n"))
324
325    if "doc" in obj:
326        lines.append(rst_subtitle("Summary"))
327        lines.append(rst_paragraph(obj["doc"], 0))
328
329    # Operations
330    if "operations" in obj:
331        lines.append(rst_subtitle("Operations"))
332        lines.append(parse_operations(obj["operations"]["list"], family))
333
334    # Multicast groups
335    if "mcast-groups" in obj:
336        lines.append(rst_subtitle("Multicast groups"))
337        lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
338
339    # Definitions
340    if "definitions" in obj:
341        lines.append(rst_subtitle("Definitions"))
342        lines.append(parse_definitions(obj["definitions"], family))
343
344    # Attributes set
345    if "attribute-sets" in obj:
346        lines.append(rst_subtitle("Attribute sets"))
347        lines.append(parse_attr_sets(obj["attribute-sets"], family))
348
349    # Sub-messages
350    if "sub-messages" in obj:
351        lines.append(rst_subtitle("Sub-messages"))
352        lines.append(parse_sub_messages(obj["sub-messages"], family))
353
354    return "\n".join(lines)
355
356
357# Main functions
358# ==============
359
360
361def parse_arguments() -> argparse.Namespace:
362    """Parse arguments from user"""
363    parser = argparse.ArgumentParser(description="Netlink RST generator")
364
365    parser.add_argument("-v", "--verbose", action="store_true")
366    parser.add_argument("-o", "--output", help="Output file name")
367
368    # Index and input are mutually exclusive
369    group = parser.add_mutually_exclusive_group()
370    group.add_argument(
371        "-x", "--index", action="store_true", help="Generate the index page"
372    )
373    group.add_argument("-i", "--input", help="YAML file name")
374
375    args = parser.parse_args()
376
377    if args.verbose:
378        logging.basicConfig(level=logging.DEBUG)
379
380    if args.input and not os.path.isfile(args.input):
381        logging.warning("%s is not a valid file.", args.input)
382        sys.exit(-1)
383
384    if not args.output:
385        logging.error("No output file specified.")
386        sys.exit(-1)
387
388    if os.path.isfile(args.output):
389        logging.debug("%s already exists. Overwriting it.", args.output)
390
391    return args
392
393
394def parse_yaml_file(filename: str) -> str:
395    """Transform the YAML specified by filename into a rst-formmated string"""
396    with open(filename, "r", encoding="utf-8") as spec_file:
397        yaml_data = yaml.safe_load(spec_file)
398        content = parse_yaml(yaml_data)
399
400    return content
401
402
403def write_to_rstfile(content: str, filename: str) -> None:
404    """Write the generated content into an RST file"""
405    logging.debug("Saving RST file to %s", filename)
406
407    with open(filename, "w", encoding="utf-8") as rst_file:
408        rst_file.write(content)
409
410
411def generate_main_index_rst(output: str) -> None:
412    """Generate the `networking_spec/index` content and write to the file"""
413    lines = []
414
415    lines.append(rst_header())
416    lines.append(rst_label("specs"))
417    lines.append(rst_title("Netlink Family Specifications"))
418    lines.append(rst_toctree(1))
419
420    index_dir = os.path.dirname(output)
421    logging.debug("Looking for .rst files in %s", index_dir)
422    for filename in sorted(os.listdir(index_dir)):
423        if not filename.endswith(".rst") or filename == "index.rst":
424            continue
425        lines.append(f"   {filename.replace('.rst', '')}\n")
426
427    logging.debug("Writing an index file at %s", output)
428    write_to_rstfile("".join(lines), output)
429
430
431def main() -> None:
432    """Main function that reads the YAML files and generates the RST files"""
433
434    args = parse_arguments()
435
436    if args.input:
437        logging.debug("Parsing %s", args.input)
438        try:
439            content = parse_yaml_file(os.path.join(args.input))
440        except Exception as exception:
441            logging.warning("Failed to parse %s.", args.input)
442            logging.warning(exception)
443            sys.exit(-1)
444
445        write_to_rstfile(content, args.output)
446
447    if args.index:
448        # Generate the index RST file
449        generate_main_index_rst(args.output)
450
451
452if __name__ == "__main__":
453    main()