1#!/usr/bin/env python3
  2# SPDX-License-Identifier: GPL-2.0
  3# -*- coding: utf-8; mode: python -*-
  4
  5"""
  6    Script to auto generate the documentation for Netlink specifications.
  7
  8    :copyright:  Copyright (C) 2023  Breno Leitao <leitao@debian.org>
  9    :license:    GPL Version 2, June 1991 see linux/COPYING for details.
 10
 11    This script performs extensive parsing to the Linux kernel's netlink YAML
 12    spec files, in an effort to avoid needing to heavily mark up the original
 13    YAML file.
 14
 15    This code is split in three big parts:
 16        1) RST formatters: Use to convert a string to a RST output
 17        2) Parser helpers: Functions to parse the YAML data structure
 18        3) Main function and small helpers
 19"""
 20
 21from typing import Any, Dict, List
 22import os.path
 23import sys
 24import argparse
 25import logging
 26import yaml
 27
 28
 29SPACE_PER_LEVEL = 4
 30
 31
 32# RST Formatters
 33# ==============
 34def headroom(level: int) -> str:
 35    """Return space to format"""
 36    return " " * (level * SPACE_PER_LEVEL)
 37
 38
 39def bold(text: str) -> str:
 40    """Format bold text"""
 41    return f"**{text}**"
 42
 43
 44def inline(text: str) -> str:
 45    """Format inline text"""
 46    return f"``{text}``"
 47
 48
 49def sanitize(text: str) -> str:
 50    """Remove newlines and multiple spaces"""
 51    # This is useful for some fields that are spread across multiple lines
 52    return str(text).replace("\n", "").strip()
 53
 54
 55def rst_fields(key: str, value: str, level: int = 0) -> str:
 56    """Return a RST formatted field"""
 57    return headroom(level) + f":{key}: {value}"
 58
 59
 60def rst_definition(key: str, value: Any, level: int = 0) -> str:
 61    """Format a single rst definition"""
 62    return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
 63
 64
 65def rst_paragraph(paragraph: str, level: int = 0) -> str:
 66    """Return a formatted paragraph"""
 67    return headroom(level) + paragraph
 68
 69
 70def rst_bullet(item: str, level: int = 0) -> str:
 71    """Return a formatted a bullet"""
 72    return headroom(level) + f"- {item}"
 73
 74
 75def rst_subsection(title: str) -> str:
 76    """Add a sub-section to the document"""
 77    return f"{title}\n" + "-" * len(title)
 78
 79
 80def rst_subsubsection(title: str) -> str:
 81    """Add a sub-sub-section to the document"""
 82    return f"{title}\n" + "~" * len(title)
 83
 84
 85def rst_section(title: str) -> str:
 86    """Add a section to the document"""
 87    return f"\n{title}\n" + "=" * len(title)
 88
 89
 90def rst_subtitle(title: str) -> str:
 91    """Add a subtitle to the document"""
 92    return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
 93
 94
 95def rst_title(title: str) -> str:
 96    """Add a title to the document"""
 97    return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
 98
 99
100def rst_list_inline(list_: List[str], level: int = 0) -> str:
101    """Format a list using inlines"""
102    return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
103
104
105def rst_header() -> str:
106    """The headers for all the auto generated RST files"""
107    lines = []
108
109    lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
110    lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
111
112    return "\n".join(lines)
113
114
115def rst_toctree(maxdepth: int = 2) -> str:
116    """Generate a toctree RST primitive"""
117    lines = []
118
119    lines.append(".. toctree::")
120    lines.append(f"   :maxdepth: {maxdepth}\n\n")
121
122    return "\n".join(lines)
123
124
125def rst_label(title: str) -> str:
126    """Return a formatted label"""
127    return f".. _{title}:\n\n"
128
129
130# Parsers
131# =======
132
133
134def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
135    """Parse 'multicast' group list and return a formatted string"""
136    lines = []
137    for group in mcast_group:
138        lines.append(rst_bullet(group["name"]))
139
140    return "\n".join(lines)
141
142
143def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
144    """Parse 'do' section and return a formatted string"""
145    lines = []
146    for key in do_dict.keys():
147        lines.append(rst_paragraph(bold(key), level + 1))
148        lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
149
150    return "\n".join(lines)
151
152
153def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
154    """Parse 'attributes' section"""
155    if "attributes" not in attrs:
156        return ""
157    lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
158
159    return "\n".join(lines)
160
161
162def parse_operations(operations: List[Dict[str, Any]]) -> str:
163    """Parse operations block"""
164    preprocessed = ["name", "doc", "title", "do", "dump"]
165    lines = []
166
167    for operation in operations:
168        lines.append(rst_section(operation["name"]))
169        lines.append(rst_paragraph(sanitize(operation["doc"])) + "\n")
170
171        for key in operation.keys():
172            if key in preprocessed:
173                # Skip the special fields
174                continue
175            lines.append(rst_fields(key, operation[key], 0))
176
177        if "do" in operation:
178            lines.append(rst_paragraph(":do:", 0))
179            lines.append(parse_do(operation["do"], 0))
180        if "dump" in operation:
181            lines.append(rst_paragraph(":dump:", 0))
182            lines.append(parse_do(operation["dump"], 0))
183
184        # New line after fields
185        lines.append("\n")
186
187    return "\n".join(lines)
188
189
190def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
191    """Parse a list of entries"""
192    lines = []
193    for entry in entries:
194        if isinstance(entry, dict):
195            # entries could be a list or a dictionary
196            lines.append(
197                rst_fields(entry.get("name", ""), sanitize(entry.get("doc", "")), level)
198            )
199        elif isinstance(entry, list):
200            lines.append(rst_list_inline(entry, level))
201        else:
202            lines.append(rst_bullet(inline(sanitize(entry)), level))
203
204    lines.append("\n")
205    return "\n".join(lines)
206
207
208def parse_definitions(defs: Dict[str, Any]) -> str:
209    """Parse definitions section"""
210    preprocessed = ["name", "entries", "members"]
211    ignored = ["render-max"]  # This is not printed
212    lines = []
213
214    for definition in defs:
215        lines.append(rst_section(definition["name"]))
216        for k in definition.keys():
217            if k in preprocessed + ignored:
218                continue
219            lines.append(rst_fields(k, sanitize(definition[k]), 0))
220
221        # Field list needs to finish with a new line
222        lines.append("\n")
223        if "entries" in definition:
224            lines.append(rst_paragraph(":entries:", 0))
225            lines.append(parse_entries(definition["entries"], 1))
226        if "members" in definition:
227            lines.append(rst_paragraph(":members:", 0))
228            lines.append(parse_entries(definition["members"], 1))
229
230    return "\n".join(lines)
231
232
233def parse_attr_sets(entries: List[Dict[str, Any]]) -> str:
234    """Parse attribute from attribute-set"""
235    preprocessed = ["name", "type"]
236    ignored = ["checks"]
237    lines = []
238
239    for entry in entries:
240        lines.append(rst_section(entry["name"]))
241        for attr in entry["attributes"]:
242            type_ = attr.get("type")
243            attr_line = attr["name"]
244            if type_:
245                # Add the attribute type in the same line
246                attr_line += f" ({inline(type_)})"
247
248            lines.append(rst_subsubsection(attr_line))
249
250            for k in attr.keys():
251                if k in preprocessed + ignored:
252                    continue
253                lines.append(rst_fields(k, sanitize(attr[k]), 0))
254            lines.append("\n")
255
256    return "\n".join(lines)
257
258
259def parse_sub_messages(entries: List[Dict[str, Any]]) -> str:
260    """Parse sub-message definitions"""
261    lines = []
262
263    for entry in entries:
264        lines.append(rst_section(entry["name"]))
265        for fmt in entry["formats"]:
266            value = fmt["value"]
267
268            lines.append(rst_bullet(bold(value)))
269            for attr in ['fixed-header', 'attribute-set']:
270                if attr in fmt:
271                    lines.append(rst_fields(attr, fmt[attr], 1))
272            lines.append("\n")
273
274    return "\n".join(lines)
275
276
277def parse_yaml(obj: Dict[str, Any]) -> str:
278    """Format the whole YAML into a RST string"""
279    lines = []
280
281    # Main header
282
283    lines.append(rst_header())
284
285    title = f"Family ``{obj['name']}`` netlink specification"
286    lines.append(rst_title(title))
287    lines.append(rst_paragraph(".. contents::\n"))
288
289    if "doc" in obj:
290        lines.append(rst_subtitle("Summary"))
291        lines.append(rst_paragraph(obj["doc"], 0))
292
293    # Operations
294    if "operations" in obj:
295        lines.append(rst_subtitle("Operations"))
296        lines.append(parse_operations(obj["operations"]["list"]))
297
298    # Multicast groups
299    if "mcast-groups" in obj:
300        lines.append(rst_subtitle("Multicast groups"))
301        lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
302
303    # Definitions
304    if "definitions" in obj:
305        lines.append(rst_subtitle("Definitions"))
306        lines.append(parse_definitions(obj["definitions"]))
307
308    # Attributes set
309    if "attribute-sets" in obj:
310        lines.append(rst_subtitle("Attribute sets"))
311        lines.append(parse_attr_sets(obj["attribute-sets"]))
312
313    # Sub-messages
314    if "sub-messages" in obj:
315        lines.append(rst_subtitle("Sub-messages"))
316        lines.append(parse_sub_messages(obj["sub-messages"]))
317
318    return "\n".join(lines)
319
320
321# Main functions
322# ==============
323
324
325def parse_arguments() -> argparse.Namespace:
326    """Parse arguments from user"""
327    parser = argparse.ArgumentParser(description="Netlink RST generator")
328
329    parser.add_argument("-v", "--verbose", action="store_true")
330    parser.add_argument("-o", "--output", help="Output file name")
331
332    # Index and input are mutually exclusive
333    group = parser.add_mutually_exclusive_group()
334    group.add_argument(
335        "-x", "--index", action="store_true", help="Generate the index page"
336    )
337    group.add_argument("-i", "--input", help="YAML file name")
338
339    args = parser.parse_args()
340
341    if args.verbose:
342        logging.basicConfig(level=logging.DEBUG)
343
344    if args.input and not os.path.isfile(args.input):
345        logging.warning("%s is not a valid file.", args.input)
346        sys.exit(-1)
347
348    if not args.output:
349        logging.error("No output file specified.")
350        sys.exit(-1)
351
352    if os.path.isfile(args.output):
353        logging.debug("%s already exists. Overwriting it.", args.output)
354
355    return args
356
357
358def parse_yaml_file(filename: str) -> str:
359    """Transform the YAML specified by filename into a rst-formmated string"""
360    with open(filename, "r", encoding="utf-8") as spec_file:
361        yaml_data = yaml.safe_load(spec_file)
362        content = parse_yaml(yaml_data)
363
364    return content
365
366
367def write_to_rstfile(content: str, filename: str) -> None:
368    """Write the generated content into an RST file"""
369    logging.debug("Saving RST file to %s", filename)
370
371    with open(filename, "w", encoding="utf-8") as rst_file:
372        rst_file.write(content)
373
374
375def generate_main_index_rst(output: str) -> None:
376    """Generate the `networking_spec/index` content and write to the file"""
377    lines = []
378
379    lines.append(rst_header())
380    lines.append(rst_label("specs"))
381    lines.append(rst_title("Netlink Family Specifications"))
382    lines.append(rst_toctree(1))
383
384    index_dir = os.path.dirname(output)
385    logging.debug("Looking for .rst files in %s", index_dir)
386    for filename in sorted(os.listdir(index_dir)):
387        if not filename.endswith(".rst") or filename == "index.rst":
388            continue
389        lines.append(f"   {filename.replace('.rst', '')}\n")
390
391    logging.debug("Writing an index file at %s", output)
392    write_to_rstfile("".join(lines), output)
393
394
395def main() -> None:
396    """Main function that reads the YAML files and generates the RST files"""
397
398    args = parse_arguments()
399
400    if args.input:
401        logging.debug("Parsing %s", args.input)
402        try:
403            content = parse_yaml_file(os.path.join(args.input))
404        except Exception as exception:
405            logging.warning("Failed to parse %s.", args.input)
406            logging.warning(exception)
407            sys.exit(-1)
408
409        write_to_rstfile(content, args.output)
410
411    if args.index:
412        # Generate the index RST file
413        generate_main_index_rst(args.output)
414
415
416if __name__ == "__main__":
417    main()