mirror of
https://github.com/torvalds/linux.git
synced 2024-11-26 06:02:05 +00:00
9104feed4c
The generated .rst has pre and post headings without any values, e.g. here: https://docs.kernel.org/6.9/networking/netlink_spec/dpll.html#device-id-get Emit keys and values in the generated .rst Signed-off-by: Donald Hunter <donald.hunter@gmail.com> Link: https://lore.kernel.org/r/20240528140652.9445-5-donald.hunter@gmail.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
454 lines
14 KiB
Python
Executable File
454 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
# -*- coding: utf-8; mode: python -*-
|
|
|
|
"""
|
|
Script to auto generate the documentation for Netlink specifications.
|
|
|
|
:copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org>
|
|
:license: GPL Version 2, June 1991 see linux/COPYING for details.
|
|
|
|
This script performs extensive parsing to the Linux kernel's netlink YAML
|
|
spec files, in an effort to avoid needing to heavily mark up the original
|
|
YAML file.
|
|
|
|
This code is split in three big parts:
|
|
1) RST formatters: Use to convert a string to a RST output
|
|
2) Parser helpers: Functions to parse the YAML data structure
|
|
3) Main function and small helpers
|
|
"""
|
|
|
|
from typing import Any, Dict, List
|
|
import os.path
|
|
import sys
|
|
import argparse
|
|
import logging
|
|
import yaml
|
|
|
|
|
|
SPACE_PER_LEVEL = 4
|
|
|
|
|
|
# RST Formatters
|
|
# ==============
|
|
def headroom(level: int) -> str:
|
|
"""Return space to format"""
|
|
return " " * (level * SPACE_PER_LEVEL)
|
|
|
|
|
|
def bold(text: str) -> str:
|
|
"""Format bold text"""
|
|
return f"**{text}**"
|
|
|
|
|
|
def inline(text: str) -> str:
|
|
"""Format inline text"""
|
|
return f"``{text}``"
|
|
|
|
|
|
def sanitize(text: str) -> str:
|
|
"""Remove newlines and multiple spaces"""
|
|
# This is useful for some fields that are spread across multiple lines
|
|
return str(text).replace("\n", " ").strip()
|
|
|
|
|
|
def rst_fields(key: str, value: str, level: int = 0) -> str:
|
|
"""Return a RST formatted field"""
|
|
return headroom(level) + f":{key}: {value}"
|
|
|
|
|
|
def rst_definition(key: str, value: Any, level: int = 0) -> str:
|
|
"""Format a single rst definition"""
|
|
return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
|
|
|
|
|
|
def rst_paragraph(paragraph: str, level: int = 0) -> str:
|
|
"""Return a formatted paragraph"""
|
|
return headroom(level) + paragraph
|
|
|
|
|
|
def rst_bullet(item: str, level: int = 0) -> str:
|
|
"""Return a formatted a bullet"""
|
|
return headroom(level) + f"- {item}"
|
|
|
|
|
|
def rst_subsection(title: str) -> str:
|
|
"""Add a sub-section to the document"""
|
|
return f"{title}\n" + "-" * len(title)
|
|
|
|
|
|
def rst_subsubsection(title: str) -> str:
|
|
"""Add a sub-sub-section to the document"""
|
|
return f"{title}\n" + "~" * len(title)
|
|
|
|
|
|
def rst_section(namespace: str, prefix: str, title: str) -> str:
|
|
"""Add a section to the document"""
|
|
return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title)
|
|
|
|
|
|
def rst_subtitle(title: str) -> str:
|
|
"""Add a subtitle to the document"""
|
|
return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
|
|
|
|
|
|
def rst_title(title: str) -> str:
|
|
"""Add a title to the document"""
|
|
return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
|
|
|
|
|
|
def rst_list_inline(list_: List[str], level: int = 0) -> str:
|
|
"""Format a list using inlines"""
|
|
return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
|
|
|
|
|
|
def rst_ref(namespace: str, prefix: str, name: str) -> str:
|
|
"""Add a hyperlink to the document"""
|
|
mappings = {'enum': 'definition',
|
|
'fixed-header': 'definition',
|
|
'nested-attributes': 'attribute-set',
|
|
'struct': 'definition'}
|
|
if prefix in mappings:
|
|
prefix = mappings[prefix]
|
|
return f":ref:`{namespace}-{prefix}-{name}`"
|
|
|
|
|
|
def rst_header() -> str:
|
|
"""The headers for all the auto generated RST files"""
|
|
lines = []
|
|
|
|
lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
|
|
lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def rst_toctree(maxdepth: int = 2) -> str:
|
|
"""Generate a toctree RST primitive"""
|
|
lines = []
|
|
|
|
lines.append(".. toctree::")
|
|
lines.append(f" :maxdepth: {maxdepth}\n\n")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def rst_label(title: str) -> str:
|
|
"""Return a formatted label"""
|
|
return f".. _{title}:\n\n"
|
|
|
|
|
|
# Parsers
|
|
# =======
|
|
|
|
|
|
def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
|
|
"""Parse 'multicast' group list and return a formatted string"""
|
|
lines = []
|
|
for group in mcast_group:
|
|
lines.append(rst_bullet(group["name"]))
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
|
|
"""Parse 'do' section and return a formatted string"""
|
|
lines = []
|
|
for key in do_dict.keys():
|
|
lines.append(rst_paragraph(bold(key), level + 1))
|
|
if key in ['request', 'reply']:
|
|
lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
|
|
else:
|
|
lines.append(headroom(level + 2) + do_dict[key] + "\n")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
|
|
"""Parse 'attributes' section"""
|
|
if "attributes" not in attrs:
|
|
return ""
|
|
lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str:
|
|
"""Parse operations block"""
|
|
preprocessed = ["name", "doc", "title", "do", "dump", "flags"]
|
|
linkable = ["fixed-header", "attribute-set"]
|
|
lines = []
|
|
|
|
for operation in operations:
|
|
lines.append(rst_section(namespace, 'operation', operation["name"]))
|
|
lines.append(rst_paragraph(operation["doc"]) + "\n")
|
|
|
|
for key in operation.keys():
|
|
if key in preprocessed:
|
|
# Skip the special fields
|
|
continue
|
|
value = operation[key]
|
|
if key in linkable:
|
|
value = rst_ref(namespace, key, value)
|
|
lines.append(rst_fields(key, value, 0))
|
|
if 'flags' in operation:
|
|
lines.append(rst_fields('flags', rst_list_inline(operation['flags'])))
|
|
|
|
if "do" in operation:
|
|
lines.append(rst_paragraph(":do:", 0))
|
|
lines.append(parse_do(operation["do"], 0))
|
|
if "dump" in operation:
|
|
lines.append(rst_paragraph(":dump:", 0))
|
|
lines.append(parse_do(operation["dump"], 0))
|
|
|
|
# New line after fields
|
|
lines.append("\n")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
|
|
"""Parse a list of entries"""
|
|
ignored = ["pad"]
|
|
lines = []
|
|
for entry in entries:
|
|
if isinstance(entry, dict):
|
|
# entries could be a list or a dictionary
|
|
field_name = entry.get("name", "")
|
|
if field_name in ignored:
|
|
continue
|
|
type_ = entry.get("type")
|
|
if type_:
|
|
field_name += f" ({inline(type_)})"
|
|
lines.append(
|
|
rst_fields(field_name, sanitize(entry.get("doc", "")), level)
|
|
)
|
|
elif isinstance(entry, list):
|
|
lines.append(rst_list_inline(entry, level))
|
|
else:
|
|
lines.append(rst_bullet(inline(sanitize(entry)), level))
|
|
|
|
lines.append("\n")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_definitions(defs: Dict[str, Any], namespace: str) -> str:
|
|
"""Parse definitions section"""
|
|
preprocessed = ["name", "entries", "members"]
|
|
ignored = ["render-max"] # This is not printed
|
|
lines = []
|
|
|
|
for definition in defs:
|
|
lines.append(rst_section(namespace, 'definition', definition["name"]))
|
|
for k in definition.keys():
|
|
if k in preprocessed + ignored:
|
|
continue
|
|
lines.append(rst_fields(k, sanitize(definition[k]), 0))
|
|
|
|
# Field list needs to finish with a new line
|
|
lines.append("\n")
|
|
if "entries" in definition:
|
|
lines.append(rst_paragraph(":entries:", 0))
|
|
lines.append(parse_entries(definition["entries"], 1))
|
|
if "members" in definition:
|
|
lines.append(rst_paragraph(":members:", 0))
|
|
lines.append(parse_entries(definition["members"], 1))
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str:
|
|
"""Parse attribute from attribute-set"""
|
|
preprocessed = ["name", "type"]
|
|
linkable = ["enum", "nested-attributes", "struct", "sub-message"]
|
|
ignored = ["checks"]
|
|
lines = []
|
|
|
|
for entry in entries:
|
|
lines.append(rst_section(namespace, 'attribute-set', entry["name"]))
|
|
for attr in entry["attributes"]:
|
|
type_ = attr.get("type")
|
|
attr_line = attr["name"]
|
|
if type_:
|
|
# Add the attribute type in the same line
|
|
attr_line += f" ({inline(type_)})"
|
|
|
|
lines.append(rst_subsubsection(attr_line))
|
|
|
|
for k in attr.keys():
|
|
if k in preprocessed + ignored:
|
|
continue
|
|
if k in linkable:
|
|
value = rst_ref(namespace, k, attr[k])
|
|
else:
|
|
value = sanitize(attr[k])
|
|
lines.append(rst_fields(k, value, 0))
|
|
lines.append("\n")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str:
|
|
"""Parse sub-message definitions"""
|
|
lines = []
|
|
|
|
for entry in entries:
|
|
lines.append(rst_section(namespace, 'sub-message', entry["name"]))
|
|
for fmt in entry["formats"]:
|
|
value = fmt["value"]
|
|
|
|
lines.append(rst_bullet(bold(value)))
|
|
for attr in ['fixed-header', 'attribute-set']:
|
|
if attr in fmt:
|
|
lines.append(rst_fields(attr,
|
|
rst_ref(namespace, attr, fmt[attr]),
|
|
1))
|
|
lines.append("\n")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_yaml(obj: Dict[str, Any]) -> str:
|
|
"""Format the whole YAML into a RST string"""
|
|
lines = []
|
|
|
|
# Main header
|
|
|
|
lines.append(rst_header())
|
|
|
|
family = obj['name']
|
|
|
|
title = f"Family ``{family}`` netlink specification"
|
|
lines.append(rst_title(title))
|
|
lines.append(rst_paragraph(".. contents:: :depth: 3\n"))
|
|
|
|
if "doc" in obj:
|
|
lines.append(rst_subtitle("Summary"))
|
|
lines.append(rst_paragraph(obj["doc"], 0))
|
|
|
|
# Operations
|
|
if "operations" in obj:
|
|
lines.append(rst_subtitle("Operations"))
|
|
lines.append(parse_operations(obj["operations"]["list"], family))
|
|
|
|
# Multicast groups
|
|
if "mcast-groups" in obj:
|
|
lines.append(rst_subtitle("Multicast groups"))
|
|
lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
|
|
|
|
# Definitions
|
|
if "definitions" in obj:
|
|
lines.append(rst_subtitle("Definitions"))
|
|
lines.append(parse_definitions(obj["definitions"], family))
|
|
|
|
# Attributes set
|
|
if "attribute-sets" in obj:
|
|
lines.append(rst_subtitle("Attribute sets"))
|
|
lines.append(parse_attr_sets(obj["attribute-sets"], family))
|
|
|
|
# Sub-messages
|
|
if "sub-messages" in obj:
|
|
lines.append(rst_subtitle("Sub-messages"))
|
|
lines.append(parse_sub_messages(obj["sub-messages"], family))
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
# Main functions
|
|
# ==============
|
|
|
|
|
|
def parse_arguments() -> argparse.Namespace:
|
|
"""Parse arguments from user"""
|
|
parser = argparse.ArgumentParser(description="Netlink RST generator")
|
|
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
parser.add_argument("-o", "--output", help="Output file name")
|
|
|
|
# Index and input are mutually exclusive
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument(
|
|
"-x", "--index", action="store_true", help="Generate the index page"
|
|
)
|
|
group.add_argument("-i", "--input", help="YAML file name")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
if args.input and not os.path.isfile(args.input):
|
|
logging.warning("%s is not a valid file.", args.input)
|
|
sys.exit(-1)
|
|
|
|
if not args.output:
|
|
logging.error("No output file specified.")
|
|
sys.exit(-1)
|
|
|
|
if os.path.isfile(args.output):
|
|
logging.debug("%s already exists. Overwriting it.", args.output)
|
|
|
|
return args
|
|
|
|
|
|
def parse_yaml_file(filename: str) -> str:
|
|
"""Transform the YAML specified by filename into a rst-formmated string"""
|
|
with open(filename, "r", encoding="utf-8") as spec_file:
|
|
yaml_data = yaml.safe_load(spec_file)
|
|
content = parse_yaml(yaml_data)
|
|
|
|
return content
|
|
|
|
|
|
def write_to_rstfile(content: str, filename: str) -> None:
|
|
"""Write the generated content into an RST file"""
|
|
logging.debug("Saving RST file to %s", filename)
|
|
|
|
with open(filename, "w", encoding="utf-8") as rst_file:
|
|
rst_file.write(content)
|
|
|
|
|
|
def generate_main_index_rst(output: str) -> None:
|
|
"""Generate the `networking_spec/index` content and write to the file"""
|
|
lines = []
|
|
|
|
lines.append(rst_header())
|
|
lines.append(rst_label("specs"))
|
|
lines.append(rst_title("Netlink Family Specifications"))
|
|
lines.append(rst_toctree(1))
|
|
|
|
index_dir = os.path.dirname(output)
|
|
logging.debug("Looking for .rst files in %s", index_dir)
|
|
for filename in sorted(os.listdir(index_dir)):
|
|
if not filename.endswith(".rst") or filename == "index.rst":
|
|
continue
|
|
lines.append(f" {filename.replace('.rst', '')}\n")
|
|
|
|
logging.debug("Writing an index file at %s", output)
|
|
write_to_rstfile("".join(lines), output)
|
|
|
|
|
|
def main() -> None:
|
|
"""Main function that reads the YAML files and generates the RST files"""
|
|
|
|
args = parse_arguments()
|
|
|
|
if args.input:
|
|
logging.debug("Parsing %s", args.input)
|
|
try:
|
|
content = parse_yaml_file(os.path.join(args.input))
|
|
except Exception as exception:
|
|
logging.warning("Failed to parse %s.", args.input)
|
|
logging.warning(exception)
|
|
sys.exit(-1)
|
|
|
|
write_to_rstfile(content, args.output)
|
|
|
|
if args.index:
|
|
# Generate the index RST file
|
|
generate_main_index_rst(args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|