cachepc-qemu

Fork of AMDESE/qemu with changes for cachepc side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-qemu
Log | Files | Refs | Submodules | LICENSE | sfeed.txt

qapidoc.py (22053B)


      1# coding=utf-8
      2#
      3# QEMU qapidoc QAPI file parsing extension
      4#
      5# Copyright (c) 2020 Linaro
      6#
      7# This work is licensed under the terms of the GNU GPLv2 or later.
      8# See the COPYING file in the top-level directory.
      9
     10"""
     11qapidoc is a Sphinx extension that implements the qapi-doc directive
     12
     13The purpose of this extension is to read the documentation comments
     14in QAPI schema files, and insert them all into the current document.
     15
     16It implements one new rST directive, "qapi-doc::".
     17Each qapi-doc:: directive takes one argument, which is the
     18pathname of the schema file to process, relative to the source tree.
     19
     20The docs/conf.py file must set the qapidoc_srctree config value to
     21the root of the QEMU source tree.
     22
     23The Sphinx documentation on writing extensions is at:
     24https://www.sphinx-doc.org/en/master/development/index.html
     25"""
     26
     27import os
     28import re
     29
     30from docutils import nodes
     31from docutils.statemachine import ViewList
     32from docutils.parsers.rst import directives, Directive
     33from sphinx.errors import ExtensionError
     34from sphinx.util.nodes import nested_parse_with_titles
     35import sphinx
     36from qapi.gen import QAPISchemaVisitor
     37from qapi.error import QAPIError, QAPISemError
     38from qapi.schema import QAPISchema
     39
     40
     41# Sphinx up to 1.6 uses AutodocReporter; 1.7 and later
     42# use switch_source_input. Check borrowed from kerneldoc.py.
     43Use_SSI = sphinx.__version__[:3] >= '1.7'
     44if Use_SSI:
     45    from sphinx.util.docutils import switch_source_input
     46else:
     47    from sphinx.ext.autodoc import AutodocReporter
     48
     49
     50__version__ = '1.0'
     51
     52
     53# Function borrowed from pydash, which is under the MIT license
     54def intersperse(iterable, separator):
     55    """Yield the members of *iterable* interspersed with *separator*."""
     56    iterable = iter(iterable)
     57    yield next(iterable)
     58    for item in iterable:
     59        yield separator
     60        yield item
     61
     62
     63class QAPISchemaGenRSTVisitor(QAPISchemaVisitor):
     64    """A QAPI schema visitor which generates docutils/Sphinx nodes
     65
     66    This class builds up a tree of docutils/Sphinx nodes corresponding
     67    to documentation for the various QAPI objects. To use it, first
     68    create a QAPISchemaGenRSTVisitor object, and call its
     69    visit_begin() method.  Then you can call one of the two methods
     70    'freeform' (to add documentation for a freeform documentation
     71    chunk) or 'symbol' (to add documentation for a QAPI symbol). These
     72    will cause the visitor to build up the tree of document
     73    nodes. Once you've added all the documentation via 'freeform' and
     74    'symbol' method calls, you can call 'get_document_nodes' to get
     75    the final list of document nodes (in a form suitable for returning
     76    from a Sphinx directive's 'run' method).
     77    """
     78    def __init__(self, sphinx_directive):
     79        self._cur_doc = None
     80        self._sphinx_directive = sphinx_directive
     81        self._top_node = nodes.section()
     82        self._active_headings = [self._top_node]
     83
     84    def _make_dlitem(self, term, defn):
     85        """Return a dlitem node with the specified term and definition.
     86
     87        term should be a list of Text and literal nodes.
     88        defn should be one of:
     89        - a string, which will be handed to _parse_text_into_node
     90        - a list of Text and literal nodes, which will be put into
     91          a paragraph node
     92        """
     93        dlitem = nodes.definition_list_item()
     94        dlterm = nodes.term('', '', *term)
     95        dlitem += dlterm
     96        if defn:
     97            dldef = nodes.definition()
     98            if isinstance(defn, list):
     99                dldef += nodes.paragraph('', '', *defn)
    100            else:
    101                self._parse_text_into_node(defn, dldef)
    102            dlitem += dldef
    103        return dlitem
    104
    105    def _make_section(self, title):
    106        """Return a section node with optional title"""
    107        section = nodes.section(ids=[self._sphinx_directive.new_serialno()])
    108        if title:
    109            section += nodes.title(title, title)
    110        return section
    111
    112    def _nodes_for_ifcond(self, ifcond, with_if=True):
    113        """Return list of Text, literal nodes for the ifcond
    114
    115        Return a list which gives text like ' (If: condition)'.
    116        If with_if is False, we don't return the "(If: " and ")".
    117        """
    118
    119        doc = ifcond.docgen()
    120        if not doc:
    121            return []
    122        doc = nodes.literal('', doc)
    123        if not with_if:
    124            return [doc]
    125
    126        nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')]
    127        nodelist.append(doc)
    128        nodelist.append(nodes.Text(')'))
    129        return nodelist
    130
    131    def _nodes_for_one_member(self, member):
    132        """Return list of Text, literal nodes for this member
    133
    134        Return a list of doctree nodes which give text like
    135        'name: type (optional) (If: ...)' suitable for use as the
    136        'term' part of a definition list item.
    137        """
    138        term = [nodes.literal('', member.name)]
    139        if member.type.doc_type():
    140            term.append(nodes.Text(': '))
    141            term.append(nodes.literal('', member.type.doc_type()))
    142        if member.optional:
    143            term.append(nodes.Text(' (optional)'))
    144        if member.ifcond.is_present():
    145            term.extend(self._nodes_for_ifcond(member.ifcond))
    146        return term
    147
    148    def _nodes_for_variant_when(self, variants, variant):
    149        """Return list of Text, literal nodes for variant 'when' clause
    150
    151        Return a list of doctree nodes which give text like
    152        'when tagname is variant (If: ...)' suitable for use in
    153        the 'variants' part of a definition list.
    154        """
    155        term = [nodes.Text(' when '),
    156                nodes.literal('', variants.tag_member.name),
    157                nodes.Text(' is '),
    158                nodes.literal('', '"%s"' % variant.name)]
    159        if variant.ifcond.is_present():
    160            term.extend(self._nodes_for_ifcond(variant.ifcond))
    161        return term
    162
    163    def _nodes_for_members(self, doc, what, base=None, variants=None):
    164        """Return list of doctree nodes for the table of members"""
    165        dlnode = nodes.definition_list()
    166        for section in doc.args.values():
    167            term = self._nodes_for_one_member(section.member)
    168            # TODO drop fallbacks when undocumented members are outlawed
    169            if section.text:
    170                defn = section.text
    171            elif (variants and variants.tag_member == section.member
    172                  and not section.member.type.doc_type()):
    173                values = section.member.type.member_names()
    174                defn = [nodes.Text('One of ')]
    175                defn.extend(intersperse([nodes.literal('', v) for v in values],
    176                                        nodes.Text(', ')))
    177            else:
    178                defn = [nodes.Text('Not documented')]
    179
    180            dlnode += self._make_dlitem(term, defn)
    181
    182        if base:
    183            dlnode += self._make_dlitem([nodes.Text('The members of '),
    184                                         nodes.literal('', base.doc_type())],
    185                                        None)
    186
    187        if variants:
    188            for v in variants.variants:
    189                if v.type.is_implicit():
    190                    assert not v.type.base and not v.type.variants
    191                    for m in v.type.local_members:
    192                        term = self._nodes_for_one_member(m)
    193                        term.extend(self._nodes_for_variant_when(variants, v))
    194                        dlnode += self._make_dlitem(term, None)
    195                else:
    196                    term = [nodes.Text('The members of '),
    197                            nodes.literal('', v.type.doc_type())]
    198                    term.extend(self._nodes_for_variant_when(variants, v))
    199                    dlnode += self._make_dlitem(term, None)
    200
    201        if not dlnode.children:
    202            return []
    203
    204        section = self._make_section(what)
    205        section += dlnode
    206        return [section]
    207
    208    def _nodes_for_enum_values(self, doc):
    209        """Return list of doctree nodes for the table of enum values"""
    210        seen_item = False
    211        dlnode = nodes.definition_list()
    212        for section in doc.args.values():
    213            termtext = [nodes.literal('', section.member.name)]
    214            if section.member.ifcond.is_present():
    215                termtext.extend(self._nodes_for_ifcond(section.member.ifcond))
    216            # TODO drop fallbacks when undocumented members are outlawed
    217            if section.text:
    218                defn = section.text
    219            else:
    220                defn = [nodes.Text('Not documented')]
    221
    222            dlnode += self._make_dlitem(termtext, defn)
    223            seen_item = True
    224
    225        if not seen_item:
    226            return []
    227
    228        section = self._make_section('Values')
    229        section += dlnode
    230        return [section]
    231
    232    def _nodes_for_arguments(self, doc, boxed_arg_type):
    233        """Return list of doctree nodes for the arguments section"""
    234        if boxed_arg_type:
    235            assert not doc.args
    236            section = self._make_section('Arguments')
    237            dlnode = nodes.definition_list()
    238            dlnode += self._make_dlitem(
    239                [nodes.Text('The members of '),
    240                 nodes.literal('', boxed_arg_type.name)],
    241                None)
    242            section += dlnode
    243            return [section]
    244
    245        return self._nodes_for_members(doc, 'Arguments')
    246
    247    def _nodes_for_features(self, doc):
    248        """Return list of doctree nodes for the table of features"""
    249        seen_item = False
    250        dlnode = nodes.definition_list()
    251        for section in doc.features.values():
    252            dlnode += self._make_dlitem([nodes.literal('', section.name)],
    253                                        section.text)
    254            seen_item = True
    255
    256        if not seen_item:
    257            return []
    258
    259        section = self._make_section('Features')
    260        section += dlnode
    261        return [section]
    262
    263    def _nodes_for_example(self, exampletext):
    264        """Return list of doctree nodes for a code example snippet"""
    265        return [nodes.literal_block(exampletext, exampletext)]
    266
    267    def _nodes_for_sections(self, doc):
    268        """Return list of doctree nodes for additional sections"""
    269        nodelist = []
    270        for section in doc.sections:
    271            snode = self._make_section(section.name)
    272            if section.name and section.name.startswith('Example'):
    273                snode += self._nodes_for_example(section.text)
    274            else:
    275                self._parse_text_into_node(section.text, snode)
    276            nodelist.append(snode)
    277        return nodelist
    278
    279    def _nodes_for_if_section(self, ifcond):
    280        """Return list of doctree nodes for the "If" section"""
    281        nodelist = []
    282        if ifcond.is_present():
    283            snode = self._make_section('If')
    284            snode += nodes.paragraph(
    285                '', '', *self._nodes_for_ifcond(ifcond, with_if=False)
    286            )
    287            nodelist.append(snode)
    288        return nodelist
    289
    290    def _add_doc(self, typ, sections):
    291        """Add documentation for a command/object/enum...
    292
    293        We assume we're documenting the thing defined in self._cur_doc.
    294        typ is the type of thing being added ("Command", "Object", etc)
    295
    296        sections is a list of nodes for sections to add to the definition.
    297        """
    298
    299        doc = self._cur_doc
    300        snode = nodes.section(ids=[self._sphinx_directive.new_serialno()])
    301        snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol),
    302                                       nodes.Text(' (' + typ + ')')])
    303        self._parse_text_into_node(doc.body.text, snode)
    304        for s in sections:
    305            if s is not None:
    306                snode += s
    307        self._add_node_to_current_heading(snode)
    308
    309    def visit_enum_type(self, name, info, ifcond, features, members, prefix):
    310        doc = self._cur_doc
    311        self._add_doc('Enum',
    312                      self._nodes_for_enum_values(doc)
    313                      + self._nodes_for_features(doc)
    314                      + self._nodes_for_sections(doc)
    315                      + self._nodes_for_if_section(ifcond))
    316
    317    def visit_object_type(self, name, info, ifcond, features,
    318                          base, members, variants):
    319        doc = self._cur_doc
    320        if base and base.is_implicit():
    321            base = None
    322        self._add_doc('Object',
    323                      self._nodes_for_members(doc, 'Members', base, variants)
    324                      + self._nodes_for_features(doc)
    325                      + self._nodes_for_sections(doc)
    326                      + self._nodes_for_if_section(ifcond))
    327
    328    def visit_alternate_type(self, name, info, ifcond, features, variants):
    329        doc = self._cur_doc
    330        self._add_doc('Alternate',
    331                      self._nodes_for_members(doc, 'Members')
    332                      + self._nodes_for_features(doc)
    333                      + self._nodes_for_sections(doc)
    334                      + self._nodes_for_if_section(ifcond))
    335
    336    def visit_command(self, name, info, ifcond, features, arg_type,
    337                      ret_type, gen, success_response, boxed, allow_oob,
    338                      allow_preconfig, coroutine):
    339        doc = self._cur_doc
    340        self._add_doc('Command',
    341                      self._nodes_for_arguments(doc,
    342                                                arg_type if boxed else None)
    343                      + self._nodes_for_features(doc)
    344                      + self._nodes_for_sections(doc)
    345                      + self._nodes_for_if_section(ifcond))
    346
    347    def visit_event(self, name, info, ifcond, features, arg_type, boxed):
    348        doc = self._cur_doc
    349        self._add_doc('Event',
    350                      self._nodes_for_arguments(doc,
    351                                                arg_type if boxed else None)
    352                      + self._nodes_for_features(doc)
    353                      + self._nodes_for_sections(doc)
    354                      + self._nodes_for_if_section(ifcond))
    355
    356    def symbol(self, doc, entity):
    357        """Add documentation for one symbol to the document tree
    358
    359        This is the main entry point which causes us to add documentation
    360        nodes for a symbol (which could be a 'command', 'object', 'event',
    361        etc). We do this by calling 'visit' on the schema entity, which
    362        will then call back into one of our visit_* methods, depending
    363        on what kind of thing this symbol is.
    364        """
    365        self._cur_doc = doc
    366        entity.visit(self)
    367        self._cur_doc = None
    368
    369    def _start_new_heading(self, heading, level):
    370        """Start a new heading at the specified heading level
    371
    372        Create a new section whose title is 'heading' and which is placed
    373        in the docutils node tree as a child of the most recent level-1
    374        heading. Subsequent document sections (commands, freeform doc chunks,
    375        etc) will be placed as children of this new heading section.
    376        """
    377        if len(self._active_headings) < level:
    378            raise QAPISemError(self._cur_doc.info,
    379                               'Level %d subheading found outside a '
    380                               'level %d heading'
    381                               % (level, level - 1))
    382        snode = self._make_section(heading)
    383        self._active_headings[level - 1] += snode
    384        self._active_headings = self._active_headings[:level]
    385        self._active_headings.append(snode)
    386
    387    def _add_node_to_current_heading(self, node):
    388        """Add the node to whatever the current active heading is"""
    389        self._active_headings[-1] += node
    390
    391    def freeform(self, doc):
    392        """Add a piece of 'freeform' documentation to the document tree
    393
    394        A 'freeform' document chunk doesn't relate to any particular
    395        symbol (for instance, it could be an introduction).
    396
    397        If the freeform document starts with a line of the form
    398        '= Heading text', this is a section or subsection heading, with
    399        the heading level indicated by the number of '=' signs.
    400        """
    401
    402        # QAPIDoc documentation says free-form documentation blocks
    403        # must have only a body section, nothing else.
    404        assert not doc.sections
    405        assert not doc.args
    406        assert not doc.features
    407        self._cur_doc = doc
    408
    409        text = doc.body.text
    410        if re.match(r'=+ ', text):
    411            # Section/subsection heading (if present, will always be
    412            # the first line of the block)
    413            (heading, _, text) = text.partition('\n')
    414            (leader, _, heading) = heading.partition(' ')
    415            self._start_new_heading(heading, len(leader))
    416            if text == '':
    417                return
    418
    419        node = self._make_section(None)
    420        self._parse_text_into_node(text, node)
    421        self._add_node_to_current_heading(node)
    422        self._cur_doc = None
    423
    424    def _parse_text_into_node(self, doctext, node):
    425        """Parse a chunk of QAPI-doc-format text into the node
    426
    427        The doc comment can contain most inline rST markup, including
    428        bulleted and enumerated lists.
    429        As an extra permitted piece of markup, @var will be turned
    430        into ``var``.
    431        """
    432
    433        # Handle the "@var means ``var`` case
    434        doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext)
    435
    436        rstlist = ViewList()
    437        for line in doctext.splitlines():
    438            # The reported line number will always be that of the start line
    439            # of the doc comment, rather than the actual location of the error.
    440            # Being more precise would require overhaul of the QAPIDoc class
    441            # to track lines more exactly within all the sub-parts of the doc
    442            # comment, as well as counting lines here.
    443            rstlist.append(line, self._cur_doc.info.fname,
    444                           self._cur_doc.info.line)
    445        # Append a blank line -- in some cases rST syntax errors get
    446        # attributed to the line after one with actual text, and if there
    447        # isn't anything in the ViewList corresponding to that then Sphinx
    448        # 1.6's AutodocReporter will then misidentify the source/line location
    449        # in the error message (usually attributing it to the top-level
    450        # .rst file rather than the offending .json file). The extra blank
    451        # line won't affect the rendered output.
    452        rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line)
    453        self._sphinx_directive.do_parse(rstlist, node)
    454
    455    def get_document_nodes(self):
    456        """Return the list of docutils nodes which make up the document"""
    457        return self._top_node.children
    458
    459
    460class QAPISchemaGenDepVisitor(QAPISchemaVisitor):
    461    """A QAPI schema visitor which adds Sphinx dependencies each module
    462
    463    This class calls the Sphinx note_dependency() function to tell Sphinx
    464    that the generated documentation output depends on the input
    465    schema file associated with each module in the QAPI input.
    466    """
    467    def __init__(self, env, qapidir):
    468        self._env = env
    469        self._qapidir = qapidir
    470
    471    def visit_module(self, name):
    472        if name != "./builtin":
    473            qapifile = self._qapidir + '/' + name
    474            self._env.note_dependency(os.path.abspath(qapifile))
    475        super().visit_module(name)
    476
    477
    478class QAPIDocDirective(Directive):
    479    """Extract documentation from the specified QAPI .json file"""
    480    required_argument = 1
    481    optional_arguments = 1
    482    option_spec = {
    483        'qapifile': directives.unchanged_required
    484    }
    485    has_content = False
    486
    487    def new_serialno(self):
    488        """Return a unique new ID string suitable for use as a node's ID"""
    489        env = self.state.document.settings.env
    490        return 'qapidoc-%d' % env.new_serialno('qapidoc')
    491
    492    def run(self):
    493        env = self.state.document.settings.env
    494        qapifile = env.config.qapidoc_srctree + '/' + self.arguments[0]
    495        qapidir = os.path.dirname(qapifile)
    496
    497        try:
    498            schema = QAPISchema(qapifile)
    499
    500            # First tell Sphinx about all the schema files that the
    501            # output documentation depends on (including 'qapifile' itself)
    502            schema.visit(QAPISchemaGenDepVisitor(env, qapidir))
    503
    504            vis = QAPISchemaGenRSTVisitor(self)
    505            vis.visit_begin(schema)
    506            for doc in schema.docs:
    507                if doc.symbol:
    508                    vis.symbol(doc, schema.lookup_entity(doc.symbol))
    509                else:
    510                    vis.freeform(doc)
    511            return vis.get_document_nodes()
    512        except QAPIError as err:
    513            # Launder QAPI parse errors into Sphinx extension errors
    514            # so they are displayed nicely to the user
    515            raise ExtensionError(str(err))
    516
    517    def do_parse(self, rstlist, node):
    518        """Parse rST source lines and add them to the specified node
    519
    520        Take the list of rST source lines rstlist, parse them as
    521        rST, and add the resulting docutils nodes as children of node.
    522        The nodes are parsed in a way that allows them to include
    523        subheadings (titles) without confusing the rendering of
    524        anything else.
    525        """
    526        # This is from kerneldoc.py -- it works around an API change in
    527        # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use
    528        # sphinx.util.nodes.nested_parse_with_titles() rather than the
    529        # plain self.state.nested_parse(), and so we can drop the saving
    530        # of title_styles and section_level that kerneldoc.py does,
    531        # because nested_parse_with_titles() does that for us.
    532        if Use_SSI:
    533            with switch_source_input(self.state, rstlist):
    534                nested_parse_with_titles(self.state, rstlist, node)
    535        else:
    536            save = self.state.memo.reporter
    537            self.state.memo.reporter = AutodocReporter(
    538                rstlist, self.state.memo.reporter)
    539            try:
    540                nested_parse_with_titles(self.state, rstlist, node)
    541            finally:
    542                self.state.memo.reporter = save
    543
    544
    545def setup(app):
    546    """ Register qapi-doc directive with Sphinx"""
    547    app.add_config_value('qapidoc_srctree', None, 'env')
    548    app.add_directive('qapi-doc', QAPIDocDirective)
    549
    550    return dict(
    551        version=__version__,
    552        parallel_read_safe=True,
    553        parallel_write_safe=True
    554    )