Document tables/database structure

.circleci/config.yml

@@ -132,6 +132,8 @@ jobs:
            at: ~/.local
        - run:
            command: |
+             sudo apt update
+             sudo apt install graphviz
              rm -f ./setup.cfg # So sphinx-build doesn't go in ~/dbUtils
              pip install --user "docutils<0.16"
              pip install --user sphinx numpy

developer/scripts/table_docs.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+"""Create the skeleton of the rst documentation for tables
+
+Using the tables module, prints to stdout the table names, column names,
+and SQL types for each column."""
+
+import dbprocessing.tables
+import sqlalchemy.schema
+import sqlalchemy.types
+
+
+if __name__ == '__main__':
+    names = sorted(dbprocessing.tables.names)
+    namelen = max([len(n) for n in names])
+    namecolumnlen = namelen + len(':sql:table:``')
+    overline = '=' * namecolumnlen + ' ' + '=' * (79 - namecolumnlen)
+    print(overline)
+    for n in names:
+        print(':sql:table:`{name}`{spaces} {name} summary'.format(
+            spaces = ' ' * (namelen - len(n)), name=n))
+    print(overline)
+    for n in names:
+        print('\n.. sql:table:: {}'.format(n))
+        for c in dbprocessing.tables.definition(n):
+            if not isinstance(c, sqlalchemy.schema.Column):
+                continue
+            print('\n.. sql:column:: {}\n'.format(c.name))
+            info = [':py:class:`~sqlalchemy.types.{}`'.format(
+                repr(c.type).split('(')[0])]
+            if c.primary_key:
+                info.append(
+                    ':py:class:`PK <sqlalchemy.schema.PrimaryKeyConstraint>`')
+            if not c.nullable:
+                # This doesn't actually link, but does render the way I want
+                # and provides a hint in the source at least...
+                info.append(':py:obj:`NOT NULL <sqlalchemy.schema.Column'
+                            '.params.nullable>`')
+            for fk in c.foreign_keys:
+                info.append(':py:class:`FK <sqlalchemy.schema'
+                            '.ForeignKeyConstraint>`\n   :sql:column:`{}`'
+                            .format(fk.target_fullname))
+            print('   ({})'.format(',\n   '.join(info)))
+

developer/scripts/table_graph.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+"""Create the diagram of the database layout
+
+Using the tables module, prints to stdout a GraphViz dot file showing the
+table relationships."""
+
+import collections
+
+import dbprocessing.tables
+import sqlalchemy.schema
+import sqlalchemy.types
+
+
+tablestart = '''
+"{tablename}" [
+    shape=none
+    label=<
+        <table border="0" cellpadding="5" cellspacing="0" cellborder="1">
+            <tr><td colspan="3" bgcolor="#E0B010" valign="BOTTOM"><font point-size="18" face="bold">{tablename}</font></td></tr>'''
+tableend = '''
+        </table>
+>];'''
+column = '            <tr><td align="center" port="{columnname}_in" sides="ltb"  bgcolor="#FFF0B0">{annotation}</td><td align="left" sides="rtb" bgcolor="#FFF0B0"><b>{columnname}</b></td><td bgcolor="#FFE080" align="left" port="{columnname}_out">{columntype}</td></tr>'''
+
+if __name__ == '__main__':
+    print('digraph g { graph [ rankdir = "LR" ];\nsplines=compound;')
+    names = sorted(dbprocessing.tables.names)
+    foreign_keys = collections.defaultdict(list)
+    for n in names:
+        print(tablestart.format(tablename=n))
+        for c in dbprocessing.tables.definition(n):
+            if not isinstance(c, sqlalchemy.schema.Column):
+                continue
+            print(column.format(
+                annotation='&#9919;' if c.primary_key else '',
+                columnname=c.name,
+                columntype=repr(c.type).split('(')[0]))
+            qualname = '{}.{}'.format(n, c.name)
+            if c.primary_key:
+                pass
+            if not c.nullable:
+                pass
+            for fk in c.foreign_keys:
+                foreign_keys[(n, c.name)].append(fk.target_fullname.split('.'))
+        print(tableend)
+    for s in sorted(foreign_keys):
+        for t in sorted(foreign_keys[s]):
+            print('{}:{}_out:e -> {}:{}_in:w'.format(s[0], s[1], t[0], t[1]))
+    print('}')

docs/_ext/sql.py

@@ -0,0 +1,153 @@
+import docutils.parsers.rst
+import sphinx
+import sphinx.addnodes
+import sphinx.directives
+import sphinx.domains
+import sphinx.locale
+import sphinx.roles
+import sphinx.util
+import sphinx.util.nodes
+
+
+class ColumnDirective(sphinx.directives.ObjectDescription):
+    """Description of a database column (.. column)."""
+
+    has_content = True
+    required_arguments = 1  # Name of the item
+
+    def handle_signature(self, sig, signode):
+        signode.clear()
+        # Alternatives: desc_addname (monospace), desc_annotation (italic)
+        signode += sphinx.addnodes.desc_type(text=sig)
+        # Remove whitespace in target
+        name = sphinx.util.ws_re.sub('', sig)
+        return name
+
+    def add_target_and_index(self, name, sig, signode):
+        table_ws = self.env.ref_context.get('sql:table')
+        table = sphinx.util.ws_re.sub('', table_ws) if table_ws else None
+        targetname = '%s-%s-%s' % (self.objtype, table, name) if table\
+                     else '%s-%s' % (self.objtype, name)
+        signode['ids'].append(targetname)
+        self.state.document.note_explicit_target(signode)
+        self.indexnode['entries'].append((
+            'single',
+            '{}; ({}column)'.format(sig, table_ws + ' ' if table_ws else ''),
+            targetname, '', None))
+        qualname = '{}.{}'.format(table, name) if table else name
+        self.env.domaindata['sql']['objects'][self.objtype, qualname] = \
+            self.env.docname, targetname
+
+
+class ColumnXRefRole(sphinx.roles.XRefRole):
+    def process_link(self, env, refnode, has_explicit_title, title, target):
+        table = env.ref_context.get('sql:table')
+        if table:
+            sphinx.util.ws_re.sub('', table)
+        # TODO need better way of distinguishing between qualified and not
+        if table and '.' not in target:
+            target = '{}.{}'.format(table, target)
+        if target.startswith('~') and '.' in title and not has_explicit_title:
+            target = target[1:]
+            title = title.split('.', 1)[1]
+        return title, sphinx.util.ws_re.sub('', target)
+
+
+class CurrentTableDirective(docutils.parsers.rst.Directive):
+    """Change the current table, no output"""
+
+    has_content = False
+    required_arguments = 0
+    optional_arguments = 1
+    final_argument_whitespace = True
+    option_spec = {}  # type: Dict
+
+    def run(self):
+        env = self.state.document.settings.env
+        if self.arguments:
+            self.state.document.settings.env.ref_context['sql:table']\
+                = self.arguments[0]
+        else:
+            self.state.document.settings.env.ref_context.pop(
+                'sql:table', None)
+        return []
+
+
+class TableDirective(sphinx.directives.ObjectDescription):
+    """Description of a database table (.. table)."""
+
+    has_content = True
+    required_arguments = 1  # Name of the item
+
+    def handle_signature(self, sig, signode):
+        signode.clear()
+        signode += sphinx.addnodes.desc_name(text=sig)
+        name = sphinx.util.ws_re.sub('', sig)
+        # Store table name w/any whitespace, so human-readable
+        self.state.document.settings.env.ref_context['sql:table'] = sig
+        return name
+
+    def add_target_and_index(self, name, sig, signode):
+        targetname = '%s-%s' % (self.objtype, name)
+        signode['ids'].append(targetname)
+        self.state.document.note_explicit_target(signode)
+        self.indexnode['entries'].append((
+            'single', '{}; (table)'.format(name), targetname, '', None))
+        self.env.domaindata['sql']['objects'][self.objtype, name] = \
+            self.env.docname, targetname
+
+
+class TableXRefRole(sphinx.roles.XRefRole):
+    def process_link(self, env, refnode, has_explicit_title, title, target):
+        return title, sphinx.util.ws_re.sub('', target)
+
+
+class SqlDomain(sphinx.domains.Domain):
+
+    name = 'sql'
+    label = 'SQL'
+    directives = {
+        'column': ColumnDirective,
+        'currenttable': CurrentTableDirective,
+        'table': TableDirective,
+    }
+    roles = {
+        'column': ColumnXRefRole(warn_dangling=True),
+        'table': TableXRefRole(warn_dangling=True),
+    }
+    object_types = {
+        'column': sphinx.domains.ObjType('column', 'column'),
+        'table': sphinx.domains.ObjType('table', 'table'),
+    }
+
+    initial_data = {
+        'objects': {},
+    }
+
+    def resolve_xref(self, env, fromdocname, builder, typ, target,
+                     node, contnode):
+        # Cribbed straight from the standard domain
+        objtypes = self.objtypes_for_role(typ) or []
+        for objtype in objtypes:
+            if (objtype, target) in self.data['objects']:
+                docname, labelid = self.data['objects'][objtype, target]
+                break
+        else:
+            docname, labelid = '', ''
+        if not docname:
+            return None
+        return sphinx.util.nodes.make_refnode(builder, fromdocname, docname,
+                                              labelid, contnode)
+
+    def merge_domaindata(self, docnames, otherdata):
+        for key, data in otherdata['objects'].items():
+            if data[0] in docnames:
+                self.data['objects'] = data
+        super(SqlDomain, self).merge_domaindata(docnames, otherdata)
+
+def setup(app):
+    app.add_domain(SqlDomain)
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        }

docs/conf.py

@@ -33,6 +33,10 @@
 thisdir = os.path.abspath(os.path.dirname(__file__))
 if not thisdir in sys.path:
     sys.path.insert(0, thisdir)
+# And extentions under this directory
+_extdir = os.path.join(thisdir, '_ext')
+if not _extdir in sys.path:
+    sys.path.insert(0, _extdir)
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -42,22 +46,17 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 
-try: #Sphinx 1.4 and later replaces pngmath with imgmath
-    import sphinx.ext.imgmath
-    imgmath = 'sphinx.ext.imgmath'
-except ImportError:
-    imgmath = 'sphinx.ext.pngmath'
 extensions = [
     'sphinx.ext.autodoc',
+    'sphinx.ext.graphviz',
     'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
-    imgmath,
     'sphinx.ext.viewcode',
     'numpydoc',
-    'sphinx.ext.inheritance_diagram',
     'sphinx.ext.autosummary',
     'sphinx.ext.extlinks',
+    'sql',
 ]
 
 autosummary_generate = True

docs/developer/documentation.rst

@@ -67,9 +67,8 @@ documentation (except Python sample code).
 All documentation must be in rST (reStructuredText) format except where
 another format is explicitly required.
 
-Section headings should follow the `recommended convention from Sphinx
-<https://www.sphinx-doc.org/en/master/usage/restructuredtext/
-basics.html#sections>`_ with the clarification that most files should be
+Section headings should follow the :ref:`recommended convention from Sphinx
+<sphinx:rst-sections>` with the clarification that most files should be
 considered chapters. Parts are very rare and may involve a separate directory,
 e.g. "User guide", "Developer Guide."
 
@@ -100,8 +99,8 @@ in the "docs" directory, which github treats specially. Files that are parsed
 by github must still end with the ``.rst`` extension (and github should
 render them correctly as rST), but must also be `parseable as Markdown
 <https://gist.github.com/dupuy/1855764>`_. ``dbprocessing`` may eventually
-`enable Sphinx parsing of Markdown <https://www.sphinx-doc.org/en/1.8/usage/
-markdown.html>`_ if necessary to support these files.
+:doc:`enable Sphinx parsing of Markdown <sphinx:usage/markdown>` if necessary
+to support these files.
 
 See also :doc:`github`.
 

docs/developer/index.rst

@@ -12,8 +12,8 @@ Developer Information
    github
    pull_requests
    repository
-   structure
    release
-   notes	      
+   notes
+   tables
    todo
    ../CONTRIBUTING