Dispatch commands: reviewed wiki layout and content

The wiki page
https://wiki.documentfoundation.org/Development/DispatchCommands
is generated by the execution of the python script:
/bin/list-dispatch-commands.py

Layout and content changes:
- better commands list coverage
Scan of .xcu, .sdi and .hxx files
The list is the "union" of the found commands
=> Base and Charts commands are part of the list
- new classification based in the 1st place on the .xcu files
i.o. the .hxx (or "slots") names
=> Commands are listed only once
- The sources providing the listed info are referenced for
each command with a direct link to the opengrok sources
- New information when available
A subclassification: the group
The potential arguments of the command

Change-Id: I54dd8c219d0e7a00fd346faa7577e181a068bbaa
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/128254
Tested-by: Jenkins
Tested-by: Jean-Pierre Ledure <jp@ledure.be>
Reviewed-by: Jean-Pierre Ledure <jp@ledure.be>

1 files changed, 388 insertions, 97 deletions

diff --git a/bin/list-dispatch-commands.py b/bin/list-dispatch-commands.py
index 0b13f89e891b..fb3d60fb5720 100755
--- a/bin/list-dispatch-commands.py
+++ b/bin/list-dispatch-commands.py
@@ -8,123 +8,414 @@
 
 """
 Script to generate https://wiki.documentfoundation.org/Development/DispatchCommands
+3 types of source files are scanned to identify and describe a list of relevant UNO commands:
+- .hxx files: containing the symbolic and numeric id's, and the respective modes and groups
+- .xcu files; containing several english labels as they appear in menus or tooltips
+- .sdi files: containing a list of potential arguments for the commands, and their types
 """
 
-import argparse
 import os
-import sys
 
+REPO = 'https://opengrok.libreoffice.org/xref/core/'
 
-def get_files_list(directory, extension):
-    array_items = []
+BLACKLIST = ('_SwitchViewShell0', '_SwitchViewShell1', '_SwitchViewShell2', '_SwitchViewShell3', '_SwitchViewShell4')
 
-    dh = os.scandir(directory)
-    for entry in dh:
-        if entry.is_dir():
-            array_items += get_files_list(entry.path, extension)
-        elif entry.is_file():
-            if entry.name.endswith(extension):
-                array_items.append(entry.path)
+XCU_DIR = 'officecfg/registry/data/org/openoffice/Office/UI/'
+XCU_FILES = (   XCU_DIR + 'BasicIDECommands.xcu',
+                XCU_DIR + 'CalcCommands.xcu',
+                XCU_DIR + 'ChartCommands.xcu',
+                XCU_DIR + 'DbuCommands.xcu',
+                XCU_DIR + 'DrawImpressCommands.xcu',
+                XCU_DIR + 'GenericCommands.xcu',
+                XCU_DIR + 'MathCommands.xcu',
+                XCU_DIR + 'ReportCommands.xcu',
+                XCU_DIR + 'WriterCommands.xcu')
 
-    return array_items
+HXX_DIR = './workdir/SdiTarget/'
+HXX_FILES = (   HXX_DIR + 'basctl/sdi/basslots.hxx',
+                HXX_DIR + 'sc/sdi/scslots.hxx',
+                HXX_DIR + 'sd/sdi/sdgslots.hxx',
+                HXX_DIR + 'sd/sdi/sdslots.hxx',
+                HXX_DIR + 'sfx2/sdi/sfxslots.hxx',
+                HXX_DIR + 'starmath/sdi/smslots.hxx',
+                HXX_DIR + 'svx/sdi/svxslots.hxx',
+                HXX_DIR + 'sw/sdi/swslots.hxx')
 
+SDI_FILES = (   'sc/sdi/scalc.sdi',
+                'sd/sdi/sdraw.sdi',
+                'sfx2/sdi/sfx.sdi',
+                'starmath/sdi/smath.sdi',
+                'svx/sdi/svx.sdi',
+                'sw/sdi/swriter.sdi')
 
-def analyze_file(filename, all_slots):
-    with open(filename) as fh:
-        for line in fh:
-            if not line.startswith('// Slot Nr. '):
-                continue
+# Category is defined by the 1st file where the command has been found. Precedence: 1. xcu, 2. hxx, 3. sdi.
+MODULES = {'BasicIDE':      'Basic IDE, Forms, Dialogs',
+            'Calc':         'Calc',
+            'Chart':        'Charts',
+            'Dbu':          'Base',
+            'DrawImpress':  'Draw / Impress',
+            'Generic':      'Global',
+            'Math':         'Math',
+            'Report':       'Reports',
+            'Writer':       'Writer',
+            'basslots':     'Basic IDE, Forms, Dialogs',
+            'scslots':      'Calc',
+            'sdgslots':     'Draw / Impress',
+            'sdslots':      'Draw / Impress',
+            'sfxslots':     'Global',
+            'smslots':      'Math',
+            'svxslots':     'Global',
+            'swslots':      'Writer',
+            'scalc':        'Calc',
+            'sdraw':        'Draw / Impress',
+            'sfx':          'Global',
+            'smath':        'Math',
+            'svx':          'Global',
+            'swriter':      'Writer'}
 
-            tmp = line.split(':')
-            slot_id = tmp[1].strip()
+def newcommand(unocommand):
+    cmd = {'unocommand': unocommand,
+           'module': '',
+           'xcufile': -1,
+           'xculinenumber': 0,
+           'xcuoccurs': 0,
+           'label': '',
+           'contextlabel': '',
+           'tooltiplabel': '',
+           'hxxfile': -1,
+           'hxxoccurs': 0,
+           'hxxlinenumber': 0,
+           'resourceid': '',
+           'numericid': '',
+           'group': '',
+           'sdifile': -1,
+           'sdioccurs': 0,
+           'sdilinenumber': 0,
+           'mode': '',
+           'arguments': ''}
+    return cmd
 
-            line = next(fh)
-            tmp = line.split(',')
-            slot_rid = tmp[1]
 
-            next(fh)
-            next(fh)
-            line = next(fh)
-            mode = 'C' if 'CACHABLE' in line else ' '
-            mode += 'U' if 'AUTOUPDATE' in line else ' '
-            mode += 'M' if 'MENUCONFIG' in line else ' '
-            mode += 'T' if 'TOOLBOXCONFIG' in line else ' '
-            mode += 'A' if 'ACCELCONFIG' in line else ' '
+def analyze_xcu(all_commands):
+    for filename in XCU_FILES:
+        ln = 0
+        with open(filename) as fh:
+            popups = False
+            for line in fh:
+                ln += 1
+                if '<node oor:name="Popups">' in line:
+                    popups = True
+                    continue
+                elif popups is True and line == '    </node>':
+                    popups = False
+                    continue
+                if '<node oor:name=".uno:' not in line:
+                    continue
+
+                cmdln = ln
+                tmp = line.split('"')
+                command_name = tmp[1]
+                command_ok = True
+
+                while '</node>' not in line:
+                    try:
+                        line = next(fh)
+                        ln += 1
+                    except StopIteration:
+                        print("Warning: couldn't find '</node>' line in %s" % filename,
+                            file=sys.stderr)
+                        break
+                    if '<prop oor:name="Label"' in line:
+                        label = 'label'
+                    elif '<prop oor:name="ContextLabel"' in line:
+                        label = 'contextlabel'
+                    elif '<prop oor:name="Label"' in line:
+                        label = 'tooltiplabel'
+                    elif '<value xml:lang="en-US">' in line:
+                        labeltext = line.replace('<value xml:lang="en-US">', '').replace('</value>', '').strip()
+                    elif '<prop oor:name="TargetURL"' in line:
+                        command_ok = False
+
+                if command_ok is True and popups is False:
+                    if command_name not in all_commands:
+                        all_commands[command_name] = newcommand(command_name)
+                    #
+                    all_commands[command_name]['xcufile'] = XCU_FILES.index(filename)
+                    all_commands[command_name]['xculinenumber'] = cmdln
+                    all_commands[command_name][label] = labeltext.replace('~', '')
+                    all_commands[command_name]['xcuoccurs'] += 1
+
+
+def analyze_hxx(all_commands):
+    for filename in HXX_FILES:
+        with open(filename) as fh:
+            ln = 0
+            mode = ''
+            for line in fh:
+                ln += 1
+                if not line.startswith('// Slot Nr. '):
+                    continue
+
+                # Parse sth like
+                # // Slot Nr. 0 : 5502
+                # SFX_NEW_SLOT_ARG( basctl_Shell,SID_SAVEASDOC,SfxGroupId::Document,
+                cmdln = ln
+                tmp = line.split(':')
+                command_id = tmp[1].strip()
 
-            next(fh)
-            next(fh)
-            line = next(fh)
-            if '"' not in line:
                 line = next(fh)
-            tmp = line.split('"')
-            try:
-                slot_name = '.uno:' + tmp[1]
-            except IndexError:
-                print("Warning: expected \" in line '%s' from file %s" % (line.strip(), filename),
-                      file=sys.stderr)
-                slot_name = '.uno:'
-
-            if slot_name not in all_slots:
-                all_slots[slot_name] = {'slot_id': slot_id,
-                                        'slot_rid': slot_rid,
-                                        'mode': mode,
-                                        'slot_description': ''}
-
-
-def analyze_xcu(filename, all_slots):
-    with open(filename) as fh:
-        for line in fh:
-            if '<node oor:name=".uno:' not in line:
-                continue
-
-            tmp = line.split('"')
-            slot_name = tmp[1]
-
-            while '<value xml:lang="en-US">' not in line:
-                try:
+                ln += 1
+                tmp = line.split(',')
+                command_rid = tmp[1]
+                command_group = tmp[2].split('::')[1]
+
+                next(fh)
+                ln += 1
+                next(fh)
+                ln += 1
+                line = next(fh)
+                ln += 1
+                mode += 'U' if 'AUTOUPDATE' in line else ''
+                mode += 'M' if 'MENUCONFIG' in line else ''
+                mode += 'T' if 'TOOLBOXCONFIG' in line else ''
+                mode += 'A' if 'ACCELCONFIG' in line else ''
+
+                next(fh)
+                ln += 1
+                next(fh)
+                ln += 1
+                line = next(fh)
+                ln += 1
+                if '"' not in line:
                     line = next(fh)
-                except StopIteration:
-                    print("Warning: couldn't find '<value xml:lang=\"en-US\">' line in %s" % filename,
-                          file=sys.stderr)
-                    break
+                tmp = line.split('"')
+                try:
+                    command_name = '.uno:' + tmp[1]
+                except IndexError:
+                    print("Warning: expected \" in line '%s' from file %s" % (line.strip(), filename),
+                            file=sys.stderr)
+                    command_name = '.uno:'
 
-            line = line.replace('<value xml:lang="en-US">', '')
-            line = line.replace('</value>', '').strip()
+                if command_name not in all_commands:
+                    all_commands[command_name] = newcommand(command_name)
+                #
+                all_commands[command_name]['hxxfile'] = HXX_FILES.index(filename)
+                all_commands[command_name]['hxxlinenumber'] = cmdln
+                all_commands[command_name]['numericid'] = command_id
+                all_commands[command_name]['resourceid'] = command_rid
+                all_commands[command_name]['group'] = command_group
+                all_commands[command_name]['mode'] = mode
+                all_commands[command_name]['hxxoccurs'] += 1
+                mode = ''
 
-            if slot_name in all_slots:
-                all_slots[slot_name]['slot_description'] = line.replace('~', '')
+
+def analyze_sdi(all_commands):
+    def SplitArguments(params):
+        # Split a string like : SfxStringItem Name SID_CHART_NAME,SfxStringItem Range SID_CHART_SOURCE,SfxBoolItem ColHeaders FN_PARAM_1,SfxBoolItem RowHeaders FN_PARAM_2
+        # in : Name (string)\nRange (string)\nRowHeaders (bool)
+        CR = '<br>'
+        split = ''
+        params = params.strip(' ,').replace(', ', ',')  #   At least 1 case of ', ' in svx/sdi/svx.sdi line 3592
+        if len(params) > 0:
+            for p in params.split(','):
+                if len(split) > 0:
+                    split += CR
+                elems = p.split()
+                if len(elems) >= 2:
+                    split += elems[1]
+                    if 'String' in elems[0]:
+                        split += ' (string)'
+                    elif 'Bool' in elems[0]:
+                        split += ' (bool)'
+                    elif 'Int16' in elems[0]:
+                        split += ' (integer)'
+                    elif 'Int32' in elems[0]:
+                        split += ' (long)'
+                    else:
+                        split += ' (' + elems[0].replace('Sfx', '').replace('Svx', '').replace('Item', '').lower() + ')'
+        return split
+
+    for filename in SDI_FILES:
+        ln = 0
+        comment, square, command, param = False, False, False, False
+        with open(filename) as fh:
+            for line in fh:
+                ln += 1
+                line = line.replace('  ', ' ').strip()       #   Anomaly met in svx/sdi/svx.sdi
+                if line.startswith('//'):
+                    pass
+                elif comment is False and line.startswith('/*') and not line.endswith('*/'):
+                    comment = True
+                elif comment is True and line.endswith('*/'):
+                    comment = False
+                elif comment is False and line.startswith('/*') and line.endswith('*/'):
+                    pass
+                elif comment is True:
+                    pass
+                elif square is False and line.startswith('['):
+                    square = True
+                    mode = ''
+                    command = False
+                elif square is True and line.endswith(']'):
+                    all_commands[command_name]['mode'] = mode
+                    square = False
+                elif square is True:
+                    squaremode = line.strip(',;').split()
+                    if len(squaremode) == 3:
+                        mode += 'U' if squaremode[0] == 'AutoUpdate' and squaremode[2] == 'TRUE' else ''
+                        mode += 'M' if squaremode[0] == 'MenuConfig' and squaremode[2] == 'TRUE' else ''
+                        mode += 'T' if squaremode[0] == 'ToolBoxConfig' and squaremode[2] == 'TRUE' else ''
+                        mode += 'A' if squaremode[0] == 'AccelConfig' and squaremode[2] == 'TRUE' else ''
+                elif comment is False and square is False and command is False and len(line) == 0:
+                    pass
+                elif command is False:
+                    command_name = '.uno:' + line.split(' ')[1]
+                    if command_name not in all_commands:
+                        all_commands[command_name] = newcommand(command_name)
+                    all_commands[command_name]['sdifile'] = SDI_FILES.index(filename)
+                    all_commands[command_name]['sdilinenumber'] = ln
+                    all_commands[command_name]['sdioccurs'] += 1
+                    if len(all_commands[command_name]['resourceid']) == 0:
+                        all_commands[command_name]['resourceid'] = line.split(' ')[2]
+                    command = True
+                elif command is True and (line == '' or line == '()'):
+                    command = False
+                elif command is True and (param is True or line.startswith('(')) and line.endswith(')'):
+                    if param:
+                        params += line.strip(' (),').replace(', ', ',') #   At least 1 case of ", " in svx/sdi/svx.sdi line 8767
+                                                                        #   At least 1 case of "( " in sw/sdi/swriter.sdi line 5477
+                    else:
+                        params = line.strip(' (),').replace(', ', ',')  #   At least 1 case in sw/sdi/swriter.sdi line 7083
+                    all_commands[command_name]['arguments'] = SplitArguments(params)
+                    command = False
+                    param = False
+                elif command is True and line.startswith('('):  #   Arguments always on 1 line, except in some cases (cfr.BasicIDEAppear)
+                    params = line.strip(' ()').replace(', ', ',')
+                    param = True
+                elif param is True:
+                    params += line
+
+
+def categorize(all_commands):
+    # Clean black listed commands
+    for command in BLACKLIST:
+        cmd = '.uno:' + command
+        if cmd in all_commands:
+            del all_commands[cmd]
+    # Set category based on the file name where the command was found first
+    for cmd in all_commands:
+        command = all_commands[cmd]
+        cxcu, chxx, csdi = '', '', ''
+        fxcu = command['xcufile']
+        if fxcu > -1:
+            cxcu = os.path.basename(XCU_FILES[fxcu]).split('.')[0].replace('Commands', '')
+        fhxx = command['hxxfile']
+        if fhxx > -1:
+            chxx = os.path.basename(HXX_FILES[fhxx]).split('.')[0]
+        fsdi = command['sdifile']
+        if fsdi > -1:
+            csdi = os.path.basename(SDI_FILES[fsdi]).split('.')[0]
+        # General rule:
+        if len(cxcu) > 0:
+            cat = cxcu
+        elif len(chxx) > 0:
+            cat = chxx
+        else:
+            cat = csdi
+        # Exceptions on general rule
+        if cat == 'Generic' and chxx == 'basslots':
+            cat = chxx
+        command['module'] = MODULES[cat]
+
+
+def print_output(all_commands):
+    def longest(*args):
+        # Return the longest string among the arguments
+        return max(args, key = len)
+    #
+    def sources(cmd):
+        # Build string identifying the sources
+        xcufile, xculinenumber, hxxfile, hxxlinenumber, sdifile, sdilinenumber = 2, 3, 8, 10, 14, 16
+        src = ''
+        if cmd[xcufile] >= 0:
+            src += '[' + REPO + XCU_FILES[cmd[xcufile]] + '#' + str(cmd[xculinenumber]) + ' XCU]'
+        if cmd[sdifile] >= 0:
+            src += ' [' + REPO + SDI_FILES[cmd[sdifile]] + '#' + str(cmd[sdilinenumber]) + ' SDI]'
+        if cmd[hxxfile] >= 0:
+            file = str(cmd[hxxfile] + 1 + len(XCU_FILES) + len(SDI_FILES))
+            src += ' <span title="File (' + file + ') line ' + str(cmd[hxxlinenumber]) + '">[[#hxx' + file + '|HXX]]</span>'
+        return src.strip()
+    #
+    # Sort by category and command name
+    commands_list = []
+    for cmd in all_commands:
+        cmdlist = tuple(all_commands[cmd].values())
+        commands_list.append(cmdlist)
+    sorted_by_command = sorted(commands_list, key = lambda cmd: cmd[0])
+    sorted_by_module = sorted(sorted_by_command, key = lambda cmd: cmd[1])
+    #
+    # Produce tabular output
+    unocommand, module, label, contextlabel, tooltiplabel, arguments, resourceid, numericid, group, mode = 0, 1, 5, 6, 7, 18, 11, 12, 13, 17
+    lastmodule = ''
+    for cmd in sorted_by_module:
+        # Format bottom and header
+        if lastmodule != cmd[module]:
+            if len(lastmodule) > 0:
+                print('\n|-\n|}\n')
+                print('</small>')
+            lastmodule = cmd[module]
+            print('=== %s ===\n' % lastmodule)
+            print('<small>')
+            print('{| class="wikitable sortable" width="100%"')
+            print('|-')
+            print('! scope="col" | Dispatch command')
+            print('! scope="col" | Description')
+            print('! scope="col" | Group')
+            print('! scope="col" | Arguments')
+            print('! scope="col" | Internal<br>name (value)')
+            print('! scope="col" | Mode')
+            print('! scope="col" | Source<br>files')
+        print('|-\n')
+        print('| ' + cmd[unocommand].replace('&amp;', '\n&'))
+        print('| ' + longest(cmd[label], cmd[contextlabel], cmd[tooltiplabel]))
+        print('| ' + cmd[group])
+        print('| ' + cmd[arguments].replace('\\n', '\n'))
+        if len(cmd[numericid]) == 0:
+            print('| ' + cmd[resourceid])
+        else:
+            print('| ' + cmd[resourceid] + ' (' + cmd[numericid] + ')')
+        print('| ' + cmd[mode])
+        print('| ' + sources(cmd))
+    print('|-\n|}\n')
+    # List the source files
+    print('== Source files ==\n')
+    fn = 0
+    for i in range(len(XCU_FILES)):
+        fn += 1
+        print(f'({fn}) {REPO}{XCU_FILES[i]}\n')
+    print('\n')
+    for i in range(len(SDI_FILES)):
+        fn += 1
+        print(f'({fn}) {REPO}{SDI_FILES[i]}\n')
+    print('\n')
+    for i in range(len(HXX_FILES)):
+        fn += 1
+        print(f'<span id="hxx{fn}">({fn}) {HXX_FILES[i][2:]}</span>\n')
+    print('</small>')
 
 
 def main():
-    modules = ['basslots', 'scslots', 'sdgslots', 'sdslots', 'sfxslots', 'smslots', 'svxslots', 'swslots']
-    sdi_dir = './workdir/SdiTarget'
-    sdi_ext = '.hxx'
-    xcu_dir = 'officecfg/registry/data/org/openoffice/Office/UI'
-    xcu_ext = '.xcu'
-    all_slots = {}
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('module', choices=modules)
-    args = parser.parse_args()
-
-    module_filename = args.module + sdi_ext
-
-    sdi_files = get_files_list(sdi_dir, sdi_ext)
-    for sdi_file in sdi_files:
-        sdi_file_basename = os.path.basename(sdi_file)
-        if sdi_file_basename == module_filename:
-            analyze_file(sdi_file, all_slots)
-
-    xcu_files = get_files_list(xcu_dir, xcu_ext)
-    for xcu_file in xcu_files:
-        analyze_xcu(xcu_file, all_slots)
-
-    for name in sorted(all_slots.keys()):
-        props = all_slots[name]
-        print('|-\n| %s' % name)
-        print('| %(slot_rid)s\n| %(slot_id)s\n| %(mode)s\n| %(slot_description)s' % props)
-
-    print("|-")
+    all_commands = {}
+
+    analyze_xcu(all_commands)
+
+    analyze_hxx(all_commands)
+
+    analyze_sdi(all_commands)
+
+    categorize(all_commands)
+
+    print_output(all_commands)
 
 if __name__ == '__main__':
     main()