From 693042ec5deed3b0f846c13be7299d022bb3b4eb Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Thu, 8 Aug 2024 23:51:17 +0900 Subject: [PATCH 1/3] build-sys: use th prefix "x-" for header files declaring the interface for subparsers Signed-off-by: Masatake YAMATO --- parsers/#vim.c# | 1090 ++++++ parsers/ansibleplaybook.c | 2 +- parsers/ant.c | 2 +- parsers/args.ctags | 0 parsers/asm.c | 2 +- parsers/autoconf.c | 4 +- parsers/automake.c | 2 +- parsers/bats.c | 2 +- parsers/biblatex.c | 2 +- parsers/bibtex.c | 2 +- parsers/c-based.c | 2 +- parsers/cpreprocessor.c | 2 +- parsers/cxx/cxx_parser.c | 2 +- parsers/cxx/cxx_parser_block.c | 2 +- parsers/cxx/cxx_parser_function.c | 2 +- parsers/cxx/cxx_parser_lambda.c | 2 +- parsers/cxx/cxx_parser_module.c | 2 +- parsers/cxx/cxx_parser_namespace.c | 2 +- parsers/cxx/cxx_parser_template.c | 2 +- parsers/cxx/cxx_parser_tokenizer.c | 2 +- parsers/cxx/cxx_tag.c | 2 +- parsers/cython.c | 34 + parsers/dbusintrospect.c | 2 +- parsers/desktopentry.c | 121 + parsers/dts.c | 2 +- parsers/frontmatter.c | 2 +- parsers/gemspec.c | 2 +- parsers/glade.c | 2 +- parsers/i18nrubygem.c | 2 +- parsers/iniconf.c | 2 +- parsers/itcl.c | 2 +- parsers/jscript.c | 2 +- parsers/ldscript.c | 2 +- parsers/m4.c | 2 +- parsers/make.c | 4 +- parsers/markdown.c | 2 +- parsers/maven2.c | 2 +- parsers/openapi.c | 2 +- parsers/perl-function-parameters.c | 2 +- parsers/perl-moose.c | 2 +- parsers/perl.c | 2 +- parsers/plist.c | 2 +- parsers/protobuf.c | 2 +- parsers/pythonloggingconfig.c | 2 +- parsers/quarto.c | 2 +- parsers/r-r6class.c | 2 +- parsers/r-s4class.c | 2 +- parsers/r.c | 2 +- parsers/rake.c | 2 +- parsers/relaxng.c | 2 +- parsers/rmarkdown.c | 2 +- parsers/rpmspec.c | 2 +- parsers/rspec.c | 2 +- parsers/ruby.c | 2 +- parsers/sh.c | 2 +- parsers/svg.c | 2 +- parsers/systemdunit.c | 2 +- parsers/tcl.c | 2 +- parsers/tcloo.c | 2 +- parsers/tex-beamer.c | 2 +- parsers/tex.c | 2 +- parsers/v.c | 2 +- parsers/v.c.orig | 2926 +++++++++++++++++ parsers/vera.c | 2 +- parsers/{autoconf.h => x-autoconf.h} | 0 parsers/{bibtex.h => x-bibtex.h} | 0 .../{cpreprocessor.h => x-cpreprocessor.h} | 0 parsers/{frontmatter.h => x-frontmatter.h} | 0 parsers/{iniconf.h => x-iniconf.h} | 0 parsers/{jscript.h => x-jscript.h} | 0 parsers/{m4.h => x-m4.h} | 0 parsers/{make.h => x-make.h} | 0 parsers/{markdown.h => x-markdown.h} | 0 parsers/{perl.h => x-perl.h} | 0 parsers/{r.h => x-r.h} | 0 parsers/{ruby.h => x-ruby.h} | 0 parsers/{sh.h => x-sh.h} | 0 parsers/{tcl.h => x-tcl.h} | 0 parsers/{tex.h => x-tex.h} | 0 parsers/{xml.h => x-xml.h} | 0 parsers/{yaml.h => x-yaml.h} | 0 parsers/xml.c | 2 +- parsers/xrc.c | 2 +- parsers/xslt.c | 2 +- parsers/yaml.c | 2 +- parsers/yamlfrontmatter.c | 4 +- parsers/yumrepo.c | 2 +- source.mak | 34 +- win32/ctags_vs2013.vcxproj | 30 +- win32/ctags_vs2013.vcxproj.filters | 42 +- 90 files changed, 4292 insertions(+), 121 deletions(-) create mode 100644 parsers/#vim.c# create mode 100644 parsers/args.ctags create mode 100644 parsers/cython.c create mode 100644 parsers/desktopentry.c create mode 100644 parsers/v.c.orig rename parsers/{autoconf.h => x-autoconf.h} (100%) rename parsers/{bibtex.h => x-bibtex.h} (100%) rename parsers/{cpreprocessor.h => x-cpreprocessor.h} (100%) rename parsers/{frontmatter.h => x-frontmatter.h} (100%) rename parsers/{iniconf.h => x-iniconf.h} (100%) rename parsers/{jscript.h => x-jscript.h} (100%) rename parsers/{m4.h => x-m4.h} (100%) rename parsers/{make.h => x-make.h} (100%) rename parsers/{markdown.h => x-markdown.h} (100%) rename parsers/{perl.h => x-perl.h} (100%) rename parsers/{r.h => x-r.h} (100%) rename parsers/{ruby.h => x-ruby.h} (100%) rename parsers/{sh.h => x-sh.h} (100%) rename parsers/{tcl.h => x-tcl.h} (100%) rename parsers/{tex.h => x-tex.h} (100%) rename parsers/{xml.h => x-xml.h} (100%) rename parsers/{yaml.h => x-yaml.h} (100%) diff --git a/parsers/#vim.c# b/parsers/#vim.c# new file mode 100644 index 0000000000..2a7a72cfbb --- /dev/null +++ b/parsers/#vim.c# @@ -0,0 +1,1090 @@ +/* +* Copyright (c) 2000-2003, Darren Hiebert +* +* This source code is released for free distribution under the terms of the +* GNU General Public License version 2 or (at your option) any later version. +* +* Thanks are due to Jay Glanville for significant improvements. +* +* This module contains functions for generating tags for user-defined +* functions for the Vim editor. +* +* references: +* - https://vim-jp.org/vimdoc-en/ +* - https://vim-jp.org/vimdoc-ja/ +* +*/ + +/* + * INCLUDE FILES + */ +#include "general.h" /* must always come first */ + +#include +#ifdef DEBUG +#include +#endif + +#include "debug.h" +#include "entry.h" +#include "parse.h" +#include "read.h" +#include "routines.h" +#include "vstring.h" + +#if 0 +typedef struct sLineInfo { + tokenType type; + keywordId keyword; + vString *string; + vString *scope; + unsigned long lineNumber; + MIOPos filePosition; +} lineInfo; +#endif + +/* + * DATA DEFINITIONS + */ +typedef enum { + K_AUGROUP, + K_COMMAND, + K_FUNCTION, + K_MAP, + K_VARIABLE, + K_FILENAME, + K_CONST, + K_HEREDOC, + K_CLASS, +} vimKind; + +typedef enum { + R_HEREDOC_ENDLABEL, +} vimHeredocRole; + +static roleDefinition VimHeredocRoles [] = { + { true, "endmarker", "end marker" }, +}; + +static kindDefinition VimKinds [] = { + { true, 'a', "augroup", "autocommand groups" }, + { true, 'c', "command", "user-defined commands" }, + { true, 'f', "function", "function definitions" }, + { true, 'm', "map", "maps" }, + { true, 'v', "variable", "variable definitions" }, + { true, 'n', "filename", "vimball filename" }, + { true, 'C', "constant", "constant definitions" }, + { false, 'h', "heredoc", "marker for here document", + .referenceOnly = false, ATTACH_ROLES (VimHeredocRoles) }, + { true, 'k', "class", "vim9script classes" }, +}; + +/* + * DATA DECLARATIONS + */ + +/* + * DATA DEFINITIONS + */ +static bool vim9script; + +/* + * FUNCTION DEFINITIONS + */ + +static bool parseVimLine (const unsigned char *line, int parent); + +/* This function takes a char pointer, tries to find a scope separator in the + * string, and if it does, returns a pointer to the character after the colon, + * and the character defining the scope. + * If a colon is not found, it returns the original pointer. + */ +static const unsigned char *skipPrefix (const unsigned char *name, int *scope) +{ + const unsigned char *result = name; + int counter; + size_t length; + length = strlen ((const char *) name); + if (scope != NULL) + *scope = '\0'; + if (length > 3 && name[1] == ':') + { + if (scope != NULL) + *scope = *name; + result = name + 2; + } + else if (length > 5 && strncasecmp ((const char *) name, "", (size_t) 5) == 0) + { + if (scope != NULL) + *scope = *name; + result = name + 5; + } + else + { + /* + * Vim7 check for dictionaries or autoload function names + */ + counter = 0; + do + { + switch (name[counter]) + { + case '.': + /* Set the scope to d - Dictionary */ + *scope = 'd'; + break; + case '#': + /* Set the scope to a - autoload */ + *scope = 'a'; + break; + } + ++counter; + } while (isalnum (name[counter]) || + name[counter] == '_' || + name[counter] == '.' || + name[counter] == '#' + ); + } + return result; +} + +static bool isWordChar (const unsigned char c) +{ + return (isalnum (c) || c == '_'); +} + +/* checks if a word at the start of `p` matches at least `min_len` first + * characters from `word` */ +static bool wordMatchLen (const unsigned char *p, const char *const word, size_t min_len) +{ + const unsigned char *w = (const unsigned char *) word; + size_t n = 0; + + while (*p && *p == *w) + { + p++; + w++; + n++; + } + + if (isWordChar (*p)) + return false; + + return n >= min_len; +} + +static const unsigned char *skipWord (const unsigned char *p) +{ + while (*p && isWordChar (*p)) + p++; + return p; +} + +static bool isMap (const unsigned char *line) +{ + /* + * There are many different short cuts for specifying a map. + * This routine should capture all the permutations. + */ + return (wordMatchLen (line, "map", 3) || + wordMatchLen (line, "nmap", 2) || + wordMatchLen (line, "vmap", 2) || + wordMatchLen (line, "xmap", 2) || + wordMatchLen (line, "smap", 4) || + wordMatchLen (line, "omap", 2) || + wordMatchLen (line, "imap", 2) || + wordMatchLen (line, "lmap", 2) || + wordMatchLen (line, "cmap", 2) || + wordMatchLen (line, "noremap", 2) || + wordMatchLen (line, "nnoremap", 2) || + wordMatchLen (line, "vnoremap", 2) || + wordMatchLen (line, "xnoremap", 2) || + wordMatchLen (line, "snoremap", 4) || + wordMatchLen (line, "onoremap", 3) || + wordMatchLen (line, "inoremap", 3) || + wordMatchLen (line, "lnoremap", 2) || + wordMatchLen (line, "cnoremap", 3)); +} + +static const unsigned char *readVimLineRaw (void) +{ + return readLineFromInputFile (); +} + +static const unsigned char *readVimLine (void) +{ + const unsigned char *line; + + while ((line = readLineFromInputFile ()) != NULL) + { + while (isspace (*line)) + ++line; + + if ((int) *line == (vim9script? '#': '"')) + continue; /* skip comment */ + + break; + } + + return line; +} + +static const unsigned char *readVimballLine (void) +{ + const unsigned char *line; + + while ((line = readLineFromInputFile ()) != NULL) + { + break; + } + + return line; +} + +static const unsigned char *parseRettype (const unsigned char *cp, tagEntryInfo *e) +{ + while (*cp && isspace (*cp)) + ++cp; + + if (!*cp) + return cp; + + vString *buf = vStringNew (); + while (*cp && *cp != '#' && *cp != '=') + { + if (isspace (*cp) + && !vStringIsEmpty(buf) + && isspace (vStringLast(buf))) + { + ++cp; + continue; + } + vStringPut (buf, *cp); + ++cp; + } + + if (vStringIsEmpty(buf)) + return cp; + + vStringStripTrailing (buf); + e->extensionFields.typeRef[0] = eStrdup ("typename"); + e->extensionFields.typeRef[1] = vStringDeleteUnwrap (buf); + + return cp; +} + +static vString *parseSignatureAndRettype (const unsigned char *cp, + tagEntryInfo *e, + vString *buf, + bool extractRettype) +{ + /* TODO capture parameters */ + + Assert (e); + Assert (cp); + + if (!buf) + { + buf = vStringNew (); + vStringPut (buf, *cp); + ++cp; + } + + while (*cp != '\0') + { + if (isspace (*cp) + && vStringLast (buf) == ',') + { + ++cp; + continue; + } + vStringPut (buf, *cp); + if (*cp == ')') + break; + ++cp; + } + + if (*cp == ')') + { + e->extensionFields.signature = vStringDeleteUnwrap (buf); + buf = NULL; + + if (extractRettype) + { + ++cp; + while (*cp && isspace (*cp)) + ++cp; + if (*cp == ':') + parseRettype (++cp, e); + } + } + + return buf; +} + +static void parseFunction (const unsigned char *line, int parent, bool definedWithDEF, + bool isPublic) +{ + vString *name = vStringNew (); + vString *signature = NULL; + /* bool inFunction = false; */ + int scope; + const unsigned char *cp = line; + int index = CORK_NIL; + tagEntryInfo *e = NULL; + + if (*cp == '!') + ++cp; + if (isspace (*cp)) + { + while (*cp && isspace (*cp)) + ++cp; + + if (*cp) + { + cp = skipPrefix (cp, &scope); + if ((definedWithDEF && (*cp == '_' || isalpha (*cp))) || + isupper (*cp) || + scope == 's' || /* script scope */ + scope == '<' || /* script scope */ + scope == 'g' || /* global scope */ + scope == 'd' || /* dictionary */ + scope == 'a') /* autoload */ + { + char prefix[3] = { [0] = (char)scope, [1] = ':', [2] = '\0' }; + if (scope == 's') + vStringCatS (name, prefix); + + do + { + vStringPut (name, *cp); + ++cp; + } while (isalnum (*cp) || *cp == '_' || *cp == '.' || *cp == '#'); + index = makeSimpleTag (name, K_FUNCTION); + vStringClear (name); + + e = getEntryInCorkQueue (index); + if (e) + { + e->extensionFields.scopeIndex = parent; + if (isFieldEnabled (FIELD_SIGNATURE)) + { + while (*cp && isspace (*cp)) + ++cp; + if (*cp == '(') + signature = parseSignatureAndRettype (cp, e, NULL, + definedWithDEF); + } + if (isPublic) + e->extensionFields.access = eStrdup("public"); + } + } + } + } + + /* TODO - update struct to indicate inside function */ + while ((line = readVimLine ()) != NULL) + { + if (signature) + { + cp = line; + while (*cp && isspace (*cp)) + ++cp; + /* A backslash at the start of a line stands for a line continuation. + * https://vimhelp.org/repeat.txt.html#line-continuation */ + if (*cp == '\\') + signature = parseSignatureAndRettype (++cp, e, signature, + definedWithDEF); + } + + if (wordMatchLen (line, "endfunction", 4) || wordMatchLen (line, "enddef", 6)) + { + if (e) + setTagEndLine (e, getInputLineNumber ()); + break; + } + + parseVimLine (line, index); + } + if (signature) + vStringDelete (signature); + vStringDelete (name); +} + +static void parseClass (const unsigned char *line, int parent, + bool isPublic, bool isAbstract) +{ + vString *name = vStringNew (); + vString *super = NULL; + const unsigned char *cp = line; + int index = CORK_NIL; + + if (isspace (*cp)) + { + while (*cp && isspace (*cp)) + ++cp; + + while (isalnum (*cp) || *cp == '_') + { + vStringPut (name, *cp); + ++cp; + } + + while (*cp && isspace (*cp)) + ++cp; + + if (wordMatchLen (cp, "extends", 7)) + { + cp += 7; + while (*cp && isspace (*cp)) + ++cp; + super = vStringNew (); + while (*cp && (isalnum (*cp) || *cp == '_' + /* ??? */ + || *cp == '.')) + { + vStringPut (super, *cp); + cp++; + } + } + + if (!vStringIsEmpty (name)) + { + tagEntryInfo *e; + + index = makeSimpleTag (name, K_CLASS); + e = getEntryInCorkQueue (index); + if (e) + { + e->extensionFields.scopeIndex = parent; + if (isPublic) + e->extensionFields.access = eStrdup ("public"); + if (isAbstract) + e->extensionFields.implementation = eStrdup ("abstract"); + if (super && !vStringIsEmpty (super)) + { + e->extensionFields.inheritance = vStringDeleteUnwrap(super); + super = NULL; + } + } + while ((line = readVimLine ()) != NULL) + { + if (wordMatchLen (line, "endclass", 8)) + { + if (e) + setTagEndLine (e, getInputLineNumber ()); + break; + } + parseVimLine (line, index); + } + } + } + + vStringDelete (super); /* NULL ia acceptable. */ + vStringDelete (name); +} + +static void parseAutogroup (const unsigned char *line) +{ + vString *name = vStringNew (); + + /* Found Autocommand Group (augroup) */ + const unsigned char *cp = line; + if (isspace (*cp)) + { + while (*cp && isspace (*cp)) + ++cp; + + if (*cp) + { + const unsigned char *end = skipWord (cp); + + /* "end" (caseless) has a special meaning and should not generate a tag */ + if (end > cp && strncasecmp ((const char *) cp, "end", end - cp) != 0) + { + vStringNCatS (name, (const char *) cp, end - cp); + makeSimpleTag (name, K_AUGROUP); + vStringClear (name); + } + } + } + vStringDelete (name); +} + +static bool parseCommand (const unsigned char *line) +{ + vString *name = vStringNew (); + bool cmdProcessed = true; + + /* + * Found a user-defined command + * + * They can have many options preceded by a dash + * command! -nargs=+ -complete Select :call s:DB_execSql("select " . ) + * The name of the command should be the first word not preceded by a dash + * + */ + const unsigned char *cp = line; + + if (cp && (*cp == '\\')) + { + /* + * We are recursively calling this function is the command + * has been continued on to the next line + * + * Vim statements can be continued onto a newline using a \ + * to indicate the previous line is continuing. + * + * com -nargs=1 -bang -complete=customlist,EditFileComplete + * \ EditFile edit + * + * If the following lines do not have a line continuation + * the command must not be spanning multiple lines and should + * be syntactically incorrect. + */ + if (*cp == '\\') + ++cp; + + while (*cp && isspace (*cp)) + ++cp; + } + else if (line && wordMatchLen (cp, "command", 3)) + { + cp = skipWord (cp); + + if (*cp == '!') + ++cp; + + if (*cp != ' ') + { + /* + * :command must be followed by a space. If it is not, it is + * not a valid command. + * Treat the line as processed and continue. + */ + cmdProcessed = true; + goto cleanUp; + } + + while (*cp && isspace (*cp)) + ++cp; + } + else + { + /* + * We are recursively calling this function. If it does not start + * with "com" or a line continuation character, we have moved off + * the command line and should let the other routines parse this file. + */ + cmdProcessed = false; + goto cleanUp; + } + + /* + * Strip off any spaces and options which are part of the command. + * These should precede the command name. + */ + do + { + if (isspace (*cp)) + { + ++cp; + } + else if (*cp == '-') + { + /* + * Read until the next space which separates options or the name + */ + while (*cp && !isspace (*cp)) + ++cp; + } + else if (!isalnum (*cp)) + { + /* + * Broken syntax: throw away this line + */ + cmdProcessed = true; + goto cleanUp; + } + } while (*cp && !isalnum (*cp)); + + if (!*cp) + { + /* + * We have reached the end of the line without finding the command name. + * Read the next line and continue processing it as a command. + */ + if ((line = readVimLine ()) != NULL) + cmdProcessed = parseCommand (line); + else + cmdProcessed = false; + goto cleanUp; + } + + do + { + vStringPut (name, *cp); + ++cp; + } while (isalnum (*cp) || *cp == '_'); + + makeSimpleTag (name, K_COMMAND); + vStringClear (name); + +cleanUp: + vStringDelete (name); + + return cmdProcessed; +} + +/* =<< trim {endmarker} */ +static int parseHeredocMarker(const unsigned char *cp) +{ + while (*cp && isspace (*cp)) + ++cp; + + if (! (cp[0] == '=' && cp[1] == '<' && cp[2] == '<')) + return CORK_NIL; + + cp += 3; + + while (*cp && isspace (*cp)) + ++cp; + + if (wordMatchLen (cp, "trim", 4)) + { + cp += 4; + while (*cp && isspace (*cp)) + ++cp; + } + + if (!('A' <= *cp && *cp <= 'Z')) + return CORK_NIL; + + vString *heredoc = vStringNew (); + do + vStringPut (heredoc, *cp++); + while (*cp != '\0' && !isspace (*cp)); + + if (vStringIsEmpty (heredoc)) + { + vStringDelete (heredoc); + return CORK_NIL; + } + + int r = makeSimpleTag (heredoc, K_HEREDOC); + vStringDelete (heredoc); + return r; +} + +static bool isFunction(int index) +{ + tagEntryInfo *e = getEntryInCorkQueue(index); + if (!e) + return false; + return (e->kindIndex == K_FUNCTION); +} + +/* + * If we have a heredoc end marker at the end of LINE, + * parseVariableOrConstant returns the tag cork index for the marker. + */ +static int parseVariableOrConstant (const unsigned char *line, int parent, int kindIndex, + bool isPublic) +{ + vString *name = vStringNew (); + int heredoc = CORK_NIL; + + const unsigned char *cp = line; + const unsigned char *np = line; + /* get the name */ + if (isspace (*cp)) + { + while (*cp && isspace (*cp)) + ++cp; + + /* + * Ignore lets which set: + * & - local buffer vim settings + * @ - registers + * [ - Lists or Dictionaries + */ + if (!*cp || *cp == '&' || *cp == '@' || *cp == '[') + goto cleanUp; + + /* + * Ignore vim variables which are read only + * v: - Vim variables. + */ + np = cp; + ++np; + if (*cp == 'v' && *np == ':') + goto cleanUp; + + /* Skip non-global vars in functions */ + bool inFunction = isFunction(parent); + if (parent != CORK_NIL && inFunction && (*np != ':' || *cp != 'g')) + goto cleanUp; + + /* deal with spaces, $, @ and & */ + while (*cp && *cp != '$' && !isalnum (*cp) + && !(vim9script && *cp == '_')) + ++cp; + + if (!*cp) + goto cleanUp; + + /* cp = skipPrefix (cp, &scope); */ + do + { + if (!*cp) + break; + + vStringPut (name, *cp); + ++cp; + } while (isalnum (*cp) || *cp == '_' || *cp == '#' || *cp == ':' || *cp == '$'); + + bool hasType = false; + if (*cp && vim9script && !vStringIsEmpty (name) && (vStringLast (name) == ':')) + { + Assert(line != cp); + Assert(*(cp - 1) == ':'); + vStringChop (name); + hasType = true; + } + + if (!vStringIsEmpty (name)) + { + int r = makeSimpleTag (name, kindIndex); + vStringClear (name); + + tagEntryInfo *e = getEntryInCorkQueue (r); + if (e) + { + if (hasType) + cp = parseRettype (cp, e); + if (!inFunction) + e->extensionFields.scopeIndex = parent; + if (isPublic) + e->extensionFields.access = eStrdup("public"); + } + + heredoc = parseHeredocMarker(cp); + } + } + +cleanUp: + vStringDelete (name); + + return heredoc; +} + +static bool parseMap (const unsigned char *line) +{ + vString *name = vStringNew (); + const unsigned char *cp = line; + + if (*cp == '!') + ++cp; + + /* + * Maps follow this basic format + * map + * nnoremap :Tlist + * map scdt GetColumnDataType + * inoremap ,,, diwi<pa>pa>kA + * inoremap ( =PreviewFunctionSignature() + * + * The Vim help shows the various special arguments available to a map: + * 1.2 SPECIAL ARGUMENTS *:map-arguments* + * + * + * + *