Merge pull request #4059 from masatake/main--extras-and-fields-in-for…

…eign-language

Main:  use extras and fields in the foreign language specified in {_language=...} flag

Tmain/list-mline-regex-flags.d/stdout-expected.txt

@@ -9,8 +9,8 @@ p       pcre2                                         use pcre2 regex engine
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-mtable-regex-flags.d/stdout-expected.txt

@@ -15,8 +15,8 @@ p       pcre2                                         use pcre2 regex engine
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-regex-flags.d/stdout-expected.txt

@@ -10,8 +10,8 @@ x       exclusive                                     skip testing the other pat
 -       scope=ACTION                                  use scope stack: ACTION = ref|push|pop|clear|set|replace|intervaltab
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
--       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
+-       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/option-no-such-foreign-extra.d/run.sh

@@ -0,0 +1,17 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+${V} ${CTAGS} --quiet --options=NONE \
+	 \
+	 --langdef=NOSUCHLANG'{_foreignLanguage=Kconfig}' \
+	 --_extradef-NOSUCHLANG='NOSUCHEXTRA,but this is not the part of Kconfig' \
+     --regex-NOSUCHLANG='/^\# (CONFIG_[^ ]+) is not set/\1/c/{_language=Kconfig}{_extra=NOSUCHEXTRA}{exclusive}' \
+	 \
+	 --_force-quit=0

Tmain/option-no-such-foreign-extra.d/stderr-expected.txt

@@ -0,0 +1 @@
+ctags: Warning: no such extra "NOSUCHEXTRA" in Kconfig

Tmain/option-no-such-foreign-field.d/run.sh

@@ -0,0 +1,17 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+${V} ${CTAGS} --quiet --options=NONE \
+	 \
+	 --langdef=NOSUCHLANG'{_foreignLanguage=Kconfig}' \
+	 --_fielddef-NOSUCHLANG='NOSUCHFIELD,but this is not the part of Kconfig' \
+     --regex-NOSUCHLANG='/^\# (CONFIG_[^ ]+) is (not set)/\1/c/{_language=Kconfig}{_field=NOSUCHFIELD:\1}{exclusive}' \
+	 \
+	 --_force-quit=0

Tmain/option-no-such-foreign-field.d/stderr-expected.txt

@@ -0,0 +1 @@
+ctags: Warning: no such field "NOSUCHFIELD" in Kconfig

Tmain/parser-own-extras-for-foreign-lang.d/input-0.x1

@@ -0,0 +1,3 @@
+D:def00
+d:def01
+v:var0

Tmain/parser-own-extras-for-foreign-lang.d/input-1.x1

@@ -0,0 +1,3 @@
+D:def10
+d:def11
+v:var1

Tmain/parser-own-extras-for-foreign-lang.d/run.sh

@@ -0,0 +1,17 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+printf "# %s\n" --extras-X0=+'{iname}'
+${V} ${CTAGS} --quiet --options=NONE --options=./x0.ctags --options=./x1.ctags \
+	 --extras-X0=+'{iname}' --fields=+'{extras}{language}' -o - input-0.x1
+
+printf "# %s\n" --extras-X0=-'{iname}'
+${V} ${CTAGS} --quiet --options=NONE --options=./x0.ctags --options=./x1.ctags \
+	 --extras-X0=-'{iname}' --fields=+'{extras}{language}' -o - input-1.x1

Tmain/parser-own-extras-for-foreign-lang.d/stdout-expected.txt

@@ -0,0 +1,7 @@
+# --extras-X0=+{iname}
+__def01__	input-0.x1	/^d:def01$/;"	d	language:X0	extras:iname
+def00	input-0.x1	/^D:def00$/;"	d	language:X0
+var0	input-0.x1	/^v:var0$/;"	v	language:X1
+# --extras-X0=-{iname}
+def10	input-1.x1	/^D:def10$/;"	d	language:X0
+var1	input-1.x1	/^v:var1$/;"	v	language:X1

Tmain/parser-own-extras-for-foreign-lang.d/x0.ctags

@@ -0,0 +1,3 @@
+--langdef=X0
+--kinddef-X0=d,def,definitions
+--_extradef-X0=iname,internal name like __x__

Tmain/parser-own-extras-for-foreign-lang.d/x1.ctags

@@ -0,0 +1,6 @@
+--langdef=X1{_foreignLanguage=X0}
+--map-X1=+.x1
+--kinddef-X1=v,var,variables
+--regex-X1=/D:([a-z0-9]+)$/\1/d/{_language=X0}
+--regex-X1=/d:([a-z0-9]+)$/__\1__/d/{_language=X0}{_extra=iname}
+--regex-X1=/v:([a-z0-9]+)$/\1/v/

Tmain/parser-own-fields-for-foreign-lang.d/input.unknownx

@@ -0,0 +1,5 @@
+public func foo(n, m);
+protected func bar(n);
+private func baz(n,...);
+X:tagme@iamowner
+Y:iamowner2=tagme2

Tmain/parser-own-fields-for-foreign-lang.d/knownz.ctags

@@ -0,0 +1,4 @@
+--langdef=knownz
+--kinddef-knownz=m,mark,makers
+
+--_fielddef-knownz=owner,the owner of the markers

Tmain/parser-own-fields-for-foreign-lang.d/run.sh

@@ -0,0 +1,15 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+${V} ${CTAGS} --options=NONE --options=./knownz.ctags --options=./unknownx.ctags \
+	 --fields=+l \
+	 --fields-unknownx=+'{protection}{signature}' \
+	 --fields-knownz=+'{owner}' \
+	 -o - input.unknownx

Tmain/parser-own-fields-for-foreign-lang.d/stderr-expected.txt

@@ -0,0 +1 @@
+ctags: Notice: No options will be read from files or environment

Tmain/parser-own-fields-for-foreign-lang.d/stdout-expected.txt

@@ -0,0 +1,5 @@
+bar	input.unknownx	/^protected func bar(n);$/;"	f	language:unknownx	protection:protected 	signature:(n)
+baz	input.unknownx	/^private func baz(n,...);$/;"	f	language:unknownx	protection:private 	signature:(n,...)
+foo	input.unknownx	/^public func foo(n, m);$/;"	f	language:unknownx	protection:public 	signature:(n, m)
+tagme	input.unknownx	/^X:tagme@iamowner$/;"	m	language:knownz	owner:iamowner
+tagme2	input.unknownx	/^Y:iamowner2=tagme2$/;"	m	language:knownz	owner:iamowner2

Tmain/parser-own-fields-for-foreign-lang.d/unknownx.ctags

@@ -0,0 +1,10 @@
+--langdef=unknownx{_foreignLanguage=knownz}
+--kinddef-unknownx=f,func,functions
+--map-unknownx=+.unknownx
+
+--_fielddef-unknownx=protection,protections
+--_fielddef-unknownx=signature,signatures
+
+--regex-unknownx=/^((public|protected|private) +)?func ([^\(]+)\((.*)\)/\3/f/{_field=protection:\1}{_field=signature:(\4)}
+--regex-unknownx=/^X:([a-z]+)@([a-z]+)/\1/m/{_language=knownz}{_field=owner:\2}
+--regex-unknownx=/^Y:([a-z0-9]+)=([a-z0-9]+)/\2/m/{_field=owner:\1}{_language=knownz}

docs/optlib.rst

@@ -371,6 +371,10 @@ The pattern matching is done only when the ``main`` is enabled.
 	$ ctags --options=python-main.ctags -o - --extras-Python='+{main}' input.py
 	__main__	input.py	/^if __name__ == '__main__':$/;"	f
 
+By default, ctags assumes the extra is a part of the language specified
+with `<LANG>` in ``--regex-<LANG>``. Together with ``{_language=<LANG>}``
+flag, you can switch the language of the extra. See ":ref:`foreigntag`".
+The combination of these flags is new in version 6.2.0.
 
 .. TODO: this "fields" section should probably be moved up this document, as a
 	subsection in the "Regex option argument flags" section
@@ -463,6 +467,10 @@ about ``--fields-<LANG>`` option.
 
 `passwd` parser is a simple example that uses ``--fields-<LANG>`` option.
 
+By default, ctags assumes the field is a part of the language specified
+with `<LANG>` in ``--regex-<LANG>``. Together with ``{_language=<LANG>}``
+flag, you can switch the language of the field. See ":ref:`foreigntag`".
+The combination of these flags is new in version 6.2.0.
 
 .. _roles:
 
@@ -1961,6 +1969,15 @@ the output for input.docc:
 	compress	input.docc	/^- function: compress(const char *plain_text, enum algorithm alg) => char *$/;"	function	language:C	roles:documented
 	decompress	input.docc	/^- function: decompress(const char *compressed_byteseq) => char *$/;"	function	language:C	roles:documented
 
+
+.. TESTCASE: Tmain/parser-own-fields-for-foreign-lang.d
+
+``{_language=<LANG>}`` flag affects ``{_field=FIELDNAME:GROUP}`` flag; ctags looks up
+the field defintion in `<LANG>`.
+
+``{_language=<LANG>}`` flag affects ``{_extra=XNAME}`` flag; ctags looks up
+the extra defintion in `<LANG>`.
+
 .. END: NOT REVIEWED YET
 
 .. _optlib2c:

main/lregex.c

@@ -1084,9 +1084,12 @@ static void common_flag_extra_long (const char* const s, const char* const v, vo
 		return;
 	}
 
-	cdata->ptrn->xtagType = getXtagTypeForNameAndLanguage (v, cdata->owner);
+	langType lang = (cdata->ptrn->foreign_lang == LANG_IGNORE)
+		? cdata->owner
+		: cdata->ptrn->foreign_lang;
+	cdata->ptrn->xtagType = getXtagTypeForNameAndLanguage (v, lang);
 	if (cdata->ptrn->xtagType == XTAG_UNKNOWN)
-		error (WARNING, "no such extra \"%s\" in %s", v, getLanguageName(cdata->owner));
+		error (WARNING, "no such extra \"%s\" in %s", v, getLanguageName(lang));
 }
 
 
@@ -1134,10 +1137,14 @@ static void common_flag_field_long (const char* const s, const char* const v, vo
 	}
 
 	fname = eStrndup (v, tmp - v);
-	ftype = getFieldTypeForNameAndLanguage (fname, cdata->owner);
+
+	langType lang = (ptrn->foreign_lang == LANG_IGNORE)
+		? cdata->owner
+		: ptrn->foreign_lang;
+	ftype = getFieldTypeForNameAndLanguage (fname, lang);
 	if (ftype == FIELD_UNKNOWN)
 	{
-		error (WARNING, "no such field \"%s\" in %s", fname, getLanguageName(cdata->owner));
+		error (WARNING, "no such field \"%s\" in %s", fname, getLanguageName(lang));
 		eFree (fname);
 		return;
 	}
@@ -1267,9 +1274,9 @@ static flagDefinition commonSpecFlagDef[] = {
 	  "\"MESSAGE\"", "print the given MESSAGE at WARNING level"},
 #define EXPERIMENTAL "_"
 	{ '\0',  EXPERIMENTAL "extra", NULL, common_flag_extra_long ,
-	  "EXTRA", "record the tag only when the extra is enabled"},
+	  "EXTRA", "record the tag only when the (foreign) extra is enabled"},
 	{ '\0',  EXPERIMENTAL "field", NULL, common_flag_field_long ,
-	  "FIELD:VALUE", "record the matched string(VALUE) to parser own FIELD of the tag"},
+	  "FIELD:VALUE", "record the matched string(VALUE) to the (foreign) language specific FIELD of the tag"},
 	{ '\0',  EXPERIMENTAL "role", NULL, common_flag_role_long,
 	  "ROLE", "set the given ROLE to the roles field"},
 	{ '\0',  EXPERIMENTAL "anonymous", NULL, common_flag_anonymous_long,