Merge pull request #4178 from masatake/main--datatype-in-fielddef

main:  provide the way to specify data taype in --_fielddef option

Tmain/flags-fielddef-datatype.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/flags-fielddef-datatype.d/run.sh

@@ -0,0 +1,22 @@
+# Copyright: 2023 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+		 --_fielddef-FIELDTEST=strfield,"a field having string value"'{datatype=str}' \
+		 --_fielddef-FIELDTEST=boolfield,"a field having boolean value"'{datatype=bool}' \
+		 --_fielddef-FIELDTEST=intfield,"a field having integer value"'{datatype=int}' \
+		 --_fielddef-FIELDTEST=strboolfield,"a field having string value or false"'{datatype=str+bool}' \
+		 --_fielddef-FIELDTEST=deffield,"a field that type is not specified" \
+		 --list-fields=FIELDTEST && \
+! ${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+  --_fielddef-FIELDTEST=fooA,"unexpected data type"'{datatype=bar}' &&
+! ${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+  --_fielddef-FIELDTEST=fooB,"unexpected data type"'{datatype=int+baz}' &&
+! ${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+  --_fielddef-FIELDTEST=fooC,"unexpected data type"'{datatype=}' &&
+! ${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+  --_fielddef-FIELDTEST=fooD,"unexpected data type"'{datatype=+baz}' &&
+! ${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+  --_fielddef-FIELDTEST=fooE,"unexpected data type"'{datatype=bool+str}'

Tmain/flags-fielddef-datatype.d/stderr-expected.txt

@@ -0,0 +1,5 @@
+ctags: unknown datatype for field "fooA": "bar"
+ctags: unknown datatype for field "fooB": "int+baz"
+ctags: no datatype given for field: "fooC"
+ctags: unknown datatype for field "fooD": "+baz"
+ctags: unknown datatype for field "fooE": "bool+str"

Tmain/flags-fielddef-datatype.d/stdout-expected.txt

@@ -0,0 +1,6 @@
+#LETTER NAME         ENABLED LANGUAGE  JSTYPE FIXED OP DESCRIPTION
+-       boolfield    no      FIELDTEST --b    no    -- a field having boolean value
+-       deffield     no      FIELDTEST s--    no    -- a field that type is not specified
+-       intfield     no      FIELDTEST -i-    no    -- a field having integer value
+-       strboolfield no      FIELDTEST s-b    no    -- a field having string value or false
+-       strfield     no      FIELDTEST s--    no    -- a field having string value

Tmain/list-fielddef-flags.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/list-fielddef-flags.d/run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Copyright: 2023 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+${CTAGS} --quiet --options=NONE --_list-fielddef-flags

Tmain/list-fielddef-flags.d/stdout-expected.txt

@@ -0,0 +1,2 @@
+#LETTER NAME          DESCRIPTION
+-       datatype=TYPE acceaptable datatype of the field ([str]|bool|int|str+bool)

Tmain/parser-specific-fields-with-datatype.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/parser-specific-fields-with-datatype.d/input.testlang

@@ -0,0 +1,17 @@
+str:s0
+str:s1
+str:s2
+
+int:i0
+int:i1
+int:i2
+int:i3
+int:i4
+
+bool:b0
+bool:b1
+bool:b2
+
+strbool:sb0
+strbool:sb1
+strbool:sb2

Tmain/parser-specific-fields-with-datatype.d/run.sh

@@ -0,0 +1,8 @@
+# Copyright: 2025 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+${CTAGS} --quiet --options=NONE --options=testlang.ctags -o - \
+		 --output-format=u-ctags \
+		 input.testlang

Tmain/parser-specific-fields-with-datatype.d/stdout-expected.txt

@@ -0,0 +1,14 @@
+s0	input.testlang	/^str:s0$/;"	o	str:string
+s1	input.testlang	/^str:s1$/;"	o	str:
+s2	input.testlang	/^str:s2$/;"	o
+i0	input.testlang	/^int:i0$/;"	o	int:10
+i1	input.testlang	/^int:i1$/;"	o	int:-1
+i2	input.testlang	/^int:i2$/;"	o	int:1
+i3	input.testlang	/^int:i3$/;"	o	int:0
+i4	input.testlang	/^int:i4$/;"	o
+b0	input.testlang	/^bool:b0$/;"	o	bool:
+b1	input.testlang	/^bool:b1$/;"	o	bool:
+b2	input.testlang	/^bool:b2$/;"	o
+sb0	input.testlang	/^strbool:sb0$/;"	o	strbool:abc
+sb1	input.testlang	/^strbool:sb1$/;"	o	strbool:
+sb2	input.testlang	/^strbool:sb2$/;"	o

Tmain/parser-specific-fields-with-datatype.d/testlang.ctags

@@ -0,0 +1,33 @@
+--sort=no
+
+--langdef=Testlang
+--map-Testlang=+.testlang
+
+--kinddef-Testlang=o,object,objects
+
+--_fielddef-Testlang=str,String{datatype=str}
+--fields-Testlang=+{str}
+--_fielddef-Testlang=int,Integer{datatype=int}
+--fields-Testlang=+{int}
+--_fielddef-Testlang=bool,Boolean{datatype=bool}
+--fields-Testlang=+{bool}
+--_fielddef-Testlang=strbool,String or Boolean{datatype=str+bool}
+--fields-Testlang=+{strbool}
+
+--regex-Testlang=/^str:(s0)/\1/o/{_field=str:string}
+--regex-Testlang=/^str:(s1)/\1/o/{_field=str:}
+--regex-Testlang=/^str:(s2)/\1/o/
+
+--regex-Testlang=/^int:(i0)/\1/o/{_field=int:10}
+--regex-Testlang=/^int:(i1)/\1/o/{_field=int:-1}
+--regex-Testlang=/^int:(i2)/\1/o/{_field=int:abc}
+--regex-Testlang=/^int:(i3)/\1/o/{_field=int:}
+--regex-Testlang=/^int:(i4)/\1/o/
+
+--regex-Testlang=/^bool:(b0)/\1/o/{_field=bool:abc}
+--regex-Testlang=/^bool:(b1)/\1/o/{_field=bool:}
+--regex-Testlang=/^bool:(b2)/\1/o/
+
+--regex-Testlang=/^strbool:(sb0)/\1/o/{_field=strbool:abc}
+--regex-Testlang=/^strbool:(sb1)/\1/o/{_field=strbool:}
+--regex-Testlang=/^strbool:(sb2)/\1/o/

docs/optlib.rst

@@ -1970,7 +1970,7 @@ the output for input.docc:
 	decompress	input.docc	/^- function: decompress(const char *compressed_byteseq) => char *$/;"	function	language:C	roles:documented
 
 
-.. TESTCASE: Tmain/parser-own-fields-for-foreign-lang.d
+.. TESTCASE: Tmain/parser-specific-fields-for-foreign-lang.d
 
 ``{_language=<LANG>}`` flag affects ``{_field=FIELDNAME:GROUP}`` flag; ctags looks up
 the field defintion in `<LANG>`.

main/entry.h

@@ -141,7 +141,7 @@ struct sTagEntryInfo {
 		short nth;
 	} extensionFields;  /* list of extension fields*/
 
-	/* `usedParserFields' tracks how many parser own fields are
+	/* `usedParserFields' tracks how many parser specific fields are
 	   used. If it is a few (less than PRE_ALLOCATED_PARSER_FIELDS),
 	   statically allocated parserFields is used. If more fields than
 	   PRE_ALLOCATED_PARSER_FIELDS is defined and attached, parserFieldsDynamic

main/options.c

@@ -464,6 +464,8 @@ static optionDescription LongOptionDescription [] = {
  {1,0,"       --list-{aliases,extras,features,fields,kind-full,langdef-flags,params," },
  {1,0,"       pseudo-tags,regex-flags,roles,subparsers} support this option."},
  {1,0,"       Specify before --list-* option."},
+ {1,1,"  --_list-fielddef-flags"},
+ {1,1,"       Output list of flags which can be used with --fielddef option."},
  {1,1,"  --_list-kinddef-flags"},
  {1,1,"       Output list of flags which can be used with --kinddef option."},
  {1,1,"  --_list-langdef-flags"},
@@ -2234,6 +2236,14 @@ static void processListKinddefFlagsOptions (
 	exit (0);
 }
 
+static void processListFielddefFlagsOptions (
+		const char *const option CTAGS_ATTR_UNUSED,
+		const char *const parameter CTAGS_ATTR_UNUSED)
+{
+	printFielddefFlags (localOption.withListHeader, localOption.machinable, stdout);
+	exit (0);
+}
+
 attr__noreturn
 static void processListRolesOptions (const char *const option CTAGS_ATTR_UNUSED,
 				     const char *const parameter)
@@ -2935,6 +2945,7 @@ static parametricOption ParametricOptions [] = {
 #ifdef HAVE_JANSSON
 	{ "_interactive",           processInteractiveOption,       true,   STAGE_ANY },
 #endif
+	{ "_list-fielddef-flags",   processListFielddefFlagsOptions,true,   STAGE_ANY },
 	{ "_list-kinddef-flags",    processListKinddefFlagsOptions, true,   STAGE_ANY },
 	{ "_list-langdef-flags",    processListLangdefFlagsOptions, true,   STAGE_ANY },
 	{ "_list-mtable-regex-flags", processListMultitableRegexFlagsOptions, true, STAGE_ANY },

main/parse.c

@@ -4001,6 +4001,33 @@ static void fieldDefinitionDestroy (fieldDefinition *fdef)
 	eFree (fdef);
 }
 
+static void field_def_flag_datatype_long (const char *const optflag CTAGS_ATTR_UNUSED,
+										  const char* const param,
+										  void *data)
+{
+	fieldDefinition *fdef = data;
+
+	if (*param == '\0')
+		error (FATAL, "no datatype given for field: \"%s\"", fdef->name);
+
+	fdef->dataType = 0;
+	if (strcmp (param, "int") == 0)
+		fdef->dataType |= FIELDTYPE_INTEGER;
+	else if (strcmp (param, "str") == 0)
+		fdef->dataType |= FIELDTYPE_STRING;
+	else if (strcmp (param, "bool") == 0)
+		fdef->dataType |= FIELDTYPE_BOOL;
+	else if (strcmp (param, "str+bool") == 0)
+		fdef->dataType |= FIELDTYPE_STRING|FIELDTYPE_BOOL;
+	else
+		error (FATAL, "unknown datatype for field \"%s\": \"%s\"", fdef->name, param);
+}
+
+static flagDefinition FieldDefFlagDef [] = {
+	{ '\0', "datatype", NULL, field_def_flag_datatype_long,
+	  "TYPE", "acceaptable datatype of the field ([str]|bool|int|str+bool)" },
+};
+
 static bool processLangDefineField (const langType language,
 									const char *const option,
 									const char *const parameter)
@@ -4047,13 +4074,14 @@ static bool processLangDefineField (const langType language,
 	fdef->setValueObject = NULL;
 	fdef->setterValueType = NULL;
 	fdef->checkValueForSetter = NULL;
-	fdef->dataType = FIELDTYPE_STRING; /* TODO */
+	fdef->dataType = 0;
+	if (flags)
+		flagsEval (flags, FieldDefFlagDef, ARRAY_SIZE (FieldDefFlagDef), fdef);
+	if (!fdef->dataType)
+		fdef->dataType = FIELDTYPE_STRING;
 	fdef->ftype = FIELD_UNKNOWN;
 	DEFAULT_TRASH_BOX(fdef, fieldDefinitionDestroy);
 
-	if (flags)
-		flagsEval (flags, NULL, 0, fdef);
-
 	defineField (fdef, language);
 
 	return true;
@@ -5362,6 +5390,18 @@ extern void printKinddefFlags (bool withListHeader, bool machinable, FILE *fp)
 	colprintTableDelete(table);
 }
 
+extern void printFielddefFlags (bool withListHeader, bool machinable, FILE *fp)
+{
+	struct colprintTable * table;
+
+	table = flagsColprintTableNew ();
+
+	flagsColprintAddDefinitions (table, FieldDefFlagDef, ARRAY_SIZE (FieldDefFlagDef));
+
+	flagsColprintTablePrint (table, withListHeader, machinable, fp);
+	colprintTableDelete(table);
+}
+
 extern void printLanguageMultitableStatistics (langType language)
 {
 	parserObject* const parser = LanguageTable + language;

main/parse_p.h

@@ -130,6 +130,7 @@ extern void printLanguageSubparsers (const langType language,
 									 bool withListHeader, bool machinable, FILE *fp);
 extern void printLangdefFlags (bool withListHeader, bool machinable, FILE *fp);
 extern void printKinddefFlags (bool withListHeader, bool machinable, FILE *fp);
+extern void printFielddefFlags (bool withListHeader, bool machinable, FILE *fp);
 extern bool doesParserRequireMemoryStream (const langType language);
 extern bool parseFile (const char *const fileName);
 extern bool parseFileWithMio (const char *const fileName, MIO *mio, void *clientData);

misc/optlib2c

@@ -58,6 +58,36 @@ my $langdef_flags =
        } ],
     ];
 
+my $fielddef_flags =
+    [
+     [ qr/\{datatype=([^\}]+)\}/, sub {
+	 my $datatype = "";
+
+	 if ($1 eq 'str')
+	 {
+	     $datatype .= "FIELDTYPE_STRING";
+	 }
+	 elsif ($1 eq 'int')
+	 {
+	     $datatype .= "FIELDTYPE_INTEGER";
+	 }
+	 elsif ($1 eq 'bool')
+	 {
+	     $datatype .= "FIELDTYPE_BOOL";
+	 }
+	 elsif (($1 eq 'bool+str') || ($1 eq 'str+bool'))
+	 {
+	     $datatype .= "FIELDTYPE_STRING|FIELDTYPE_BOOL";
+	 }
+	 else
+	 {
+	     die "Unknown datatype specification: \"{datatype=$1}\" in \"--_fielddef-<LANG>=...\"";
+	 }
+
+	 $_[0]->{'datatype'} = $datatype;
+       } ],
+    ];
+
 my $options =
     [
      [ qr/^--options=(.*)/, sub {
@@ -88,7 +118,7 @@ my $options =
 	   $_[0]->{'versionCurrent'} = 0;
 	   $_[0]->{'versionAge'} = 0;
 	   $_[0]->{'foreignLanguages'} = [];
-	   parse_flags ($rest, $_[0], $langdef_flags,);
+	   parse_flags ($rest, $_[0], $langdef_flags);
 
 	   return 1;
        } ],
@@ -135,18 +165,22 @@ my $options =
 	 push @{$_[0]->{'extradefs'}}, { name => $name, desc => $desc };
 	 return 1;
        } ],
-     [ qr/^--_fielddef-(.*)=([^,]+),([^\{]+)/, sub {
+     [ qr/^--_fielddef-(.*)=([^,]+),([^\{]+)(.*)/, sub {
 	 die "Don't use --_fielddef-<LANG>=+ option before defining the language"
 	     if (! defined $_[0]->{'langdef'});
 	 die "Adding a field is allowed only to the language specified with --langdef: $1"
 	     unless ($_[0]->{'langdef'} eq $1);
 
 	 my $name = $2;
 	 my $desc = $3;
+	 my $rest = $4;
 	 die "unacceptable character is used for field name: $name"
 	     unless ($name =~ /^[a-zA-Z]+$/);
 
-	 push @{$_[0]->{'fielddefs'}}, { name => $name, desc => $desc };
+	 my $fdef = { name => $name, desc => $desc };
+	 push @{$_[0]->{'fielddefs'}}, $fdef;
+	 parse_flags ($rest, $fdef, $fielddef_flags);
+
 	 return 1;
        } ],
      [ qr/^--_roledef-([^.]*)\.(?:([a-zA-Z])|\{([a-zA-Z][a-zA-Z0-9]*)\})=([a-zA-Z0-9]+),([^\{]+)/, sub {
@@ -1062,6 +1096,13 @@ EOF
 		  .enabled     = $enabled,
 		  .name        = "$_->{'name'}",
 		  .description = "$desc",
+EOF
+      if (defined $_->{'datatype'}) {
+	  print <<EOF;
+		  .dataType = $_->{'datatype'},
+EOF
+      }
+      print <<EOF;
 		},
 EOF
     }

optlib/inko.c

@@ -178,6 +178,7 @@ extern parserDefinition* InkoParser (void)
 		  .enabled     = true,
 		  .name        = "implements",
 		  .description = "Trait being implemented",
+		  .dataType = FIELDTYPE_STRING,
 		},
 	};
 

optlib/inko.ctags

@@ -29,7 +29,7 @@
 --kinddef-Inko=c,constant,Constant definition
 --kinddef-Inko=r,reopen,Reopen class
 
---_fielddef-Inko=implements,Trait being implemented
+--_fielddef-Inko=implements,Trait being implemented{datatype=str}
 
 --fields-Inko=+{implements}
 

optlib/passwd.c

@@ -37,11 +37,13 @@ extern parserDefinition* PasswdParser (void)
 		  .enabled     = true,
 		  .name        = "home",
 		  .description = "home directory",
+		  .dataType = FIELDTYPE_STRING,
 		},
 		{
 		  .enabled     = true,
 		  .name        = "shell",
 		  .description = "login shell",
+		  .dataType = FIELDTYPE_STRING,
 		},
 	};
 	static tagRegexTable PasswdTagRegexTable [] = {

optlib/passwd.ctags

@@ -26,8 +26,8 @@
 --langdef=Passwd
 --map-Passwd=+(passwd)
 --kinddef-Passwd=u,username,user names
---_fielddef-Passwd=home,home directory
---_fielddef-Passwd=shell,login shell
+--_fielddef-Passwd=home,home directory{datatype=str}
+--_fielddef-Passwd=shell,login shell{datatype=str}
 --fields-Passwd=+{home}
 --fields-Passwd=+{shell}
 --regex-Passwd=/^([^:]+):([^:]+):([^:]+):([^:]+):([^:]*):([^:]+):([^:]+)/\1/u/{_field=home:\6}{_field=shell:\7}