Add unit test to test various special characters in tag values

idx/memory/tag_query_test.go

@@ -559,6 +559,108 @@ func TestExpressionParsing(t *testing.T) {
 	}
 }
 
+func TestSpecialCharactersInTags(t *testing.T) {
+	type testCase struct {
+		id         schema.MKey
+		lastUpdate int64
+		tags       []string
+	}
+
+	ids = []schema.MKey{
+		test.GetMKey(0),
+		test.GetMKey(1),
+		test.GetMKey(2),
+		test.GetMKey(3),
+		test.GetMKey(4),
+		test.GetMKey(5),
+		test.GetMKey(6),
+	}
+
+	data := []testCase{}
+
+	for i := 0; i < 6; i++ {
+		data = append(data, testCase{ids[i], 1, []string{}})
+	}
+
+	blackList := []rune{
+		';', // not allowed in values
+		'~', // can't query values starting with ~ because =~ looks like a regular expression query
+	}
+
+	isBlacklisted := func(char rune) bool {
+		for i := 0; i < len(blackList); i++ {
+			if char == blackList[i] {
+				return true
+			}
+		}
+		return false
+	}
+
+	for i := 0; i < 128; i++ {
+		firstChar := i
+		for isBlacklisted(rune(firstChar)) {
+			firstChar = (firstChar + 1) % 128
+			i++
+			if i >= 127 {
+				break
+			}
+		}
+		secondChar := (firstChar + 1) % 128
+		for isBlacklisted(rune(secondChar)) {
+			secondChar = (secondChar + 1) % 128
+		}
+		thirdChar := (secondChar + 1) % 128
+		for isBlacklisted(rune(thirdChar)) {
+			thirdChar = (thirdChar + 1) % 128
+		}
+
+		tagValue := string(firstChar) + string(secondChar) + string(thirdChar)
+		data[i%6].tags = append(data[i%6].tags, fmt.Sprintf("testTag%d=%s", i, tagValue))
+	}
+
+	data = append(data, testCase{ids[6], 1, []string{"key1=你好", "tiếng chào=value2", "key1=مرحبا", "123해답=value3"}})
+
+	tagIdx := make(TagIndex)
+	byId := make(map[schema.MKey]*idx.Archive)
+
+	for i, d := range data {
+		byId[d.id] = &idx.Archive{}
+		byId[d.id].Name = fmt.Sprintf("metric%d", i)
+		byId[d.id].Tags = d.tags
+		byId[d.id].LastUpdate = d.lastUpdate
+		for _, tag := range d.tags {
+			tagSplits := strings.SplitN(tag, "=", 2)
+			tagIdx.addTagId(tagSplits[0], tagSplits[1], d.id)
+		}
+		tagIdx.addTagId("name", byId[d.id].Name, d.id)
+	}
+
+	queryAndCompare := func(query []string, id int) {
+		q, _ := NewTagQuery(query, 0)
+		expect := make(IdSet)
+		expect[ids[id]] = struct{}{}
+
+		// uncomment to get details about queries
+		// fmt.Println(fmt.Sprintf("querying for tag value %d %d %d", int(query[0][len(query[0])-3]), int(query[0][len(query[0])-2]), int(query[0][len(query[0])-1])))
+		res := q.Run(tagIdx, byId)
+
+		if !reflect.DeepEqual(expect, res) {
+			t.Fatalf("Returned data does not match expected data:\nExpected: %s\nGot: %s", expect, res)
+		}
+	}
+
+	for i := 0; i < 6; i++ {
+		for tag := 0; tag < len(data[i].tags); tag++ {
+			queryAndCompare([]string{data[i].tags[tag]}, i)
+		}
+	}
+
+	queryAndCompare([]string{"key1=~.*好$"}, 6)
+	queryAndCompare([]string{"tiếng chào!="}, 6)
+	queryAndCompare([]string{"123해답=~val.*"}, 6)
+	queryAndCompare([]string{"key1=مرحبا"}, 6)
+}
+
 func BenchmarkExpressionParsing(b *testing.B) {
 	expressions := [][]string{
 		{"key=value", "key!=value"},