Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split ES Types into separate indices #263

Closed
wants to merge 4 commits into from
+243 −192
Closed

Split ES Types into separate indices #263

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db202fc
refactor writer to have split schema
mh-park Jul 11, 2017
545c3c3
refactor reader to have split schema
mh-park Jul 11, 2017
dcdc19b
dry types in schema
mh-park Jul 13, 2017
abddac6
fix bad string error
mh-park Jul 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
refactor writer to have split schema
@mh-park
mh-park committed Jul 11, 2017

Verified

This commit was signed with the committer’s verified signature.
targos Michaël Zasso
GPG key ID: 770F7A9A5AE15600
Verified
Learn about vigilant mode
commit db202fc8f895d2830b7f7417c79b0581b1642059
272 changes: 142 additions & 130 deletions plugin/storage/es/spanstore/schema.go
View file
Original file line number Diff line number Diff line change
@@ -20,9 +20,10 @@

package spanstore

// TODO: divide the span and service type into separate indices
import "fmt"

// TODO: resolve traceID concerns (may not require any changes here)
const spanMapping = `{
const mapping = `{
"settings":{
"index.mapping.nested_fields.limit":50,
"index.requests.cache.enable":true,
@@ -34,133 +35,144 @@ const spanMapping = `{
"enabled":false
}
},
"span":{
"properties":{
"traceID":{
"type":"keyword",
"ignore_above":256
},
"parentSpanID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
},
"startTime":{
"type":"long"
},
"duration":{
"type":"long"
},
"flags":{
"type":"integer"
},
"logs":{
"properties":{
"timestamp":{
"type":"long"
},
"fields":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"process":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"references":{
"type":"nested",
"dynamic":false,
"properties":{
"refType":{
"type":"keyword",
"ignore_above":256
},
"traceID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
}
}
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"service":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
}
}
}
%s
}
}`

var (
spanMapping = fmt.Sprintf(
mapping,
`"span":{
"properties":{
"traceID":{
"type":"keyword",
"ignore_above":256
},
"parentSpanID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
},
"startTime":{
"type":"long"
},
"duration":{
"type":"long"
},
"flags":{
"type":"integer"
},
"logs":{
"properties":{
"timestamp":{
"type":"long"
},
"fields":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"process":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"references":{
"type":"nested",
"dynamic":false,
"properties":{
"refType":{
"type":"keyword",
"ignore_above":256
},
"traceID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
}
}
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
}`,
)

serviceMapping = fmt.Sprintf(
mapping,
`"service":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
}
}
}`,
)
)
24 changes: 14 additions & 10 deletions plugin/storage/es/spanstore/writer.go
View file
Original file line number Diff line number Diff line change
@@ -63,38 +63,42 @@ func NewSpanWriter(client es.Client, logger *zap.Logger) *SpanWriter {

// WriteSpan writes a span and its corresponding service:operation in ElasticSearch
func (s *SpanWriter) WriteSpan(span *model.Span) error {
jaegerIndexName := spanIndexName(span)
spanIndexName, serviceIndexName := indexNames(span)
// Convert model.Span into json.Span
jsonSpan := json.FromDomainEmbedProcess(span)

if err := s.checkAndCreateIndex(jaegerIndexName, jsonSpan); err != nil {
if err := s.checkAndCreateIndex(spanIndexName, serviceIndexName, jsonSpan); err != nil {
return err
}
if err := s.writeService(jaegerIndexName, jsonSpan); err != nil {
if err := s.writeService(serviceIndexName, jsonSpan); err != nil {
return err
}
if err := s.writeSpan(jaegerIndexName, jsonSpan); err != nil {
if err := s.writeSpan(spanIndexName, jsonSpan); err != nil {
return err
}
return nil
}

func spanIndexName(span *model.Span) string {
func indexNames(span *model.Span) (string, string) {
spanDate := span.StartTime.Format("2006-01-02")
return "jaeger-" + spanDate
return "jaeger-span-" + spanDate, "jaeger-service-" + spanDate
}

// Check if index exists, and create index if it does not.
func (s *SpanWriter) checkAndCreateIndex(indexName string, jsonSpan *jModel.Span) error {
func (s *SpanWriter) checkAndCreateIndex(spanIndexName string, serviceIndexName string, jsonSpan *jModel.Span) error {
// TODO: We don't need to check every write. Try to pull this out of WriteSpan.
exists, err := s.client.IndexExists(indexName).Do(s.ctx)
exists, err := s.client.IndexExists(spanIndexName).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to find index", s.logger)
}
if !exists {
_, err = s.client.CreateIndex(indexName).Body(spanMapping).Do(s.ctx)
_, err = s.client.CreateIndex(spanIndexName).Body(spanMapping).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to create index", s.logger)
return s.logError(jsonSpan, err, "Failed to create span index", s.logger)
}
_, err = s.client.CreateIndex(serviceIndexName).Body(serviceMapping).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to create service:operation index", s.logger)
}
}
return nil
Loading