-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Relates: #4341 This commit adds the Csv processor to the collection of ingest processors. (cherry picked from commit f91ccda)
- Loading branch information
Showing
4 changed files
with
150 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
using System; | ||
using System.Linq.Expressions; | ||
using System.Runtime.Serialization; | ||
using Elasticsearch.Net; | ||
using Elasticsearch.Net.Utf8Json; | ||
|
||
|
||
namespace Nest | ||
{ | ||
/// <summary> | ||
/// Extracts fields from CSV line out of a single text field within a document. | ||
/// Any empty field in CSV will be skipped. | ||
/// <para></para> | ||
/// Available in Elasticsearch 7.6.0+ | ||
/// </summary> | ||
[InterfaceDataContract] | ||
public interface ICsvProcessor : IProcessor | ||
{ | ||
/// <summary> | ||
/// The field to extract data from | ||
/// </summary> | ||
[DataMember(Name ="field")] | ||
Field Field { get; set; } | ||
|
||
/// <summary> | ||
/// The array of fields to assign extracted values to. | ||
/// </summary> | ||
[DataMember(Name ="target_fields")] | ||
Fields TargetFields { get; set; } | ||
|
||
/// <summary> | ||
/// Separator used in CSV, has to be single character string. Defaults to <c>,</c> | ||
/// </summary> | ||
[DataMember(Name = "separator")] | ||
string Separator { get; set; } | ||
|
||
/// <summary> | ||
/// Quote used in CSV, has to be single character string. Defaults to <c>"</c> | ||
/// </summary> | ||
[DataMember(Name = "quote")] | ||
string Quote { get; set; } | ||
|
||
/// <summary> | ||
/// If <c>true</c> and <see cref="Field" /> does not exist or is null, | ||
/// the processor quietly exits without modifying the document. Default is <c>false</c> | ||
/// </summary> | ||
[DataMember(Name = "ignore_missing")] | ||
bool? IgnoreMissing { get; set; } | ||
|
||
/// <summary> | ||
/// Trim whitespaces in unquoted fields. Default is <c>false</c>; | ||
/// </summary> | ||
[DataMember(Name = "trim")] | ||
bool? Trim { get; set; } | ||
} | ||
|
||
/// <inheritdoc cref="ICsvProcessor"/> | ||
public class CsvProcessor : ProcessorBase, ICsvProcessor | ||
{ | ||
/// <inheritdoc /> | ||
public Field Field { get; set; } | ||
/// <inheritdoc /> | ||
public Fields TargetFields { get; set; } | ||
/// <inheritdoc /> | ||
public string Separator { get; set; } | ||
/// <inheritdoc /> | ||
public string Quote { get; set; } | ||
/// <inheritdoc /> | ||
public bool? IgnoreMissing { get; set; } | ||
/// <inheritdoc /> | ||
public bool? Trim { get; set; } | ||
/// <inheritdoc /> | ||
protected override string Name => "csv"; | ||
} | ||
|
||
/// <inheritdoc cref="ICsvProcessor"/> | ||
public class CsvProcessorDescriptor<T> : ProcessorDescriptorBase<CsvProcessorDescriptor<T>, ICsvProcessor>, ICsvProcessor | ||
where T : class | ||
{ | ||
protected override string Name => "csv"; | ||
Field ICsvProcessor.Field { get; set; } | ||
Fields ICsvProcessor.TargetFields { get; set; } | ||
bool? ICsvProcessor.IgnoreMissing { get; set; } | ||
string ICsvProcessor.Quote { get; set; } | ||
string ICsvProcessor.Separator { get; set; } | ||
bool? ICsvProcessor.Trim { get; set; } | ||
|
||
/// <inheritdoc cref="ICsvProcessor.Field" /> | ||
public CsvProcessorDescriptor<T> Field(Field field) => Assign(field, (a, v) => a.Field = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.Field" /> | ||
public CsvProcessorDescriptor<T> Field<TValue>(Expression<Func<T, TValue>> objectPath) => | ||
Assign(objectPath, (a, v) => a.Field = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.TargetFields" /> | ||
public CsvProcessorDescriptor<T> TargetFields(Func<FieldsDescriptor<T>, IPromise<Fields>> targetFields) => | ||
Assign(targetFields, (a, v) => a.TargetFields = v?.Invoke(new FieldsDescriptor<T>())?.Value); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.TargetFields" /> | ||
public CsvProcessorDescriptor<T> TargetFields(Fields targetFields) => Assign(targetFields, (a, v) => a.TargetFields = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.IgnoreMissing" /> | ||
public CsvProcessorDescriptor<T> IgnoreMissing(bool? ignoreMissing = true) => Assign(ignoreMissing, (a, v) => a.IgnoreMissing = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.Trim" /> | ||
public CsvProcessorDescriptor<T> Trim(bool? trim = true) => Assign(trim, (a, v) => a.Trim = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.Quote" /> | ||
public CsvProcessorDescriptor<T> Quote(string quote) => Assign(quote, (a, v) => a.Quote = v); | ||
|
||
/// <inheritdoc cref="ICsvProcessor.Separator" /> | ||
public CsvProcessorDescriptor<T> Separator(string separator) => Assign(separator, (a, v) => a.Separator = v); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters