Ability to set an inclusions regex list (#162)

* Ability to set an inclusions regex list

* slightly change docs

* priorities switch - exclusions are applied after inclusions

* update example

* dot to comma

docs/configuration.md

@@ -72,6 +72,8 @@ sites:
   s3_ops_retries: 10
   exclusions:
   - logs
+  inclusions:
+  - "subpath/*"
 ```
 
 # Global configuration options
@@ -106,6 +108,7 @@ sites:
 | secret_access_key | Site AWS Secret Access Key | `global.secret_access_key` | no |
 | watch_interval | Interval for file system watcher in format of number and a unit suffix. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". | `global.watch_interval` | no |
 | exclusions | List of regex filters for exclusions, please consult with [Go regex syntax](https://github.com/google/re2/wiki/Syntax) for more details | n/a | no |
+| inclusions | List of regex filters for inclusions, please consult with [Go regex syntax](https://github.com/google/re2/wiki/Syntax) for more details, **`exclusions` have higher priority.** | `[".*"]` | no |
 | s3_ops_retries | Number of retries for upload and delete operations | `global.s3_ops_retries` | no |
 
 ---

service/config.go

@@ -54,6 +54,7 @@ type Site struct {
 	SecretAccessKey string        `yaml:"secret_access_key"`
 	RetireDeleted   bool          `yaml:"retire_deleted"`
 	Exclusions      []string      `yaml:",flow"`
+	Inclusions      []string      `yaml:",flow"`
 	WatchInterval   time.Duration `yaml:"watch_interval"`
 	S3OpsRetries    int           `yaml:"s3_ops_retries"`
 }
@@ -118,6 +119,10 @@ func (site *Site) setDefaults(cfg *Config) {
 	if site.S3OpsRetries == 0 {
 		site.S3OpsRetries = cfg.S3OpsRetries
 	}
+	// Set site S3OpsRetries
+	if len(site.Inclusions) == 0 {
+		site.Inclusions = []string{".*"}
+	}
 }
 
 func configProcessError(err error) {

service/config_test.go

@@ -84,6 +84,8 @@ sites:
   secret_access_key: je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY
   watch_interval: 5m
   s3_ops_retries: 5
+  inclusions:
+  - test
 `,
 			true,
 		},

service/local.go

@@ -33,8 +33,15 @@ import (
 )
 
 // IsExcluded check if a path is excluded
-func IsExcluded(path string, exclusions []string) bool {
-	excluded := false
+func IsExcluded(path string, exclusions []string, inclusions []string) bool {
+	var excluded bool
+
+	for _, inclusion := range inclusions {
+		re := regexp.MustCompile(inclusion)
+		if re.FindAll([]byte(path), -1) != nil {
+			excluded = false
+		}
+	}
 
 	for _, exclusion := range exclusions {
 		re := regexp.MustCompile(exclusion)
@@ -60,7 +67,7 @@ func FilePathWalkDir(site Site, awsItems map[string]string, s3Service *s3.S3, up
 			if d.Type() == fs.ModeSymlink {
 				logger.Infof("%s is a symlink, skipping", path)
 			} else {
-				excluded := IsExcluded(path, site.Exclusions)
+				excluded := IsExcluded(path, site.Exclusions, site.Inclusions)
 				s3Key := generateS3Key(site.BucketPath, site.LocalPath, path)
 				if excluded {
 					logger.Debugf("skipping without errors: %+v", path)

service/local_test.go

@@ -25,6 +25,7 @@ import (
 type exclusionsTest struct {
 	path       string
 	exclusions []string
+	inclusions []string
 	expected   bool
 }
 
@@ -37,17 +38,20 @@ type checksumComparisonTest struct {
 
 func TestIsExcluded(t *testing.T) {
 	var exclusionsTestData = []exclusionsTest{
-		{"/etc/init.d", []string{"etc", "init.d"}, true},
-		{"/var/log/messages", []string{}, false},
-		{"", []string{"etc"}, false},
+		{"/etc/init.d", []string{"etc", "init.d"}, []string{".*"}, true},
+		{"/some/file.txt", []string{".*"}, []string{"txt"}, true},
+		{"/some/file.txt", []string{}, []string{".*"}, false},
+		{"/var/log/messages", []string{}, []string{".*"}, false},
+		{"", []string{"etc"}, []string{".*"}, false},
 	}
 
 	for _, testSet := range exclusionsTestData {
-		result := IsExcluded(testSet.path, testSet.exclusions)
+		result := IsExcluded(testSet.path, testSet.exclusions, testSet.inclusions)
 		if result != testSet.expected {
 			t.Error(
 				"For path", testSet.path,
 				"with exclusions", testSet.exclusions,
+				"with inclusions", testSet.inclusions,
 				"expected", testSet.expected,
 				"got", result,
 			)

service/watcher.go

@@ -43,7 +43,7 @@ func watch(s3Service *s3.S3, site Site, uploadCh chan<- UploadCFG,
 					logger.Debugln(event)
 					// Convert filepath to string
 					filepath := fmt.Sprint(event.Path)
-					excluded := IsExcluded(filepath, site.Exclusions)
+					excluded := IsExcluded(filepath, site.Exclusions, site.Inclusions)
 					if excluded {
 						logger.Debugf("skipping without errors: %+v", filepath)
 					} else {