Merge branch 'check-test-results' into next

adapters/folder/readFolderWithFIles_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/simulot/immich-go/helpers/configuration"
 	cliflags "github.com/simulot/immich-go/internal/cliFlags"
 	"github.com/simulot/immich-go/internal/fileevent"
+	"github.com/simulot/immich-go/internal/filters"
 	"github.com/simulot/immich-go/internal/metadata"
 	"github.com/simulot/immich-go/internal/tzone"
 )
@@ -37,6 +38,9 @@ func TestLocalAssets(t *testing.T) {
 						TZ: time.Local,
 					},
 				},
+				ManageBurst:   filters.BurstNothing,
+				ManageRawJPG:  filters.RawJPGNothing,
+				ManageHEICJPG: filters.HeicJpgNothing,
 			},
 			fsys: []fs.FS{
 				os.DirFS("DATA/date-range"),
@@ -52,6 +56,9 @@ func TestLocalAssets(t *testing.T) {
 		{
 			name: "date on name",
 			flags: ImportFolderOptions{
+				ManageBurst:    filters.BurstNothing,
+				ManageRawJPG:   filters.RawJPGNothing,
+				ManageHEICJPG:  filters.HeicJpgNothing,
 				SupportedMedia: metadata.DefaultSupportedMedia,
 				DateHandlingFlags: cliflags.DateHandlingFlags{
 					Method: cliflags.DateMethodName,
@@ -88,19 +95,29 @@ func TestLocalAssets(t *testing.T) {
 					UseExifTool: true,
 					Timezone:    tzone.Timezone{TZ: time.Local},
 				},
+				ManageBurst:   filters.BurstNothing,
+				ManageRawJPG:  filters.RawJPGNothing,
+				ManageHEICJPG: filters.HeicJpgNothing,
 			},
 			fsys: []fs.FS{
 				os.DirFS("DATA/date-range"),
 			},
 			expectedFiles: []string{
 				"photo1_w_exif.jpg",
 				"photo1_2024-10-06_w_exif.jpg",
+				"photo1_2023-10-06_wo_exif.jpg",
 			},
-			expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Set(fileevent.Uploaded, 2).Value(),
+			expectedCounts: fileevent.NewCounts().
+				Set(fileevent.DiscoveredImage, 4).
+				Set(fileevent.DiscoveredDiscarded, 1).
+				Set(fileevent.Uploaded, 3).Value(),
 		},
 		{
 			name: "select exif date using exiftool",
 			flags: ImportFolderOptions{
+				ManageBurst:    filters.BurstNothing,
+				ManageRawJPG:   filters.RawJPGNothing,
+				ManageHEICJPG:  filters.HeicJpgNothing,
 				SupportedMedia: metadata.DefaultSupportedMedia,
 				DateHandlingFlags: cliflags.DateHandlingFlags{
 					Method: cliflags.DateMethodEXIF,
@@ -122,8 +139,12 @@ func TestLocalAssets(t *testing.T) {
 			expectedFiles: []string{
 				"photo1_w_exif.jpg",
 				"photo1_2024-10-06_w_exif.jpg",
+				"photo1_2023-10-06_wo_exif.jpg",
 			},
-			expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Set(fileevent.Uploaded, 2).Value(),
+			expectedCounts: fileevent.NewCounts().
+				Set(fileevent.DiscoveredImage, 4).
+				Set(fileevent.DiscoveredDiscarded, 1).
+				Set(fileevent.Uploaded, 3).Value(),
 		},
 		{
 			name: "select exif date using exiftool then date",
@@ -215,7 +236,7 @@ func TestLocalAssets(t *testing.T) {
 
 			log := application.Log{
 				File:  logFile,
-				Level: "INFO",
+				Level: "DEBUG",
 			}
 			err := log.OpenLogFile()
 			if err != nil {

internal/assets/asset.go

@@ -70,6 +70,9 @@ type Asset struct {
 
 func (l *Asset) SetNameInfo(ni filenames.NameInfo) {
 	l.nameInfo = ni
+	if l.CaptureDate.IsZero() {
+		l.CaptureDate = ni.Taken
+	}
 }
 
 func (l *Asset) NameInfo() filenames.NameInfo {

internal/groups/burst/burst.go

@@ -5,16 +5,24 @@ import (
 	"time"
 
 	"github.com/simulot/immich-go/internal/assets"
+	"github.com/simulot/immich-go/internal/filenames"
+	"github.com/simulot/immich-go/internal/metadata"
 	"golang.org/x/exp/constraints"
 )
 
-const frameInterval = 1 * time.Second
+const frameInterval = 500 * time.Millisecond
 
-// Group groups assets taken within a period of less than 1 second.
+// Group groups photos taken within a period of less than 1 second with a digital camera.
+// This addresses photos taken with a digital camera when there isn't any burst indication in the file namee
+//
+// Ex: IMG_0001.JPG, IMG_0002.JPG, etc. and the date taken is different by a fraction of second
+// Ex: IMG_0001.JPG, IMG_0001.RAW, IMG_0002.JPG, IMG_0002.RAW, etc.
+//
+// Edited images, images identified as as burst already are not considered.
 // The in channel receives assets sorted by date taken.
 func Group(ctx context.Context, in <-chan *assets.Asset, out chan<- *assets.Asset, gOut chan<- *assets.Group) {
 	var currentGroup []*assets.Asset
-	var lastTimestamp time.Time
+	var lastTaken time.Time
 
 	for {
 		select {
@@ -35,13 +43,25 @@ func Group(ctx context.Context, in <-chan *assets.Asset, out chan<- *assets.Asse
 				continue
 			}
 
-			if len(currentGroup) == 0 && abs(a.DateTaken().Sub(lastTimestamp)) > frameInterval {
-				sendBurstGroup(ctx, out, gOut, currentGroup)
+			// exclude movies, edited or burst images
+			// exclude images without a date taken
+			// exclude images taken more than 500ms apart
+			ni := a.NameInfo()
+			dontGroupMe := ni.Type != metadata.TypeImage ||
+				a.CaptureDate.IsZero() ||
+				ni.Kind == filenames.KindBurst ||
+				ni.Kind == filenames.KindEdited ||
+				abs(a.DateTaken().Sub(lastTaken)) > frameInterval
+
+			if dontGroupMe {
+				if len(currentGroup) > 0 {
+					sendBurstGroup(ctx, out, gOut, currentGroup)
+				}
 				currentGroup = []*assets.Asset{a}
-				lastTimestamp = a.DateTaken()
+				lastTaken = a.DateTaken()
 			} else {
 				currentGroup = append(currentGroup, a)
-				lastTimestamp = a.DateTaken()
+				lastTaken = a.DateTaken()
 			}
 		}
 	}

internal/groups/burst/burst_test.go

@@ -29,23 +29,23 @@ func TestGroup(t *testing.T) {
 	// Create assets with a DateTaken interval of 200 milliseconds
 	testAssets := []*assets.Asset{
 		mockAsset(ic, "IMG_001.jpg", baseTime),
-		mockAsset(ic, "IMG_002.jpg", baseTime.Add(200*time.Millisecond)),
-		mockAsset(ic, "IMG_003.jpg", baseTime.Add(400*time.Millisecond)),
-		mockAsset(ic, "IMG_004.jpg", baseTime.Add(600*time.Millisecond)),
-		mockAsset(ic, "IMG_005.jpg", baseTime.Add(800*time.Millisecond)),
-		mockAsset(ic, "IMG_006.jpg", baseTime.Add(1000*time.Millisecond)),
-		mockAsset(ic, "IMG_007.jpg", baseTime.Add(1200*time.Millisecond)),
-		mockAsset(ic, "IMG_008.jpg", baseTime.Add(1400*time.Millisecond)),
+		mockAsset(ic, "IMG_002.jpg", baseTime.Add(200*time.Millisecond)),  // group 1
+		mockAsset(ic, "IMG_003.jpg", baseTime.Add(400*time.Millisecond)),  // group 1
+		mockAsset(ic, "IMG_004.jpg", baseTime.Add(600*time.Millisecond)),  // group 1
+		mockAsset(ic, "IMG_005.jpg", baseTime.Add(800*time.Millisecond)),  // group 1
+		mockAsset(ic, "IMG_006.jpg", baseTime.Add(1000*time.Millisecond)), // group 1
+		mockAsset(ic, "IMG_007.jpg", baseTime.Add(1200*time.Millisecond)), // group 1
+		mockAsset(ic, "IMG_008.jpg", baseTime.Add(1400*time.Millisecond)), // group 1
 		mockAsset(ic, "IMG_009.jpg", baseTime.Add(1600*time.Millisecond)),
 		mockAsset(ic, "IMG_010.jpg", baseTime.Add(5*time.Second)),
 		mockAsset(ic, "IMG_011.jpg", baseTime.Add(10*time.Second)),
-		mockAsset(ic, "IMG_012.jpg", baseTime.Add(10*time.Second+200*time.Millisecond)),
-		mockAsset(ic, "IMG_013.jpg", baseTime.Add(10*time.Second+400*time.Millisecond)),
+		mockAsset(ic, "IMG_012.jpg", baseTime.Add(10*time.Second+200*time.Millisecond)), // group 2
+		mockAsset(ic, "IMG_013.jpg", baseTime.Add(10*time.Second+400*time.Millisecond)), // group 2
 		mockAsset(ic, "IMG_014.jpg", baseTime.Add(15*time.Second)),
 		mockAsset(ic, "IMG_015.jpg", baseTime.Add(20*time.Second)),
 		mockAsset(ic, "IMG_016.jpg", baseTime.Add(30*time.Second)),
-		mockAsset(ic, "IMG_017.jpg", baseTime.Add(30*time.Second+200*time.Millisecond)),
-		mockAsset(ic, "IMG_018.jpg", baseTime.Add(30*time.Second+400*time.Millisecond)),
+		mockAsset(ic, "IMG_017.jpg", baseTime.Add(30*time.Second+200*time.Millisecond)), // group 3
+		mockAsset(ic, "IMG_018.jpg", baseTime.Add(30*time.Second+400*time.Millisecond)), // group 3
 	}
 
 	expectedAssets := []*assets.Asset{

internal/groups/groups_test.go

@@ -118,7 +118,7 @@ func TestGroup(t *testing.T) {
 	var gotAssets []*assets.Asset
 	ctx := context.Background()
 
-	gOut := groups.NewGrouperPipeline(ctx, series.Group, burst.Group).PipeGrouper(ctx, in)
+	gOut := groups.NewGrouperPipeline(ctx, burst.Group, series.Group).PipeGrouper(ctx, in)
 	for g := range gOut {
 		switch g.Grouping {
 		case assets.GroupByNone:

internal/metadata/exiftool.go

@@ -53,6 +53,26 @@ var dateKeys = []struct {
 
 // GetMetadata returns the metadata of the file. The date of capture is searched in the preferred tags first.
 // missing tags or tags  with incorrect dates are skipped
+//
+// TODO: make a better use of time offset taken on the exif fields
+// ```
+// Modify Date                     : 2023:10:06 08:30:00
+// Date/Time Original              : 2023:10:06 08:30:00
+// Create Date                     : 2023:10:06 08:30:00
+// Offset Time                     : +02:00
+// Offset Time Original            : +02:00
+// Offset Time Digitized           : +02:00
+// Sub Sec Time                    : 139
+// Sub Sec Time Original           : 139
+// Sub Sec Time Digitized          : 139
+// GPS Time Stamp                  : 06:29:56
+// GPS Date Stamp                  : 2023:10:06
+// Profile Date Time               : 2023:03:09 10:57:00
+// Create Date                     : 2023:10:06 08:30:00.139+02:00
+// Date/Time Original              : 2023:10:06 08:30:00.139+02:00
+// Modify Date                     : 2023:10:06 08:30:00.139+02:00
+// GPS Date/Time                   : 2023:10:06 06:29:56Z
+// ```
 
 func (et *ExifTool) ReadMetaData(fileName string) (*Metadata, error) {
 	ms := et.eTool.ExtractMetadata(fileName)

internal/metadata/exiftool_test.go

@@ -19,7 +19,7 @@ func TestExifTool_ReadMetaData(t *testing.T) {
 			name:     "read JPG",
 			fileName: "DATA/PXL_20231006_063000139.jpg",
 			want: &Metadata{
-				DateTaken: time.Date(2023, 10, 6, 8, 30, 0, 0, time.Local),
+				DateTaken: time.Date(2023, 10, 6, 6, 29, 56, 0, time.UTC), // 2023:10:06 06:29:56Z
 				Latitude:  +48.8583736,
 				Longitude: +2.2919010,
 			},