Merge pull request #285 from Receipt-Wrangler/enhancement/tie-system-…

…tasks-to-receipt

Enhancement/tie system tasks to receipt

internal/commands/check_receipt_processing_settings_connectivity_command.go

@@ -40,7 +40,7 @@ func (command *CheckReceiptProcessingSettingsCommand) Validate() structs.Validat
 	}
 
 	if !settingsEmpty {
-		settingsErrors := command.UpsertReceiptProcessingSettingsCommand.Validate()
+		settingsErrors := command.UpsertReceiptProcessingSettingsCommand.Validate(false)
 		for k, v := range settingsErrors.Errors {
 			vErrs.Errors[k] = v
 		}

internal/commands/upsert_receipt_processing_settings_command.go

@@ -34,7 +34,7 @@ func (command *UpsertReceiptProcessingSettingsCommand) LoadDataFromRequest(w htt
 	return nil
 }
 
-func (command *UpsertReceiptProcessingSettingsCommand) Validate() structs.ValidatorError {
+func (command *UpsertReceiptProcessingSettingsCommand) Validate(updateKey bool) structs.ValidatorError {
 	vErrs := structs.ValidatorError{}
 	errors := map[string]string{}
 	vErrs.Errors = errors
@@ -59,7 +59,7 @@ func (command *UpsertReceiptProcessingSettingsCommand) Validate() structs.Valida
 	}
 
 	if command.AiType == models.OPEN_AI_NEW || command.AiType == models.GEMINI_NEW {
-		if command.Key == "" {
+		if command.Key == "" && updateKey {
 			errors["key"] = "key is required"
 		}
 

internal/email/email.go

@@ -236,18 +236,18 @@ func processEmails(metadataList []structs.EmailMetadata, groupSettings []models.
 						return err
 					}
 
-					systemTaskCommand := commands.UpsertSystemTaskCommand{
-						Type:                 models.EMAIL_READ,
-						Status:               models.SYSTEM_TASK_SUCCEEDED,
-						AssociatedEntityType: models.SYSTEM_EMAIL,
-						AssociatedEntityId:   groupSettingsToUse.SystemEmail.ID,
-						StartedAt:            emailProcessStart,
-						EndedAt:              &emailProcessEnd,
-						RanByUserId:          nil,
-						ResultDescription:    string(metadataBytes),
-					}
-
-					createdSystemTask, err := systemTaskRepository.CreateSystemTask(systemTaskCommand)
+					emailReadSystemTask, err := systemTaskRepository.CreateSystemTask(
+						commands.UpsertSystemTaskCommand{
+							Type:                 models.EMAIL_READ,
+							Status:               models.SYSTEM_TASK_SUCCEEDED,
+							AssociatedEntityType: models.SYSTEM_EMAIL,
+							AssociatedEntityId:   groupSettingsToUse.SystemEmail.ID,
+							StartedAt:            emailProcessStart,
+							EndedAt:              &emailProcessEnd,
+							RanByUserId:          nil,
+							ResultDescription:    string(metadataBytes),
+						},
+					)
 					if err != nil {
 						return err
 					}
@@ -259,12 +259,13 @@ func processEmails(metadataList []structs.EmailMetadata, groupSettings []models.
 						models.EMAIL_UPLOAD,
 						nil,
 						func(command commands.UpsertSystemTaskCommand) *uint {
-							return &createdSystemTask.ID
+							return &emailReadSystemTask.ID
 						},
 					)
 
+					createReceiptStart := time.Now()
 					createdReceipt, err := receiptRepository.CreateReceipt(command, 0)
-					taskErr := systemTaskService.CreateReceiptUploadedSystemTask(
+					_, taskErr := systemTaskService.CreateReceiptUploadedSystemTask(
 						err,
 						createdReceipt,
 						processingSystemTasks,
@@ -277,6 +278,7 @@ func processEmails(metadataList []structs.EmailMetadata, groupSettings []models.
 						tx.Commit()
 						return err
 					}
+					createReceiptEnd := time.Now()
 
 					fileData := models.FileData{
 						ReceiptId: createdReceipt.ID,
@@ -290,6 +292,17 @@ func processEmails(metadataList []structs.EmailMetadata, groupSettings []models.
 						return err
 					}
 
+					err = systemTaskService.AssociateSystemTasksToReceipt(
+						createdReceipt.ID,
+						emailReadSystemTask.ID,
+						createReceiptStart,
+						createReceiptEnd,
+					)
+					if err != nil {
+						tx.Commit()
+						return err
+					}
+
 					return nil
 				})
 			}

internal/handlers/receipt_processing_settings.go

@@ -77,7 +77,7 @@ func CreateReceiptProcessingSettings(w http.ResponseWriter, r *http.Request) {
 				return http.StatusInternalServerError, err
 			}
 
-			vErr := command.Validate()
+			vErr := command.Validate(false)
 			if len(vErr.Errors) > 0 {
 				structs.WriteValidatorErrorResponse(w, vErr, http.StatusBadRequest)
 				return 0, nil
@@ -153,7 +153,7 @@ func UpdateReceiptProcessingSettingsById(w http.ResponseWriter, r *http.Request)
 				return http.StatusInternalServerError, err
 			}
 
-			vErr := command.Validate()
+			vErr := command.Validate(updateKey)
 			if len(vErr.Errors) > 0 {
 				structs.WriteValidatorErrorResponse(w, vErr, http.StatusBadRequest)
 				return 0, nil

internal/models/system_task.go

@@ -44,6 +44,8 @@ func (self SystemTaskStatus) Value() (driver.Value, error) {
 type SystemTaskType string
 
 const (
+	META_COMBINE_QUICK_SCAN                        SystemTaskType = "META_COMBINE_QUICK_SCAN"
+	META_ASSOCIATE_TASKS_TO_RECEIPT                SystemTaskType = "META_ASSOCIATE_TASKS_TO_RECEIPT"
 	RECEIPT_UPLOADED                               SystemTaskType = "RECEIPT_UPLOADED"
 	OCR_PROCESSING                                 SystemTaskType = "OCR_PROCESSING"
 	CHAT_COMPLETION                                SystemTaskType = "CHAT_COMPLETION"
@@ -71,6 +73,8 @@ func (self SystemTaskType) Value() (driver.Value, error) {
 		self != CHAT_COMPLETION &&
 		self != OCR_PROCESSING &&
 		self != RECEIPT_UPLOADED &&
+		self != META_COMBINE_QUICK_SCAN &&
+		self != META_ASSOCIATE_TASKS_TO_RECEIPT &&
 		self != PROMPT_GENERATED {
 		return nil, errors.New("invalid SystemTaskType")
 	}
@@ -80,9 +84,11 @@ func (self SystemTaskType) Value() (driver.Value, error) {
 type AssociatedEntityType string
 
 const (
+	RECEIPT                     AssociatedEntityType = "RECEIPT"
 	SYSTEM_EMAIL                AssociatedEntityType = "SYSTEM_EMAIL"
 	PROMPT                      AssociatedEntityType = "PROMPT"
 	RECEIPT_PROCESSING_SETTINGS AssociatedEntityType = "RECEIPT_PROCESSING_SETTINGS"
+	NOOP_ENTITY_TYPE            AssociatedEntityType = "NOOP_ENTITY_TYPE"
 )
 
 func (self *AssociatedEntityType) Scan(value string) error {
@@ -92,7 +98,9 @@ func (self *AssociatedEntityType) Scan(value string) error {
 
 func (self AssociatedEntityType) Value() (driver.Value, error) {
 	if self != SYSTEM_EMAIL &&
+		self != NOOP_ENTITY_TYPE &&
 		self != RECEIPT_PROCESSING_SETTINGS &&
+		self != RECEIPT &&
 		self != PROMPT {
 		return nil, errors.New("invalid AssociatedEntityType")
 	}

internal/services/import.go

@@ -134,7 +134,7 @@ func (service ImportService) importAiSettings(tx *gorm.DB, aiSettings structs.Ai
 		OcrEngine: ocrEngine,
 		PromptId:  prompt.ID,
 	}
-	vErrs := command.Validate()
+	vErrs := command.Validate(false)
 	if len(vErrs.Errors) > 0 {
 		logging.LogStd(logging.LOG_LEVEL_ERROR, "Unable to import invalid AI settings: ", vErrs.Errors)
 		return models.ReceiptProcessingSettings{},

internal/services/receipts.go

@@ -79,6 +79,7 @@ func QuickScan(
 ) (models.Receipt, error) {
 	db := repositories.GetDB()
 	systemTaskService := NewSystemTaskService(nil)
+	systemTaskRepository := repositories.NewSystemTaskRepository(nil)
 	var createdReceipt models.Receipt
 
 	fileRepository := repositories.NewFileRepository(nil)
@@ -109,13 +110,25 @@ func QuickScan(
 	receiptCommand, receiptProcessingMetadata, err := MagicFillFromImage(magicFillCommand, groupIdString)
 	finishedAt := time.Now()
 
+	metaCombineSystemTask, err := systemTaskRepository.CreateSystemTask(commands.UpsertSystemTaskCommand{
+		Type:                 models.META_COMBINE_QUICK_SCAN,
+		Status:               models.SYSTEM_TASK_SUCCEEDED,
+		AssociatedEntityType: models.NOOP_ENTITY_TYPE,
+		AssociatedEntityId:   0,
+	})
+	if err != nil {
+		return models.Receipt{}, err
+	}
+
 	quickScanSystemTasks, taskErr := systemTaskService.CreateSystemTasksFromMetadata(
 		receiptProcessingMetadata,
 		now,
 		finishedAt,
 		models.QUICK_SCAN,
 		&token.UserId,
-		nil)
+		func(command commands.UpsertSystemTaskCommand) *uint {
+			return &metaCombineSystemTask.ID
+		})
 	if taskErr != nil {
 		return models.Receipt{}, taskErr
 	}
@@ -141,7 +154,8 @@ func QuickScan(
 		uploadStart := time.Now()
 
 		createdReceipt, err = receiptRepository.CreateReceipt(receiptCommand, token.UserId)
-		taskErr := systemTaskService.CreateReceiptUploadedSystemTask(
+		uploadEnd := time.Now()
+		_, taskErr := systemTaskService.CreateReceiptUploadedSystemTask(
 			err,
 			createdReceipt,
 			quickScanSystemTasks,
@@ -161,12 +175,21 @@ func QuickScan(
 			ReceiptId: createdReceipt.ID,
 			FileType:  validatedFileType,
 		}
-
 		_, err := receiptImageRepository.CreateReceiptImage(fileData, fileBytes)
 		if err != nil {
 			return err
 		}
 
+		err = systemTaskService.AssociateSystemTasksToReceipt(
+			createdReceipt.ID,
+			metaCombineSystemTask.ID,
+			uploadStart,
+			uploadEnd)
+		if err != nil {
+			tx.Commit()
+			return err
+		}
+
 		return nil
 	})
 	if err != nil {

internal/services/system_task.go

@@ -51,7 +51,12 @@ func (service SystemTaskService) BuildSuccessReceiptProcessResultDescription(met
 	)
 }
 
-func (service SystemTaskService) CreateSystemTasksFromMetadata(metadata commands.ReceiptProcessingMetadata, startDate time.Time, endDate time.Time, taskType models.SystemTaskType, userId *uint, parentAssociatedSystemTaskId func(command commands.UpsertSystemTaskCommand) *uint) (structs.ReceiptProcessingSystemTasks, error) {
+func (service SystemTaskService) CreateSystemTasksFromMetadata(metadata commands.ReceiptProcessingMetadata,
+	startDate time.Time,
+	endDate time.Time,
+	taskType models.SystemTaskType,
+	userId *uint,
+	parentAssociatedSystemTaskId func(command commands.UpsertSystemTaskCommand) *uint) (structs.ReceiptProcessingSystemTasks, error) {
 	systemTaskRepository := repositories.NewSystemTaskRepository(service.TX)
 	result := structs.ReceiptProcessingSystemTasks{}
 
@@ -130,7 +135,6 @@ func (service SystemTaskService) CreateChildSystemTasks(parentSystemTask models.
 		systemTasks = append(systemTasks, ocrSystemTask)
 	}
 
-	// TODO: create tasks for prompt,
 	if !isFallback && len(metadata.PromptSystemTaskCommand.Type) > 0 {
 		metadata.PromptSystemTaskCommand.AssociatedSystemTaskId = &parentSystemTask.ID
 		promptSystemTask, err := systemTaskRepository.CreateSystemTask(metadata.PromptSystemTaskCommand)
@@ -196,7 +200,7 @@ func (service SystemTaskService) CreateReceiptUploadedSystemTask(
 	createdReceipt models.Receipt,
 	quickScanSystemTasks structs.ReceiptProcessingSystemTasks,
 	uploadStart time.Time,
-) error {
+) (models.SystemTask, error) {
 	systemTaskRepository := repositories.NewSystemTaskRepository(service.GetDB())
 	receiptProcessingSettingsId := quickScanSystemTasks.SystemTask.AssociatedEntityId
 	systemTaskId := quickScanSystemTasks.SystemTask.ID
@@ -211,7 +215,7 @@ func (service SystemTaskService) CreateReceiptUploadedSystemTask(
 
 	receiptBytes, err := json.Marshal(createdReceipt)
 	if err != nil {
-		return err
+		return models.SystemTask{}, err
 	}
 
 	resultDescription = string(receiptBytes)
@@ -221,7 +225,7 @@ func (service SystemTaskService) CreateReceiptUploadedSystemTask(
 		resultDescription = createReceiptError.Error()
 	}
 
-	_, err = systemTaskRepository.CreateSystemTask(commands.UpsertSystemTaskCommand{
+	systemTask, err := systemTaskRepository.CreateSystemTask(commands.UpsertSystemTaskCommand{
 		Type:                   models.RECEIPT_UPLOADED,
 		Status:                 status,
 		AssociatedEntityType:   models.RECEIPT_PROCESSING_SETTINGS,
@@ -231,9 +235,61 @@ func (service SystemTaskService) CreateReceiptUploadedSystemTask(
 		ResultDescription:      resultDescription,
 		AssociatedSystemTaskId: &systemTaskId,
 	})
+	if err != nil {
+		return models.SystemTask{}, err
+	}
+
+	return systemTask, nil
+}
+
+func (service SystemTaskService) AssociateSystemTasksToReceipt(
+	createdReceiptId uint,
+	parentProcessingSystemTaskId uint,
+	uploadStart time.Time,
+	uploadEnd time.Time,
+) error {
+	db := service.GetDB()
+
+	err := db.Transaction(func(tx *gorm.DB) error {
+		systemTaskRepository := repositories.NewSystemTaskRepository(tx)
+
+		associateTasksToReceipt, txErr := systemTaskRepository.CreateSystemTask(
+			service.BuildAssociateSystemTasksToReceiptTask(
+				createdReceiptId,
+				uploadStart,
+				uploadEnd,
+			))
+		if txErr != nil {
+			return txErr
+		}
+
+		txErr = tx.Model(models.SystemTask{}).
+			Where("id = ?", parentProcessingSystemTaskId).
+			Update("associated_system_task_id", associateTasksToReceipt.ID).Error
+		if txErr != nil {
+			return txErr
+		}
+
+		return nil
+	})
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+
+func (service SystemTaskService) BuildAssociateSystemTasksToReceiptTask(
+	createdReceiptId uint,
+	uploadStart time.Time,
+	uploadEnd time.Time,
+) commands.UpsertSystemTaskCommand {
+	return commands.UpsertSystemTaskCommand{
+		Type:                 models.META_ASSOCIATE_TASKS_TO_RECEIPT,
+		Status:               models.SYSTEM_TASK_SUCCEEDED,
+		AssociatedEntityType: models.RECEIPT,
+		AssociatedEntityId:   createdReceiptId,
+		StartedAt:            uploadStart,
+		EndedAt:              &uploadEnd,
+	}
+}

swagger.yml

@@ -2248,11 +2248,13 @@ components:
     SystemTaskType:
       type: string
       enum:
+        - "META_ASSOCIATE_TASKS_TO_RECEIPT"
         - "OCR_PROCESSING"
         - "CHAT_COMPLETION"
         - "MAGIC_FILL"
         - "QUICK_SCAN"
         - "EMAIL_READ"
+        - "META_COMBINE_QUICK_SCAN"
         - "EMAIL_UPLOAD"
         - "SYSTEM_EMAIL_CONNECTIVITY_CHECK"
         - "RECEIPT_PROCESSING_SETTINGS_CONNECTIVITY_CHECK"
@@ -2261,6 +2263,8 @@ components:
     AssociatedEntityType:
       type: string
       enum:
+        - "NOOP_ENTITY_TYPE"
+        - "RECEIPT"
         - "SYSTEM_EMAIL"
         - "RECEIPT_PROCESSING_SETTINGS"
         - "PROMPT"