Improve VQ measurment methodology

common.go

@@ -6,20 +6,15 @@
 package main
 
 import (
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/evolution-gaming/ease/internal/encoding"
 	"github.com/evolution-gaming/ease/internal/logging"
-	"github.com/evolution-gaming/ease/internal/vqm"
-	"github.com/jszwec/csvutil"
 )
 
 // Commander interface should be implemented by commands and sub-commands.
@@ -53,165 +48,6 @@ func printSubCommandUsage(longHelp string, fs *flag.FlagSet) {
 	fs.PrintDefaults()
 }
 
-// namedVqmResult is structure that wraps vqm.Result with a name.
-type namedVqmResult struct {
-	Name string
-	vqm.Result
-}
-
-// report contains application execution result.
-type report struct {
-	EncodingResult encoding.PlanResult
-	VQMResults     []namedVqmResult
-}
-
-// WriteJSON writes application execution result as JSON.
-func (r *report) WriteJSON(w io.Writer) error {
-	// Write Plan execution result to JSON (for now)
-	res, err := json.MarshalIndent(r, "", "  ")
-	if err != nil {
-		return fmt.Errorf("marshaling encoding result to JSON: %w", err)
-	}
-	_, err = w.Write(res)
-	if err != nil {
-		return fmt.Errorf("writing encoding result %w", err)
-	}
-	return nil
-}
-
-// csvRecord contains result fields from one encode.
-type csvRecord struct {
-	Name             string
-	SourceFile       string
-	CompressedFile   string
-	Cmd              string
-	HStime           string
-	HUtime           string
-	HElapsed         string
-	Stime            time.Duration
-	Utime            time.Duration
-	Elapsed          time.Duration
-	MaxRss           int64
-	VideoDuration    float64
-	AvgEncodingSpeed float64
-	PSNR             float64
-	MS_SSIM          float64
-	VMAF             float64
-}
-
-// Wrap rows of csvRecords mainly to attach relevant methods.
-type csvReport struct {
-	rows []csvRecord
-}
-
-func newCsvReport(r *report) (*csvReport, error) {
-	size := len(r.EncodingResult.RunResults)
-	if size != len(r.VQMResults) {
-		return nil, errors.New("Encoding result and VQM result size do not match")
-	}
-
-	var report csvReport
-	report.rows = make([]csvRecord, 0, size)
-
-	// Need to create an intermediate mapping from CompressedFile to VQM metrics to make
-	// merging fields from two sources easier (we cannot rely on order). CompressedFile
-	// being a unique identifier (Name does not work when there are multiple input files).
-	tVqms := make(map[string]vqm.VideoQualityMetrics, size)
-	for _, v := range r.VQMResults {
-		tVqms[v.CompressedFile] = v.Metrics
-	}
-
-	// Final loop to merge into a single report.
-	for _, v := range r.EncodingResult.RunResults {
-		vqm, ok := tVqms[v.CompressedFile]
-		if !ok {
-			return nil, fmt.Errorf("no VQMs for map key: %s", v.CompressedFile)
-		}
-		report.rows = append(report.rows, csvRecord{
-			Name:             v.Name,
-			SourceFile:       v.SourceFile,
-			CompressedFile:   v.CompressedFile,
-			Cmd:              v.Cmd,
-			HStime:           v.Stats.HStime,
-			HUtime:           v.Stats.HUtime,
-			HElapsed:         v.Stats.HElapsed,
-			Stime:            v.Stats.Stime,
-			Utime:            v.Stats.Utime,
-			Elapsed:          v.Stats.Elapsed,
-			MaxRss:           v.Stats.MaxRss,
-			VideoDuration:    v.VideoDuration,
-			AvgEncodingSpeed: v.AvgEncodingSpeed,
-			PSNR:             vqm.PSNR,
-			MS_SSIM:          vqm.MS_SSIM,
-			VMAF:             vqm.VMAF,
-		})
-	}
-
-	return &report, nil
-}
-
-// WriteCSV saves flat application report representation to io.Writer.
-func (r *csvReport) WriteCSV(w io.Writer) error {
-	data, err := csvutil.Marshal(r.rows)
-	if err != nil {
-		return err
-	}
-	_, err2 := w.Write(data)
-	if err2 != nil {
-		return err2
-	}
-	return nil
-}
-
-// parseReportFile is a helper to read and parse report JSON file into report type.
-func parseReportFile(fPath string) *report {
-	var r report
-
-	b, err := os.ReadFile(fPath)
-	if err != nil {
-		log.Panicf("Unable to read file %s: %v", fPath, err)
-	}
-
-	if err := json.Unmarshal(b, &r); err != nil {
-		log.Panic(err)
-	}
-
-	return &r
-}
-
-// sourceData is a helper data structure with fields related to single encoded file.
-type sourceData struct {
-	CompressedFile string
-	WorkDir        string
-	VqmResultFile  string
-}
-
-// extractSourceData create mapping from compressed file to sourceData.
-//
-// Since in report file we have separate keys RunResults and VQMResults and we
-// need to merge fields from both, we create mapping from unique CompressedFile
-// field to sourceData.
-func extractSourceData(r *report) map[string]sourceData {
-	s := make(map[string]sourceData)
-	// Create map to sourceData (incomplete at this point) from RunResults
-	for i := range r.EncodingResult.RunResults {
-		v := &r.EncodingResult.RunResults[i]
-		sd := s[v.CompressedFile]
-		sd.WorkDir = v.WorkDir
-		sd.CompressedFile = v.CompressedFile
-		s[v.CompressedFile] = sd
-	}
-
-	// Fill-in missing VqmResultFile field from VQMResult.
-	for i := range r.VQMResults {
-		v := &r.VQMResults[i]
-		sd := s[v.CompressedFile]
-		sd.VqmResultFile = v.ResultFile
-		s[v.CompressedFile] = sd
-	}
-	return s
-}
-
 // unrollResultErrors helper to unroll all errors from RunResults into a string.
 func unrollResultErrors(results []encoding.RunResult) string {
 	sb := strings.Builder{}

common_test.go

@@ -6,83 +6,11 @@
 package main
 
 import (
-	"bytes"
-	"os"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_extractSourceData(t *testing.T) {
-	given := parseReportFile("testdata/encoding_artifacts/report.json")
-	want := map[string]sourceData{
-		"out/testsrc01_libx264.mp4": {
-			CompressedFile: "out/testsrc01_libx264.mp4",
-			WorkDir:        "/tmp",
-			VqmResultFile:  "out/testsrc01_libx264_vqm.json",
-		},
-		"out/testsrc01_libx265.mp4": {
-			CompressedFile: "out/testsrc01_libx265.mp4",
-			WorkDir:        "/tmp",
-			VqmResultFile:  "out/testsrc01_libx265_vqm.json",
-		},
-		"out/testsrc02_libx264.mp4": {
-			CompressedFile: "out/testsrc02_libx264.mp4",
-			WorkDir:        "/tmp",
-			VqmResultFile:  "out/testsrc02_libx264_vqm.json",
-		},
-		"out/testsrc02_libx265.mp4": {
-			CompressedFile: "out/testsrc02_libx265.mp4",
-			WorkDir:        "/tmp",
-			VqmResultFile:  "out/testsrc02_libx265_vqm.json",
-		},
-	}
-
-	got := extractSourceData(given)
-	assert.Equal(t, want, got)
-}
-
-func Test_parseReportFile(t *testing.T) {
-	got := parseReportFile("testdata/encoding_artifacts/report.json")
-	t.Log("Should have RunResults")
-	assert.Len(t, got.EncodingResult.RunResults, 4)
-
-	t.Log("Should have VQMResults")
-	assert.Len(t, got.VQMResults, 4)
-}
-
-func Test_report_WriteJSON(t *testing.T) {
-	// Do the round-trip of JOSN report unmarshalling-marshalling.
-	reportFile := "testdata/encoding_artifacts/report.json"
-	parsedReport := parseReportFile(reportFile)
-
-	var got bytes.Buffer
-	err := parsedReport.WriteJSON(&got)
-	assert.NoError(t, err)
-
-	want, err := os.ReadFile(reportFile)
-	assert.NoError(t, err)
-	wantStr := strings.TrimRight(string(want), "\n")
-
-	assert.Equal(t, wantStr, got.String())
-}
-
-func Test_csvReport_writeCSV(t *testing.T) {
-	// Create report from fixture data.
-	reportFile := "testdata/encoding_artifacts/report.json"
-	parsedReport := parseReportFile(reportFile)
-
-	csvReport, err := newCsvReport(parsedReport)
-	assert.NoError(t, err)
-
-	var b bytes.Buffer
-	err = csvReport.WriteCSV(&b)
-	assert.NoError(t, err)
-
-	assert.Len(t, b.Bytes(), 1332)
-}
-
 func Test_all_Positive(t *testing.T) {
 	floatTests := map[string]struct {
 		given []float64

config.go

@@ -23,7 +23,7 @@ import (
 
 var (
 	ErrInvalidConfig  = errors.New("invalid configuration")
-	defaultReportFile = "report.json"
+	defaultReportFile = "report.csv"
 )
 
 // Config represent application configuration.

ease_test.go

@@ -6,13 +6,16 @@
 package main
 
 import (
+	"encoding/csv"
 	"fmt"
 	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"testing"
 
+	"github.com/evolution-gaming/ease/internal/encoding"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,24 +26,33 @@ func Test_RunApp_Run(t *testing.T) {
 	ePlan := fixPlanConfig(t)
 	outDir := path.Join(tempDir, "out")
 
-	t.Log("Should succeed execution with -plan flag")
-	// Run command will generate encoding artifacts and analysis artifacts.
-	err := CreateRunCommand().Run([]string{"-plan", ePlan, "-out-dir", outDir})
-	assert.NoError(t, err, "Unexpected error running encode")
+	t.Run("Should succeed execution with -plan flag", func(t *testing.T) {
+		// Run command will generate encoding artifacts and analysis artifacts.
+		app := CreateRunCommand()
+		err := app.Run([]string{"-plan", ePlan, "-out-dir", outDir})
+		assert.NoError(t, err, "Unexpected error running encode")
+	})
 
-	buf, err2 := os.ReadFile(path.Join(outDir, "report.json"))
-	assert.NoError(t, err2, "Unexpected error reading report.json")
-	assert.Greater(t, len(buf), 0, "No data in report file")
+	t.Run("Should have a CSV report file", func(t *testing.T) {
+		fd, err2 := os.Open(path.Join(outDir, "report.csv"))
+		assert.NoError(t, err2, "Unexpected error opening report.csv")
+		defer fd.Close()
+		records, err3 := csv.NewReader(fd).ReadAll()
+		assert.NoError(t, err3, "Unexpected error reading CSV records")
+		// Expect 2 records: CSV header + record for 1 encoding.
+		assert.Len(t, records, 2, "Unexpected number of records in report file")
+	})
 
-	t.Log("Analyse should create bitrate, VMAF, PSNR and SSIM plots")
-	bitratePlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*bitrate.png", outDir))
-	assert.Len(t, bitratePlots, 1, "Expecting one file for bitrate plot")
+	t.Run("Should create plots", func(t *testing.T) {
+		bitratePlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*bitrate.png", outDir))
+		assert.Len(t, bitratePlots, 1, "Expecting one file for bitrate plot")
 
-	vmafPlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*vmaf.png", outDir))
-	assert.Len(t, vmafPlots, 1, "Expecting one file for VMAF plot")
+		vmafPlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*vmaf.png", outDir))
+		assert.Len(t, vmafPlots, 1, "Expecting one file for VMAF plot")
 
-	psnrPlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*psnr.png", outDir))
-	assert.Len(t, psnrPlots, 1, "Expecting one file for PSNR plot")
+		psnrPlots, _ := filepath.Glob(fmt.Sprintf("%s/*/*psnr.png", outDir))
+		assert.Len(t, psnrPlots, 1, "Expecting one file for PSNR plot")
+	})
 }
 
 /*************************************
@@ -165,6 +177,16 @@ func Test_RunApp_Run_WithInvalidApplicationConfig(t *testing.T) {
 	assert.ErrorAs(t, gotErr, &expErr, "Expecting error of type AppError")
 }
 
+func Test_RunApp_Run_MisalignedFrames(t *testing.T) {
+	plan := fixPlanConfigMisalignedFrames(t)
+	app := CreateRunCommand()
+	gotErr := app.Run([]string{"-plan", plan, "-out-dir", t.TempDir()})
+
+	var expErr *AppError
+	assert.ErrorAs(t, gotErr, &expErr, "Expecting error of type AppError")
+	assert.ErrorContains(t, gotErr, "VQM calculations had errors, see log for reasons")
+}
+
 // Functional tests for other sub-commands..
 func TestIntegration_AllSubcommands(t *testing.T) {
 	tempDir := t.TempDir()
@@ -176,7 +198,7 @@ func TestIntegration_AllSubcommands(t *testing.T) {
 	err := CreateRunCommand().Run([]string{"-plan", ePlan, "-out-dir", outDir})
 	require.NoError(t, err)
 
-	t.Run("Vqmplot should create plots", func(t *testing.T) {
+	t.Run("vqmplot should create plots", func(t *testing.T) {
 		var vqmFile string
 		// Need to get file with VQMs from encode stage.
 		m, _ := filepath.Glob(fmt.Sprintf("%s/*vqm.json", outDir))
@@ -193,7 +215,7 @@ func TestIntegration_AllSubcommands(t *testing.T) {
 		}
 	})
 
-	t.Run("Bitrate should create bitrate plot", func(t *testing.T) {
+	t.Run("bitrate should create bitrate plot", func(t *testing.T) {
 		var compressedFile string
 		// Need to get compressed file from encode stage.
 		m, _ := filepath.Glob(fmt.Sprintf("%s/*.mp4", outDir))
@@ -205,4 +227,19 @@ func TestIntegration_AllSubcommands(t *testing.T) {
 		assert.NoError(t, err, "Unexpected error running bitrate")
 		assert.FileExists(t, outFile, "bitrate plot file missing")
 	})
+
+	t.Run("new-plan should create plan template", func(t *testing.T) {
+		planFile := path.Join(t.TempDir(), "plan.json")
+		err := CreateNewPlanCommand().Run([]string{"-i", "video1.mp4", "-o", planFile})
+		assert.NoError(t, err)
+
+		b, err := os.ReadFile(planFile)
+		assert.NoError(t, err)
+		pc, err := encoding.NewPlanConfigFromJSON(b)
+		assert.NoError(t, err)
+
+		assert.Len(t, pc.Inputs, 1)
+		assert.Equal(t, pc.Inputs[0], "video1.mp4")
+		assert.True(t, len(pc.Schemes) > 0)
+	})
 }