MM-23110 CSV Exporter

go.mod

@@ -1,10 +1,11 @@
-module github.com/mattermost/mattermost-plugin-starter-template
+module github.com/mattermost/mattermost-plugin-channel-export
 
-go 1.12
+go 1.14
 
 require (
 	github.com/mattermost/mattermost-plugin-api v0.0.9
 	github.com/mattermost/mattermost-server/v5 v5.3.2-0.20200313113657-e2883bfe5f37
 	github.com/mholt/archiver/v3 v3.3.0
 	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.5.1
 )

server/command_hooks.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 
@@ -65,9 +64,14 @@ func (p *Plugin) executeCommandExport(args *model.CommandArgs) *model.CommandRes
 		}
 	}
 
+	exporter := CSVExporter{}
+	fileName := exporter.FileName(channelToExport.Name)
+
+	exportedFileReader, exportedFileWriter := io.Pipe()
 	go func() {
-		fileName := fmt.Sprintf("%d_%s.json", time.Now().Unix(), channelToExport.Name)
-		fileContents, err := p.exportChannel(channelToExport)
+		defer exportedFileWriter.Close()
+
+		err := exporter.Export(p.channelPostsIterator(channelToExport), exportedFileWriter)
 		if err != nil {
 			p.client.Post.CreatePost(&model.Post{
 				UserId:    p.botID,
@@ -77,8 +81,10 @@ func (p *Plugin) executeCommandExport(args *model.CommandArgs) *model.CommandRes
 
 			return
 		}
+	}()
 
-		file, err := p.uploadExportedChannelTo(fileName, fileContents, args.UserId)
+	go func() {
+		file, err := p.uploadFileTo(fileName, exportedFileReader, channelDM.Id)
 		if err != nil {
 			p.client.Post.CreatePost(&model.Post{
 				UserId:    p.botID,
@@ -104,17 +110,11 @@ func (p *Plugin) executeCommandExport(args *model.CommandArgs) *model.CommandRes
 	}
 }
 
-// Export an empty JSON file for now. The actual implementation will come later.
-func (p *Plugin) exportChannel(channel *model.Channel) (io.Reader, error) {
-	fileContents := strings.NewReader("{}")
-	return fileContents, nil
-}
-
-func (p *Plugin) uploadExportedChannelTo(fileName string, fileContents io.Reader, receiverID string) (*model.FileInfo, error) {
-	file, err := p.client.File.Upload(fileContents, fileName, receiverID)
+func (p *Plugin) uploadFileTo(fileName string, contents io.Reader, channelID string) (*model.FileInfo, error) {
+	file, err := p.client.File.Upload(contents, fileName, channelID)
 	if err != nil {
 		p.client.Log.Error("unable to upload the exported file to the channel",
-			"Channel ID", receiverID, "Error", err)
+			"Channel ID", channelID, "Error", err)
 		return nil, fmt.Errorf("unable to upload the exported file")
 	}
 

server/csvExporter.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strconv"
+)
+
+// CSVExporter exports all the posts in a channel to a chronollogically
+// ordered file in CSV format
+type CSVExporter struct{}
+
+// FileName returns the passed name with the .csv extension added
+func (e *CSVExporter) FileName(name string) string {
+	return fmt.Sprintf("%s.csv", name)
+}
+
+// Export consumes all the posts returned by the iterator and writes them in
+// CSV format to the writer
+func (e *CSVExporter) Export(nextPosts PostIterator, writer io.Writer) error {
+	csvWriter := csv.NewWriter(writer)
+	err := csvWriter.Write([]string{
+		"Post Creation Time",
+		"User Id",
+		"User Email",
+		"User Type",
+		"User Name",
+		"Post Id",
+		"Parent Post Id",
+		"Post Message",
+		"Post Type",
+	})
+
+	if err != nil {
+		return fmt.Errorf("Unable to create a CSV file: %w", err)
+	}
+
+	for {
+		posts, err := nextPosts()
+		if err != nil {
+			return fmt.Errorf("unable to retrieve next posts: %w", err)
+		}
+
+		for _, post := range posts {
+			csvWriter.Write([]string{
+				strconv.FormatInt(post.CreateAt, 10),
+				post.UserID,
+				post.UserEmail,
+				post.UserType,
+				post.UserName,
+				post.ID,
+				post.ParentPostID,
+				post.Message,
+				post.Type,
+			})
+		}
+
+		if len(posts) == 0 {
+			break
+		}
+
+	}
+
+	csvWriter.Flush()
+
+	return nil
+}

server/csvExporter_test.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileName(t *testing.T) {
+	exporter := CSVExporter{}
+
+	testCases := []struct {
+		testName         string
+		name             string
+		expectedFilename string
+	}{
+		{"Empty name", "", ".csv"},
+		{"Normal name", "name", "name.csv"},
+		{"Name with unicode chars", "αβ", "αβ.csv"},
+		{"Name with digits", "1", "1.csv"},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.testName, func(*testing.T) {
+			require.Equal(t, exporter.FileName(test.name), test.expectedFilename)
+		})
+	}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+
+	return b
+}
+
+var exportedPost = &ExportedPost{
+	CreateAt:     1,
+	UserID:       "dummyUserID",
+	UserEmail:    "[email protected]",
+	UserType:     "user",
+	UserName:     "dummy",
+	ID:           "dummyPostID",
+	ParentPostID: "",
+	Message:      "Lorem ipsum",
+	Type:         "message",
+}
+
+func exportedPostToCSV(post *ExportedPost) string {
+	fields := []string{
+		strconv.FormatInt(post.CreateAt, 10),
+		post.UserID,
+		post.UserEmail,
+		post.UserType,
+		post.UserName,
+		post.ID,
+		post.ParentPostID,
+		post.Message,
+		post.Type,
+	}
+	return strings.Join(fields, ",") + "\n"
+}
+
+func TestExport(t *testing.T) {
+	header := []string{
+		"Post Creation Time",
+		"User Id",
+		"User Email",
+		"User Type",
+		"User Name",
+		"Post Id",
+		"Parent Post Id",
+		"Post Message",
+		"Post Type",
+	}
+	headerCSV := strings.Join(header, ",") + "\n"
+
+	genIterator := func(numPosts, batchSize int) PostIterator {
+		sent := 0
+		return func() ([]*ExportedPost, error) {
+			if sent >= numPosts {
+				return nil, nil
+			}
+
+			length := min(numPosts-sent, batchSize)
+
+			posts := make([]*ExportedPost, length)
+			for i := 0; i < length; i++ {
+				posts[i] = exportedPost
+			}
+
+			sent += length
+			return posts, nil
+		}
+	}
+
+	exporter := CSVExporter{}
+
+	t.Run("Empty iterator", func(t *testing.T) {
+		var actualString strings.Builder
+
+		err := exporter.Export(genIterator(0, 0), &actualString)
+
+		require.Nil(t, err)
+		require.Equal(t, headerCSV, actualString.String())
+	})
+
+	t.Run("One post", func(t *testing.T) {
+		var actualString strings.Builder
+
+		err := exporter.Export(genIterator(1, 1), &actualString)
+
+		require.Nil(t, err)
+		require.Equal(t, headerCSV+exportedPostToCSV(exportedPost), actualString.String())
+	})
+
+	t.Run("Several posts", func(t *testing.T) {
+		var actualString strings.Builder
+
+		err := exporter.Export(genIterator(10, 4), &actualString)
+
+		expected := headerCSV
+		for i := 0; i < 10; i++ {
+			expected += exportedPostToCSV(exportedPost)
+		}
+
+		require.Nil(t, err)
+		require.Equal(t, expected, actualString.String())
+	})
+
+	t.Run("Wrong iterator", func(t *testing.T) {
+		var actualString strings.Builder
+
+		err := exporter.Export(
+			func() ([]*ExportedPost, error) { return nil, fmt.Errorf("Forcing an error") },
+			&actualString,
+		)
+
+		require.Error(t, err)
+	})
+}

server/exporter.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/pkg/errors"
+)
+
+// PostIterator returns the next batch of posts when called
+type PostIterator func() ([]*ExportedPost, error)
+
+// Exporter processes a list of posts and writes them to a writer
+type Exporter interface {
+	// FileName returns the name of the exported file, given the core name passed
+	FileName(name string) string
+
+	// Export processes the posts returned by nextPosts and exports them to writer
+	Export(nextPosts PostIterator, writer io.Writer) error
+}
+
+// ExportedPost contains all the information from a post needed in
+// an export, with all the relevant information already resolved
+type ExportedPost struct {
+	CreateAt     int64
+	UserID       string
+	UserEmail    string
+	UserType     string
+	UserName     string
+	ID           string
+	ParentPostID string
+	Message      string
+	Type         string
+}
+
+// channelPostsIterator returns a function that returns, every time it is
+// called, a new batch of posts from the channel, chronollogically ordered
+// (most recent first), until all posts have been consumed.
+func (p *Plugin) channelPostsIterator(channel *model.Channel) PostIterator {
+	page := 0
+	perPage := 100
+	return func() ([]*ExportedPost, error) {
+		postList, err := p.client.Post.GetPostsForChannel(channel.Id, page, perPage)
+		if err != nil {
+			return nil, err
+		}
+
+		exportedPostList := make([]*ExportedPost, 0, len(postList.Order))
+		for _, key := range postList.Order {
+			post := postList.Posts[key]
+
+			// Ignore posts that have been edited; exporting only what's visible in the channel
+			if post.OriginalId != "" {
+				continue
+			}
+
+			exportedPost, err := p.toExportedPost(post)
+			if err != nil {
+				return nil, fmt.Errorf("Unable to export post: %w", err)
+			}
+
+			exportedPostList = append(exportedPostList, exportedPost)
+		}
+
+		page++
+		return exportedPostList, nil
+	}
+}
+
+// toExportedPost resolves all the data from post that is needed in
+// ExportedPost, as the user information and the type of message
+func (p *Plugin) toExportedPost(post *model.Post) (*ExportedPost, error) {
+	user, err := p.client.User.Get(post.UserId)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed retrieving post's author information")
+	}
+
+	userType := "user"
+	if user.IsBot {
+		userType = "bot"
+	}
+
+	postType := "message"
+	if post.Type != "" {
+		postType = post.Type
+		userType = "system"
+	}
+
+	return &ExportedPost{
+		CreateAt:     post.CreateAt,
+		UserID:       post.UserId,
+		UserEmail:    user.Email,
+		UserType:     userType,
+		UserName:     user.Nickname,
+		ID:           post.Id,
+		ParentPostID: post.ParentId,
+		Message:      post.Message,
+		Type:         postType,
+	}, nil
+}