Skip to content

Commit

Permalink
work
Browse files Browse the repository at this point in the history
  • Loading branch information
mcantelon committed May 16, 2024
1 parent 95b103e commit 70f536c
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 0 deletions.
4 changes: 4 additions & 0 deletions cmd/worker/workercmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ func (m *Main) Run(ctx context.Context) error {
activities.NewMetadataValidationActivity().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.MetadataValidationName},
)
w.RegisterActivityWithOptions(
activities.NewCombinePREMISActivity().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.CombinePREMISName},
)

Check warning on line 77 in cmd/worker/workercmd/cmd.go

View check run for this annotation

Codecov / codecov/patch

cmd/worker/workercmd/cmd.go#L74-L77

Added lines #L74 - L77 were not covered by tests
w.RegisterActivityWithOptions(
activities.NewSipCreationActivity().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.SipCreationName},
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.22.3

require (
github.com/artefactual-sdps/temporal-activities v0.0.0-20240513093038-77e9f8382ca9
github.com/beevik/etree v1.4.0
github.com/go-logr/logr v1.4.1
github.com/nyudlts/go-bagit v0.3.0-alpha
github.com/otiai10/copy v1.14.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/artefactual-sdps/temporal-activities v0.0.0-20240513093038-77e9f8382ca9 h1:JhybsG9MteMDlkXH/dHsENUon/8l9KY7VUc6F4XOOts=
github.com/artefactual-sdps/temporal-activities v0.0.0-20240513093038-77e9f8382ca9/go.mod h1:uf0jIGyZGHi3oTfhg+QkwCkyTaGhtAwromhwouC1FhU=
github.com/beevik/etree v1.4.0 h1:oz1UedHRepuY3p4N5OjE0nK1WLCqtzHf25bxplKOHLs=
github.com/beevik/etree v1.4.0/go.mod h1:cyWiXwGoasx60gHvtnEh5x8+uIjUVnjWqBvEnhnqKDA=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
Expand Down
5 changes: 5 additions & 0 deletions hack/sampledata/xsd/empty_premis.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<premis version="3.0"
xmlns="http://www.loc.gov/premis/v3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/premis/v3 http://www.loc.gov/standards/premis/premis.xsd">
</premis>
138 changes: 138 additions & 0 deletions internal/activities/combine_premis.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package activities

import (
etree "github.com/beevik/etree"
"context"
"errors"
"os"
"path"
"strings"
)

const CombinePREMISName = "combine-premis"

type CombinePREMISActivity struct{}

func NewCombinePREMISActivity() *CombinePREMISActivity {
return &CombinePREMISActivity{}

Check warning on line 17 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L16-L17

Added lines #L16 - L17 were not covered by tests
}

type CombinePREMISParams struct {
Path string
}

type CombinePREMISResult struct {
Out string
}

func (md *CombinePREMISActivity) Execute(
ctx context.Context,
params *CombinePREMISParams,
) (*CombinePREMISResult, error) {

Check warning on line 31 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L31

Added line #L31 was not covered by tests
// Get transfer's PREMIS file paths
file_paths, err := CombinePREMISGetPaths(params.Path)
if err != nil {
return nil, err

Check warning on line 35 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L33-L35

Added lines #L33 - L35 were not covered by tests
}

// Write elements from transfer's PREMIS files to combined PREMIS file
for _, file_path := range file_paths {
err := CombinePREMISCopy(file_path, "metadata/premis.xml")
if err != nil {
return nil, err

Check warning on line 42 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L39-L42

Added lines #L39 - L42 were not covered by tests
}
}

res := &CombinePREMISResult{}
res.Out = "OK"
return res, nil

Check warning on line 48 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L46-L48

Added lines #L46 - L48 were not covered by tests
}

func CombinePREMISGetPaths(transfer_dir string) ([]string, error) {
content_subdir := "content"
file_dir := path.Join(transfer_dir, content_subdir)

Check warning on line 53 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L51-L53

Added lines #L51 - L53 were not covered by tests

dir_items, err := os.ReadDir(file_dir)
if err != nil {
return nil, err

Check warning on line 57 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L55-L57

Added lines #L55 - L57 were not covered by tests
}

file_paths := []string{}
for _, dir_item := range dir_items {
if dir_item.IsDir() {
subdir := path.Join(transfer_dir, content_subdir, dir_item.Name())

Check warning on line 63 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L60-L63

Added lines #L60 - L63 were not covered by tests

sub_items, err := os.ReadDir(subdir)
if err != nil {
return nil, err

Check warning on line 67 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L65-L67

Added lines #L65 - L67 were not covered by tests
}

for _, subdir_item := range sub_items {
if !subdir_item.IsDir() {
if strings.HasSuffix(strings.ToLower(subdir_item.Name()), "_premis.xml") {
file_paths = append(file_paths, path.Join(subdir, subdir_item.Name()))

Check warning on line 73 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L70-L73

Added lines #L70 - L73 were not covered by tests
}
}
}
}
}

return file_paths, nil

Check warning on line 80 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L80

Added line #L80 was not covered by tests
}

func CombinePREMISCopy(source_filepath string, destination_filepath string) error {

Check warning on line 83 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L83

Added line #L83 was not covered by tests
// Parse source document and get root PREMIS element
source_doc := etree.NewDocument()

Check warning on line 85 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L85

Added line #L85 was not covered by tests

if err := source_doc.ReadFromFile(source_filepath); err != nil {
return err

Check warning on line 88 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L87-L88

Added lines #L87 - L88 were not covered by tests
}

source_premis_element := source_doc.FindElement("/premis")
if source_premis_element == nil {
return errors.New("no root premis element found in source document")

Check warning on line 93 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L91-L93

Added lines #L91 - L93 were not covered by tests
}

// Read source child PREMIS elements
source_premis_object_elements := source_premis_element.FindElements("object")
source_premis_event_elements := source_premis_element.FindElements("event")
source_premis_agent_elements := source_premis_element.FindElements("agent")

Check warning on line 99 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L97-L99

Added lines #L97 - L99 were not covered by tests

// Parse destination document and get root PREMIS element
dest_doc := etree.NewDocument()
if err := dest_doc.ReadFromFile(destination_filepath); err != nil {
return err

Check warning on line 104 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L102-L104

Added lines #L102 - L104 were not covered by tests
}

dest_premis_element := dest_doc.FindElement("/premis")
if dest_premis_element == nil {
return errors.New("no root premis element found in destination document")

Check warning on line 109 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L107-L109

Added lines #L107 - L109 were not covered by tests
}

// Update PREMIS originalname child elements of PREMIS object elements
for _, premis_object_element := range source_premis_object_elements {
objectname_element := premis_object_element.FindElement("originalName")
if objectname_element != nil {
objectname_element.SetText("data/" + objectname_element.Text())

Check warning on line 116 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L113-L116

Added lines #L113 - L116 were not covered by tests
}
}

// Write destination child PREMIS elements
CombinePREMISAddChildElements(dest_premis_element, source_premis_object_elements)
CombinePREMISAddChildElements(dest_premis_element, source_premis_event_elements)
CombinePREMISAddChildElements(dest_premis_element, source_premis_agent_elements)

Check warning on line 123 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L121-L123

Added lines #L121 - L123 were not covered by tests

dest_doc.Indent(2)
err := dest_doc.WriteToFile(destination_filepath)
if err != nil {
return err

Check warning on line 128 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L125-L128

Added lines #L125 - L128 were not covered by tests
}

return nil

Check warning on line 131 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L131

Added line #L131 was not covered by tests
}

func CombinePREMISAddChildElements(parent_element *etree.Element, new_child_elements []*etree.Element) {
for _, child_element := range new_child_elements {
parent_element.AddChild(child_element)

Check warning on line 136 in internal/activities/combine_premis.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/combine_premis.go#L134-L136

Added lines #L134 - L136 were not covered by tests
}
}
11 changes: 11 additions & 0 deletions internal/workflow/preprocessing.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,17 @@ func (w *PreprocessingWorkflow) Execute(
return nil, e
}

// Combine PREMIS files into one.
var combinePREMIS activities.CombinePREMISResult
e = temporalsdk_workflow.ExecuteActivity(
withLocalActOpts(ctx),
activities.CombinePREMISName,
&activities.CombinePREMISParams{Path: localPath},
).Get(ctx, &combinePREMIS)
if e != nil {
return nil, e

Check warning on line 143 in internal/workflow/preprocessing.go

View check run for this annotation

Codecov / codecov/patch

internal/workflow/preprocessing.go#L143

Added line #L143 was not covered by tests
}

// Remove PREMIS XML files.
var removeFilesResult removefiles.ActivityResult
e = temporalsdk_workflow.ExecuteActivity(
Expand Down
11 changes: 11 additions & 0 deletions internal/workflow/preprocessing_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ func (s *PreprocessingTestSuite) SetupTest(cfg config.Configuration) {
activities.NewTransformVecteurAIPActivity().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.TransformVecteurAIPName},
)
s.env.RegisterActivityWithOptions(
activities.NewCombinePREMISActivity().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.CombinePREMISName},
)
s.env.RegisterActivityWithOptions(
removefiles.NewActivity(removefiles.Config{}).Execute,
temporalsdk_activity.RegisterOptions{Name: removefiles.ActivityName},
Expand Down Expand Up @@ -174,6 +178,13 @@ func (s *PreprocessingTestSuite) TestVecteurAIP() {
).Return(
&activities.TransformVecteurAIPResult{}, nil,
)
s.env.OnActivity(
activities.CombinePREMISName,
sessionCtx,
&activities.CombinePREMISParams{Path: sipPath},
).Return(
&activities.CombinePREMISResult{}, nil,
)
s.env.OnActivity(
removefiles.ActivityName,
sessionCtx,
Expand Down

0 comments on commit 70f536c

Please sign in to comment.