Skip to content

Commit

Permalink
Add Bagit validation (#93)
Browse files Browse the repository at this point in the history 
[skip codecov]
  • Loading branch information
@mcantelon
mcantelon committed Dec 18, 2024
1 parent a682cec commit 208b73c
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 1 deletion.
4 changes: 4 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.23.2

require (
github.com/antchfx/xmlquery v1.4.2
github.com/artefactual-labs/bagit-gython v0.2.0
github.com/artefactual-sdps/temporal-activities v0.0.0-20241105002718-bc4a9d85ce42
github.com/beevik/etree v1.4.0
github.com/deckarep/golang-set/v2 v2.6.0
Expand Down Expand Up @@ -58,6 +59,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kluctl/go-embed-python v0.0.0-3.12.3-20240415-1 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/nyudlts/go-bagit v0.3.0-alpha.0.20240515212815-8dab411c23af // indirect
Expand All @@ -71,10 +73,12 @@ require (
github.com/richardlehane/msoleps v1.0.3 // indirect
github.com/richardlehane/xmldetect v1.0.2 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/ross-spencer/spargo v0.4.1 // indirect
github.com/ross-spencer/wikiprov v0.2.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ github.com/antchfx/xmlquery v1.4.2 h1:MZKd9+wblwxfQ1zd1AdrTsqVaMjMCwow3IqkCSe00K
github.com/antchfx/xmlquery v1.4.2/go.mod h1:QXhvf5ldTuGqhd1SHNvvtlhhdQLks4dD0awIVhXIDTA=
github.com/antchfx/xpath v1.3.2 h1:LNjzlsSjinu3bQpw9hWMY9ocB80oLOWuQqFvO6xt51U=
github.com/antchfx/xpath v1.3.2/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/artefactual-labs/bagit-gython v0.2.0 h1:Zje4Lb1goZVUPoxpc/k65sWtYpNgK9Rvphvaok5cYzE=
github.com/artefactual-labs/bagit-gython v0.2.0/go.mod h1:C+hFZQMDnji1hjGt3nrlMK3BahaBhvo/hU2uqd+Q9Z4=
github.com/artefactual-sdps/temporal-activities v0.0.0-20241105002718-bc4a9d85ce42 h1:0Ymucvkou8aiZkQrVgZsTODGeGoQHVNV414IFOFRxX0=
github.com/artefactual-sdps/temporal-activities v0.0.0-20241105002718-bc4a9d85ce42/go.mod h1:hV4rUdJ8FYqlEkvT0OqWuHj8DNc1v9j5/Dv8VsrYXLU=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
Expand Down Expand Up @@ -92,6 +94,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -154,6 +158,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kluctl/go-embed-python v0.0.0-3.12.3-20240415-1 h1:IX2O3LJUL0AjYsROGZ4aNENTEb3c/Ll/0b9Yb/8P61Q=
github.com/kluctl/go-embed-python v0.0.0-3.12.3-20240415-1/go.mod h1:9kqX8IjRCNh4ppXxlKGtLN+QFuvsdSsNGKsTLgdSNRw=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
Expand Down Expand Up @@ -210,6 +216,8 @@ github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgY
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
Expand Down Expand Up @@ -345,6 +353,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
20 changes: 19 additions & 1 deletion internal/workflow/preprocessing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

"github.com/artefactual-sdps/temporal-activities/bagcreate"
"github.com/artefactual-sdps/temporal-activities/bagvalidate"
"github.com/artefactual-sdps/temporal-activities/ffvalidate"
"github.com/artefactual-sdps/temporal-activities/xmlvalidate"
"go.artefactual.dev/tools/temporal"
Expand Down Expand Up @@ -124,7 +125,24 @@ func (w *PreprocessingWorkflow) Execute(

// Unbag the SIP if it is a bag.
if isBag.IsBag {
ev := result.newEvent(ctx, "Unbag SIP")
ev := result.newEvent(ctx, "Validate Bag")
var bagValidateResult bagvalidate.Result
e = temporalsdk_workflow.ExecuteActivity(
withFilesysActOpts(ctx),
bagvalidate.Name,
&bagvalidate.Params{Path: localPath},
).Get(ctx, &bagValidateResult)
if e != nil {
result.systemError(ctx, e, ev, "Error attempting to validate the Bag")
return result, nil
}
if bagValidateResult.Error != "" {
result.validationError(ctx, ev, "Bag validation has failed", []string{bagValidateResult.Error})
} else {
ev.Succeed(ctx, "Bag validated")
}

ev = result.newEvent(ctx, "Unbag SIP")
var unbagResult activities.UnbagResult
e = temporalsdk_workflow.ExecuteActivity(
withFilesysActOpts(ctx),
Expand Down
24 changes: 24 additions & 0 deletions internal/workflow/preprocessing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"testing"
"time"

bagit_gython "github.com/artefactual-labs/bagit-gython"
"github.com/artefactual-sdps/temporal-activities/bagcreate"
"github.com/artefactual-sdps/temporal-activities/bagvalidate"
"github.com/artefactual-sdps/temporal-activities/ffvalidate"
"github.com/artefactual-sdps/temporal-activities/xmlvalidate"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -135,6 +137,14 @@ func (s *PreprocessingTestSuite) SetupTest(cfg *config.Configuration) {
s.sipPath = sp

// Register activities.
validator, err := bagit_gython.NewBagIt()
if err != nil {
s.T().Fatalf("initialize bagit: %v", err)
}
s.env.RegisterActivityWithOptions(
bagvalidate.New(validator).Execute,
temporalsdk_activity.RegisterOptions{Name: bagvalidate.Name},
)
s.env.RegisterActivityWithOptions(
activities.NewUnbag().Execute,
temporalsdk_activity.RegisterOptions{Name: activities.UnbagName},
Expand Down Expand Up @@ -266,6 +276,13 @@ func (s *PreprocessingTestSuite) TestPreprocessingWorkflowSuccess() {
).Return(
&localact.IsBagResult{IsBag: true}, nil,
)
s.env.OnActivity(
bagvalidate.Name,
sessionCtx,
&bagvalidate.Params{Path: s.sipPath},
).Return(
&bagvalidate.Result{Valid: true}, nil,
)
s.env.OnActivity(
activities.UnbagName,
sessionCtx,
Expand Down Expand Up @@ -425,6 +442,13 @@ func (s *PreprocessingTestSuite) TestPreprocessingWorkflowSuccess() {
Outcome: workflow.OutcomeSuccess,
RelativePath: relPath,
PreservationTasks: []*eventlog.Event{
{
Name: "Validate Bag",
Message: "Bag validated",
Outcome: enums.EventOutcomeSuccess,
StartedAt: testTime,
CompletedAt: testTime,
},
{
Name: "Unbag SIP",
Message: "SIP unbagged",
Expand Down

0 comments on commit 208b73c

Please sign in to comment.