diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6509ac0..ce543d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,8 @@ jobs: name: Set up Go uses: actions/setup-go@v3 with: - go-version-file: 'go.mod' + # go-version-file: 'go.mod' + go-version: v1.20.6 - name: install syft run: go install github.com/anchore/syft/cmd/syft@latest - diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a4bfe50..76e6f9b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,7 +17,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version-file: 'go.mod' + # go-version-file: 'go.mod' + go-version: v1.20.6 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: @@ -28,7 +29,7 @@ jobs: # working-directory: somedir # Optional: golangci-lint command line arguments. - args: --enable gosec + args: --enable gosec --timeout 3m0s # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true diff --git a/db/webpage-repository.go b/db/webpage-repository.go index 8830579..48ec8ce 100644 --- a/db/webpage-repository.go +++ b/db/webpage-repository.go @@ -31,17 +31,20 @@ func (repos *Repository) FindWebpageByURL(ctx context.Context, url string) (*mod } // InsertAPODData method inserts APOD data to database. -func (repos *Repository) InsertWebpage(ctx context.Context, data []model.Webpage) error { +func (repos *Repository) InsertWebpage(ctx context.Context, datalist []model.Webpage) error { if repos == nil { return errs.Wrap(ecode.ErrNullPointer) } - if err := repos.Db().WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if t := tx.Create(data); t.Error != nil { - return errs.Wrap(t.Error) + for _, data := range datalist { + data := data + if err := repos.Db().WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if t := tx.Create(&data); t.Error != nil { + return errs.Wrap(t.Error) + } + return nil + }); err != nil { + return err } - return nil - }); err != nil { - return err } return nil } diff --git a/dependency.png b/dependency.png index 86f2f7e..b954b7b 100644 Binary files a/dependency.png and b/dependency.png differ diff --git a/go.mod b/go.mod index 594f9dc..f3f9d4a 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.20 require ( github.com/PuerkitoBio/goquery v1.8.1 - github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705 - github.com/glebarez/sqlite v1.8.0 + github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65 + github.com/glebarez/sqlite v1.9.0 github.com/goark/errs v1.3.2 github.com/goark/errs/zapobject v0.1.3 github.com/goark/fetch v0.4.1 @@ -22,7 +22,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 - gorm.io/gorm v1.25.1 + gorm.io/gorm v1.25.2 moul.io/zapgorm2 v1.3.0 ) @@ -31,7 +31,7 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/glebarez/go-sqlite v1.21.1 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -94,8 +94,8 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect - modernc.org/libc v1.22.3 // indirect + modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.5.0 // indirect - modernc.org/sqlite v1.21.1 // indirect + modernc.org/sqlite v1.23.1 // indirect ) diff --git a/go.sum b/go.sum index 7a41002..33cbde0 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705 h1:MK11a0stoFLXKVbAlvsZUZPgf12skLJKNUEBnTTVyaE= -github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705/go.mod h1:oDI5NiD0XzShv5VITWyUJNP3pSh4prTDEzhKbkdKORA= +github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65 h1:0z5rF7oA9eqqoAFoiHTfioBK8U4hzMmyGMoajEqJVFE= +github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65/go.mod h1:oDI5NiD0XzShv5VITWyUJNP3pSh4prTDEzhKbkdKORA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -73,10 +73,10 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/glebarez/go-sqlite v1.21.1 h1:7MZyUPh2XTrHS7xNEHQbrhfMZuPSzhkm2A1qgg0y5NY= -github.com/glebarez/go-sqlite v1.21.1/go.mod h1:ISs8MF6yk5cL4n/43rSOmVMGJJjHYr7L2MbZZ5Q4E2E= -github.com/glebarez/sqlite v1.8.0 h1:02X12E2I/4C1n+v90yTqrjRa8yuo7c3KeHI3FRznCvc= -github.com/glebarez/sqlite v1.8.0/go.mod h1:bpET16h1za2KOOMb8+jCp6UBP/iahDpfPQqSaYLTLx8= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.9.0 h1:Aj6bPA12ZEx5GbSF6XADmCkYXlljPNUY+Zf1EQxynXs= +github.com/glebarez/sqlite v1.9.0/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -732,8 +732,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64= -gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= +gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -743,14 +743,14 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= -modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU= -modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/webpage/feed.go b/webpage/feed.go index f11f0e8..58765a0 100644 --- a/webpage/feed.go +++ b/webpage/feed.go @@ -40,7 +40,12 @@ func (cfg *Config) getNewDataList(ctx context.Context, items []*feed.Item) { if cfg.itemPool == nil { cfg.CreatePool() } + urls := map[string]bool{} for _, item := range items { + if urls[item.Link] { + continue + } + urls[item.Link] = true if page, err := cfg.find(ctx, item.Link); err != nil || page == nil { cfg.itemPool.putFeedItem(ctx, item) cfg.Logger().Debug("new item", zap.Any("item", item))