diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index f1b4cf06f..c15fa90cc 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -1,14 +1,14 @@ package douyin import ( + _ "embed" "encoding/json" "fmt" - "github.com/dop251/goja" "net/http" netURL "net/url" - "os" "strings" + "github.com/dop251/goja" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" @@ -22,6 +22,9 @@ func init() { extractors.Register("iesdouyin", e) } +//go:embed sign.js +var script string + type extractor struct{} // New returns a douyin extractor. @@ -58,13 +61,6 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor } itemId := itemIds[len(itemIds)-1] - // init JavaScripts runtime - vm := goja.New() - // read sign file - script, err := os.ReadFile("./script/douyin-sign.js") - if err != nil { - return nil, errors.WithStack(extractors.ErrSignScriptsRead) - } api := "https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=" + itemId // parse api query params string query, err := netURL.Parse(api) @@ -75,10 +71,16 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor headers := map[string]string{} headers["Cookie"] = "s_v_web_id=verify_leytkxgn_kvO5kOmO_SdMs_4t1o_B5ml_BUqtWM1mP6BF;" agent := "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66" + + // init JavaScripts runtime + vm := goja.New() // load sign scripts - _, _ = vm.RunString(string(script)) + _, _ = vm.RunString(script) // sign sign, err := vm.RunString(fmt.Sprintf("sign('%s', '%s')", query.RawQuery, agent)) + if err != nil { + return nil, errors.WithStack(err) + } api = fmt.Sprintf("%s&X-Bogus=%s", api, sign) jsonData, err := request.Get(api, url, headers) diff --git a/script/douyin-sign.js b/extractors/douyin/sign.js similarity index 100% rename from script/douyin-sign.js rename to extractors/douyin/sign.js diff --git a/extractors/errors.go b/extractors/errors.go index 97ff6b9b8..9b57c107c 100644 --- a/extractors/errors.go +++ b/extractors/errors.go @@ -9,5 +9,4 @@ var ( ErrURLParseFailed = errors.New("url parse failed") ErrInvalidRegularExpression = errors.New("invalid regular expression") ErrURLQueryParamsParseFailed = errors.New("url query params parse failed") - ErrSignScriptsRead = errors.New("sign scripts read failed") ) diff --git a/go.mod b/go.mod index 43e7ab8ab..3b7f2acff 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 github.com/PuerkitoBio/goquery v1.8.0 github.com/cheggaaa/pb/v3 v3.0.8 + github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c github.com/fatih/color v1.13.0 github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b github.com/itchyny/gojq v0.12.7 @@ -23,7 +24,6 @@ require ( github.com/bitly/go-simplejson v0.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect - github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect diff --git a/go.sum b/go.sum index 55bdf8f8f..3568a6f85 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,6 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20221115122301-6c0d9883792e h1:Uo51nR73BJlci20AE5tXT5qiLSGZy5LHnRlKt7VkcUM= -github.com/dop251/goja v0.0.0-20221115122301-6c0d9883792e/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c h1:/utv6nmTctV6OVgfk5+O6lEMEWL+6KJy4h9NZ5fnkQQ= github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=