add handling for twitter api error

downloading/features.go

@@ -55,9 +55,6 @@ func downloadTweetMedia(ctx context.Context, client *resty.Client, dir string, t
 		if err != nil {
 			return err
 		}
-		if err := utils.CheckRespStatus(resp); err != nil {
-			return err
-		}
 
 		mutex.Lock()
 		path, err := utils.UniquePath(filepath.Join(dir, text+ext))
@@ -501,11 +498,11 @@ func BatchUserDownload(ctx context.Context, client *resty.Client, db *sqlx.DB, u
 
 			user := uidToUser[entity.Uid()]
 			tweets, err := user.GetMeidas(ctx, client, &utils.TimeRange{Min: entity.LatestReleaseTime()})
-			if utils.IsStatusCode(err, 429) {
-				// json 版本的响应 {"errors":[{"code":88,"message":"Rate limit exceeded."}]} 代表达到看帖上限
-				// text 版本的响应 Rate limit exceeded. 代表暂时达到速率限制
-				v := err.(*utils.HttpStatusError)
-				if v.Msg[0] == '{' && v.Msg[len(v.Msg)-1] == '}' {
+			if v, ok := err.(*twitter.TwitterApiError); ok {
+				if v.Code == twitter.ErrDependency {
+					cancel(fmt.Errorf("maybe account is locked"))
+					continue
+				} else if v.Code == twitter.ErrExceedPostLimit {
 					cancel(fmt.Errorf("reached the limit for seeing posts today"))
 					continue
 				}

internal/utils/http.go

@@ -8,7 +8,7 @@ import (
 )
 
 func CheckRespStatus(resp *resty.Response) error {
-	if resp.StatusCode() != 200 {
+	if resp.StatusCode() >= 400 {
 		return &HttpStatusError{Code: resp.StatusCode(), Msg: resp.String()}
 	}
 	return nil
@@ -37,7 +37,7 @@ type HttpStatusError struct {
 }
 
 func (err *HttpStatusError) Error() string {
-	return fmt.Sprintf("%d %s", err.Code, err.Msg)
+	return fmt.Sprintf("HTTP Error: %d %s", err.Code, err.Msg)
 }
 
 func IsStatusCode(err error, code int) bool {

main.go

@@ -373,7 +373,7 @@ func connectDatabase(path string) (*sqlx.DB, error) {
 		return nil, err
 	}
 
-	dsn := fmt.Sprintf("file:%s?cache=shared&_journal_mode=WAL&busy_timeout=10000", path)
+	dsn := fmt.Sprintf("file:%s?cache=shared&_journal_mode=WAL&busy_timeout=100000", path)
 	db, err := sqlx.Connect("sqlite3", dsn)
 	if err != nil {
 		return nil, err

twitter/api.go

@@ -3,9 +3,6 @@ package twitter
 import (
 	"fmt"
 	"net/url"
-
-	"github.com/go-resty/resty/v2"
-	"github.com/tidwall/gjson"
 )
 
 const HOST = "https://x.com"
@@ -184,11 +181,3 @@ func (l *likes) QueryParam() url.Values {
 func (l *likes) SetCursor(cursor string) {
 	l.cursor = cursor
 }
-
-func CheckApiResp(resp *resty.Response) error {
-	errors := gjson.GetBytes(resp.Body(), "errors")
-	if errors.Exists() {
-		return fmt.Errorf(errors.String())
-	}
-	return nil
-}

twitter/client.go

@@ -47,19 +47,34 @@ func Login(ctx context.Context, authToken string, ct0 string) (*resty.Client, st
 	// 鉴权
 	SetClientAuth(client, authToken, ct0)
 
+	// 错误检查
+	client.OnAfterResponse(func(c *resty.Client, r *resty.Response) error {
+		if err := CheckApiResp(r.Body()); err != nil {
+			return err
+		}
+		if err := utils.CheckRespStatus(r); err != nil {
+			return err
+		}
+		return nil
+	})
+
 	// 重试
 	client.SetRetryCount(5)
-	client.AddRetryAfterErrorCondition()
 	client.AddRetryCondition(func(r *resty.Response, err error) bool {
-		return !strings.HasSuffix(r.Request.RawRequest.Host, "twimg.com") && err != nil
+		// For TCP Error
+		_, ok := err.(*TwitterApiError)
+		_, ok2 := err.(*utils.HttpStatusError)
+		return !ok && !ok2 && err != nil && !strings.HasSuffix(r.Request.RawRequest.Host, "twimg.com")
 	})
 	client.AddRetryCondition(func(r *resty.Response, err error) bool {
-		// For OverCapacity
-		return r.Request.RawRequest.Host == "x.com" && r.StatusCode() == 400
+		// For Twitter API Error
+		v, ok := err.(*TwitterApiError)
+		return ok && r.Request.RawRequest.Host == "x.com" && (v.Code == ErrTimeout || v.Code == ErrOverCapacity)
 	})
 	client.AddRetryCondition(func(r *resty.Response, err error) bool {
-		// 仅重试 429 Rate Limit Exceed
-		return r.Request.RawRequest.Host == "x.com" && r.StatusCode() == 429 && CheckApiResp(r) == nil
+		// For Http 429
+		v, ok := err.(*utils.HttpStatusError)
+		return ok && r.Request.RawRequest.Host == "x.com" && v.Code == 429
 	})
 
 	client.SetTransport(&http.Transport{

twitter/errors.go

@@ -0,0 +1,39 @@
+package twitter
+
+import (
+	"github.com/tidwall/gjson"
+)
+
+const (
+	ErrTimeout         = 29
+	ErrDependency      = 0
+	ErrExceedPostLimit = 88
+	ErrOverCapacity    = 130
+)
+
+func CheckApiResp(body []byte) error {
+	errors := gjson.GetBytes(body, "errors")
+	if !errors.Exists() {
+		return nil
+	}
+
+	codej := errors.Get("0.code")
+	code := -1
+	if codej.Exists() {
+		code = int(codej.Int())
+	}
+	return NewTwitterApiError(code, string(body))
+}
+
+type TwitterApiError struct {
+	Code int
+	raw  string
+}
+
+func (err *TwitterApiError) Error() string {
+	return err.raw
+}
+
+func NewTwitterApiError(code int, raw string) *TwitterApiError {
+	return &TwitterApiError{Code: code, raw: raw}
+}

twitter/list.go

@@ -7,7 +7,6 @@ import (
 	"github.com/go-resty/resty/v2"
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
-	"github.com/unkmonster/tmd2/internal/utils"
 )
 
 type ListBase interface {
@@ -33,11 +32,6 @@ func GetLst(ctx context.Context, client *resty.Client, id uint64) (*List, error)
 		return nil, err
 	}
 
-	err = utils.CheckRespStatus(resp)
-	if err != nil {
-		return nil, err
-	}
-
 	list := gjson.GetBytes(resp.Body(), "data.list")
 	return parseList(&list)
 }

twitter/timeline.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/go-resty/resty/v2"
 	"github.com/tidwall/gjson"
-	"github.com/unkmonster/tmd2/internal/utils"
 )
 
 const (
@@ -92,12 +91,6 @@ func getTimelineResp(ctx context.Context, api timelineApi, client *resty.Client)
 	if err != nil {
 		return nil, err
 	}
-	if err = utils.CheckRespStatus(resp); err != nil {
-		return nil, err
-	}
-	if err = CheckApiResp(resp); err != nil {
-		return nil, err
-	}
 	return resp.Body(), nil
 }
 

twitter/twitter_test.go

@@ -284,3 +284,57 @@ func TestGetMember(t *testing.T) {
 // 		return
 // 	}
 // }
+
+func TestApiError(t *testing.T) {
+	resp := `{
+  "errors": [
+    {
+      "message": "Dependency: Unspecified",
+      "locations": [
+        {
+          "line": 12,
+          "column": 11
+        }
+      ],
+      "path": [
+        "user",
+        "result",
+        "timeline_v2",
+        "timeline"
+      ],
+      "extensions": {
+        "name": "DependencyError",
+        "source": "Server",
+        "retry_after": 0,
+        "code": 0,
+        "kind": "Operational",
+        "tracing": {
+          "trace_id": "8110de4fb50877e1"
+        }
+      },
+      "code": 0,
+      "kind": "Operational",
+      "name": "DependencyError",
+      "source": "Server",
+      "retry_after": 0,
+      "tracing": {
+        "trace_id": "8110de4fb50877e1"
+      }
+    }
+  ],
+  "data": {
+    "user": {
+      "result": {
+        "__typename": "User",
+        "timeline_v2": {}
+      }
+    }
+  }
+}`
+
+	err := CheckApiResp([]byte(resp))
+	if err == nil {
+		t.Errorf("err = nil, want TwitterApiError")
+		return
+	}
+}

twitter/user.go

@@ -54,9 +54,6 @@ func getUser(ctx context.Context, client *resty.Client, url string) (*User, erro
 	if err != nil {
 		return nil, err
 	}
-	if err := utils.CheckRespStatus(resp); err != nil {
-		return nil, err
-	}
 	return parseRespJson(resp.Body())
 }