update mocks; add tests for liked_tweets method; update readme

.github/workflows/test.yml

@@ -2,6 +2,7 @@ name: test
 
 on:
   push:
+  pull_request:
 
 env:
   PIP_ROOT_USER_ACTION: ignore

Makefile

@@ -1,7 +1,7 @@
 all:
 	@echo "hi"
 
-install:
+deps:
 	@pip install -e .[dev]
 
 build:
@@ -56,14 +56,16 @@ test-sq-matrix:
 	@make test-sq y=2023 v=3440000
 
 update-mocks:
-	twscrape user_by_id --raw 2244994945 | jq > ./tests/mocked-data/user_by_id_raw.json
-	twscrape user_by_login --raw xdevelopers | jq > ./tests/mocked-data/user_by_login_raw.json
-	twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/followers_raw.json
-	twscrape following --raw --limit 10  2244994945 | jq > ./tests/mocked-data/following_raw.json
-	twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/tweet_details_raw.json
-	twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/retweeters_raw.json
-	twscrape favoriters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/favoriters_raw.json
-	twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/user_tweets_raw.json
-	twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/user_tweets_and_replies_raw.json
-	twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/search_raw.json
-	twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/list_timeline_raw.json
+	@rm -rf ./tests/mocked-data/raw_*.json
+	twscrape user_by_id --raw 2244994945 | jq > ./tests/mocked-data/raw_user_by_id.json
+	twscrape user_by_login --raw xdevelopers | jq > ./tests/mocked-data/raw_user_by_login.json
+	twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_followers.json
+	twscrape following --raw --limit 10  2244994945 | jq > ./tests/mocked-data/raw_following.json
+	twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_details.json
+	twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_retweeters.json
+	twscrape favoriters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_favoriters.json
+	twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets.json
+	twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets_and_replies.json
+	twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/raw_search.json
+	twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/raw_list_timeline.json
+	twscrape likes --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_likes.json

_get_gql_ops.py

@@ -7,21 +7,9 @@
 
 client = httpx.Client(headers={"user-agent": UserAgent().chrome})
 
-ops = """
-SearchTimeline
-UserByRestId
-UserByScreenName
-TweetDetail
-Followers
-Following
-Retweeters
-Favoriters
-UserTweets
-UserTweetsAndReplies
-ListLatestTweetsTimeline
-"""
-
-ops = [op.strip() for op in ops.split("\n") if op.strip()]
+with open("./twscrape/api.py") as fp:
+    ops = [x.strip() for x in fp.read().split("\n")]
+    ops = [x.split("=")[0].removeprefix("OP_").strip() for x in ops if x.startswith("OP_")]
 
 
 def script_url(k: str, v: str):

pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-  "pyright>=1.1.344",
+  "pyright>=1.1.350",
   "pytest-asyncio>=0.23.3",
   "pytest-cov>=4.1.0",
   "pytest-httpx>=0.28.0",

readme.md

@@ -89,6 +89,7 @@ async def main():
     await gather(api.following(user_id, limit=20))  # list[User]
     await gather(api.user_tweets(user_id, limit=20))  # list[Tweet]
     await gather(api.user_tweets_and_replies(user_id, limit=20))  # list[Tweet]
+    await gather(api.liked_tweets(user_id, limit=20))  # list[Tweet]
 
     # list info
     list_id = 123456789
@@ -139,36 +140,60 @@ twscrape
 twscrape search --help
 ```
 
-### Add accounts & login
+### Add accounts
 
-First add accounts from file:
+To add accounts use `add_accounts` command. Command syntax is:
+```sh
+twscrape add_accounts <file_path> <line_format>
+```
+
+Where:
+`<line_format>` is format of line if accounts file splited by delimeter. Possible tokens:
+- `username` – required
+- `password` – required
+- `email` – required
+- `email_password` – to receive email code (you can use `--manual` mode to get code)
+- `cookies` – can be any parsable format (string, json, base64 string, etc)
+- `_` – skip column from parse
+
+Tokens should be splited by delimeter, usually "`:`" used.
+
+Example:
 
+I have account files named `order-12345.txt` with format:
+```text
+username:password:email:email password:user_agent:cookies
+```
+
+Command to add accounts will be (user_agent column skiped with `_`):
 ```sh
-# twscrape add_accounts <file_path> <line_format>
-# line_format should have "username", "password", "email", "email_password" tokens
-# note: tokens delimeter should be same as an file
-twscrape add_accounts ./accounts.txt username:password:email:email_password
+twscrape add_accounts ./order-12345.txt username:password:email:email_password:_:cookies
 ```
 
-Then call login:
+### Login accounts
+
+_Note: If you added accounts with cookies, login not required._
+
+Run:
 
 ```sh
 twscrape login_accounts
 ```
 
-Accounts and their sessions will be saved, so they can be reused for future requests
+`twscrape` will start login flow for each new account. If X will ask to verify email and you provided `email_password` in `add_account`, then `twscrape` will try to receive verification map by IMAP protocol. After success login account cookies will be saved to db file for future use.
 
-Note: Possible to use `_` in `line_format` to skip some value
+#### Manual email verefication
 
-### Add accounts with cookies
+In case your email provider not support IMAP protocol (ProtonMail, Tutanota, etc) or IMAP is disabled in settings, you can enter email verification code manually. To do this run login command with `--manual` flag.
 
-Use `cookies` param in `line_format`, e.g.:
+Example:
 
 ```sh
-twscrape add_accounts ./accounts.txt username:password:email:email_password:cookies
+twscrape login_accounts --manual
+twscrape relogin user1 user2 --manual
+twscrape relogin_failed --manual
 ```
 
-In this case login not required.
 
 ### Get list of accounts and their statuses
 
@@ -196,16 +221,6 @@ Or retry login for all failed logins:
 twscrape relogin_failed
 ```
 
-### Enter email verification code manually
-
-twscrape may not be able to access some emails (if imap is disabled or does not exist at all - eg in protonmail), in such cases you can use `--manual` flag, which allows to enter the verification code manually.
-
-```sh
-twscrape login_accounts --manual
-twscrape relogin user1 user2 --manual
-twscrape relogin_failed --manual
-```
-
 ### Use different accounts file
 
 Useful if using a different set of accounts for different actions
@@ -227,6 +242,7 @@ twscrape followers USER_ID --limit=20
 twscrape following USER_ID --limit=20
 twscrape user_tweets USER_ID --limit=20
 twscrape user_tweets_and_replies USER_ID --limit=20
+twscrape liked_tweets USER_ID --limit=20
 ```
 
 The default output is in the console (stdout), one document per line. So it can be redirected to the file.