Skip to content

Commit

Permalink
fix: wip fix setting CONSENT cookie in lynx for google search to work
Browse files Browse the repository at this point in the history
  • Loading branch information
ErikBjare committed Nov 1, 2024
1 parent 40d1a5d commit 7b8201e
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion gptme/tools/_browser_lynx.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,21 @@
import subprocess


def read_url(url):
def read_url(url, cookies: dict | None = None) -> str:
# TODO: create and set LYNX_CFG to use custom lynx config file (needed to save cookies, which I need to debug how cookies should be read)
if cookies:
# save them to file to be read by lynx
with open("cookies.txt", "w") as f:
for k, v in cookies.items():
f.write(f"{k}\t{v}\n")
return subprocess.run(
["lynx", "--dump", url, "--display_charset=utf-8"], stdout=subprocess.PIPE
).stdout.decode("utf-8")


def search(query, engine="google"):
if engine == "google":
# TODO: needs a CONSENT cookie
return read_url(f"https://www.google.com/search?q={query}")
elif engine == "duckduckgo":
return read_url(f"https://duckduckgo.com/?q={query}")
Expand Down

0 comments on commit 7b8201e

Please sign in to comment.