fix auth handling for quote calls

v3rtumnus · Jun 25, 2023 · a961b19 · a961b19
1 parent 1d409f9
commit a961b19
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 37 deletions.
diff --git a/src/main/java/yahoofinance/YahooFinance.java b/src/main/java/yahoofinance/YahooFinance.java
@@ -45,14 +45,14 @@
 public class YahooFinance {
 
     public static final String QUOTES_BASE_URL = System.getProperty("yahoofinance.baseurl.quotes", "http://download.finance.yahoo.com/d/quotes.csv");
-    public static final String QUOTES_QUERY1V7_BASE_URL = System.getProperty("yahoofinance.baseurl.quotesquery1v7", "https://query1.finance.yahoo.com/v7/finance/quote");
+    public static final String QUOTES_QUERY1V7_BASE_URL = System.getProperty("yahoofinance.baseurl.quotesquery1v7", "https://query2.finance.yahoo.com/v7/finance/quote");
     public static final String QUOTES_QUERY1V7_ENABLED = System.getProperty("yahoofinance.quotesquery1v7.enabled", "true");
     public static final String HISTQUOTES_BASE_URL = System.getProperty("yahoofinance.baseurl.histquotes", "https://ichart.yahoo.com/table.csv");
     public static final String HISTQUOTES2_ENABLED = System.getProperty("yahoofinance.histquotes2.enabled", "true");
     public static final String HISTQUOTES2_BASE_URL = System.getProperty("yahoofinance.baseurl.histquotes2", "https://query1.finance.yahoo.com/v7/finance/download/");
     public static final String HISTQUOTES_QUERY2V8_BASE_URL = System.getProperty("yahoofinance.baseurl.histquotesquery2v8", "https://query2.finance.yahoo.com/v8/finance/chart/");
     public static final String HISTQUOTES2_SCRAPE_URL = System.getProperty("yahoofinance.scrapeurl.histquotes2", "https://finance.yahoo.com/quote/%5EGSPC/options");
-    public static final String HISTQUOTES2_CRUMB_URL = System.getProperty("yahoofinance.crumburl.histquotes2", "https://query1.finance.yahoo.com/v1/test/getcrumb");
+    public static final String HISTQUOTES2_CRUMB_URL = System.getProperty("yahoofinance.crumburl.histquotes2", "https://query2.finance.yahoo.com/v1/test/getcrumb");
     public static final String HISTQUOTES2_CRUMB = System.getProperty("yahoofinance.crumb", "");
     public static final String HISTQUOTES2_COOKIE = System.getProperty("yahoofinance.cookie", "");
     public static final String HISTQUOTES2_COOKIE_NAMESPACE = System.getProperty("yahoofinance.cookie.namespace", "yahoo");

diff --git a/src/main/java/yahoofinance/histquotes2/CrumbManager.java b/src/main/java/yahoofinance/histquotes2/CrumbManager.java
@@ -4,17 +4,11 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
-import java.net.CookieHandler;
-import java.net.CookieManager;
-import java.net.CookieStore;
-import java.net.HttpCookie;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.net.URLEncoder;
+import java.net.*;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -41,24 +35,22 @@ private static void setCookie() throws IOException {
             return;
         }
 
-        URL request = new URL(YahooFinance.HISTQUOTES2_SCRAPE_URL);
+        URL request = new URL("https://fc.yahoo.com");
         RedirectableRequest redirectableRequest = new RedirectableRequest(request, 5);
         redirectableRequest.setConnectTimeout(YahooFinance.CONNECTION_TIMEOUT);
         redirectableRequest.setReadTimeout(YahooFinance.CONNECTION_TIMEOUT);
-
+
+        Map<String, String> requestProperties = new TreeMap<>();
+        requestProperties.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36");
 
-        URLConnection connection = redirectableRequest.openConnection();
+        URLConnection connection = redirectableRequest.openConnection(requestProperties);
 
         for(String headerKey : connection.getHeaderFields().keySet()) {        	
             if("Set-Cookie".equalsIgnoreCase(headerKey)) {
-                for(String cookieField : connection.getHeaderFields().get(headerKey)) {                	
-                    for(String cookieValue : cookieField.split(";")) {
-                        if(cookieValue.matches("B=.*")) {
-                            cookie = cookieValue;
-                            log.debug("Set cookie from http request: {}", cookie);
-                            return;
-                        }
-                    }
+                for(String cookieField : connection.getHeaderFields().get(headerKey)) {
+                    cookie = cookieField;
+                    log.debug("Set cookie from http request: {}", cookie);
+                    return;
                 }
             }
         }
@@ -67,7 +59,7 @@ private static void setCookie() throws IOException {
         InputStreamReader is = new InputStreamReader(connection.getInputStream());
         BufferedReader br = new BufferedReader(is);
         String line;
-        Pattern patternPostForm = Pattern.compile("action=\"/consent\"");
+        Pattern patternPostForm = Pattern.compile("class=\"consent-form\"");
         Pattern patternInput = Pattern.compile("(<input type=\"hidden\" name=\")(.*?)(\" value=\")(.*?)(\">)");
         Matcher matcher;
         Map<String,String> datas = new HashMap<String,String>();
@@ -137,12 +129,10 @@ private static void setCookie() throws IOException {
         // Then Set the cookie with the cookieJar
         CookieStore cookieJar =  ((CookieManager)CookieHandler.getDefault()).getCookieStore();
         List <HttpCookie> cookies = cookieJar.getCookies();
-        for (HttpCookie hcookie: cookies) {        	
-        	if(hcookie.toString().matches("B=.*")) {
-                 cookie = hcookie.toString();
-                 log.debug("Set cookie from http request: {}", cookie);
-                 return;
-             }
+        for (HttpCookie hcookie: cookies) {
+            cookie = hcookie.toString();
+            log.debug("Set cookie from http request: {}", cookie);
+            return;
         }
 
         log.debug("Failed to set cookie from http request. Historical quote requests will most likely fail.");

diff --git a/src/main/java/yahoofinance/quotes/query1v7/QuotesRequest.java b/src/main/java/yahoofinance/quotes/query1v7/QuotesRequest.java
@@ -12,10 +12,8 @@
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.net.URLConnection;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+
 import yahoofinance.histquotes2.CrumbManager;
 
 /**
@@ -74,7 +72,11 @@ public List<T> getResult() throws IOException {
         RedirectableRequest redirectableRequest = new RedirectableRequest(request, 5);
         redirectableRequest.setConnectTimeout(YahooFinance.CONNECTION_TIMEOUT);
         redirectableRequest.setReadTimeout(YahooFinance.CONNECTION_TIMEOUT);
-        URLConnection connection = redirectableRequest.openConnection();
+
+        Map<String, String> requestProperties = new HashMap<String, String>();
+        requestProperties.put("Cookie", CrumbManager.getCookie());
+
+        URLConnection connection = redirectableRequest.openConnection(requestProperties);
 
         InputStreamReader is = new InputStreamReader(connection.getInputStream());
         JsonNode node = objectMapper.readTree(is);

diff --git a/src/main/java/yahoofinance/util/RedirectableRequest.java b/src/main/java/yahoofinance/util/RedirectableRequest.java
@@ -17,10 +17,6 @@ public class RedirectableRequest {
     private int connectTimeout = 10000;
     private int readTimeout = 10000;
 
-    static {
-        CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL));
-    }
-
     public RedirectableRequest(URL request) {
         this(request, 2);
     }