0

I'm trying to download the barchart data table from https://www.barchart.com/investing-ideas/ai-stocks using Excel VBA in similar manner as the python script in Automatic file downloading on Barchart.com using python

I'm not sure why I'm getting a 401 Authorization error using the following VBA code. Any suggestions?

Sub GetBarchart()
    Dim Cookies As Dictionary, token As String
    Dim url As String
    Dim HttpObj As Object
    Set HttpObj = CreateObject("WinHttp.WinHttpRequest.5.1")
    
    Set Cookies = GetCookies("https://www.barchart.com/investing-ideas/ai-stocks")
    token = Cookies("XSRF-TOKEN")
    Debug.Print "token", token
    
    url = "https://www.barchart.com/proxies/core-api/v1/quotes/get?list=stocks.us.fixed.ai&fields=symbol,symbolName,lastPrice&hasOptions=true&page=1&limit=100&raw=1"
    HttpObj.Open "GET", url, False
    ' SetHeaders
    HttpObj.setRequestHeader "Accept", "application/json"
    HttpObj.setRequestHeader "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
    HttpObj.setRequestHeader "X-XSRF-TOKEN", token
    HttpObj.send
    Debug.Print "status", HttpObj.status
    Debug.Print HttpObj.responseText    
End Sub

Function GetCookies(sURL As String, Optional dbg As Boolean = False) As Variant
    Dim winHttp As Object
    Dim sResponseHeaders As String, sCookie As String
    Dim aHeaders As Variant
    Dim i As Long
    Dim cookieParts() As String, cookieNameVal() As String, cookieName As String, cookieVal As String
    Dim Cookies As New Dictionary
  
    Set winHttp = CreateObject("winHttp.winHttpRequest.5.1")
   
    winHttp.Open "GET", sURL, False
    winHttp.setRequestHeader "User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36 Mozilla/5.0"
    winHttp.send

    ' Get all response headers
    sResponseHeaders = winHttp.getAllResponseHeaders
    aHeaders = Split(sResponseHeaders, vbCrLf)
    For i = LBound(aHeaders) To UBound(aHeaders)
        ' get all the cookies
        If InStr(1, aHeaders(i), "Set-Cookie:", vbTextCompare) > 0 Then
            sCookie = Replace(aHeaders(i), "Set-Cookie: ", "", 1, 1, vbTextCompare)
            cookieParts = Split(sCookie, ";")
            If UBound(cookieParts) >= 1 Then
                cookieNameVal = Split(cookieParts(0), "=")
                If UBound(cookieNameVal) >= 1 Then
                    cookieName = cookieNameVal(0)
                    cookieVal = Replace(cookieNameVal(1), "%3D", "=")
                    Cookies.Add cookieName, cookieVal
                End If
            End If
        End If
    Next i

    Set winHttp = Nothing
    Set GetCookies = Cookies
End Function

The GetCookies function finds XRSF-TOKEN as shown in the VBA output

token         eyJpdiI6ImhwN1FvN25GS2YyYW9PV1VaSkhQNFE9PSIsInZhbHVlIjoicXd6THFvU3hqbHhUU0Y3MDU0Q3U0WnRIMVF1TDRmUmhKeW1hTGJ4VzllSUN6VG5TOWJKKzNTK1Y4dHFOY054RVVRb1dUQjlLa1RWZzd6SkZrLzF2ckUzd0JxZFFsL09VWkFIVFlGL3FwK001Qk5qTDVubFYzWFl0a2ZjM3ZtOUUiLCJtYWMiOiJmM2RjNGY0YzZlMzA0N2I1ZjY1YzRlZGU4NWY4ODE2NDFiN2E4OTUzODRlM2I0ZDBhYThmMWNjNmZmMDJkN2Q5IiwidGFnIjoiIn0=
status         401

This the corresponding python code that works

import json
import requests
from urllib.parse import unquote

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Gecko/20100101 Firefox/86.0"
}

url = "https://www.barchart.com/proxies/core-api/v1/quotes/get?list=stocks.us.fixed.ai&fields=symbol,symbolName,lastPrice&hasOptions=true&page=1&limit=100&raw=1"

with requests.Session() as s:
    # get all cookies
    s.get(
        "https://www.barchart.com/investing-ideas/ai-stocks",
        headers=headers,
    )
    # use one cookie as HTTP header
    headers["X-XSRF-TOKEN"] = unquote(s.cookies["XSRF-TOKEN"])
    print("headers",  headers)

    response = s.get(url, headers=headers)
    data = response.json()
    #data = s.get(url, headers=headers).json()
    print ("status", response.status_code)

for d in data["data"]:
    print("{:<8}{:<50}{}".format(d["symbol"], d["symbolName"], d["lastPrice"]))

Python output

headers {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Gecko/20100101 Firefox/86.0', 'X-XSRF-TOKEN': 'eyJpdiI6IkN4YVN3ZTFHdktMMkQrYVpYcUEzVXc9PSIsInZhbHVlIjoiWFBmM1R4M1Zvay9Rb0ZrbTRJcFI5YUl2UGRPYmtqTFpYaTlYTTVMUFBHL21FTXJVa2tCNGc1VVdmbmoxajBVNjJtK3FaNGdieE9mVmxFYkdkNVBMbWh2MWhNREQxcG0vSGNER3F6Z2k5NkhaVmlyUjFSb2ppbVBWc0Y5eWVwc1EiLCJtYWMiOiI1ODJmN2QzMDhiZTRjYWYyZDRiMDk0OGJmYjljOTU2Mjk3YzEzMzgwZDk0YmU4ODc2ZmQ5ODhiOTczYWU5MjIwIiwidGFnIjoiIn0='}
status 200
AAPL    Apple Inc                                         245.27
ACN     Accenture Plc                                     240.94
ADBE    Adobe Systems Inc                                 337.51
AI      C3.Ai Inc Cl A                                    17.91
AMD     Adv Micro Devices                                 214.90
AMZN    Amazon.com Inc                                    216.37
ANET    Arista Networks Inc                               154.10 

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.