How do you import simple data from a webpage using Javascript

Question

The problem I have run into seems like it has a simple answer. I want to have my website do the simple task of grabbing a piece of text from a webpage.

I basically want my code to look like...

var str = TextFromWebPage;

The text I want to "grab" is on http://dasd-sharepoint.dasd.org/Schools/STEMAcademy/Pages/default.aspx

If you look on the page (try Control F and search for the current date, Ex: "May-9-2013:") for a list of the next five dates. I want to take that list and use it in variables in Javascript.

So my prefered end result would be

var str = new Array("May-9-2013: “F” Day", "May-10-2013: “G” Day", "May-13-2013: “H” Day", "May-14-2013: “I” Day", "May-15-2013: “J” Day", "May-16-2013: “K” Day");

Purpose: Basically for school I'm always checking the time. So instead of doing the math I make a quick little program that look at the current time vs the time the current class ends. In my school however we have different "letter days" which means class lengths are different on different days. Letter days are based on a 12 day cycle which means. Monday through Friday could be A-E but that would mean the following week Monday would then be F-J. Not to mention if there is a snow day or a day off. So instead of using a calendar I figured there has to be a way I can grab this data from that district webpage and utilize it in the code to automatically check what letter day it is.

Thanks for any and all help!

Here is my code for what I'm doing...

<!DOCTYPE html>
<html>
     <head>
        //<link rel="stylesheet" type="text/css" href="graphics.css">
        <script>
            //==========|  Data  |==========
            //These array lists contain the times the class periods start, end, and their title
            var Ptest = new Array("starts: 07:25 ends: 15:00 title: Testing", "starts: 07:25 ends: 30:00 title: Testing");
            var P14 = new Array("starts: 07:25 ends: 09:00 title: Pd: 1", "starts: 09:04 ends: 10:34 title: Pd: 2", "starts: 10:38 ends: 11:08 title: Lunch", "starts: 11:12 ends: 12:42 title: Pd: 3", "starts: 12:46 ends: 14:23 title: Pd: 4");
            var P17 = new Array("starts: 07:25 ends: 08:20 title: Pd: 1", "starts: 08:24 ends: 09:14 title: Pd: 2", "starts: 09:18 ends: 10:08 title: Pd: 3", "starts: 10:12 ends: 11:02 title: Pd: 4", "starts: 11:06 ends: 11:36 title: Lunch", "starts: 11:40 ends: 12:30 title: Pd: 5", "starts: 12:34 ends: 13:24 title: Pd: 6", "starts: 13:28 ends: 14:23 title: Pd: 7");
            var P57 = new Array("starts: 07:25 ends: 09:00 title: Pd: 5", "starts: 09:04 ends: 10:34 title: Pd: 6", "starts: 10:38 ends: 11:08 title: Lunch", "starts: 11:12 ends: 12:42 title: Pd: 7", "starts: 12:46 ends: 13:36 title: Advisory", "starts: 13:40 ends: 14:23 title: Seminar");
            //---------------------------------

            //==========|  Variables  |==========
            var input = P17; //What set of data to use
            var currentSlot = 0; //What the current Period is
            var over = false; //If the current school day if over
            //---------------------------------

            //==========|  Actions  |==========
            window.onload=function() {
                setInterval(function() {refresh();}, 1); //Loop
            }

            //---------------------------------


            //==========|  Core Functions  |==========
            function refresh() {
                display();
                updateClock();
            }

            function display() {
                var currentTime = getTime();
                var timeInSec = ((parseInt(currentTime.slice(0,2)))*3600)+((parseInt(currentTime.slice(3,5)))*60)+(parseInt(currentTime.slice(6,8)));

                if (currentSlot >= input.length) {over = true;}
                if (over == false) {
                    var data = input[currentSlot];
                    var nextSlot = input[currentSlot + 1];

                    var PdEndTime = ((parseInt(data.slice(20,23)))*3600)+((parseInt(data.slice(23,25)))*60);

                    var PdStart = data.slice(8,13);
                    var PdEnd = data.slice(20,25);
                    var PdTitle = data.slice(32,data.length);

                    if (nextSlot!==undefined) {
                        var NPdStart = nextSlot.slice(8,13);
                        var NPdEnd = nextSlot.slice(20,25);
                        var NPdTitle = nextSlot.slice(32,nextSlot.length);
                    }

                    var timeLeft = PdEndTime - timeInSec;
                    var hLeft = Math.floor(timeLeft / 3600);
                    var mLeft = Math.floor(timeLeft / 60);
                    var sLeft = Math.floor(timeLeft % 60);

                    if (hLeft > 0) {mLeft = mLeft - (hLeft*60);}

                    hLeft = add0(hLeft);
                    mLeft = add0(mLeft);
                    sLeft = add0(sLeft);

                    if (timeInSec > PdEndTime) {currentSlot++;}

                    document.getElementById("display").innerHTML = PdTitle+" ends at "+PdEnd+", in "+hLeft+":"+mLeft+":"+sLeft;
                }
                else {document.getElementById("display").innerHTML = "School is over!";}
            }

            //---------------------------------

            //==========|  Misc Functions  |==========


            function getTime() {
                var clock = new Date();
                var h = clock.getHours();
                var m = clock.getMinutes();
                var s = clock.getSeconds();

                h = add0(h);
                m = add0(m);
                s = add0(s);

                return h+":"+m+":"+s;
            }

            function updateClock() {
                document.getElementById("clock").innerHTML = getTime();
            }

            function add0(i) {
                if (i < 10) {
                    i  = "0"+i;
                }
                return i;
            }

            function changeInput(newInput) {
                input = newInput;
                currentSlot = 0;
            }
            //---------------------------------
        </script>
     </head>
    <body>
        <p id="clock" class="textCenter"></p>
        <p id="display" class="textCenter"></p>
        <p class="textCenter">
        <input type="button" onClick="changeInput(P14)" value="1-4 Block">
        <input type="button" onClick="changeInput(P17)" value="1-7 Normal">
        <input type="button" onClick="changeInput(P57)" value="5-7 Block">
        </p>
    </body>
</html>

I want it to simply automatically check what letter day it is so You don't have to have the users click the buttons. For example A-day = 1-7, B-Day = 1-4, C-day = 1-7, D-day = 5-7

I'm guessing that this is'nt your page, and so it's a cross domain request to scrape data, and clientside javascript is'nt the right tool for that job. — adeneo
– adeneo, Commented May 9, 2013 at 15:25
If it's a personal project you could install forcecors plugin in firefox, with that you can make xmlhttprequest from any site to any site. After installing it go to view => toolbars => add-on bar. In the right bottom corcer you can see "cors" click on it and it'll turn red so you can use it. But based on the JS code you posted it's going to be painfully hard to do this since you have to have some JS experience to pull it off or invest a lot of time in learning it. — HMR
– HMR, Commented May 9, 2013 at 15:30
I'll look into that. I feel like it should be simple since I can copy and paste the text with my mouse. it should just be like and Img. Every img has a link. By using the <img url""> tag you can get any image based off it's url. In an exel document you can import data off a website like that table. In fact I have... Could I make my webpage look at an xcel document then? — CTOverton
– CTOverton, Commented May 9, 2013 at 16:01

Destreyf · Accepted Answer · 2013-05-09 17:20:35Z

1

If you are doing this from another website, your best bet is to use a secondary page on your website that uses PHP or ASP, or any other server side programming language to grab that page and return its contents, while you're at you can use the powerful regular expressions in that server side language and return a JSON object of what you need. The real problem here is that you can't go from yoursite.com to dasd-sharepoint.dasd.org/Schools/STEMAcademy/Pages/default.aspx this is simply not possible in a traditional java script environment.

The reason for my conclusion is that when executing XMLHttpRequest (XHR) the browsers are designed to not access another website. And even if it were, technically their webserver should deny access via XHR from another domain, this is known as a Cross Domain Request and is not natively possible within javascript.

Using a secondary page to go fetch the contents of this, would be far more efficient and require no add-on to use this function/website. This gives you precise control over your functionality and easy portability to most hosts. It also allows any client to be able to access this page and utilize its functionality.

Using Add-ons is never a proper way to do something, there is always a better easier way.

Now onto the HTML5 goodies for doing this from another website

If you have some say in the headers sent by the remote host, you can have them send a "Access-Control-Allow-Credentials: true" header this will allow XMLHttpRequest2 aka CORS requests to process, you can see more information regarding this subject at the following url: http://www.html5rocks.com/en/tutorials/cors/

Unfortunately i don't have any experience with CORS so i don't have any helpful how-to's or anything else, its pretty similar to a typical ajax request it just supports Cross Domain as long as the remote party allows it (Control-Allow-Credentials: True) and the system is properly formated of course this doesn't really leave you with a lot of help, but hopefully gives you the information to look into making a proper system.

If you are doing this on the same website (so say a different page of the same website/domain dasd-sharepoint.dasd.org Then there are easy ways to get this information. Parsing through all the information on the web page can be a difficult process when getting started but it should not be terrible. I will write more on this subject later today as i have to get going.

answered May 9, 2013 at 17:20

Destreyf

4615 silver badges7 bronze badges

Sign up to request clarification or add additional context in comments.

3 Comments

CTOverton Over a year ago

Thanks for explaining it! I edited the question to explain the purpose I have behind this.

HMR Over a year ago

Why a browser refuses to make XHR requests from A to B or sub.A to A or from your local computer to Server A or from A:80 to A:808 is called same origin policy Wikipedia has great article on it if you're interested and explains cors/JSONP (note the P in the end so it's not the same as JSON). If you don't have control over the server you want to connect to then you can install a plugin in Firefox that will add cors headers and trick FF into allowing the request. You can't use that publicly as you can't expect users to install the plugin first but for personal projects it's fine.

Destreyf Over a year ago

HMR Love your feedback on this. Rext from what i understand you're looking for something a little bit different. What platforms are you working on, and what format are you attempting to achieve, If you're running this on your own server i can get you very simple snippets to get this information so you can use JSON to return an array/object that is parse-able for utilization in your setup.

HMR · Accepted Answer · 2013-05-09 15:51:14Z

1

Here is code that'll pick up the information you need (if the page doesn't change). You have to use forcecors (and activate it) in Firefox, press F12 to see the array of the values logged to the console.

[update] Forgot to mention that you have to use jQuery as well. Put the jquery-1.9.0.js in the same directory as the html.

<!DOCTYPE html>
<html>
 <head>
<meta content="text/html; charset=UTF-8" http-equiv="content-type">
<title>Example</title>
<script type="text/javascript" src="jquery-1.9.0.js"></script>
<style type="text/css">
</style>
<script type="text/javascript">
$(document).ready(function(){
  var url="http://dasd-sharepoint.dasd.org/Schools/STEMAcademy/Pages/default.aspx";
  $.get(url,function(res){
    var p = new DOMParser();
    var doc = p.parseFromString(res, "text/html");
    var anchors=$(doc).find(".dfwp-column")[0].getElementsByTagName("a");
    var arr=[];
    for(var i=0;i<anchors.length;i++){
      arr.push(anchors[i].textContent);
    }
    console.log(arr);
  });

});
</script>
</head>
 <body>
<div class="bigDiv">Big</div>
<div class="smallerDiv">Small</div>
</html>

answered May 9, 2013 at 15:51

HMR

39.5k26 gold badges99 silver badges175 bronze badges

2 Comments

CTOverton Over a year ago

Thanks so much! I am very new to Javascript and this is exactly (well I hope) what I needed. I'll try to figure it out but I have no experience with forcecors but Thanks for the help!

HMR Over a year ago

You're welcome. For this code to work you have to use Firefox and install the forcecors plugin. If you want a page on a public website where anyone can visit that page and scrape the data then this code is not realy good for that (users have to use firefox and install forcecors first). In that case you can use curl with PHP.

Collectives™ on Stack Overflow

How do you import simple data from a webpage using Javascript

2 Answers 2

3 Comments

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

3 Comments

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related