Skip to content

John McEleney / casperjs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 133310d8 133310d814d79db08c3982ee4af31d0a71813b8c by Nicolas Perriault
Options

Initial import.

0 parents
Inline Side-by-side
Showing 4 changed files with 691 additions and 0 deletions
1 Copyright (c) 2011 Nicolas Perriault
2
3 Permission is hereby granted, free of charge, to any person obtaining a copy
4 of this software and associated documentation files (the "Software"), to deal
5 in the Software without restriction, including without limitation the rights
6 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 copies of the Software, and to permit persons to whom the Software is furnished
8 to do so, subject to the following conditions:
9
10 The above copyright notice and this permission notice shall be included in all
11 copies or substantial portions of the Software.
12
13 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 THE SOFTWARE.
1 # Casper.js
2
3 Casper is a navigation utility for [PhantomJS](http://www.phantomjs.org/).
4
5 More documentation to come soon, I swear. If you just can't wait, here's a sample script:
6
7 phantom.injectJs('casper.js');
8
9 // User defined functions
10 function q() {
11 document.querySelector('input[name="q"]').setAttribute('value', '%term%');
12 document.querySelector('form[name="f"]').submit();
13 }
14
15 function getLinks() {
16 return Array.prototype.map.call(document.querySelectorAll('h3.r a'), function(e) {
17 return e.getAttribute('href');
18 });
19 }
20
21 // Casper suite
22 var links = [];
23 var casper = new phantom.Casper()
24 .start('http://google.fr/')
25 .thenEvaluate(q, {
26 term: 'casper',
27 })
28 .then(function(self) {
29 links = self.evaluate(getLinks);
30 })
31 .thenEvaluate(q, {
32 term: 'homer',
33 })
34 .then(function(self) {
35 links = links.concat(self.evaluate(getLinks));
36 })
37 .run(function(self) {
38 self.echo(JSON.stringify({
39 result: self.result,
40 links: links
41 }, null, ' '));
42 self.exit();
43 })
44 ;
45
46 Run it:
47
48 $ phantomjs example.js
49 {
50 "result": {
51 "log": [
52 {
53 "level": "info",
54 "space": "phantom",
55 "message": "Starting…",
56 "date": "Mon Sep 05 2011 16:10:56 GMT+0200 (CEST)"
57 },
58 {
59 "level": "info",
60 "space": "phantom",
61 "message": "Running suite: 4 steps",
62 "date": "Mon Sep 05 2011 16:10:56 GMT+0200 (CEST)"
63 },
64 {
65 "level": "info",
66 "space": "phantom",
67 "message": "Step 1/4: http://www.google.fr/ (HTTP 301)",
68 "date": "Mon Sep 05 2011 16:10:57 GMT+0200 (CEST)"
69 },
70 {
71 "level": "info",
72 "space": "phantom",
73 "message": "Step 1/4: done in 1259ms.",
74 "date": "Mon Sep 05 2011 16:10:57 GMT+0200 (CEST)"
75 },
76 {
77 "level": "info",
78 "space": "phantom",
79 "message": "Step 2/4: http://www.google.fr/search?sclient=psy&hl=fr&site=&source=hp&q=casper&pbx=1&oq=&aq=&aqi=&aql=&gs_sm=&gs_upl= (HTTP 301)",
80 "date": "Mon Sep 05 2011 16:10:58 GMT+0200 (CEST)"
81 },
82 {
83 "level": "info",
84 "space": "phantom",
85 "message": "Step 2/4: done in 2145ms.",
86 "date": "Mon Sep 05 2011 16:10:58 GMT+0200 (CEST)"
87 },
88 {
89 "level": "info",
90 "space": "phantom",
91 "message": "Step 3/4: http://www.google.fr/search?sclient=psy&hl=fr&site=&source=hp&q=casper&pbx=1&oq=&aq=&aqi=&aql=&gs_sm=&gs_upl= (HTTP 301)",
92 "date": "Mon Sep 05 2011 16:10:58 GMT+0200 (CEST)"
93 },
94 {
95 "level": "info",
96 "space": "phantom",
97 "message": "Step 3/4: done in 2390ms.",
98 "date": "Mon Sep 05 2011 16:10:58 GMT+0200 (CEST)"
99 },
100 {
101 "level": "info",
102 "space": "phantom",
103 "message": "Step 4/4: http://www.google.fr/search?sclient=psy&hl=fr&source=hp&q=homer&pbx=1&oq=&aq=&aqi=&aql=&gs_sm=&gs_upl= (HTTP 301)",
104 "date": "Mon Sep 05 2011 16:10:59 GMT+0200 (CEST)"
105 },
106 {
107 "level": "info",
108 "space": "phantom",
109 "message": "Step 4/4: done in 3077ms.",
110 "date": "Mon Sep 05 2011 16:10:59 GMT+0200 (CEST)"
111 },
112 {
113 "level": "info",
114 "space": "phantom",
115 "message": "Done 4 steps in 3077ms.",
116 "date": "Mon Sep 05 2011 16:10:59 GMT+0200 (CEST)"
117 }
118 ],
119 "status": "success",
120 "time": 3077
121 },
122 "links": [
123 "http://fr.wikipedia.org/wiki/Casper_le_gentil_fant%C3%B4me",
124 "http://fr.wikipedia.org/wiki/Casper",
125 "http://casperflights.com/",
126 "http://www.allocine.fr/film/fichefilm_gen_cfilm=13018.html",
127 "/search?q=casper&hl=fr&prmd=ivns&tbm=isch&tbo=u&source=univ&sa=X&ei=cdhkTurpFa364QTB5uGeCg&ved=0CFkQsAQ",
128 "http://www.youtube.com/watch?v=Kuvo0QMiNEE",
129 "http://www.youtube.com/watch?v=W7cW5YlHaeQ",
130 "http://www.imdb.com/title/tt0112642/",
131 "http://blog.caspie.net/",
132 "http://www.casperwy.gov/",
133 "http://www.lequipe.fr/Cyclisme/CyclismeFicheCoureur147.html",
134 "http://homer-simpson-tv.blog4ever.com/",
135 "http://fr.wikipedia.org/wiki/Homer_Simpson",
136 "http://en.wikipedia.org/wiki/Homer",
137 "/search?q=homer&hl=fr&prmd=ivnsb&tbm=isch&tbo=u&source=univ&sa=X&ei=cthkTr73Hefh4QSUmt3UCg&ved=0CEQQsAQ",
138 "http://www.youtube.com/watch?v=Ajd08hgerRo",
139 "http://www.koreus.com/video/homer-simpson-photo-39-ans.html",
140 "http://www.nrel.gov/homer/",
141 "http://www.luds.net/homer.php",
142 "http://www.thesimpsons.com/bios/bios_family_homer.htm",
143 "http://www.homeralaska.org/",
144 "http://homeralaska.com/"
145 ]
146 }
147
148 ## Now what
149
150 Feel free to play with the code and report an issue on github. I'm also reachable [on twitter](https://twitter.com/n1k0).
1 /*!
2 * Casper is a navigator for PhantomJS - http://github.com/n1k0/casperjs
3 *
4 * Copyright (c) 2011 Nicolas Perriault
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 */
25 (function(phantom) {
26 /**
27 * Main Casper class. Available options are:
28 *
29 * Type Name Default Description
30 * - Array clientScripts ([]): A collection of script filepaths to include to every page loaded
31 * - String logLevel ("error") Logging level (see logLevels for available values)
32 * - Object pageSettings ({}): PhantomJS's WebPage settings object
33 * - WebPage page (null): An existing WebPage instance
34 * - Boolean verbose (false): Realtime output of log messages
35 *
36 * @param Object options Casper options
37 * @return Casper
38 */
39 phantom.Casper = function(options) {
40 const DEFAULT_DIE_MESSAGE = "Suite explicitely interrupted without any message given.";
41 const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1";
42 // init & checks
43 if (!(this instanceof arguments.callee)) {
44 return new Casper(options);
45 }
46 // default options
47 this.defaults = {
48 clientScripts: [],
49 logLevel: "error",
50 onDie: null,
51 page: null,
52 pageSettings: {
53 userAgent: DEFAULT_USER_AGENT,
54 },
55 verbose: false
56 };
57 // local properties
58 this.checker = null;
59 this.currentHTTPStatus = 200;
60 this.loadInProgress = false;
61 this.logLevels = ["debug", "info", "warning", "error"];
62 this.options = mergeObjects(this.defaults, options);
63 this.page = null;
64 this.requestUrl = 'about:blank';
65 this.result = {
66 log: [],
67 status: "success",
68 time: 0
69 }
70 this.started = false;
71 this.step = 0;
72 this.steps = [];
73 };
74
75 /**
76 * Casper prototype
77 */
78 phantom.Casper.prototype = {
79 /**
80 * Proxy method for WebPage#render. Adds a clipRect parameter for
81 * automatically set page clipRect setting values and sets it back once
82 * done.
83 *
84 * @param string targetFile A target filename
85 * @param mixed clipRect An optional clipRect object
86 * @return Casper
87 */
88 capture: function(targetFile, clipRect) {
89 var previousClipRect = this.page.clipRect;
90 if (clipRect) {
91 this.page.clipRect = clipRect;
92 }
93 if (!this.page.render(targetFile)) {
94 this.log('Failed to capture screenshot as ' + targetFile, "error");
95 }
96 this.page.clipRect = previousClipRect;
97 return this;
98 },
99
100 /**
101 * Checks for any further navigation step to process.
102 *
103 * @param Casper self A self reference
104 * @param function onComplete An options callback to apply on completion
105 */
106 checkStep: function(self, onComplete) {
107 if (!self.loadInProgress && typeof(self.steps[self.step]) === "function") {
108 var curStepNum = self.step + 1
109 , stepInfo = "Step " + curStepNum + "/" + self.steps.length + ": ";
110 self.log(stepInfo + self.page.evaluate(function() {
111 return document.location.href;
112 }) + ' (HTTP ' + self.currentHTTPStatus + ')', "info");
113 try {
114 self.steps[self.step](self);
115 } catch (e) {
116 self.log("Fatal: " + e, "error");
117 }
118 var time = new Date().getTime() - self.startTime;
119 self.log(stepInfo + "done in " + time + "ms.", "info");
120 self.step++;
121 }
122 if (typeof(self.steps[self.step]) !== "function") {
123 self.result.time = new Date().getTime() - self.startTime;
124 self.log("Done " + self.steps.length + " steps in " + self.result.time + 'ms.', "info");
125 clearInterval(self.checker);
126 if (typeof(onComplete) === "function") {
127 onComplete(self);
128 } else {
129 // default behavior is to exit phantom
130 self.exit();
131 }
132 }
133 },
134
135 /**
136 * Logs the HTML code of the current page.
137 *
138 * @return Casper
139 */
140 debugHTML: function() {
141 this.echo(this.page.evaluate(function() {
142 return document.body.innerHTML;
143 }));
144 return this;
145 },
146
147 /**
148 * Logs the textual contents of the current page.
149 *
150 * @return Casper
151 */
152 debugPage: function() {
153 this.echo(this.page.evaluate(function() {
154 return document.body.innerText;
155 }));
156 return this;
157 },
158
159 /**
160 * Exit phantom on failure, with a logged error message.
161 *
162 * @param string message An optional error message
163 * @param Number status An optional exit status code (must be > 0)
164 * @return Casper
165 */
166 die: function(message, status) {
167 this.result.status = 'error';
168 message = typeof(message) === "string" && message.length > 0 ? message : DEFAULT_DIE_MESSAGE;
169 this.log(message, "error");
170 if (typeof(this.options.onDie) === "function") {
171 this.options.onDie(this, status);
172 }
173 return this.exit(Number(status) > 0 ? Number(status) : 1);
174 },
175
176 /**
177 * Prints something to stdout.
178 *
179 * @param string text A string to echo to stdout
180 * @return Casper
181 */
182 echo: function(text) {
183 console.log(text);
184 return this;
185 },
186
187 /**
188 * Evaluates an expression in the page context, a bit like what
189 * WebPage#evaluate does, but can also replace values by their
190 * placeholer names:
191 *
192 * navigator.evaluate(function() {
193 * document.querySelector('#username').setAttribute('value', '%username%');
194 * document.querySelector('#password').setAttribute('value', '%password%');
195 * document.querySelector('#submit').click();
196 * }, {
197 * username: 'Bazoonga',
198 * password: 'baz00nga'
199 * })
200 *
201 * FIXME: waiting for a patch of PhantomJS to allow direct passing of
202 * arguments to the function.
203 * TODO: don't forget to keep this backward compatible.
204 *
205 * @param function fn The function to be evaluated within current page DOM
206 * @param object replacements Optional replacements to performs, eg. for '%foo%' => {foo: 'bar'}
207 * @return mixed
208 * @see WebPage#evaluate
209 */
210 evaluate: function(fn, replacements) {
211 if (replacements && typeof replacements === "object") {
212 fn = fn.toString();
213 for (var p in replacements) {
214 var match = '%' + p + '%';
215 do {
216 fn = fn.replace(match, replacements[p]);
217 } while(fn.indexOf(match) !== -1);
218 }
219 }
220 return this.page.evaluate(fn);
221 },
222
223 /**
224 * Evaluates an expression within the current page DOM and die() if it
225 * returns false.
226 *
227 * @param function fn Expression to evaluate
228 * @param string message Error message to log
229 * @return Casper
230 */
231 evaluateOrDie: function(fn, message) {
232 if (!this.evaluate(fn)) {
233 return this.die(message);
234 }
235 return this;
236 },
237
238 /**
239 * Exits phantom.
240 *
241 * @param Number status Status
242 * @return Casper
243 */
244 exit: function(status) {
245 phantom.exit(status);
246 return this;
247 },
248
249 /**
250 * Logs a message.
251 *
252 * @param string message The message to log
253 * @param string level The log message level (from Casper.logLevels property)
254 * @param string space Space from where the logged event occured (default: "phantom")
255 * @return Casper
256 */
257 log: function(message, level, space) {
258 level = level && this.logLevels.indexOf(level) > -1 ? level : "debug";
259 space = space ? space : "phantom";
260 if (this.logLevels.indexOf(level) < this.logLevels.indexOf(this.options.logLevel)) {
261 return this; // skip logging
262 }
263 if (this.options.verbose) {
264 this.echo('[' + level + '] [' + space + '] ' + message); // direct output
265 }
266 this.result.log.push({
267 level: level,
268 space: space,
269 message: message,
270 date: new Date().toString(),
271 });
272 return this;
273 },
274
275 /**
276 * Opens a page. Takes only one argument, the url to open (using the
277 * callback argument would defeat the whole purpose of Casper
278 * actually).
279 *
280 * @param string location The url to open
281 * @return Casper
282 */
283 open: function(location) {
284 this.requestUrl = location;
285 this.page.open(location);
286 return this;
287 },
288
289 /**
290 * Runs the whole suite of steps.
291 *
292 * @param function onComplete an optional callback
293 * @param Number time an optional amount of milliseconds for interval checking
294 * @return Casper
295 */
296 run: function(onComplete, time) {
297 if (!this.steps || this.steps.length < 1) {
298 this.log("No steps defined, aborting", "error");
299 return this;
300 }
301 this.log("Running suite: " + this.steps.length + " step" + (this.steps.length > 1 ? "s" : ""), "info");
302 this.checker = setInterval(this.checkStep, (time ? time: 250), this, onComplete);
303 return this;
304 },
305
306 /**
307 * Configures and start the Casper.
308 *
309 * @param string location An optional location to open on start
310 * @param function then Next step function to execute on page loaded (optional)
311 * @return Casper
312 */
313 start: function(location, then) {
314 this.log('Starting…', "info");
315 this.startTime = new Date().getTime();
316 this.steps = [];
317 this.step = 0;
318 // Option checks
319 if (this.logLevels.indexOf(this.options.logLevel) < 0) {
320 this.log("Unknown log level '" + this.options.logLevel + "', defaulting to 'warning'", "warning");
321 this.options.logLevel = "warning";
322 }
323 // WebPage
324 if (!(this.page instanceof WebPage)) {
325 if (this.options.page instanceof WebPage) {
326 this.page = this.options.page;
327 } else {
328 this.page = createPage(this);
329 }
330 }
331 this.page.settings = mergeObjects(this.page.settings, this.options.pageSettings);
332 this.started = true;
333 if (typeof(location) === "string" && location.length > 0) {
334 if (typeof(then) === "function") {
335 return this.open(location).then(then);
336 } else {
337 return this.open(location);
338 }
339 }
340 return this;
341 },
342
343 /**
344 * Schedules the next step in the navigation process.
345 *
346 * @param function step A function to be called as a step
347 * @return Casper
348 */
349 then: function(step) {
350 if (!this.started) {
351 throw "Casper not started; please use Casper#start";
352 }
353 if (typeof(step) !== "function") {
354 throw "You can only define a step as a function";
355 }
356 this.steps.push(step);
357 return this;
358 },
359
360 /**
361 * Adds a new navigation step to perform code evaluation within the
362 * current retrieved page DOM.
363 *
364 * @param function fn The function to be evaluated within current page DOM
365 * @param object replacements Optional replacements to performs, eg. for '%foo%' => {foo: 'bar'}
366 * @return Casper
367 * @see Casper#evaluate
368 */
369 thenEvaluate: function(fn, replacements) {
370 return this.then(function(self) {
371 self.evaluate(fn, replacements);
372 });
373 },
374
375 /**
376 * Adds a new navigation step depending on a condition to be evaluated
377 * within current page DOM. Dies on precondition failure with an
378 * optional message to be added to the results.errors Array.
379 *
380 * @param function condition An expression to be evaluated as a Boolean
381 * @param function then The next step to add if precondition succeeded
382 * @param string
383 */
384 thenIf: function(condition, then, message) {
385 return this.then(function(self) {
386 if (self.evaluate(condition) === true) {
387 return self.then(then);
388 }
389 return self.die(message);
390 });
391 },
392
393 /**
394 * Adds a new navigation step for opening the provided location.
395 *
396 * @param string location The URL to load
397 * @param function then Next step function to execute on page loaded (optional)
398 * @return Casper
399 * @see Casper#open
400 */
401 thenOpen: function(location, then) {
402 this.then(function(self) {
403 self.open(location);
404 });
405 return typeof(then) === "function" ? this.then(then) : this;
406 },
407
408 /**
409 * Adds a new navigation step for opening and evaluate an expression
410 * against the DOM retrieved from the provided location.
411 *
412 * @param string location The url to open
413 * @param function fn The function to be evaluated within current page DOM
414 * @param object replacements Optional replacements to performs, eg. for '%foo%' => {foo: 'bar'}
415 * @return Casper
416 * @see Casper#evaluate
417 * @see Casper#open
418 */
419 thenOpenAndEvaluate: function(location, fn, replacements) {
420 return this.thenOpen(location).thenEvaluate(fn, replacements);
421 },
422 };
423
424 /**
425 * Creates a new WebPage instance for Casper use.
426 *
427 * @param Casper casper A Casper instance
428 * @return WebPage
429 */
430 function createPage(casper) {
431 var page = new WebPage();
432 page.onConsoleMessage = function(msg) {
433 casper.log(msg, "info", "remote");
434 };
435 page.onLoadStarted = function() {
436 casper.loadInProgress = true;
437 };
438 page.onLoadFinished = function(status) {
439 if (status !== "success") {
440 casper.log('Loading resource failed with status=' + status + ': ' + casper.requestUrl, "info");
441 }
442 if (casper.options.clientScripts) {
443 for (var i = 0; i < casper.options.clientScripts.length; i++) {
444 var script = casper.options.clientScripts[i];
445 if (casper.page.injectJs(script)) {
446 casper.log('Automatically injected ' + script + ' client side', "debug");
447 } else {
448 casper.log('Failed injecting ' + script + ' client side', "debug");
449 }
450 }
451 }
452 casper.loadInProgress = false;
453 };
454 page.onResourceReceived = function(resource) {
455 if (resource.url === casper.requestUrl) {
456 casper.currentHTTPStatus = resource.status;
457 }
458 };
459 return page;
460 }
461
462 /**
463 * Object recursive merging utility.
464 *
465 * @param object obj1 the destination object
466 * @param object obj2 the source object
467 * @return object
468 */
469 function mergeObjects(obj1, obj2) {
470 for (var p in obj2) {
471 try {
472 if (obj2[p].constructor == Object) {
473 obj1[p] = mergeObjects(obj1[p], obj2[p]);
474 } else {
475 obj1[p] = obj2[p];
476 }
477 } catch(e) {
478 obj1[p] = obj2[p];
479 }
480 }
481 return obj1;
482 }
483 })(phantom);
1 phantom.injectJs('casper.js');
2
3 function q() {
4 document.querySelector('input[name="q"]').setAttribute('value', '%term%');
5 document.querySelector('form[name="f"]').submit();
6 }
7
8 function getLinks() {
9 return Array.prototype.map.call(document.querySelectorAll('h3.r a'), function(e) {
10 return e.getAttribute('href');
11 });
12 }
13
14 var links = [];
15 var casper = new phantom.Casper({
16 logLevel: "info",
17 verbose: true
18 })
19 .start('http://google.fr/')
20 .thenEvaluate(q, {
21 term: 'casper',
22 })
23 .then(function(self) {
24 links = self.evaluate(getLinks);
25 })
26 .thenEvaluate(q, {
27 term: 'homer',
28 })
29 .then(function(self) {
30 links = links.concat(self.evaluate(getLinks));
31 })
32 .run(function(self) {
33 self.echo(JSON.stringify({
34 result: self.result,
35 links: links
36 }, null, ' '));
37 self.exit();
38 })
39 ;