From: Chris Jones <jones.chris.g@gmail.com>
Date: Sun, 19 Jun 2011 01:09:21 +0000 (-0700)
Subject: Initial import of first test harness
X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=7c024b89aa60d7d36f31278cf76a0b4db20fd1de;p=pdf.js.git

Initial import of first test harness

The harness (test.py) operates as follows.  First it locates executable browsers
(or symlinks or scripts) named "[browser][version]", e.g. "firefox4".
It then launches the located browsers and asks them to load the file
test_slave.html.  At the same time, test.py sets up an HTTP server on
localhost:8080 (there's a race condition here currently ;).  After
test_slave loads in the browser(s), it fetches the task manifest
(test_manifest.json).  The entries in the manifest specify which PDF
to load and how many times to cycle through page rendering.  This will
probably evolve over time.  test_slave then performs the requested
tasks and POSTs the results back to test.py, which saves them.  When
all the results of for a task are in, test.py checks them.

There are three types of tests currently.  "==" tests compare the
rendering of a PDF against a master copy.  This is not yet implemented
because setting up a master copy is complicated.  "fbf" tests render
all a PDF's pages, then go back to page 1 and render all pages a
second time.  The renderings from the first round must match the ones
from the second round.  "load" tests just check that a PDF's pages
load without errors.

Currently the test harness will only launch a "firefox4" target.  This
can be a bash script in your pdf.js checkout, pdf.js/firefox4,
something like the following

 #!/bin/bash
 dist="/path/to/firefox4/installation"
 profile=`mktemp -dt 'pdf.js-test-ff-profile-XXXXXXXXXX'`
 $dist/firefox -no-remote -profile $profile $*
 rm -rf $profile

(Yes, this script doesn't clean up properly on early termination.)
It's possible to run the tests in a normal browsing session, but that
might be annoying.  With that set up, run the harness like so

 python test.py

If all goes well, you'll see all "TEST-PASS" messages printed to
stdout.  If something goes wrong, you'll see "TEST-UNEXPECTED-FAIL"
printed to stdout.
---

diff --git a/test.py b/test.py
new file mode 100644
index 0000000..46d30fe
--- /dev/null
+++ b/test.py
@@ -0,0 +1,175 @@
+import json, os, sys, subprocess
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+ANAL = True
+VERBOSE = False
+
+MIMEs = {
+    '.css': 'text/css',
+    '.html': 'text/html',
+    '.js': 'application/json',
+    '.json': 'application/json',
+    '.pdf': 'application/pdf',
+    '.xhtml': 'application/xhtml+xml',
+}
+
+class State:
+    browsers = [ ]
+    manifest = { }
+    taskResults = { }
+    remaining = 0
+    results = { }
+    done = False
+
+class Result:
+    def __init__(self, snapshot, failure):
+        self.snapshot = snapshot
+        self.failure = failure
+
+
+class PDFTestHandler(BaseHTTPRequestHandler):
+    # Disable annoying noise by default
+    def log_request(code=0, size=0):
+        if VERBOSE:
+            BaseHTTPRequestHandler.log_request(code, size)
+
+    def do_GET(self):
+        cwd = os.getcwd()
+        path = os.path.abspath(os.path.realpath(cwd + os.sep + self.path))
+        cwd = os.path.abspath(cwd)
+        prefix = os.path.commonprefix(( path, cwd ))
+        _, ext = os.path.splitext(path)
+
+        if not (prefix == cwd
+                and os.path.isfile(path) 
+                and ext in MIMEs):
+            self.send_error(404)
+            return
+
+        if 'Range' in self.headers:
+            # TODO for fetch-as-you-go
+            self.send_error(501)
+            return
+
+        self.send_response(200)
+        self.send_header("Content-Type", MIMEs[ext])
+        self.end_headers()
+
+        # Sigh, os.sendfile() plz
+        f = open(path)
+        self.wfile.write(f.read())
+        f.close()
+
+
+    def do_POST(self):
+        numBytes = int(self.headers['Content-Length'])
+
+        self.send_response(200)
+        self.send_header('Content-Type', 'text/plain')
+        self.end_headers()
+
+        result = json.loads(self.rfile.read(numBytes))
+        browser = 'firefox4'
+        id, failure, round, page, snapshot = result['id'], result['failure'], result['round'], result['page'], result['snapshot']
+        taskResults = State.taskResults[browser][id]
+        taskResults[round][page - 1] = Result(snapshot, failure)
+
+        if result['taskDone']:
+            check(State.manifest[id], taskResults, browser)
+            State.remaining -= 1
+
+        State.done = (0 == State.remaining)
+            
+
+def set_up():
+    # Only serve files from a pdf.js clone
+    assert not ANAL or os.path.isfile('pdf.js') and os.path.isdir('.git')
+
+    testBrowsers = [ b for b in
+                     ( 'firefox4', )
+#'chrome12', 'chrome13', 'firefox5', 'firefox6','opera11' ):
+                     if os.access(b, os.R_OK | os.X_OK) ]
+
+    mf = open('test_manifest.json')
+    manifestList = json.load(mf)
+    mf.close()
+
+    for b in testBrowsers:
+        State.taskResults[b] = { }
+        for item in manifestList:
+            id, rounds = item['id'], int(item['rounds'])
+            State.manifest[id] = item
+            taskResults = [ ]
+            for r in xrange(rounds):
+                taskResults.append([ None ] * 100)
+            State.taskResults[b][id] = taskResults
+
+    State.remaining = len(manifestList)
+
+    for b in testBrowsers:
+        print 'Launching', b
+        subprocess.Popen(( os.path.abspath(os.path.realpath(b)),
+                           'http://localhost:8080/test_slave.html' ))
+
+
+def check(task, results, browser):
+    failed = False
+    for r in xrange(len(results)):
+        pageResults = results[r]
+        for p in xrange(len(pageResults)):
+            pageResult = pageResults[p]
+            if pageResult is None:
+                continue
+            failure = pageResult.failure
+            if failure:
+                failed = True
+                print 'TEST-UNEXPECTED-FAIL | test failed', task['id'], '| in', browser, '| page', p + 1, 'round', r, '|', failure
+
+    if failed:
+        return
+
+    kind = task['type']
+    if '==' == kind:
+        checkEq(task, results, browser)
+    elif 'fbf' == kind:
+        checkFBF(task, results, browser)
+    elif 'load' == kind:
+        checkLoad(task, results, browser)
+    else:
+        assert 0 and 'Unknown test type'
+
+
+def checkEq(task, results, browser):
+    print '  !!! [TODO: == tests] !!!'
+    print 'TEST-PASS | == test', task['id'], '| in', browser
+
+
+printed = [False]
+
+def checkFBF(task, results, browser):
+    round0, round1 = results[0], results[1]
+    assert len(round0) == len(round1)
+
+    for page in xrange(len(round1)):
+        r0Page, r1Page = round0[page], round1[page]
+        if r0Page is None:
+            break
+        if r0Page.snapshot != r1Page.snapshot:
+            print 'TEST-UNEXPECTED-FAIL | forward-back-forward test', task['id'], '| in', browser, '| first rendering of page', page + 1, '!= second'
+    print 'TEST-PASS | forward-back-forward test', task['id'], '| in', browser
+
+
+def checkLoad(task, results, browser):
+    # Load just checks for absence of failure, so if we got here the
+    # test has passed
+    print 'TEST-PASS | load test', task['id'], '| in', browser
+
+
+def main():
+    set_up()
+    server = HTTPServer(('127.0.0.1', 8080), PDFTestHandler)
+    while not State.done:
+        server.handle_request()
+
+if __name__ == '__main__':
+    main()
diff --git a/test_manifest.json b/test_manifest.json
new file mode 100644
index 0000000..2f45a02
--- /dev/null
+++ b/test_manifest.json
@@ -0,0 +1,17 @@
+[
+    {  "id": "tracemonkey-==",
+       "file": "tests/tracemonkey.pdf",
+       "rounds": 1,
+       "type": "=="
+    },
+    {  "id": "tracemonkey-fbf",
+       "file": "tests/tracemonkey.pdf",
+       "rounds": 2,
+       "type": "fbf"
+    },
+    {  "id": "html5-canvas-cheat-sheet-load",
+       "file": "tests/canvas.pdf",
+       "rounds": 1,
+       "type": "load"
+    }
+]
diff --git a/test_slave.html b/test_slave.html
new file mode 100644
index 0000000..c560d90
--- /dev/null
+++ b/test_slave.html
@@ -0,0 +1,149 @@
+<html>
+<head>
+  <title>pdf.js test slave</title>
+  <script type="text/javascript" src="pdf.js"></script>
+  <script type="text/javascript" src="fonts.js"></script>
+  <script type="text/javascript" src="glyphlist.js"></script>
+  <script type="application/javascript">
+var canvas, currentTask, currentTaskIdx, failure, manifest, pdfDoc, stdout;
+
+function load() {
+  canvas = document.createElement("canvas");
+  // 8.5x11in @ 100% ... XXX need something better here
+  canvas.width = 816;
+  canvas.height = 1056;
+  canvas.mozOpaque = true;
+  stdout = document.getElementById("stdout");
+
+  log("Fetching manifest ...");
+
+  var r = new XMLHttpRequest();
+  r.open("GET", "test_manifest.json", false);
+  r.onreadystatechange = function(e) {
+    if (r.readyState == 4) {
+      log("done\n");
+
+      manifest = JSON.parse(r.responseText);
+      currentTaskIdx = 0, nextTask();
+    }
+  };
+  r.send(null);
+}
+
+function nextTask() {
+  if (currentTaskIdx == manifest.length) {
+    return done();
+  }
+  currentTask = manifest[currentTaskIdx];
+  currentTask.round = 0;
+
+  log("Loading file "+ currentTask.file +"\n");
+
+  var r = new XMLHttpRequest();
+  r.open("GET", currentTask.file);
+  r.mozResponseType = r.responseType = "arraybuffer";
+  r.onreadystatechange = function() {
+    if (r.readyState == 4) {
+      var data = r.mozResponseArrayBuffer || r.mozResponse ||
+                 r.responseArrayBuffer || r.response;
+      pdfDoc = new PDFDoc(new Stream(data));
+      currentTask.pageNum = 1, nextPage();
+    }    
+  };
+  r.send(null);
+}
+
+function nextPage() {
+  if (currentTask.pageNum > pdfDoc.numPages) {
+    if (++currentTask.round < currentTask.rounds) {
+      log("  Round "+ (1 + currentTask.round) +"\n");
+      currentTask.pageNum = 1;
+    } else {
+      ++currentTaskIdx, nextTask();
+      return;
+    }
+  }
+
+  failure = '';
+  log("    drawing page "+ currentTask.pageNum +"...");
+
+  currentPage = pdfDoc.getPage(currentTask.pageNum);
+
+  var ctx = canvas.getContext("2d");
+  clear(ctx);
+
+  var fonts = [];
+  var gfx = new CanvasGraphics(ctx);
+  try {
+    currentPage.compile(gfx, fonts);
+  } catch(e) {
+    failure = 'compile: '+ e.toString();
+  }
+
+  // TODO load fonts
+  setTimeout(function() {
+      if (!failure) {
+        try {
+          currentPage.display(gfx);
+        } catch(e) {
+          failure = 'render: '+ e.toString();
+        }
+      }
+      currentTask.taskDone = (currentTask.pageNum == pdfDoc.numPages
+                              && (1 + currentTask.round) == currentTask.rounds);
+      sendTaskResult(canvas.toDataURL("image/png"));
+      log("done"+ (failure ? " (failed!)" : "") +"\n");
+
+      ++currentTask.pageNum, nextPage();
+    },
+    0
+  );
+}
+
+function done() {
+  log("Done!\n");
+  setTimeout(function() {
+      document.body.innerHTML = "Tests are finished.  <h1>CLOSE ME!</h1>";
+      window.close();
+    },
+    100
+  );
+}
+
+function sendTaskResult(snapshot) {
+  var result = { id: currentTask.id,
+                 taskDone: currentTask.taskDone,
+                 failure: failure,
+                 file: currentTask.file,
+                 round: currentTask.round,
+                 page: currentTask.pageNum,
+                 snapshot: snapshot };
+
+  var r = new XMLHttpRequest();
+  // (The POST URI is ignored atm.)
+  r.open("POST", "submit_task_results", false);
+  r.setRequestHeader("Content-Type", "application/json");
+  // XXX async
+  r.send(JSON.stringify(result));
+}
+
+function clear(ctx) {
+  var ctx = canvas.getContext("2d");
+  ctx.save();
+  ctx.fillStyle = "rgb(255, 255, 255)";
+  ctx.fillRect(0, 0, canvas.width, canvas.height);
+  ctx.restore();
+}
+
+function log(str) {
+  stdout.innerHTML += str;
+  window.scrollTo(0, stdout.getBoundingClientRect().bottom);
+}
+  </script>
+</head>
+
+<body onload="load();">
+  <pre id="stdout"></pre>
+</body>
+
+</html>
diff --git a/tests/canvas.pdf b/tests/canvas.pdf
new file mode 100644
index 0000000..900d8af
Binary files /dev/null and b/tests/canvas.pdf differ
diff --git a/tests/tracemonkey.pdf b/tests/tracemonkey.pdf
new file mode 100644
index 0000000..6557018
Binary files /dev/null and b/tests/tracemonkey.pdf differ