|
|
|
@ -1,7 +1,16 @@ |
|
|
|
import json, os, sys, subprocess |
|
|
|
import json, os, sys, subprocess, urllib2 |
|
|
|
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer |
|
|
|
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer |
|
|
|
|
|
|
|
from urlparse import urlparse |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def prompt(question): |
|
|
|
|
|
|
|
'''Return True iff the user answered "yes" to |question|.''' |
|
|
|
|
|
|
|
inp = raw_input(question +' [yes/no] > ') |
|
|
|
|
|
|
|
return inp == 'yes' |
|
|
|
|
|
|
|
|
|
|
|
ANAL = True |
|
|
|
ANAL = True |
|
|
|
|
|
|
|
DEFAULT_MANIFEST_FILE = 'test_manifest.json' |
|
|
|
|
|
|
|
REFDIR = 'ref' |
|
|
|
|
|
|
|
TMPDIR = 'tmp' |
|
|
|
VERBOSE = False |
|
|
|
VERBOSE = False |
|
|
|
|
|
|
|
|
|
|
|
MIMEs = { |
|
|
|
MIMEs = { |
|
|
|
@ -20,6 +29,12 @@ class State: |
|
|
|
remaining = 0 |
|
|
|
remaining = 0 |
|
|
|
results = { } |
|
|
|
results = { } |
|
|
|
done = False |
|
|
|
done = False |
|
|
|
|
|
|
|
masterMode = False |
|
|
|
|
|
|
|
numErrors = 0 |
|
|
|
|
|
|
|
numEqFailures = 0 |
|
|
|
|
|
|
|
numEqNoSnapshot = 0 |
|
|
|
|
|
|
|
numFBFFailures = 0 |
|
|
|
|
|
|
|
numLoadFailures = 0 |
|
|
|
|
|
|
|
|
|
|
|
class Result: |
|
|
|
class Result: |
|
|
|
def __init__(self, snapshot, failure): |
|
|
|
def __init__(self, snapshot, failure): |
|
|
|
@ -34,8 +49,11 @@ class PDFTestHandler(BaseHTTPRequestHandler): |
|
|
|
BaseHTTPRequestHandler.log_request(code, size) |
|
|
|
BaseHTTPRequestHandler.log_request(code, size) |
|
|
|
|
|
|
|
|
|
|
|
def do_GET(self): |
|
|
|
def do_GET(self): |
|
|
|
|
|
|
|
url = urlparse(self.path) |
|
|
|
|
|
|
|
# Ignore query string |
|
|
|
|
|
|
|
path, _ = url.path, url.query |
|
|
|
cwd = os.getcwd() |
|
|
|
cwd = os.getcwd() |
|
|
|
path = os.path.abspath(os.path.realpath(cwd + os.sep + self.path)) |
|
|
|
path = os.path.abspath(os.path.realpath(cwd + os.sep + path)) |
|
|
|
cwd = os.path.abspath(cwd) |
|
|
|
cwd = os.path.abspath(cwd) |
|
|
|
prefix = os.path.commonprefix(( path, cwd )) |
|
|
|
prefix = os.path.commonprefix(( path, cwd )) |
|
|
|
_, ext = os.path.splitext(path) |
|
|
|
_, ext = os.path.splitext(path) |
|
|
|
@ -69,31 +87,59 @@ class PDFTestHandler(BaseHTTPRequestHandler): |
|
|
|
self.end_headers() |
|
|
|
self.end_headers() |
|
|
|
|
|
|
|
|
|
|
|
result = json.loads(self.rfile.read(numBytes)) |
|
|
|
result = json.loads(self.rfile.read(numBytes)) |
|
|
|
browser = 'firefox4' |
|
|
|
browser, id, failure, round, page, snapshot = result['browser'], result['id'], result['failure'], result['round'], result['page'], result['snapshot'] |
|
|
|
id, failure, round, page, snapshot = result['id'], result['failure'], result['round'], result['page'], result['snapshot'] |
|
|
|
|
|
|
|
taskResults = State.taskResults[browser][id] |
|
|
|
taskResults = State.taskResults[browser][id] |
|
|
|
taskResults[round][page - 1] = Result(snapshot, failure) |
|
|
|
taskResults[round].append(Result(snapshot, failure)) |
|
|
|
|
|
|
|
assert len(taskResults[round]) == page |
|
|
|
|
|
|
|
|
|
|
|
if result['taskDone']: |
|
|
|
if result['taskDone']: |
|
|
|
check(State.manifest[id], taskResults, browser) |
|
|
|
check(State.manifest[id], taskResults, browser) |
|
|
|
|
|
|
|
# Please oh please GC this ... |
|
|
|
|
|
|
|
del State.taskResults[browser][id] |
|
|
|
State.remaining -= 1 |
|
|
|
State.remaining -= 1 |
|
|
|
|
|
|
|
|
|
|
|
State.done = (0 == State.remaining) |
|
|
|
State.done = (0 == State.remaining) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def set_up(): |
|
|
|
def setUp(manifestFile, masterMode): |
|
|
|
# Only serve files from a pdf.js clone |
|
|
|
# Only serve files from a pdf.js clone |
|
|
|
assert not ANAL or os.path.isfile('pdf.js') and os.path.isdir('.git') |
|
|
|
assert not ANAL or os.path.isfile('pdf.js') and os.path.isdir('.git') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
State.masterMode = masterMode |
|
|
|
|
|
|
|
if masterMode and os.path.isdir(TMPDIR): |
|
|
|
|
|
|
|
print 'Temporary snapshot dir tmp/ is still around.' |
|
|
|
|
|
|
|
print 'tmp/ can be removed if it has nothing you need.' |
|
|
|
|
|
|
|
if prompt('SHOULD THIS SCRIPT REMOVE tmp/? THINK CAREFULLY'): |
|
|
|
|
|
|
|
subprocess.call(( 'rm', '-rf', 'tmp' )) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
assert not os.path.isdir(TMPDIR) |
|
|
|
|
|
|
|
|
|
|
|
testBrowsers = [ b for b in |
|
|
|
testBrowsers = [ b for b in |
|
|
|
( 'firefox4', ) |
|
|
|
( 'firefox5', ) |
|
|
|
#'chrome12', 'chrome13', 'firefox5', 'firefox6','opera11' ): |
|
|
|
#'chrome12', 'chrome13', 'firefox4', 'firefox6','opera11' ): |
|
|
|
if os.access(b, os.R_OK | os.X_OK) ] |
|
|
|
if os.access(b, os.R_OK | os.X_OK) ] |
|
|
|
|
|
|
|
|
|
|
|
mf = open('test_manifest.json') |
|
|
|
mf = open(manifestFile) |
|
|
|
manifestList = json.load(mf) |
|
|
|
manifestList = json.load(mf) |
|
|
|
mf.close() |
|
|
|
mf.close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for item in manifestList: |
|
|
|
|
|
|
|
f, isLink = item['file'], item.get('link', False) |
|
|
|
|
|
|
|
if isLink and not os.access(f, os.R_OK): |
|
|
|
|
|
|
|
linkFile = open(f +'.link') |
|
|
|
|
|
|
|
link = linkFile.read() |
|
|
|
|
|
|
|
linkFile.close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sys.stdout.write('Downloading '+ link +' to '+ f +' ...') |
|
|
|
|
|
|
|
sys.stdout.flush() |
|
|
|
|
|
|
|
response = urllib2.urlopen(link) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
out = open(f, 'w') |
|
|
|
|
|
|
|
out.write(response.read()) |
|
|
|
|
|
|
|
out.close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print 'done' |
|
|
|
|
|
|
|
|
|
|
|
for b in testBrowsers: |
|
|
|
for b in testBrowsers: |
|
|
|
State.taskResults[b] = { } |
|
|
|
State.taskResults[b] = { } |
|
|
|
for item in manifestList: |
|
|
|
for item in manifestList: |
|
|
|
@ -101,15 +147,16 @@ def set_up(): |
|
|
|
State.manifest[id] = item |
|
|
|
State.manifest[id] = item |
|
|
|
taskResults = [ ] |
|
|
|
taskResults = [ ] |
|
|
|
for r in xrange(rounds): |
|
|
|
for r in xrange(rounds): |
|
|
|
taskResults.append([ None ] * 100) |
|
|
|
taskResults.append([ ]) |
|
|
|
State.taskResults[b][id] = taskResults |
|
|
|
State.taskResults[b][id] = taskResults |
|
|
|
|
|
|
|
|
|
|
|
State.remaining = len(manifestList) |
|
|
|
State.remaining = len(manifestList) |
|
|
|
|
|
|
|
|
|
|
|
for b in testBrowsers: |
|
|
|
for b in testBrowsers: |
|
|
|
print 'Launching', b |
|
|
|
print 'Launching', b |
|
|
|
|
|
|
|
qs = 'browser='+ b +'&manifestFile='+ manifestFile |
|
|
|
subprocess.Popen(( os.path.abspath(os.path.realpath(b)), |
|
|
|
subprocess.Popen(( os.path.abspath(os.path.realpath(b)), |
|
|
|
'http://localhost:8080/test_slave.html' )) |
|
|
|
'http://localhost:8080/test_slave.html?'+ qs)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check(task, results, browser): |
|
|
|
def check(task, results, browser): |
|
|
|
@ -123,13 +170,14 @@ def check(task, results, browser): |
|
|
|
failure = pageResult.failure |
|
|
|
failure = pageResult.failure |
|
|
|
if failure: |
|
|
|
if failure: |
|
|
|
failed = True |
|
|
|
failed = True |
|
|
|
|
|
|
|
State.numErrors += 1 |
|
|
|
print 'TEST-UNEXPECTED-FAIL | test failed', task['id'], '| in', browser, '| page', p + 1, 'round', r, '|', failure |
|
|
|
print 'TEST-UNEXPECTED-FAIL | test failed', task['id'], '| in', browser, '| page', p + 1, 'round', r, '|', failure |
|
|
|
|
|
|
|
|
|
|
|
if failed: |
|
|
|
if failed: |
|
|
|
return |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
kind = task['type'] |
|
|
|
kind = task['type'] |
|
|
|
if '==' == kind: |
|
|
|
if 'eq' == kind: |
|
|
|
checkEq(task, results, browser) |
|
|
|
checkEq(task, results, browser) |
|
|
|
elif 'fbf' == kind: |
|
|
|
elif 'fbf' == kind: |
|
|
|
checkFBF(task, results, browser) |
|
|
|
checkFBF(task, results, browser) |
|
|
|
@ -140,23 +188,60 @@ def check(task, results, browser): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def checkEq(task, results, browser): |
|
|
|
def checkEq(task, results, browser): |
|
|
|
print ' !!! [TODO: == tests] !!!' |
|
|
|
pfx = os.path.join(REFDIR, sys.platform, browser, task['id']) |
|
|
|
print 'TEST-PASS | == test', task['id'], '| in', browser |
|
|
|
results = results[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
passed = True |
|
|
|
|
|
|
|
for page in xrange(len(results)): |
|
|
|
|
|
|
|
snapshot = results[page].snapshot |
|
|
|
|
|
|
|
ref = None |
|
|
|
|
|
|
|
eq = True |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
path = os.path.join(pfx, str(page + 1)) |
|
|
|
|
|
|
|
if not os.access(path, os.R_OK): |
|
|
|
|
|
|
|
print 'WARNING: no reference snapshot', path |
|
|
|
|
|
|
|
State.numEqNoSnapshot += 1 |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
f = open(path) |
|
|
|
|
|
|
|
ref = f.read() |
|
|
|
|
|
|
|
f.close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eq = (ref == snapshot) |
|
|
|
|
|
|
|
if not eq: |
|
|
|
|
|
|
|
print 'TEST-UNEXPECTED-FAIL | eq', task['id'], '| in', browser, '| rendering of page', page + 1, '!= reference rendering' |
|
|
|
|
|
|
|
passed = False |
|
|
|
|
|
|
|
State.numEqFailures += 1 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if State.masterMode and (ref is None or not eq): |
|
|
|
|
|
|
|
tmpTaskDir = os.path.join(TMPDIR, sys.platform, browser, task['id']) |
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
os.makedirs(tmpTaskDir) |
|
|
|
|
|
|
|
except OSError, e: |
|
|
|
|
|
|
|
pass |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
of = open(os.path.join(tmpTaskDir, str(page + 1)), 'w') |
|
|
|
|
|
|
|
of.write(snapshot) |
|
|
|
|
|
|
|
of.close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if passed: |
|
|
|
|
|
|
|
print 'TEST-PASS | eq test', task['id'], '| in', browser |
|
|
|
|
|
|
|
|
|
|
|
printed = [False] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def checkFBF(task, results, browser): |
|
|
|
def checkFBF(task, results, browser): |
|
|
|
round0, round1 = results[0], results[1] |
|
|
|
round0, round1 = results[0], results[1] |
|
|
|
assert len(round0) == len(round1) |
|
|
|
assert len(round0) == len(round1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
passed = True |
|
|
|
for page in xrange(len(round1)): |
|
|
|
for page in xrange(len(round1)): |
|
|
|
r0Page, r1Page = round0[page], round1[page] |
|
|
|
r0Page, r1Page = round0[page], round1[page] |
|
|
|
if r0Page is None: |
|
|
|
if r0Page is None: |
|
|
|
break |
|
|
|
break |
|
|
|
if r0Page.snapshot != r1Page.snapshot: |
|
|
|
if r0Page.snapshot != r1Page.snapshot: |
|
|
|
print 'TEST-UNEXPECTED-FAIL | forward-back-forward test', task['id'], '| in', browser, '| first rendering of page', page + 1, '!= second' |
|
|
|
print 'TEST-UNEXPECTED-FAIL | forward-back-forward test', task['id'], '| in', browser, '| first rendering of page', page + 1, '!= second' |
|
|
|
print 'TEST-PASS | forward-back-forward test', task['id'], '| in', browser |
|
|
|
passed = False |
|
|
|
|
|
|
|
State.numFBFFailures += 1 |
|
|
|
|
|
|
|
if passed: |
|
|
|
|
|
|
|
print 'TEST-PASS | forward-back-forward test', task['id'], '| in', browser |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def checkLoad(task, results, browser): |
|
|
|
def checkLoad(task, results, browser): |
|
|
|
@ -165,11 +250,55 @@ def checkLoad(task, results, browser): |
|
|
|
print 'TEST-PASS | load test', task['id'], '| in', browser |
|
|
|
print 'TEST-PASS | load test', task['id'], '| in', browser |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(): |
|
|
|
def processResults(): |
|
|
|
set_up() |
|
|
|
print '' |
|
|
|
|
|
|
|
numErrors, numEqFailures, numEqNoSnapshot, numFBFFailures = State.numErrors, State.numEqFailures, State.numEqNoSnapshot, State.numFBFFailures |
|
|
|
|
|
|
|
numFatalFailures = (numErrors + numFBFFailures) |
|
|
|
|
|
|
|
if 0 == numEqFailures and 0 == numFatalFailures: |
|
|
|
|
|
|
|
print 'All tests passed.' |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
print 'OHNOES! Some tests failed!' |
|
|
|
|
|
|
|
if 0 < numErrors: |
|
|
|
|
|
|
|
print ' errors:', numErrors |
|
|
|
|
|
|
|
if 0 < numEqFailures: |
|
|
|
|
|
|
|
print ' different ref/snapshot:', numEqFailures |
|
|
|
|
|
|
|
if 0 < numFBFFailures: |
|
|
|
|
|
|
|
print ' different first/second rendering:', numFBFFailures |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if State.masterMode and (0 < numEqFailures or 0 < numEqNoSnapshot): |
|
|
|
|
|
|
|
print "Some eq tests failed or didn't have snapshots." |
|
|
|
|
|
|
|
print 'Checking to see if master references can be updated...' |
|
|
|
|
|
|
|
if 0 < numFatalFailures: |
|
|
|
|
|
|
|
print ' No. Some non-eq tests failed.' |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
' Yes! The references in tmp/ can be synced with ref/.' |
|
|
|
|
|
|
|
if not prompt('Would you like to update the master copy in ref/?'): |
|
|
|
|
|
|
|
print ' OK, not updating.' |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
sys.stdout.write(' Updating ... ') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# XXX unclear what to do on errors here ... |
|
|
|
|
|
|
|
# NB: do *NOT* pass --delete to rsync. That breaks this |
|
|
|
|
|
|
|
# entire scheme. |
|
|
|
|
|
|
|
subprocess.check_call(( 'rsync', '-arv', 'tmp/', 'ref/' )) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print 'done' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(args): |
|
|
|
|
|
|
|
masterMode = False |
|
|
|
|
|
|
|
manifestFile = DEFAULT_MANIFEST_FILE |
|
|
|
|
|
|
|
if len(args) == 1: |
|
|
|
|
|
|
|
masterMode = (args[0] == '-m') |
|
|
|
|
|
|
|
manifestFile = args[0] if not masterMode else manifestFile |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
setUp(manifestFile, masterMode) |
|
|
|
|
|
|
|
|
|
|
|
server = HTTPServer(('127.0.0.1', 8080), PDFTestHandler) |
|
|
|
server = HTTPServer(('127.0.0.1', 8080), PDFTestHandler) |
|
|
|
while not State.done: |
|
|
|
while not State.done: |
|
|
|
server.handle_request() |
|
|
|
server.handle_request() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
processResults() |
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
if __name__ == '__main__': |
|
|
|
main() |
|
|
|
main(sys.argv[1:]) |
|
|
|
|
Vivien Nicolas
14 years ago
