test/test_parser.py
author Mario D. Santana <mario@elorangutan.com>
Fri, 15 Apr 2016 18:29:24 -0600
changeset 97 44522cd37b07
parent 70 1fe2c20adeba
child 102 e8cb8d1367c0
permissions -rw-r--r--
Fixed tests. Also some UTF bugs (python2/3 hell)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
70
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     1
from __future__ import print_function
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     2
from __future__ import unicode_literals
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     3
from __future__ import division
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     4
from __future__ import absolute_import
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     5
from future import standard_library
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     6
standard_library.install_aliases()
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
     7
from builtins import *
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
     8
import unittest
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
     9
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    10
import os
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    11
import sys
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    12
sys.path.append('../')
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    13
12
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    14
import datetime
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    15
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    16
import simplejson
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    17
from glob import glob
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    18
12
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    19
from whois.parser import WhoisEntry, cast_date
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    20
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    21
class TestParser(unittest.TestCase):
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    22
    def test_com_expiration(self):
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    23
        data = """
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    24
        Status: ok
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    25
        Updated Date: 14-apr-2008
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    26
        Creation Date: 14-apr-2008
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    27
        Expiration Date: 14-apr-2009
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    28
        
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    29
        >>> Last update of whois database: Sun, 31 Aug 2008 00:18:23 UTC <<<
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    30
        """
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    31
        w = WhoisEntry.load('urlowl.com', data)
12
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    32
        expires = w.expiration_date.strftime('%Y-%m-%d')
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    33
        self.assertEqual(expires, '2009-04-14')
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    34
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    35
    def test_cast_date(self):
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    36
        dates = ['14-apr-2008', '2008-04-14']
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    37
        for d in dates:
12
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    38
            r = cast_date(d).strftime('%Y-%m-%d')
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    39
            self.assertEqual(r, '2008-04-14')
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    40
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    41
    def test_com_allsamples(self):
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    42
        """
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    43
        Iterate over all of the sample/whois/*.com files, read the data,
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    44
        parse it, and compare to the expected values in sample/expected/.
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    45
        Only keys defined in keys_to_test will be tested.
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    46
        
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    47
        To generate fresh expected value dumps, see NOTE below.
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    48
        """
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    49
        keys_to_test = ['domain_name', 'expiration_date', 'updated_date',
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    50
                        'creation_date', 'status']
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    51
        fail = 0
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    52
        total = 0
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    53
        for path in glob('test/samples/whois/*.com'):
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    54
            # Parse whois data
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    55
            domain = os.path.basename(path)
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    56
            with open(path) as whois_fp:
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    57
                data = whois_fp.read()
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    58
            
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    59
            w = WhoisEntry.load(domain, data)
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    60
            results = {key: w.get(key) for key in keys_to_test}
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    61
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    62
            # NOTE: Toggle condition below to write expected results from the
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    63
            # parse results This will overwrite the existing expected results.
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    64
            # Only do this if you've manually confirmed that the parser is
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    65
            # generating correct values at its current state.
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    66
            if False:
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    67
                def date2str4json(obj):
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    68
                    if isinstance(obj, datetime.datetime):
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    69
                        return str(obj)
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    70
                    raise TypeError(
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    71
                            '{} is not JSON serializable'.format(repr(obj)))
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    72
                outfile_name = os.path.join('test/samples/expected/', domain)
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    73
                with open(outfile_name, 'w') as outfil:
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    74
                    expected_results = simplejson.dump(results, outfil,
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    75
                                                       default=date2str4json)
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    76
                continue
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    77
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    78
            # Load expected result
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    79
            with open(os.path.join('test/samples/expected/', domain)) as infil:
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    80
                expected_results = simplejson.load(infil)
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    81
            
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    82
            # Compare each key
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    83
            for key in results:
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    84
                total += 1
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    85
                result = results.get(key)
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    86
                if isinstance(result, datetime.datetime):
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    87
                    result = str(result)
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    88
                expected = expected_results.get(key)
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    89
                if expected != result:
70
1fe2c20adeba Python3 support
Johnny Wezel<j@wezel.name>
parents: 12
diff changeset
    90
                    print("%s \t(%s):\t %s != %s" % (domain, key, result, expected))
0
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    91
                    fail += 1
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    92
            
ea0e45971cea initial commit to mercurial
Richard Baron Penman
parents:
diff changeset
    93
        if fail:
97
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    94
            self.fail("%d/%d sample whois attributes were not parsed properly!"
44522cd37b07 Fixed tests. Also some UTF bugs (python2/3 hell)
Mario D. Santana <mario@elorangutan.com>
parents: 70
diff changeset
    95
                      % (fail, total))
12
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    96
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    97
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    98
if __name__ == '__main__':
c57439b500cb fixed test cases
Richard Baron Penman
parents: 0
diff changeset
    99
    unittest.main()