blob: 1e570f37c4e293ed0f663fffba56add95d022736 [file] [log] [blame] [edit]
#!/usr/bin/python
# Copyright 2009 Google Inc. Released under the GPL v2
import unittest, cStringIO, httplib, time, os
import common
from autotest_lib.mirror import source
from autotest_lib.client.common_lib.test_utils import mock
class common_source(unittest.TestCase):
"""
Common support class for source unit tests.
"""
def setUp(self):
self.god = mock.mock_god()
self.db_mock = self.god.create_mock_class(
source.database.database, 'database')
# Set fixed timezone so parsing time values does not break.
self._old_tz = getattr(os.environ, 'TZ', '')
os.environ['TZ'] = 'America/Los_Angeles'
time.tzset()
def tearDown(self):
self.god.unstub_all()
os.environ['TZ'] = self._old_tz
time.tzset()
class rsync_source_unittest(common_source):
_cmd_template = '/usr/bin/rsync -rltz --no-motd %s %s/%s'
_prefix = 'rsync://rsync.kernel.org/pub/linux/kernel'
_path1 = 'v2.6/patch-2.6.*.bz2'
_path2 = 'v2.6/testing/patch*.bz2'
_output1 = """\
-rw-rw-r-- 10727 2003/12/17 19:04:34 patch-2.6.0.bz2
-rw-rw-r-- 777959 2004/01/08 23:31:48 patch-2.6.1.bz2
-rw-rw-r-- 4851041 2004/12/24 14:38:58 patch-2.6.10.bz2
-rw-r--r-- 713 2005/03/08 16:59:09 patch-2.6.11.1.bz2
-rw-r--r-- 15141 2005/05/16 11:17:23 patch-2.6.11.10.bz2
-rw-rw-r-- 20868 2005/05/26 22:51:21 patch-2.6.11.11.bz2
-rw-rw-r-- 23413 2005/06/11 19:57:26 patch-2.6.11.12.bz2
-rw-r--r-- 1010 2005/03/12 22:55:52 patch-2.6.11.2.bz2
"""
_output2 = """\
-rw-rw-r-- 10462721 2009/04/07 15:45:35 patch-2.6.30-rc1.bz2
-rw-rw-r-- 10815919 2009/04/14 19:01:40 patch-2.6.30-rc2.bz2
-rw-rw-r-- 11032734 2009/04/21 20:28:11 patch-2.6.30-rc3.bz2
"""
_output_excluded = """\
-rw-rw-r-- 10462721 2009/04/07 15:45:35 patch-2.6.30-rc1.bz2
-rw-rw-r-- 11032734 2009/04/21 20:28:11 patch-2.6.30-rc3.bz2
"""
_known_files = {
'v2.6/patch-2.6.1.bz2': source.database.item(
'v2.6/patch-2.6.1.bz2', 777959, 1073633508),
'v2.6/patch-2.6.11.10.bz2': source.database.item(
'v2.6/patch-2.6.11.10.bz2', 15141, 1116267443),
'v2.6/testing/patch-2.6.30-rc1.bz2': source.database.item(
'v2.6/testing/patch-2.6.30-rc1.bz2', 10462721, 1239144335),
}
_result = {
'v2.6/patch-2.6.0.bz2': source.database.item(
'v2.6/patch-2.6.0.bz2', 10727, 1071716674),
'v2.6/patch-2.6.10.bz2': source.database.item(
'v2.6/patch-2.6.10.bz2', 4851041, 1103927938),
'v2.6/patch-2.6.11.12.bz2': source.database.item(
'v2.6/patch-2.6.11.12.bz2', 23413, 1118545046),
'v2.6/patch-2.6.11.11.bz2': source.database.item(
'v2.6/patch-2.6.11.11.bz2', 20868, 1117173081),
'v2.6/patch-2.6.11.2.bz2': source.database.item(
'v2.6/patch-2.6.11.2.bz2', 1010, 1110696952),
'v2.6/patch-2.6.11.1.bz2': source.database.item(
'v2.6/patch-2.6.11.1.bz2', 713, 1110329949),
'v2.6/testing/patch-2.6.30-rc3.bz2': source.database.item(
'v2.6/testing/patch-2.6.30-rc3.bz2', 11032734, 1240370891),
'v2.6/testing/patch-2.6.30-rc2.bz2': source.database.item(
'v2.6/testing/patch-2.6.30-rc2.bz2', 10815919, 1239760900),
}
def setUp(self):
super(rsync_source_unittest, self).setUp()
self.god.stub_function(source.utils, 'system_output')
def test_simple(self):
# record
(source.utils.system_output.expect_call(
self._cmd_template % ('', self._prefix, self._path1))
.and_return(self._output1))
(source.utils.system_output.expect_call(
self._cmd_template % ('', self._prefix, self._path2))
.and_return(self._output2))
self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
# playback
s = source.rsync_source(self.db_mock, self._prefix)
s.add_path('v2.6/patch-2.6.*.bz2', 'v2.6')
s.add_path('v2.6/testing/patch*.bz2', 'v2.6/testing')
self.assertEquals(s.get_new_files(), self._result)
self.god.check_playback()
def test_exclusions(self):
# setup
exclude_str = '--exclude "2.6.30-rc2"'
excluded_result = dict(self._result)
del excluded_result['v2.6/testing/patch-2.6.30-rc2.bz2']
# record
(source.utils.system_output.expect_call(
self._cmd_template % (exclude_str, self._prefix, self._path1))
.and_return(self._output1))
(source.utils.system_output.expect_call(
self._cmd_template % (exclude_str, self._prefix, self._path2))
.and_return(self._output_excluded))
self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
# playback
s = source.rsync_source(self.db_mock, self._prefix,
excludes=('2.6.30-rc2',))
s.add_path('v2.6/patch-2.6.*.bz2', 'v2.6')
s.add_path('v2.6/testing/patch*.bz2', 'v2.6/testing')
self.assertEquals(s.get_new_files(), excluded_result)
self.god.check_playback()
class url_source_unittest(common_source):
_prefix = 'http://www.kernel.org/pub/linux/kernel/'
_path1 = 'v2.6/'
_full_path1 = '%s%s' % (_prefix, _path1)
_path2 = 'v2.6/testing'
_full_path2 = '%s%s/' % (_prefix, _path2)
_output1 = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>Index of /pub/linux/kernel/v2.6</title>
</head>
<body>
<h1>Index of /pub/linux/kernel/v2.6</h1>
<pre><a href="?C=N;O=D">Name</a> <a href="?C=M;O=A">Last modified</a> <a href="?C=S;O=A">Size</a> <hr><a href="/pub/linux/kernel/">Parent Directory</a> -
<a href="incr/">incr/</a> 23-Mar-2009 22:13 -
<a href="pre-releases/">pre-releases/</a> 18-Dec-2003 15:50 -
<a href="snapshots/">snapshots/</a> 25-Apr-2009 00:18 -
<a href="stable-review/">stable-review/</a> 23-Apr-2009 07:51 -
<a href="testing/">testing/</a> 22-Apr-2009 03:31 -
<a href="ChangeLog-2.6.0">ChangeLog-2.6.0</a> 18-Dec-2003 03:04 12K
<a href="ChangeLog-2.6.1">ChangeLog-2.6.1</a> 09-Jan-2004 07:08 189K
<a href="ChangeLog-2.6.2">ChangeLog-2.6.2</a> 04-Feb-2004 04:06 286K
<a href="patch-2.6.19.6.bz2.sign">patch-2.6.19.6.bz2.sign</a> 03-Mar-2007 01:06 248
<a href="patch-2.6.19.6.gz">patch-2.6.19.6.gz</a> 03-Mar-2007 01:06 68K
<a href="patch-2.6.19.6.gz.sign">patch-2.6.19.6.gz.sign</a> 03-Mar-2007 01:06 248
<a href="patch-2.6.19.6.sign">patch-2.6.19.6.sign</a> 03-Mar-2007 01:06 248
<a href="patch-2.6.19.7.bz2">patch-2.6.19.7.bz2</a> 03-Mar-2007 05:29 62K
<a href="patch-2.6.19.7.bz2.sign">patch-2.6.19.7.bz2.sign</a> 03-Mar-2007 05:29 248
<a href="linux-2.6.28.1.tar.sign">linux-2.6.28.1.tar.sign</a> 18-Jan-2009 18:48 248
<a href="linux-2.6.28.2.tar.bz2">linux-2.6.28.2.tar.bz2</a> 25-Jan-2009 00:47 50M
<a href="linux-2.6.28.2.tar.bz2.sign">linux-2.6.28.2.tar.bz2.sign</a> 25-Jan-2009 00:47 248
<a href="linux-2.6.28.2.tar.gz">linux-2.6.28.2.tar.gz</a> 25-Jan-2009 00:47 64M
<a href="linux-2.6.28.2.tar.gz.sign">linux-2.6.28.2.tar.gz.sign</a> 25-Jan-2009 00:47 248
<a href="linux-2.6.28.2.tar.sign">linux-2.6.28.2.tar.sign</a> 25-Jan-2009 00:47 248
<a href="linux-2.6.28.3.tar.bz2">linux-2.6.28.3.tar.bz2</a> 02-Feb-2009 18:21 50M
<hr></pre>
</body></html>
"""
_output2 = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>Index of /pub/linux/kernel/v2.6/testing</title>
</head>
<body>
<h1>Index of /pub/linux/kernel/v2.6/testing</h1>
<pre><a href="?C=N;O=D">Name</a> <a href="?C=M;O=A">Last modified</a> <a href="?C=S;O=A">Size</a> <hr><a href="/pub/linux/kernel/v2.6/">Parent Directory</a> -
<a href="cset/">cset/</a> 04-Apr-2005 17:12 -
<a href="incr/">incr/</a> 22-Apr-2009 03:30 -
<a href="old/">old/</a> 14-Jul-2003 16:06 -
<a href="v2.6.1/">v2.6.1/</a> 15-Feb-2008 21:47 -
<a href="v2.6.2/">v2.6.2/</a> 15-Feb-2008 21:47 -
<a href="LATEST-IS-2.6.30-rc3">LATEST-IS-2.6.30-rc3</a> 22-Apr-2009 03:13 0
<a href="linux-2.6.30-rc1.tar.bz2">linux-2.6.30-rc1.tar.bz2</a> 07-Apr-2009 22:43 57M
<a href="linux-2.6.30-rc1.tar.bz2.sign">linux-2.6.30-rc1.tar.bz2.sign</a> 07-Apr-2009 22:43 248
<a href="linux-2.6.30-rc3.tar.gz.sign">linux-2.6.30-rc3.tar.gz.sign</a> 22-Apr-2009 03:25 248
<a href="linux-2.6.30-rc3.tar.sign">linux-2.6.30-rc3.tar.sign</a> 22-Apr-2009 03:25 248
<a href="patch-2.6.30-rc1.bz2">patch-2.6.30-rc1.bz2</a> 07-Apr-2009 22:45 10M
<a href="patch-2.6.30-rc1.bz2.sign">patch-2.6.30-rc1.bz2.sign</a> 07-Apr-2009 22:45 248
<hr></pre>
</body></html>
"""
_extracted_links1 = (
(_full_path1 + 'patch-2.6.19.6.gz', '70021',
(2007, 3, 3, 1, 6, 0, 0, 1, 0)),
(_full_path1 + 'patch-2.6.19.7.bz2', '63424',
(2007, 3, 3, 5, 29, 0, 0, 1, 0)),
(_full_path1 + 'linux-2.6.28.2.tar.bz2', '52697313',
(2009, 1, 25, 0, 47, 0, 0, 1, 0)),
(_full_path1 + 'linux-2.6.28.2.tar.gz', '66781113',
(2009, 1, 25, 0, 47, 0, 0, 1, 0)),
(_full_path1 + 'linux-2.6.28.3.tar.bz2', '52703558',
(2009, 2, 2, 18, 21, 0, 0, 1, 0)),
)
_extracted_links2 = (
(_full_path2 + 'patch-2.6.30-rc1.bz2', '10462721',
(2009, 4, 7, 22, 43, 0, 0, 1, 0)),
)
_known_files = {
_full_path1 + 'linux-2.6.28.2.tar.gz': source.database.item(
_full_path1 + 'linux-2.6.28.2.tar.gz', 66781113, 1232873220),
}
_result = {
_full_path1 + 'linux-2.6.28.3.tar.bz2': source.database.item(
_full_path1 + 'linux-2.6.28.3.tar.bz2', 52703558, 1233627660),
_full_path2 + 'patch-2.6.30-rc1.bz2': source.database.item(
_full_path2 + 'patch-2.6.30-rc1.bz2', 10462721, 1239172980),
_full_path1 + 'patch-2.6.19.7.bz2': source.database.item(
_full_path1 + 'patch-2.6.19.7.bz2', 63424, 1172928540),
_full_path1 + 'linux-2.6.28.2.tar.bz2': source.database.item(
_full_path1 + 'linux-2.6.28.2.tar.bz2', 52697313, 1232873220),
_full_path1 + 'patch-2.6.19.6.gz': source.database.item(
_full_path1 + 'patch-2.6.19.6.gz', 70021, 1172912760),
}
def setUp(self):
super(url_source_unittest, self).setUp()
self.god.stub_function(source.urllib2, 'urlopen')
self.addinfourl_mock = self.god.create_mock_class(
source.urllib2.addinfourl, 'addinfourl')
self.mime_mock = self.god.create_mock_class(
httplib.HTTPMessage, 'HTTPMessage')
def test_get_new_files(self):
# record
(source.urllib2.urlopen.expect_call(self._full_path1)
.and_return(cStringIO.StringIO(self._output1)))
for link, size, time in self._extracted_links1:
(source.urllib2.urlopen.expect_call(link)
.and_return(self.addinfourl_mock))
self.addinfourl_mock.info.expect_call().and_return(self.mime_mock)
self.mime_mock.get.expect_call('content-length').and_return(size)
self.mime_mock.getdate.expect_call('date').and_return(time)
(source.urllib2.urlopen.expect_call(self._full_path2)
.and_return(cStringIO.StringIO(self._output2)))
for link, size, time in self._extracted_links2:
(source.urllib2.urlopen.expect_call(link)
.and_return(self.addinfourl_mock))
self.addinfourl_mock.info.expect_call().and_return(self.mime_mock)
self.mime_mock.get.expect_call('content-length').and_return(size)
self.mime_mock.getdate.expect_call('date').and_return(time)
self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
# playback
s = source.url_source(self.db_mock, self._prefix)
s.add_url(self._path1, r'.*\.(gz|bz2)$')
s.add_url(self._path2, r'.*patch-[0-9.]+(-rc[0-9]+)?\.bz2$')
self.assertEquals(s.get_new_files(), self._result)
self.god.check_playback()
class directory_source_unittest(common_source):
"""
Unit test class for directory_source.
"""
def setUp(self):
super(directory_source_unittest, self).setUp()
self.god.stub_function(os, 'listdir')
self._stat_mock = self.god.create_mock_function('stat')
@staticmethod
def _get_stat_result(mode=0644, ino=12345, dev=12345, nlink=1, uid=1000,
gid=1000, size=10, atime=123, mtime=123, ctime=123):
"""
Build an os.stat_result() instance with many default values.
@param mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime:
See help(os.stat_result).
"""
return os.stat_result((mode, ino, dev, nlink, uid, gid, size, atime,
mtime, ctime))
def test_get_new_files_invalid_path(self):
"""
Test directory_source.get_new_files() on an invalid path.
"""
path = '/some/invalid/path'
os.listdir.expect_call(path).and_raises(OSError('Error'))
s = source.directory_source(self.db_mock, path)
self.assertRaises(OSError, s.get_new_files)
self.god.check_playback()
def test_get_new_files_stat_fails(self):
"""
Test directory_source.get_new_files() when stat fails.
"""
path = '/some/valid/path'
os.listdir.expect_call(path).and_return(['file1', 'file2'])
(self._stat_mock.expect_call(os.path.join(path, 'file1'))
.and_raises(OSError('Error')))
stat_result = self._get_stat_result(size=1010, mtime=123)
file2_full_path = os.path.join(path, 'file2')
self._stat_mock.expect_call(file2_full_path).and_return(stat_result)
self.db_mock.get_dictionary.expect_call().and_return({})
s = source.directory_source(self.db_mock, path)
expected = {'file2': source.database.item(file2_full_path, 1010, 123)}
self.assertEquals(expected, s.get_new_files(_stat_func=self._stat_mock))
self.god.check_playback()
def test_get_new_files_success(self):
"""
Test directory_source.get_new_files() success.
"""
path = '/some/valid/path'
file1_full_path = os.path.join(path, 'file1')
file2_full_path = os.path.join(path, 'file2')
file3_full_path = os.path.join(path, 'file3')
os.listdir.expect_call(path).and_return(['file1', 'file2', 'file3'])
stat_result = self._get_stat_result(size=1010, mtime=123)
self._stat_mock.expect_call(file1_full_path).and_return(stat_result)
stat_result = self._get_stat_result(size=1020, mtime=1234)
self._stat_mock.expect_call(file2_full_path).and_return(stat_result)
stat_result = self._get_stat_result(size=1030, mtime=12345)
self._stat_mock.expect_call(file3_full_path).and_return(stat_result)
known_files = {
'file2': source.database.item(file2_full_path, 1020, 1234),
}
self.db_mock.get_dictionary.expect_call().and_return(known_files)
s = source.directory_source(self.db_mock, path)
expected = {
'file1': source.database.item(file1_full_path, 1010, 123),
'file3': source.database.item(file3_full_path, 1030, 12345),
}
self.assertEquals(expected, s.get_new_files(_stat_func=self._stat_mock))
self.god.check_playback()
if __name__ == "__main__":
unittest.main()