| """Test largefile support on system where this makes sense. | |
| """ | |
| import os | |
| import sys | |
| import unittest | |
| import socket | |
| import shutil | |
| import threading | |
| from test.support import requires, bigmemtest, requires_resource | |
| from test.support import SHORT_TIMEOUT | |
| from test.support import socket_helper | |
| from test.support.os_helper import TESTFN, unlink | |
| import io # C implementation of io | |
| import _pyio as pyio # Python implementation of io | |
| # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes) | |
| size = 2_500_000_000 | |
| TESTFN2 = TESTFN + '2' | |
| class LargeFileTest: | |
| def setUp(self): | |
| if os.path.exists(TESTFN): | |
| mode = 'r+b' | |
| else: | |
| mode = 'w+b' | |
| with self.open(TESTFN, mode) as f: | |
| current_size = os.fstat(f.fileno()).st_size | |
| if current_size == size+1: | |
| return | |
| if current_size == 0: | |
| f.write(b'z') | |
| f.seek(0) | |
| f.seek(size) | |
| f.write(b'a') | |
| f.flush() | |
| self.assertEqual(os.fstat(f.fileno()).st_size, size+1) | |
| @classmethod | |
| def tearDownClass(cls): | |
| with cls.open(TESTFN, 'wb'): | |
| pass | |
| if not os.stat(TESTFN).st_size == 0: | |
| raise cls.failureException('File was not truncated by opening ' | |
| 'with mode "wb"') | |
| unlink(TESTFN2) | |
| class TestFileMethods(LargeFileTest): | |
| """Test that each file function works as expected for large | |
| (i.e. > 2 GiB) files. | |
| """ | |
| @bigmemtest(size=size, memuse=1, dry_run=False) | |
| def test_large_read(self, _size): | |
| # bpo-24658: Test that a read greater than 2GB does not fail. | |
| with self.open(TESTFN, "rb") as f: | |
| self.assertEqual(len(f.read()), size + 1) | |
| self.assertEqual(f.tell(), size + 1) | |
| def test_osstat(self): | |
| self.assertEqual(os.stat(TESTFN).st_size, size+1) | |
| def test_seek_read(self): | |
| with self.open(TESTFN, 'rb') as f: | |
| self.assertEqual(f.tell(), 0) | |
| self.assertEqual(f.read(1), b'z') | |
| self.assertEqual(f.tell(), 1) | |
| f.seek(0) | |
| self.assertEqual(f.tell(), 0) | |
| f.seek(0, 0) | |
| self.assertEqual(f.tell(), 0) | |
| f.seek(42) | |
| self.assertEqual(f.tell(), 42) | |
| f.seek(42, 0) | |
| self.assertEqual(f.tell(), 42) | |
| f.seek(42, 1) | |
| self.assertEqual(f.tell(), 84) | |
| f.seek(0, 1) | |
| self.assertEqual(f.tell(), 84) | |
| f.seek(0, 2) # seek from the end | |
| self.assertEqual(f.tell(), size + 1 + 0) | |
| f.seek(-10, 2) | |
| self.assertEqual(f.tell(), size + 1 - 10) | |
| f.seek(-size-1, 2) | |
| self.assertEqual(f.tell(), 0) | |
| f.seek(size) | |
| self.assertEqual(f.tell(), size) | |
| # the 'a' that was written at the end of file above | |
| self.assertEqual(f.read(1), b'a') | |
| f.seek(-size-1, 1) | |
| self.assertEqual(f.read(1), b'z') | |
| self.assertEqual(f.tell(), 1) | |
| def test_lseek(self): | |
| with self.open(TESTFN, 'rb') as f: | |
| self.assertEqual(os.lseek(f.fileno(), 0, 0), 0) | |
| self.assertEqual(os.lseek(f.fileno(), 42, 0), 42) | |
| self.assertEqual(os.lseek(f.fileno(), 42, 1), 84) | |
| self.assertEqual(os.lseek(f.fileno(), 0, 1), 84) | |
| self.assertEqual(os.lseek(f.fileno(), 0, 2), size+1+0) | |
| self.assertEqual(os.lseek(f.fileno(), -10, 2), size+1-10) | |
| self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0) | |
| self.assertEqual(os.lseek(f.fileno(), size, 0), size) | |
| # the 'a' that was written at the end of file above | |
| self.assertEqual(f.read(1), b'a') | |
| def test_truncate(self): | |
| with self.open(TESTFN, 'r+b') as f: | |
| if not hasattr(f, 'truncate'): | |
| raise unittest.SkipTest("open().truncate() not available " | |
| "on this system") | |
| f.seek(0, 2) | |
| # else we've lost track of the true size | |
| self.assertEqual(f.tell(), size+1) | |
| # Cut it back via seek + truncate with no argument. | |
| newsize = size - 10 | |
| f.seek(newsize) | |
| f.truncate() | |
| self.assertEqual(f.tell(), newsize) # else pointer moved | |
| f.seek(0, 2) | |
| self.assertEqual(f.tell(), newsize) # else wasn't truncated | |
| # Ensure that truncate(smaller than true size) shrinks | |
| # the file. | |
| newsize -= 1 | |
| f.seek(42) | |
| f.truncate(newsize) | |
| self.assertEqual(f.tell(), 42) | |
| f.seek(0, 2) | |
| self.assertEqual(f.tell(), newsize) | |
| # XXX truncate(larger than true size) is ill-defined | |
| # across platform; cut it waaaaay back | |
| f.seek(0) | |
| f.truncate(1) | |
| self.assertEqual(f.tell(), 0) # else pointer moved | |
| f.seek(0) | |
| # Verify readall on a truncated file is well behaved. read() | |
| # without a size can be unbounded, this should get just the byte | |
| # that remains. | |
| self.assertEqual(len(f.read()), 1) # else wasn't truncated | |
| def test_seekable(self): | |
| # Issue #5016; seekable() can return False when the current position | |
| # is negative when truncated to an int. | |
| for pos in (2**31-1, 2**31, 2**31+1): | |
| with self.open(TESTFN, 'rb') as f: | |
| f.seek(pos) | |
| self.assertTrue(f.seekable()) | |
| @bigmemtest(size=size, memuse=1, dry_run=False) | |
| def test_seek_readall(self, _size): | |
| # Seek which doesn't change position should readall successfully. | |
| with self.open(TESTFN, 'rb') as f: | |
| self.assertEqual(f.seek(0, os.SEEK_CUR), 0) | |
| self.assertEqual(len(f.read()), size + 1) | |
| # Seek which changes (or might change) position should readall | |
| # successfully. | |
| with self.open(TESTFN, 'rb') as f: | |
| self.assertEqual(f.seek(20, os.SEEK_SET), 20) | |
| self.assertEqual(len(f.read()), size - 19) | |
| with self.open(TESTFN, 'rb') as f: | |
| self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) | |
| self.assertEqual(len(f.read()), 3) | |
| def skip_no_disk_space(path, required): | |
| def decorator(fun): | |
| def wrapper(*args, **kwargs): | |
| if not hasattr(shutil, "disk_usage"): | |
| raise unittest.SkipTest("requires shutil.disk_usage") | |
| if shutil.disk_usage(os.path.realpath(path)).free < required: | |
| hsize = int(required / 1024 / 1024) | |
| raise unittest.SkipTest( | |
| f"required {hsize} MiB of free disk space") | |
| return fun(*args, **kwargs) | |
| return wrapper | |
| return decorator | |
| class TestCopyfile(LargeFileTest, unittest.TestCase): | |
| open = staticmethod(io.open) | |
| # Exact required disk space would be (size * 2), but let's give it a | |
| # bit more tolerance. | |
| @skip_no_disk_space(TESTFN, size * 2.5) | |
| @requires_resource('cpu') | |
| def test_it(self): | |
| # Internally shutil.copyfile() can use "fast copy" methods like | |
| # os.sendfile(). | |
| size = os.path.getsize(TESTFN) | |
| shutil.copyfile(TESTFN, TESTFN2) | |
| self.assertEqual(os.path.getsize(TESTFN2), size) | |
| with open(TESTFN2, 'rb') as f: | |
| self.assertEqual(f.read(5), b'z\x00\x00\x00\x00') | |
| f.seek(size - 5) | |
| self.assertEqual(f.read(), b'\x00\x00\x00\x00a') | |
| @unittest.skipIf(not hasattr(os, 'sendfile'), 'sendfile not supported') | |
| class TestSocketSendfile(LargeFileTest, unittest.TestCase): | |
| open = staticmethod(io.open) | |
| timeout = SHORT_TIMEOUT | |
| def setUp(self): | |
| super().setUp() | |
| self.thread = None | |
| def tearDown(self): | |
| super().tearDown() | |
| if self.thread is not None: | |
| self.thread.join(self.timeout) | |
| self.thread = None | |
| def tcp_server(self, sock): | |
| def run(sock): | |
| with sock: | |
| conn, _ = sock.accept() | |
| conn.settimeout(self.timeout) | |
| with conn, open(TESTFN2, 'wb') as f: | |
| event.wait(self.timeout) | |
| while True: | |
| chunk = conn.recv(65536) | |
| if not chunk: | |
| return | |
| f.write(chunk) | |
| event = threading.Event() | |
| sock.settimeout(self.timeout) | |
| self.thread = threading.Thread(target=run, args=(sock, )) | |
| self.thread.start() | |
| event.set() | |
| # Exact required disk space would be (size * 2), but let's give it a | |
| # bit more tolerance. | |
| @skip_no_disk_space(TESTFN, size * 2.5) | |
| @requires_resource('cpu') | |
| def test_it(self): | |
| port = socket_helper.find_unused_port() | |
| with socket.create_server(("", port)) as sock: | |
| self.tcp_server(sock) | |
| with socket.create_connection(("127.0.0.1", port)) as client: | |
| with open(TESTFN, 'rb') as f: | |
| client.sendfile(f) | |
| self.tearDown() | |
| size = os.path.getsize(TESTFN) | |
| self.assertEqual(os.path.getsize(TESTFN2), size) | |
| with open(TESTFN2, 'rb') as f: | |
| self.assertEqual(f.read(5), b'z\x00\x00\x00\x00') | |
| f.seek(size - 5) | |
| self.assertEqual(f.read(), b'\x00\x00\x00\x00a') | |
| def setUpModule(): | |
| try: | |
| import signal | |
| # The default handler for SIGXFSZ is to abort the process. | |
| # By ignoring it, system calls exceeding the file size resource | |
| # limit will raise OSError instead of crashing the interpreter. | |
| signal.signal(signal.SIGXFSZ, signal.SIG_IGN) | |
| except (ImportError, AttributeError): | |
| pass | |
| # On Windows and Mac OSX this test consumes large resources; It | |
| # takes a long time to build the >2 GiB file and takes >2 GiB of disk | |
| # space therefore the resource must be enabled to run this test. | |
| # If not, nothing after this line stanza will be executed. | |
| if sys.platform[:3] == 'win' or sys.platform == 'darwin': | |
| requires('largefile', | |
| 'test requires %s bytes and a long time to run' % str(size)) | |
| else: | |
| # Only run if the current filesystem supports large files. | |
| # (Skip this test on Windows, since we now always support | |
| # large files.) | |
| f = open(TESTFN, 'wb', buffering=0) | |
| try: | |
| # 2**31 == 2147483648 | |
| f.seek(2147483649) | |
| # Seeking is not enough of a test: you must write and flush, too! | |
| f.write(b'x') | |
| f.flush() | |
| except (OSError, OverflowError): | |
| raise unittest.SkipTest("filesystem does not have " | |
| "largefile support") | |
| finally: | |
| f.close() | |
| unlink(TESTFN) | |
| class CLargeFileTest(TestFileMethods, unittest.TestCase): | |
| open = staticmethod(io.open) | |
| class PyLargeFileTest(TestFileMethods, unittest.TestCase): | |
| open = staticmethod(pyio.open) | |
| def tearDownModule(): | |
| unlink(TESTFN) | |
| unlink(TESTFN2) | |
| if __name__ == '__main__': | |
| unittest.main() |