| #!/usr/bin/env python |
| # coding=utf-8 |
| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Traces an executable and its child processes and extract the files accessed |
| by them. |
| |
| The implementation uses OS-specific API. The native Kernel logger and the ETL |
| interface is used on Windows. Dtrace is used on OSX. Strace is used otherwise. |
| The OS-specific implementation is hidden in an 'API' interface. |
| |
| The results are embedded in a Results instance. The tracing is done in two |
| phases, the first is to do the actual trace and generate an |
| implementation-specific log file. Then the log file is parsed to extract the |
| information, including the individual child processes and the files accessed |
| from the log. |
| """ |
| |
| import codecs |
| import csv |
| import getpass |
| import glob |
| import json |
| import logging |
| import optparse |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import threading |
| import time |
| import weakref |
| |
| ## OS-specific imports |
| |
| if sys.platform == 'win32': |
| from ctypes.wintypes import byref, create_unicode_buffer, c_int, c_wchar_p |
| from ctypes.wintypes import windll, FormatError # pylint: disable=E0611 |
| from ctypes.wintypes import GetLastError # pylint: disable=E0611 |
| elif sys.platform == 'darwin': |
| import Carbon.File # pylint: disable=F0401 |
| import MacOS # pylint: disable=F0401 |
| |
| |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
| ROOT_DIR = os.path.dirname(os.path.dirname(BASE_DIR)) |
| |
| |
| class TracingFailure(Exception): |
| """An exception occured during tracing.""" |
| def __init__(self, description, pid, line_number, line, *args): |
| super(TracingFailure, self).__init__( |
| description, pid, line_number, line, *args) |
| self.description = description |
| self.pid = pid |
| self.line_number = line_number |
| self.line = line |
| self.extra = args |
| |
| def __str__(self): |
| out = self.description |
| if self.pid: |
| out += '\npid: %d' % self.pid |
| if self.line_number: |
| out += '\nline: %d' % self.line_number |
| if self.line: |
| out += '\n%s' % self.line |
| if self.extra: |
| out += '\n' + ', '.join(map(str, filter(None, self.extra))) |
| return out |
| |
| |
| ## OS-specific functions |
| |
| if sys.platform == 'win32': |
| def QueryDosDevice(drive_letter): |
| """Returns the Windows 'native' path for a DOS drive letter.""" |
| assert re.match(r'^[a-zA-Z]:$', drive_letter), drive_letter |
| # Guesswork. QueryDosDeviceW never returns the required number of bytes. |
| chars = 1024 |
| drive_letter = unicode(drive_letter) |
| p = create_unicode_buffer(chars) |
| if 0 == windll.kernel32.QueryDosDeviceW(drive_letter, p, chars): |
| err = GetLastError() |
| if err: |
| # pylint: disable=E0602 |
| raise WindowsError( |
| err, |
| 'QueryDosDevice(%s): %s (%d)' % ( |
| str(drive_letter), FormatError(err), err)) |
| return p.value |
| |
| |
| def GetShortPathName(long_path): |
| """Returns the Windows short path equivalent for a 'long' path.""" |
| long_path = unicode(long_path) |
| # Adds '\\\\?\\' when given an absolute path so the MAX_PATH (260) limit is |
| # not enforced. |
| if os.path.isabs(long_path) and not long_path.startswith('\\\\?\\'): |
| long_path = '\\\\?\\' + long_path |
| chars = windll.kernel32.GetShortPathNameW(long_path, None, 0) |
| if chars: |
| p = create_unicode_buffer(chars) |
| if windll.kernel32.GetShortPathNameW(long_path, p, chars): |
| return p.value |
| |
| err = GetLastError() |
| if err: |
| # pylint: disable=E0602 |
| raise WindowsError( |
| err, |
| 'GetShortPathName(%s): %s (%d)' % ( |
| str(long_path), FormatError(err), err)) |
| |
| |
| def GetLongPathName(short_path): |
| """Returns the Windows long path equivalent for a 'short' path.""" |
| short_path = unicode(short_path) |
| # Adds '\\\\?\\' when given an absolute path so the MAX_PATH (260) limit is |
| # not enforced. |
| if os.path.isabs(short_path) and not short_path.startswith('\\\\?\\'): |
| short_path = '\\\\?\\' + short_path |
| chars = windll.kernel32.GetLongPathNameW(short_path, None, 0) |
| if chars: |
| p = create_unicode_buffer(chars) |
| if windll.kernel32.GetLongPathNameW(short_path, p, chars): |
| return p.value |
| |
| err = GetLastError() |
| if err: |
| # pylint: disable=E0602 |
| raise WindowsError( |
| err, |
| 'GetLongPathName(%s): %s (%d)' % ( |
| str(short_path), FormatError(err), err)) |
| |
| |
| def get_current_encoding(): |
| """Returns the 'ANSI' code page associated to the process.""" |
| return 'cp%d' % int(windll.kernel32.GetACP()) |
| |
| |
| class DosDriveMap(object): |
| """Maps \Device\HarddiskVolumeN to N: on Windows.""" |
| # Keep one global cache. |
| _MAPPING = {} |
| |
| def __init__(self): |
| """Lazy loads the cache.""" |
| if not self._MAPPING: |
| # This is related to UNC resolver on windows. Ignore that. |
| self._MAPPING['\\Device\\Mup'] = None |
| self._MAPPING['\\SystemRoot'] = os.environ['SystemRoot'] |
| |
| for letter in (chr(l) for l in xrange(ord('C'), ord('Z')+1)): |
| try: |
| letter = '%s:' % letter |
| mapped = QueryDosDevice(letter) |
| if mapped in self._MAPPING: |
| logging.warn( |
| ('Two drives: \'%s\' and \'%s\', are mapped to the same disk' |
| '. Drive letters are a user-mode concept and the kernel ' |
| 'traces only have NT path, so all accesses will be ' |
| 'associated with the first drive letter, independent of the ' |
| 'actual letter used by the code') % ( |
| self._MAPPING[mapped], letter)) |
| else: |
| self._MAPPING[mapped] = letter |
| except WindowsError: # pylint: disable=E0602 |
| pass |
| |
| def to_win32(self, path): |
| """Converts a native NT path to Win32/DOS compatible path.""" |
| match = re.match(r'(^\\Device\\[a-zA-Z0-9]+)(\\.*)?$', path) |
| if not match: |
| raise ValueError( |
| 'Can\'t convert %s into a Win32 compatible path' % path, |
| path) |
| if not match.group(1) in self._MAPPING: |
| # Unmapped partitions may be accessed by windows for the |
| # fun of it while the test is running. Discard these. |
| return None |
| drive = self._MAPPING[match.group(1)] |
| if not drive or not match.group(2): |
| return drive |
| return drive + match.group(2) |
| |
| |
| def isabs(path): |
| """Accepts X: as an absolute path, unlike python's os.path.isabs().""" |
| return os.path.isabs(path) or len(path) == 2 and path[1] == ':' |
| |
| |
| def get_native_path_case(p): |
| """Returns the native path case for an existing file. |
| |
| On Windows, removes any leading '\\?\'. |
| """ |
| if not isabs(p): |
| raise ValueError( |
| 'Can\'t get native path case for a non-absolute path: %s' % p, |
| p) |
| # Windows used to have an option to turn on case sensitivity on non Win32 |
| # subsystem but that's out of scope here and isn't supported anymore. |
| # Go figure why GetShortPathName() is needed. |
| try: |
| out = GetLongPathName(GetShortPathName(p)) |
| except OSError, e: |
| if e.args[0] in (2, 3, 5): |
| # The path does not exist. Try to recurse and reconstruct the path. |
| base = os.path.dirname(p) |
| rest = os.path.basename(p) |
| return os.path.join(get_native_path_case(base), rest) |
| raise |
| if out.startswith('\\\\?\\'): |
| out = out[4:] |
| # Always upper case the first letter since GetLongPathName() will return the |
| # drive letter in the case it was given. |
| return out[0].upper() + out[1:] |
| |
| |
| def CommandLineToArgvW(command_line): |
| """Splits a commandline into argv using CommandLineToArgvW().""" |
| # http://msdn.microsoft.com/library/windows/desktop/bb776391.aspx |
| size = c_int() |
| ptr = windll.shell32.CommandLineToArgvW(unicode(command_line), byref(size)) |
| try: |
| return [arg for arg in (c_wchar_p * size.value).from_address(ptr)] |
| finally: |
| windll.kernel32.LocalFree(ptr) |
| |
| |
| elif sys.platform == 'darwin': |
| |
| |
| # On non-windows, keep the stdlib behavior. |
| isabs = os.path.isabs |
| |
| |
| def _find_item_native_case(root_path, item): |
| """Gets the native path case of a single item based at root_path. |
| |
| There is no API to get the native path case of symlinks on OSX. So it |
| needs to be done the slow way. |
| """ |
| item = item.lower() |
| for element in os.listdir(root_path): |
| if element.lower() == item: |
| return element |
| |
| |
| def _native_case(p): |
| """Gets the native path case. Warning: this function resolves symlinks.""" |
| logging.debug('native_case(%s)' % p) |
| try: |
| rel_ref, _ = Carbon.File.FSPathMakeRef(p) |
| out = rel_ref.FSRefMakePath() |
| if p.endswith(os.path.sep) and not out.endswith(os.path.sep): |
| return out + os.path.sep |
| return out |
| except MacOS.Error, e: |
| if e.args[0] in (-43, -120): |
| # The path does not exist. Try to recurse and reconstruct the path. |
| # -43 means file not found. |
| # -120 means directory not found. |
| base = os.path.dirname(p) |
| rest = os.path.basename(p) |
| return os.path.join(_native_case(base), rest) |
| raise OSError( |
| e.args[0], 'Failed to get native path for %s' % p, p, e.args[1]) |
| |
| |
| def _split_at_symlink_native(base_path, rest): |
| """Returns the native path for a symlink.""" |
| base, symlink, rest = split_at_symlink(base_path, rest) |
| if symlink: |
| if not base_path: |
| base_path = base |
| else: |
| base_path = safe_join(base_path, base) |
| symlink = _find_item_native_case(base_path, symlink) |
| return base, symlink, rest |
| |
| |
| def get_native_path_case(path): |
| """Returns the native path case for an existing file. |
| |
| Technically, it's only HFS+ on OSX that is case preserving and |
| insensitive. It's the default setting on HFS+ but can be changed. |
| """ |
| if not isabs(path): |
| raise ValueError( |
| 'Can\'t get native path case for a non-absolute path: %s' % path, |
| path) |
| if path.startswith('/dev'): |
| # /dev is not visible from Carbon, causing an exception. |
| return path |
| |
| # Starts assuming there is no symlink along the path. |
| resolved = _native_case(path) |
| if resolved.lower() == path.lower(): |
| # This code path is incredibly faster. |
| return resolved |
| |
| # There was a symlink, process it. |
| base, symlink, rest = _split_at_symlink_native(None, path) |
| assert symlink, (path, base, symlink, rest, resolved) |
| prev = base |
| base = safe_join(_native_case(base), symlink) |
| assert len(base) > len(prev) |
| while rest: |
| prev = base |
| relbase, symlink, rest = _split_at_symlink_native(base, rest) |
| base = safe_join(base, relbase) |
| assert len(base) > len(prev), (prev, base, symlink) |
| if symlink: |
| base = safe_join(base, symlink) |
| assert len(base) > len(prev), (prev, base, symlink) |
| # Make sure no symlink was resolved. |
| assert base.lower() == path.lower(), (base, path) |
| return base |
| |
| |
| else: # OSes other than Windows and OSX. |
| |
| |
| # On non-windows, keep the stdlib behavior. |
| isabs = os.path.isabs |
| |
| |
| def get_native_path_case(path): |
| """Returns the native path case for an existing file. |
| |
| On OSes other than OSX and Windows, assume the file system is |
| case-sensitive. |
| |
| TODO(maruel): This is not strictly true. Implement if necessary. |
| """ |
| if not isabs(path): |
| raise ValueError( |
| 'Can\'t get native path case for a non-absolute path: %s' % path, |
| path) |
| # Give up on cygwin, as GetLongPathName() can't be called. |
| # Linux traces tends to not be normalized so use this occasion to normalize |
| # it. This function implementation already normalizes the path on the other |
| # OS so this needs to be done here to be coherent between OSes. |
| out = os.path.normpath(path) |
| if path.endswith(os.path.sep) and not out.endswith(os.path.sep): |
| return out + os.path.sep |
| return out |
| |
| |
| if sys.platform != 'win32': # All non-Windows OSes. |
| |
| |
| def safe_join(*args): |
| """Joins path elements like os.path.join() but doesn't abort on absolute |
| path. |
| |
| os.path.join('foo', '/bar') == '/bar' |
| but safe_join('foo', '/bar') == 'foo/bar'. |
| """ |
| out = '' |
| for element in args: |
| if element.startswith(os.path.sep): |
| if out.endswith(os.path.sep): |
| out += element[1:] |
| else: |
| out += element |
| else: |
| if out.endswith(os.path.sep): |
| out += element |
| else: |
| out += os.path.sep + element |
| return out |
| |
| |
| def split_at_symlink(base_dir, relfile): |
| """Scans each component of relfile and cut the string at the symlink if |
| there is any. |
| |
| Returns a tuple (base_path, symlink, rest), with symlink == rest == None if |
| not symlink was found. |
| """ |
| if base_dir: |
| assert relfile |
| assert os.path.isabs(base_dir) |
| index = 0 |
| else: |
| assert os.path.isabs(relfile) |
| index = 1 |
| |
| def at_root(rest): |
| if base_dir: |
| return safe_join(base_dir, rest) |
| return rest |
| |
| while True: |
| try: |
| index = relfile.index(os.path.sep, index) |
| except ValueError: |
| index = len(relfile) |
| full = at_root(relfile[:index]) |
| if os.path.islink(full): |
| # A symlink! |
| base = os.path.dirname(relfile[:index]) |
| symlink = os.path.basename(relfile[:index]) |
| rest = relfile[index:] |
| logging.debug( |
| 'split_at_symlink(%s, %s) -> (%s, %s, %s)' % |
| (base_dir, relfile, base, symlink, rest)) |
| return base, symlink, rest |
| if index == len(relfile): |
| break |
| index += 1 |
| return relfile, None, None |
| |
| |
| def fix_python_path(cmd): |
| """Returns the fixed command line to call the right python executable.""" |
| out = cmd[:] |
| if out[0] == 'python': |
| out[0] = sys.executable |
| elif out[0].endswith('.py'): |
| out.insert(0, sys.executable) |
| return out |
| |
| |
| def create_thunk(): |
| handle, name = tempfile.mkstemp(prefix='trace_inputs_thunk', suffix='.py') |
| os.write( |
| handle, |
| ( |
| 'import subprocess\n' |
| 'import sys\n' |
| 'sys.exit(subprocess.call(sys.argv[2:]))\n' |
| )) |
| os.close(handle) |
| return name |
| |
| |
| def strace_process_quoted_arguments(text): |
| """Extracts quoted arguments on a string and return the arguments as a list. |
| |
| Implemented as an automaton. Supports incomplete strings in the form |
| '"foo"...'. |
| |
| Example: |
| With text = '"foo", "bar"', the function will return ['foo', 'bar'] |
| |
| TODO(maruel): Implement escaping. |
| """ |
| # All the possible states of the DFA. |
| ( NEED_QUOTE, # Begining of a new arguments. |
| INSIDE_STRING, # Inside an argument. |
| ESCAPED, # Found a '\' inside a quote. Treat the next char as-is. |
| NEED_COMMA_OR_DOT, # Right after the closing quote of an argument. Could be |
| # a serie of 3 dots or a comma. |
| NEED_SPACE, # Right after a comma |
| NEED_DOT_2, # Found a dot, need a second one. |
| NEED_DOT_3, # Found second dot, need a third one. |
| NEED_COMMA, # Found third dot, need a comma. |
| ) = range(8) |
| |
| state = NEED_QUOTE |
| out = [] |
| for index, char in enumerate(text): |
| if char == '"': |
| if state == NEED_QUOTE: |
| state = INSIDE_STRING |
| # A new argument was found. |
| out.append('') |
| elif state == INSIDE_STRING: |
| # The argument is now closed. |
| state = NEED_COMMA_OR_DOT |
| elif state == ESCAPED: |
| out[-1] += char |
| state = INSIDE_STRING |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| elif char == ',': |
| if state in (NEED_COMMA_OR_DOT, NEED_COMMA): |
| state = NEED_SPACE |
| elif state == INSIDE_STRING: |
| out[-1] += char |
| elif state == ESCAPED: |
| out[-1] += char |
| state = INSIDE_STRING |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| elif char == ' ': |
| if state == NEED_SPACE: |
| state = NEED_QUOTE |
| elif state == INSIDE_STRING: |
| out[-1] += char |
| elif state == ESCAPED: |
| out[-1] += char |
| state = INSIDE_STRING |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| elif char == '.': |
| if state == NEED_COMMA_OR_DOT: |
| # The string is incomplete, this mean the strace -s flag should be |
| # increased. |
| state = NEED_DOT_2 |
| elif state == NEED_DOT_2: |
| state = NEED_DOT_3 |
| elif state == NEED_DOT_3: |
| state = NEED_COMMA |
| elif state == INSIDE_STRING: |
| out[-1] += char |
| elif state == ESCAPED: |
| out[-1] += char |
| state = INSIDE_STRING |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| elif char == '\\': |
| if state == ESCAPED: |
| out[-1] += char |
| state = INSIDE_STRING |
| elif state == INSIDE_STRING: |
| state = ESCAPED |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| else: |
| if state == INSIDE_STRING: |
| out[-1] += char |
| else: |
| raise ValueError( |
| 'Can\'t process char at column %d for: %r' % (index, text), |
| index, |
| text) |
| if state not in (NEED_COMMA, NEED_COMMA_OR_DOT): |
| raise ValueError( |
| 'String is incorrectly terminated: %r' % text, |
| text) |
| return out |
| |
| |
| def read_json(filepath): |
| with open(filepath, 'r') as f: |
| return json.load(f) |
| |
| |
| def write_json(filepath_or_handle, data, dense): |
| """Writes data into filepath or file handle encoded as json. |
| |
| If dense is True, the json is packed. Otherwise, it is human readable. |
| """ |
| if hasattr(filepath_or_handle, 'write'): |
| if dense: |
| filepath_or_handle.write( |
| json.dumps(data, sort_keys=True, separators=(',',':'))) |
| else: |
| filepath_or_handle.write(json.dumps(data, sort_keys=True, indent=2)) |
| else: |
| with open(filepath_or_handle, 'wb') as f: |
| if dense: |
| json.dump(data, f, sort_keys=True, separators=(',',':')) |
| else: |
| json.dump(data, f, sort_keys=True, indent=2) |
| |
| |
| class Results(object): |
| """Results of a trace session.""" |
| |
| class _TouchedObject(object): |
| """Something, a file or a directory, that was accessed.""" |
| def __init__(self, root, path, tainted, size, nb_files): |
| logging.debug( |
| '%s(%s, %s, %s, %s, %s)' % |
| (self.__class__.__name__, root, path, tainted, size, nb_files)) |
| self.root = root |
| self.path = path |
| self.tainted = tainted |
| self.nb_files = nb_files |
| # Can be used as a cache or a default value, depending on context. |
| self._size = size |
| # These are cache only. |
| self._real_path = None |
| |
| # Check internal consistency. |
| assert path, path |
| assert tainted or bool(root) != bool(isabs(path)), (root, path) |
| assert tainted or ( |
| not os.path.exists(self.full_path) or |
| (self.full_path == get_native_path_case(self.full_path))), ( |
| tainted, self.full_path, get_native_path_case(self.full_path)) |
| |
| @property |
| def existent(self): |
| return self.size != -1 |
| |
| @property |
| def full_path(self): |
| if self.root: |
| return os.path.join(self.root, self.path) |
| return self.path |
| |
| @property |
| def real_path(self): |
| """Returns the path with symlinks resolved.""" |
| if not self._real_path: |
| self._real_path = os.path.realpath(self.full_path) |
| return self._real_path |
| |
| @property |
| def size(self): |
| """File's size. -1 is not existent.""" |
| if self._size is None and not self.tainted: |
| try: |
| self._size = os.stat(self.full_path).st_size |
| except OSError: |
| self._size = -1 |
| return self._size |
| |
| def flatten(self): |
| """Returns a dict representing this object. |
| |
| A 'size' of 0 means the file was only touched and not read. |
| """ |
| return { |
| 'path': self.path, |
| 'size': self.size, |
| } |
| |
| def replace_variables(self, variables): |
| """Replaces the root of this File with one of the variables if it matches. |
| |
| If a variable replacement occurs, the cloned object becomes tainted. |
| """ |
| for variable, root_path in variables.iteritems(): |
| if self.path.startswith(root_path): |
| return self._clone( |
| self.root, variable + self.path[len(root_path):], True) |
| # No need to clone, returns ourself. |
| return self |
| |
| def strip_root(self, root): |
| """Returns a clone of itself with 'root' stripped off.""" |
| # Check internal consistency. |
| assert self.tainted or (isabs(root) and root.endswith(os.path.sep)), root |
| if not self.full_path.startswith(root): |
| # Now try to resolve the symlinks to see if it can be reached this way. |
| # Only try *after* trying without resolving symlink. |
| if not self.real_path.startswith(root): |
| return None |
| path = self.real_path |
| else: |
| path = self.full_path |
| return self._clone(root, path[len(root):], self.tainted) |
| |
| def _clone(self, new_root, new_path, tainted): |
| raise NotImplementedError(self.__class__.__name__) |
| |
| class File(_TouchedObject): |
| """A file that was accessed. May not be present anymore. |
| |
| If tainted is true, it means it is not a real path anymore as a variable |
| replacement occured. |
| |
| If touched_only is True, this means the file was probed for existence, and |
| it is existent, but was never _opened_. If touched_only is True, the file |
| must have existed. |
| """ |
| def __init__(self, root, path, tainted, size): |
| super(Results.File, self).__init__(root, path, tainted, size, 1) |
| |
| def _clone(self, new_root, new_path, tainted): |
| """Clones itself keeping meta-data.""" |
| # Keep the self.size and self._real_path caches for performance reason. It |
| # is also important when the file becomes tainted (with a variable instead |
| # of the real path) since self.path is not an on-disk path anymore so |
| # out._size cannot be updated. |
| out = self.__class__(new_root, new_path, tainted, self.size) |
| out._real_path = self._real_path |
| return out |
| |
| class Directory(_TouchedObject): |
| """A directory of files. Must exist.""" |
| def __init__(self, root, path, tainted, size, nb_files): |
| """path='.' is a valid value and must be handled appropriately.""" |
| assert not path.endswith(os.path.sep), path |
| super(Results.Directory, self).__init__( |
| root, path + os.path.sep, tainted, size, nb_files) |
| # For a Directory instance, self.size is not a cache, it's an actual value |
| # that is never modified and represents the total size of the files |
| # contained in this directory. It is possible that the directory is empty |
| # so that size == 0; this happens if there's only an invalid symlink in |
| # it. |
| |
| def flatten(self): |
| out = super(Results.Directory, self).flatten() |
| out['nb_files'] = self.nb_files |
| return out |
| |
| def _clone(self, new_root, new_path, tainted): |
| """Clones itself keeping meta-data.""" |
| out = self.__class__( |
| new_root, |
| new_path.rstrip(os.path.sep), |
| tainted, |
| self.size, |
| self.nb_files) |
| out._real_path = self._real_path |
| return out |
| |
| class Process(object): |
| """A process that was traced. |
| |
| Contains references to the files accessed by this process and its children. |
| """ |
| def __init__(self, pid, files, executable, command, initial_cwd, children): |
| logging.debug('Process(%s, %d, ...)' % (pid, len(files))) |
| self.pid = pid |
| self.files = sorted(files, key=lambda x: x.path) |
| self.children = children |
| self.executable = executable |
| self.command = command |
| self.initial_cwd = initial_cwd |
| |
| # Check internal consistency. |
| assert len(set(f.path for f in self.files)) == len(self.files), sorted( |
| f.path for f in self.files) |
| assert isinstance(self.children, list) |
| assert isinstance(self.files, list) |
| |
| @property |
| def all(self): |
| for child in self.children: |
| for i in child.all: |
| yield i |
| yield self |
| |
| def flatten(self): |
| return { |
| 'children': [c.flatten() for c in self.children], |
| 'command': self.command, |
| 'executable': self.executable, |
| 'files': [f.flatten() for f in self.files], |
| 'initial_cwd': self.initial_cwd, |
| 'pid': self.pid, |
| } |
| |
| def strip_root(self, root): |
| assert isabs(root) and root.endswith(os.path.sep), root |
| # Loads the files after since they are constructed as objects. |
| out = self.__class__( |
| self.pid, |
| filter(None, (f.strip_root(root) for f in self.files)), |
| self.executable, |
| self.command, |
| self.initial_cwd, |
| [c.strip_root(root) for c in self.children]) |
| logging.debug( |
| 'strip_root(%s) %d -> %d' % (root, len(self.files), len(out.files))) |
| return out |
| |
| def __init__(self, process): |
| self.process = process |
| # Cache. |
| self._files = None |
| |
| def flatten(self): |
| return { |
| 'root': self.process.flatten(), |
| } |
| |
| @property |
| def files(self): |
| if self._files is None: |
| self._files = sorted( |
| sum((p.files for p in self.process.all), []), |
| key=lambda x: x.path) |
| return self._files |
| |
| @property |
| def existent(self): |
| return [f for f in self.files if f.existent] |
| |
| @property |
| def non_existent(self): |
| return [f for f in self.files if not f.existent] |
| |
| def strip_root(self, root): |
| """Returns a clone with all the files outside the directory |root| removed |
| and converts all the path to be relative paths. |
| """ |
| # Resolve any symlink |
| root = os.path.realpath(root) |
| root = get_native_path_case(root).rstrip(os.path.sep) + os.path.sep |
| logging.debug('strip_root(%s)' % root) |
| return Results(self.process.strip_root(root)) |
| |
| |
| class ApiBase(object): |
| """OS-agnostic API to trace a process and its children.""" |
| class Context(object): |
| """Processes one log line at a time and keeps the list of traced processes. |
| |
| The parsing is complicated by the fact that logs are traced out of order for |
| strace but in-order for dtrace and logman. In addition, on Windows it is |
| very frequent that processids are reused so a flat list cannot be used. But |
| at the same time, it is impossible to faithfully construct a graph when the |
| logs are processed out of order. So both a tree and a flat mapping are used, |
| the tree is the real process tree, while the flat mapping stores the last |
| valid process for the corresponding processid. For the strace case, the |
| tree's head is guessed at the last moment. |
| """ |
| class Process(object): |
| """Keeps context for one traced child process. |
| |
| Logs all the files this process touched. Ignores directories. |
| """ |
| def __init__(self, blacklist, pid, initial_cwd): |
| # Check internal consistency. |
| assert isinstance(pid, int), repr(pid) |
| self.pid = pid |
| # children are Process instances. |
| self.children = [] |
| self.initial_cwd = initial_cwd |
| self.cwd = None |
| self.files = set() |
| self.only_touched = set() |
| self.executable = None |
| self.command = None |
| self._blacklist = blacklist |
| |
| def to_results_process(self): |
| """Resolves file case sensitivity and or late-bound strings.""" |
| # When resolving files, it's normal to get dupe because a file could be |
| # opened multiple times with different case. Resolve the deduplication |
| # here. |
| def fix_path(x): |
| """Returns the native file path case. |
| |
| Converts late-bound strings. |
| """ |
| if not x: |
| # Do not convert None instance to 'None'. |
| return x |
| # TODO(maruel): Do not upconvert to unicode here, on linux we don't |
| # know the file path encoding so they must be treated as bytes. |
| x = unicode(x) |
| if os.path.isabs(x): |
| # If the path is not absolute, which tends to happen occasionally on |
| # Windows, it is not possible to get the native path case so ignore |
| # that trace. It mostly happens for 'executable' value. |
| x = get_native_path_case(x) |
| return x |
| |
| def fix_and_blacklist_path(x): |
| x = fix_path(x) |
| if not x: |
| return |
| # The blacklist needs to be reapplied, since path casing could |
| # influence blacklisting. |
| if self._blacklist(x): |
| return |
| return x |
| |
| # Filters out directories. Some may have passed through. |
| files = set(f for f in map(fix_and_blacklist_path, self.files) if f) |
| only_touched = set( |
| f for f in map(fix_and_blacklist_path, self.only_touched) if f) |
| only_touched -= files |
| |
| files = [ |
| Results.File(None, f, False, None) for f in files |
| if not os.path.isdir(f) |
| ] |
| # Using 0 as size means the file's content is ignored since the file was |
| # never opened for I/O. |
| files.extend( |
| Results.File(None, f, False, 0) for f in only_touched |
| if not os.path.isdir(f) |
| ) |
| return Results.Process( |
| self.pid, |
| files, |
| fix_path(self.executable), |
| self.command, |
| fix_path(self.initial_cwd), |
| [c.to_results_process() for c in self.children]) |
| |
| def add_file(self, filepath, touch_only): |
| """Adds a file if it passes the blacklist.""" |
| if self._blacklist(unicode(filepath)): |
| return |
| logging.debug('add_file(%d, %s, %s)' % (self.pid, filepath, touch_only)) |
| # Note that filepath and not unicode(filepath) is added. It is because |
| # filepath could be something else than a string, like a RelativePath |
| # instance for dtrace logs. |
| if touch_only: |
| self.only_touched.add(filepath) |
| else: |
| self.files.add(filepath) |
| |
| def __init__(self, blacklist): |
| self.blacklist = blacklist |
| # Initial process. |
| self.root_process = None |
| # dict to accelerate process lookup, to not have to lookup the whole graph |
| # each time. |
| self._process_lookup = {} |
| |
| class Tracer(object): |
| """During it's lifetime, the tracing subsystem is enabled.""" |
| def __init__(self, logname): |
| self._logname = logname |
| self._lock = threading.Lock() |
| self._traces = [] |
| self._initialized = True |
| |
| def trace(self, cmd, cwd, tracename, output): |
| """Runs the OS-specific trace program on an executable. |
| |
| Arguments: |
| - cmd: The command (a list) to run. |
| - cwd: Current directory to start the child process in. |
| - tracename: Name of the trace in the logname file. |
| - output: If False, redirects output to PIPEs. |
| |
| Returns a tuple (resultcode, output) and updates the internal trace |
| entries. |
| """ |
| # The implementation adds an item to self._traces. |
| raise NotImplementedError(self.__class__.__name__) |
| |
| def close(self, _timeout=None): |
| """Saves the meta-data in the logname file. |
| |
| For kernel-based tracing, stops the tracing subsystem. |
| |
| Must not be used manually when using 'with' construct. |
| """ |
| with self._lock: |
| assert self._initialized |
| try: |
| data = { |
| 'traces': self._traces, |
| } |
| write_json(self._logname, data, False) |
| finally: |
| self._initialized = False |
| |
| def post_process_log(self): |
| """Post-processes the log so it becomes faster to load afterward. |
| |
| Must not be used manually when using 'with' construct. |
| """ |
| assert not self._initialized, 'Must stop tracing first.' |
| |
| def __enter__(self): |
| """Enables 'with' statement.""" |
| return self |
| |
| def __exit__(self, exc_type, exc_value, traceback): |
| """Enables 'with' statement.""" |
| self.close() |
| # If an exception was thrown, do not process logs. |
| if not exc_type: |
| self.post_process_log() |
| |
| def get_tracer(self, logname): |
| """Returns an ApiBase.Tracer instance. |
| |
| Initializes the tracing subsystem, which is a requirement for kernel-based |
| tracers. Only one tracer instance should be live at a time! |
| |
| logname is the filepath to the json file that will contain the meta-data |
| about the logs. |
| """ |
| return self.Tracer(logname) |
| |
| @staticmethod |
| def clean_trace(logname): |
| """Deletes an old log.""" |
| raise NotImplementedError() |
| |
| @classmethod |
| def parse_log(cls, logname, blacklist): |
| """Processes trace logs and returns the files opened and the files that do |
| not exist. |
| |
| It does not track directories. |
| |
| Most of the time, files that do not exist are temporary test files that |
| should be put in /tmp instead. See http://crbug.com/116251. |
| |
| Returns a list of dict with keys: |
| - results: A Results instance. |
| - trace: The corresponding tracename parameter provided to |
| get_tracer().trace(). |
| - output: Output gathered during execution, if get_tracer().trace(..., |
| output=False) was used. |
| """ |
| raise NotImplementedError(cls.__class__.__name__) |
| |
| |
| class Strace(ApiBase): |
| """strace implies linux.""" |
| class Context(ApiBase.Context): |
| """Processes a strace log line and keeps the list of existent and non |
| existent files accessed. |
| |
| Ignores directories. |
| |
| Uses late-binding to processes the cwd of each process. The problem is that |
| strace generates one log file per process it traced but doesn't give any |
| information about which process was started when and by who. So we don't |
| even know which process is the initial one. So process the logs out of |
| order and use late binding with RelativePath to be able to deduce the |
| initial directory of each process once all the logs are parsed. |
| """ |
| class Process(ApiBase.Context.Process): |
| """Represents the state of a process. |
| |
| Contains all the information retrieved from the pid-specific log. |
| """ |
| # Function names are using ([a-z_0-9]+) |
| # This is the most common format. function(args) = result |
| RE_HEADER = re.compile(r'^([a-z_0-9]+)\((.+?)\)\s+= (.+)$') |
| # An interrupted function call, only grab the minimal header. |
| RE_UNFINISHED = re.compile(r'^([^\(]+)(.*) \<unfinished \.\.\.\>$') |
| # A resumed function call. |
| RE_RESUMED = re.compile(r'^<\.\.\. ([^ ]+) resumed> (.+)$') |
| # A process received a signal. |
| RE_SIGNAL = re.compile(r'^--- SIG[A-Z]+ .+ ---') |
| # A process didn't handle a signal. Ignore any junk appearing before, |
| # because the process was forcibly killed so it won't open any new file. |
| RE_KILLED = re.compile( |
| r'^.*\+\+\+ killed by ([A-Z]+)( \(core dumped\))? \+\+\+$') |
| # The process has exited. |
| RE_PROCESS_EXITED = re.compile(r'^\+\+\+ exited with (\d+) \+\+\+') |
| # A call was canceled. Ignore any prefix. |
| RE_UNAVAILABLE = re.compile(r'^.*\)\s*= \? <unavailable>$') |
| # Happens when strace fails to even get the function name. |
| UNNAMED_FUNCTION = '????' |
| |
| # Corner-case in python, a class member function decorator must not be |
| # @staticmethod. |
| def parse_args(regexp, expect_zero): # pylint: disable=E0213 |
| """Automatically convert the str 'args' into a list of processed |
| arguments. |
| |
| Arguments: |
| - regexp is used to parse args. |
| - expect_zero: one of True, False or None. |
| - True: will check for result.startswith('0') first and will ignore |
| the trace line completely otherwise. This is important because for |
| many functions, the regexp will not process if the call failed. |
| - False: will check for not result.startswith(('?', '-1')) for the |
| same reason than with True. |
| - None: ignore result. |
| """ |
| def meta_hook(function): |
| assert function.__name__.startswith('handle_') |
| def hook(self, args, result): |
| if expect_zero is True and not result.startswith('0'): |
| return |
| if expect_zero is False and result.startswith(('?', '-1')): |
| return |
| match = re.match(regexp, args) |
| if not match: |
| raise TracingFailure( |
| 'Failed to parse %s(%s) = %s' % |
| (function.__name__[len('handle_'):], args, result), |
| None, None, None) |
| return function(self, match.groups(), result) |
| return hook |
| return meta_hook |
| |
| class RelativePath(object): |
| """A late-bound relative path.""" |
| def __init__(self, parent, value): |
| self.parent = parent |
| self.value = value |
| |
| def render(self): |
| """Returns the current directory this instance is representing. |
| |
| This function is used to return the late-bound value. |
| """ |
| if self.value and self.value.startswith(u'/'): |
| # An absolute path. |
| return self.value |
| parent = self.parent.render() if self.parent else u'<None>' |
| if self.value: |
| return os.path.normpath(os.path.join(parent, self.value)) |
| return parent |
| |
| def __unicode__(self): |
| """Acts as a string whenever needed.""" |
| return unicode(self.render()) |
| |
| def __str__(self): |
| """Acts as a string whenever needed.""" |
| return str(self.render()) |
| |
| def __init__(self, root, pid): |
| """Keeps enough information to be able to guess the original process |
| root. |
| |
| strace doesn't store which process was the initial process. So more |
| information needs to be kept so the graph can be reconstructed from the |
| flat map. |
| """ |
| logging.info('%s(%d)' % (self.__class__.__name__, pid)) |
| super(Strace.Context.Process, self).__init__(root.blacklist, pid, None) |
| assert isinstance(root, ApiBase.Context) |
| self._root = weakref.ref(root) |
| # The dict key is the function name of the pending call, like 'open' |
| # or 'execve'. |
| self._pending_calls = {} |
| self._line_number = 0 |
| # Current directory when the process started. |
| self.initial_cwd = self.RelativePath(self._root(), None) |
| self.parentid = None |
| |
| def get_cwd(self): |
| """Returns the best known value of cwd.""" |
| return self.cwd or self.initial_cwd |
| |
| def render(self): |
| """Returns the string value of the RelativePath() object. |
| |
| Used by RelativePath. Returns the initial directory and not the |
| current one since the current directory 'cwd' validity is time-limited. |
| |
| The validity is only guaranteed once all the logs are processed. |
| """ |
| return self.initial_cwd.render() |
| |
| def on_line(self, line): |
| self._line_number += 1 |
| if self.RE_SIGNAL.match(line): |
| # Ignore signals. |
| return |
| |
| try: |
| match = self.RE_KILLED.match(line) |
| if match: |
| # Converts a '+++ killed by Foo +++' trace into an exit_group(). |
| self.handle_exit_group(match.group(1), None) |
| return |
| |
| match = self.RE_PROCESS_EXITED.match(line) |
| if match: |
| # Converts a '+++ exited with 1 +++' trace into an exit_group() |
| self.handle_exit_group(match.group(1), None) |
| return |
| |
| match = self.RE_UNFINISHED.match(line) |
| if match: |
| if match.group(1) in self._pending_calls: |
| raise TracingFailure( |
| 'Found two unfinished calls for the same function', |
| None, None, None, |
| self._pending_calls) |
| self._pending_calls[match.group(1)] = ( |
| match.group(1) + match.group(2)) |
| return |
| |
| match = self.RE_UNAVAILABLE.match(line) |
| if match: |
| # This usually means a process was killed and a pending call was |
| # canceled. |
| # TODO(maruel): Look up the last exit_group() trace just above and |
| # make sure any self._pending_calls[anything] is properly flushed. |
| return |
| |
| match = self.RE_RESUMED.match(line) |
| if match: |
| if match.group(1) not in self._pending_calls: |
| raise TracingFailure( |
| 'Found a resumed call that was not logged as unfinished', |
| None, None, None, |
| self._pending_calls) |
| pending = self._pending_calls.pop(match.group(1)) |
| # Reconstruct the line. |
| line = pending + match.group(2) |
| |
| match = self.RE_HEADER.match(line) |
| if not match: |
| raise TracingFailure( |
| 'Found an invalid line: %s' % line, |
| None, None, None) |
| if match.group(1) == self.UNNAMED_FUNCTION: |
| return |
| |
| # It's a valid line, handle it. |
| handler = getattr(self, 'handle_%s' % match.group(1), None) |
| if not handler: |
| self._handle_unknown(match.group(1), match.group(2), match.group(3)) |
| return handler(match.group(2), match.group(3)) |
| except TracingFailure, e: |
| # Hack in the values since the handler could be a static function. |
| e.pid = self.pid |
| e.line = line |
| e.line_number = self._line_number |
| # Re-raise the modified exception. |
| raise |
| except (KeyError, NotImplementedError, ValueError), e: |
| raise TracingFailure( |
| 'Trace generated a %s exception: %s' % ( |
| e.__class__.__name__, str(e)), |
| self.pid, |
| self._line_number, |
| line, |
| e) |
| |
| @parse_args(r'^\"(.+?)\", [FKORWX_|]+$', True) |
| def handle_access(self, args, _result): |
| self._handle_file(args[0], True) |
| |
| @parse_args(r'^\"(.+?)\"$', True) |
| def handle_chdir(self, args, _result): |
| """Updates cwd.""" |
| self.cwd = self.RelativePath(self, args[0]) |
| logging.debug('handle_chdir(%d, %s)' % (self.pid, self.cwd)) |
| |
| def handle_clone(self, _args, result): |
| """Transfers cwd.""" |
| if result.startswith(('?', '-1')): |
| # The call failed. |
| return |
| # Update the other process right away. |
| childpid = int(result) |
| child = self._root().get_or_set_proc(childpid) |
| if child.parentid is not None or childpid in self.children: |
| raise TracingFailure( |
| 'Found internal inconsitency in process lifetime detection ' |
| 'during a clone() call', |
| None, None, None) |
| |
| # Copy the cwd object. |
| child.initial_cwd = self.get_cwd() |
| child.parentid = self.pid |
| # It is necessary because the logs are processed out of order. |
| self.children.append(child) |
| |
| def handle_close(self, _args, _result): |
| pass |
| |
| def handle_chmod(self, _args, _result): |
| pass |
| |
| def handle_creat(self, _args, _result): |
| # Ignore files created, since they didn't need to exist. |
| pass |
| |
| @parse_args(r'^\"(.+?)\", \[(.+)\], \[\/\* \d+ vars? \*\/\]$', True) |
| def handle_execve(self, args, _result): |
| # Even if in practice execve() doesn't returns when it succeeds, strace |
| # still prints '0' as the result. |
| filepath = args[0] |
| self._handle_file(filepath, False) |
| self.executable = self.RelativePath(self.get_cwd(), filepath) |
| self.command = strace_process_quoted_arguments(args[1]) |
| |
| def handle_exit_group(self, _args, _result): |
| """Removes cwd.""" |
| self.cwd = None |
| |
| def handle_fork(self, args, result): |
| self._handle_unknown('fork', args, result) |
| |
| def handle_getcwd(self, _args, _result): |
| pass |
| |
| @parse_args(r'^\"(.+?)\", \"(.+?)\"$', True) |
| def handle_link(self, args, _result): |
| self._handle_file(args[0], False) |
| self._handle_file(args[1], False) |
| |
| @parse_args(r'\"(.+?)\", \{.+?, \.\.\.\}', True) |
| def handle_lstat(self, args, _result): |
| self._handle_file(args[0], True) |
| |
| def handle_mkdir(self, _args, _result): |
| pass |
| |
| @parse_args(r'^\"(.*?)\", ([A-Z\_\|]+)(|, \d+)$', False) |
| def handle_open(self, args, _result): |
| if 'O_DIRECTORY' in args[1]: |
| return |
| self._handle_file(args[0], False) |
| |
| @parse_args(r'^(\d+|AT_FDCWD), \"(.*?)\", ([A-Z\_\|]+)(|, \d+)$', False) |
| def handle_openat(self, args, _result): |
| if 'O_DIRECTORY' in args[2]: |
| return |
| if args[1] == 'AT_FDCWD': |
| self._handle_file(args[1], False) |
| else: |
| # TODO(maruel): Implement relative open if necessary instead of the |
| # AT_FDCWD flag, let's hope not since this means tracking all active |
| # directory handles. |
| raise Exception('Relative open via openat not implemented.') |
| |
| @parse_args(r'^\"(.+?)\", \".+?\"(\.\.\.)?, \d+$', False) |
| def handle_readlink(self, args, _result): |
| self._handle_file(args[0], False) |
| |
| @parse_args(r'^\"(.+?)\", \"(.+?)\"$', True) |
| def handle_rename(self, args, _result): |
| self._handle_file(args[0], False) |
| self._handle_file(args[1], False) |
| |
| def handle_rmdir(self, _args, _result): |
| pass |
| |
| def handle_setxattr(self, _args, _result): |
| pass |
| |
| @parse_args(r'\"(.+?)\", \{.+?, \.\.\.\}', True) |
| def handle_stat(self, args, _result): |
| self._handle_file(args[0], True) |
| |
| def handle_symlink(self, _args, _result): |
| pass |
| |
| def handle_unlink(self, _args, _result): |
| # In theory, the file had to be created anyway. |
| pass |
| |
| def handle_statfs(self, _args, _result): |
| pass |
| |
| def handle_vfork(self, args, result): |
| self._handle_unknown('vfork', args, result) |
| |
| @staticmethod |
| def _handle_unknown(function, args, result): |
| raise TracingFailure( |
| 'Unexpected/unimplemented trace %s(%s)= %s' % |
| (function, args, result), |
| None, None, None) |
| |
| def _handle_file(self, filepath, touch_only): |
| filepath = self.RelativePath(self.get_cwd(), filepath) |
| #assert not touch_only, unicode(filepath) |
| self.add_file(filepath, touch_only) |
| |
| def __init__(self, blacklist, initial_cwd): |
| super(Strace.Context, self).__init__(blacklist) |
| self.initial_cwd = initial_cwd |
| |
| def render(self): |
| """Returns the string value of the initial cwd of the root process. |
| |
| Used by RelativePath. |
| """ |
| return self.initial_cwd |
| |
| def on_line(self, pid, line): |
| """Transfers control into the Process.on_line() function.""" |
| self.get_or_set_proc(pid).on_line(line.strip()) |
| |
| def to_results(self): |
| """Finds back the root process and verify consistency.""" |
| # TODO(maruel): Absolutely unecessary, fix me. |
| root = [p for p in self._process_lookup.itervalues() if not p.parentid] |
| if len(root) != 1: |
| raise TracingFailure( |
| 'Found internal inconsitency in process lifetime detection ' |
| 'while finding the root process', |
| None, |
| None, |
| None, |
| sorted(p.pid for p in root)) |
| self.root_process = root[0] |
| process = self.root_process.to_results_process() |
| if sorted(self._process_lookup) != sorted(p.pid for p in process.all): |
| raise TracingFailure( |
| 'Found internal inconsitency in process lifetime detection ' |
| 'while looking for len(tree) == len(list)', |
| None, |
| None, |
| None, |
| sorted(self._process_lookup), |
| sorted(p.pid for p in process.all)) |
| return Results(process) |
| |
| def get_or_set_proc(self, pid): |
| """Returns the Context.Process instance for this pid or creates a new one. |
| """ |
| if not pid or not isinstance(pid, int): |
| raise TracingFailure( |
| 'Unpexpected value for pid: %r' % pid, |
| pid, |
| None, |
| None, |
| pid) |
| if pid not in self._process_lookup: |
| self._process_lookup[pid] = self.Process(self, pid) |
| return self._process_lookup[pid] |
| |
| @classmethod |
| def traces(cls): |
| """Returns the list of all handled traces to pass this as an argument to |
| strace. |
| """ |
| prefix = 'handle_' |
| return [i[len(prefix):] for i in dir(cls.Process) if i.startswith(prefix)] |
| |
| class Tracer(ApiBase.Tracer): |
| MAX_LEN = 256 |
| |
| def trace(self, cmd, cwd, tracename, output): |
| """Runs strace on an executable.""" |
| logging.info('trace(%s, %s, %s, %s)' % (cmd, cwd, tracename, output)) |
| assert os.path.isabs(cmd[0]), cmd[0] |
| assert os.path.isabs(cwd), cwd |
| assert os.path.normpath(cwd) == cwd, cwd |
| with self._lock: |
| if not self._initialized: |
| raise TracingFailure( |
| 'Called Tracer.trace() on an unitialized object', |
| None, None, None, tracename) |
| assert tracename not in (i['trace'] for i in self._traces) |
| stdout = stderr = None |
| if output: |
| stdout = subprocess.PIPE |
| stderr = subprocess.STDOUT |
| # Ensure all file related APIs are hooked. |
| traces = ','.join(Strace.Context.traces() + ['file']) |
| trace_cmd = [ |
| 'strace', |
| '-ff', |
| '-s', '%d' % self.MAX_LEN, |
| '-e', 'trace=%s' % traces, |
| '-o', self._logname + '.' + tracename, |
| ] |
| child = subprocess.Popen( |
| trace_cmd + cmd, |
| cwd=cwd, |
| stdin=subprocess.PIPE, |
| stdout=stdout, |
| stderr=stderr) |
| out = child.communicate()[0] |
| # TODO(maruel): Walk the logs and figure out the root process would |
| # simplify parsing the logs a *lot*. |
| with self._lock: |
| assert tracename not in (i['trace'] for i in self._traces) |
| self._traces.append( |
| { |
| 'cmd': cmd, |
| 'cwd': cwd, |
| # The pid of strace process, not very useful. |
| 'pid': child.pid, |
| 'trace': tracename, |
| 'output': out, |
| }) |
| return child.returncode, out |
| |
| @staticmethod |
| def clean_trace(logname): |
| if os.path.isfile(logname): |
| os.remove(logname) |
| # Also delete any pid specific file from previous traces. |
| for i in glob.iglob(logname + '.*'): |
| if i.rsplit('.', 1)[1].isdigit(): |
| os.remove(i) |
| |
| @classmethod |
| def parse_log(cls, logname, blacklist): |
| logging.info('parse_log(%s, %s)' % (logname, blacklist)) |
| data = read_json(logname) |
| out = [] |
| for item in data['traces']: |
| result = { |
| 'trace': item['trace'], |
| 'output': item['output'], |
| } |
| try: |
| context = cls.Context(blacklist, item['cwd']) |
| for pidfile in glob.iglob('%s.%s.*' % (logname, item['trace'])): |
| pid = pidfile.rsplit('.', 1)[1] |
| if pid.isdigit(): |
| pid = int(pid) |
| # TODO(maruel): Load as utf-8 |
| for line in open(pidfile, 'rb'): |
| context.on_line(pid, line) |
| result['results'] = context.to_results() |
| except TracingFailure, e: |
| result['exception'] = e |
| out.append(result) |
| return out |
| |
| |
| class Dtrace(ApiBase): |
| """Uses DTrace framework through dtrace. Requires root access. |
| |
| Implies Mac OSX. |
| |
| dtruss can't be used because it has compatibility issues with python. |
| |
| Also, the pid->cwd handling needs to be done manually since OSX has no way to |
| get the absolute path of the 'cwd' dtrace variable from the probe. |
| |
| Also, OSX doesn't populate curpsinfo->pr_psargs properly, see |
| https://discussions.apple.com/thread/1980539. So resort to handling execve() |
| manually. |
| |
| errno is not printed in the log since this implementation currently only cares |
| about files that were successfully opened. |
| """ |
| class Context(ApiBase.Context): |
| # Format: index pid function(args) |
| RE_HEADER = re.compile(r'^\d+ (\d+) ([a-zA-Z_\-]+)\((.*?)\)$') |
| |
| # Arguments parsing. |
| RE_DTRACE_BEGIN = re.compile(r'^\"(.+?)\"$') |
| RE_CHDIR = re.compile(r'^\"(.+?)\"$') |
| RE_EXECVE = re.compile(r'^\"(.+?)\", \[(\d+), (.+)\]$') |
| RE_OPEN = re.compile(r'^\"(.+?)\", (0x[0-9a-z]+), (0x[0-9a-z]+)$') |
| RE_PROC_START = re.compile(r'^(\d+), \"(.+?)\", (\d+)$') |
| RE_RENAME = re.compile(r'^\"(.+?)\", \"(.+?)\"$') |
| |
| O_DIRECTORY = 0x100000 |
| |
| class Process(ApiBase.Context.Process): |
| def __init__(self, *args): |
| super(Dtrace.Context.Process, self).__init__(*args) |
| self.cwd = self.initial_cwd |
| |
| def __init__(self, blacklist, tracer_pid, initial_cwd): |
| logging.info( |
| '%s(%d, %s)' % (self.__class__.__name__, tracer_pid, initial_cwd)) |
| super(Dtrace.Context, self).__init__(blacklist) |
| # Process ID of the temporary script created by create_thunk(). |
| self._tracer_pid = tracer_pid |
| self._initial_cwd = initial_cwd |
| self._line_number = 0 |
| |
| def on_line(self, line): |
| self._line_number += 1 |
| match = self.RE_HEADER.match(line) |
| if not match: |
| raise TracingFailure( |
| 'Found malformed line: %s' % line, |
| None, |
| self._line_number, |
| line) |
| fn = getattr( |
| self, |
| 'handle_%s' % match.group(2).replace('-', '_'), |
| self._handle_ignored) |
| # It is guaranteed to succeed because of the regexp. Or at least I thought |
| # it would. |
| pid = int(match.group(1)) |
| try: |
| return fn(pid, match.group(3)) |
| except TracingFailure, e: |
| # Hack in the values since the handler could be a static function. |
| e.pid = pid |
| e.line = line |
| e.line_number = self._line_number |
| # Re-raise the modified exception. |
| raise |
| except (KeyError, NotImplementedError, ValueError), e: |
| raise TracingFailure( |
| 'Trace generated a %s exception: %s' % ( |
| e.__class__.__name__, str(e)), |
| pid, |
| self._line_number, |
| line, |
| e) |
| |
| def to_results(self): |
| process = self.root_process.to_results_process() |
| # Internal concistency check. |
| if sorted(self._process_lookup) != sorted(p.pid for p in process.all): |
| raise TracingFailure( |
| 'Found internal inconsitency in process lifetime detection ' |
| 'while looking for len(tree) == len(list)', |
| None, |
| None, |
| None, |
| sorted(self._process_lookup), |
| sorted(p.pid for p in process.all)) |
| return Results(process) |
| |
| def handle_dtrace_BEGIN(self, _pid, args): |
| if not self.RE_DTRACE_BEGIN.match(args): |
| raise TracingFailure( |
| 'Found internal inconsitency in dtrace_BEGIN log line', |
| None, None, None) |
| |
| def handle_proc_start(self, pid, args): |
| """Transfers cwd. |
| |
| The dtrace script already takes care of only tracing the processes that |
| are child of the traced processes so there is no need to verify the |
| process hierarchy. |
| """ |
| if pid in self._process_lookup: |
| raise TracingFailure( |
| 'Found internal inconsitency in proc_start: %d started two times' % |
| pid, |
| None, None, None) |
| match = self.RE_PROC_START.match(args) |
| if not match: |
| raise TracingFailure( |
| 'Failed to parse arguments: %s' % args, |
| None, None, None) |
| ppid = int(match.group(1)) |
| if ppid == self._tracer_pid and not self.root_process: |
| proc = self.root_process = self.Process( |
| self.blacklist, pid, self._initial_cwd) |
| elif ppid in self._process_lookup: |
| proc = self.Process(self.blacklist, pid, self._process_lookup[ppid].cwd) |
| self._process_lookup[ppid].children.append(proc) |
| else: |
| # Another process tree, ignore. |
| return |
| self._process_lookup[pid] = proc |
| logging.debug( |
| 'New child: %s -> %d cwd:%s' % |
| (ppid, pid, unicode(proc.initial_cwd))) |
| |
| def handle_proc_exit(self, pid, _args): |
| """Removes cwd.""" |
| if pid in self._process_lookup: |
| # self._tracer_pid is not traced itself and other traces run neither. |
| self._process_lookup[pid].cwd = None |
| |
| def handle_execve(self, pid, args): |
| """Sets the process' executable. |
| |
| TODO(maruel): Read command line arguments. See |
| https://discussions.apple.com/thread/1980539 for an example. |
| https://gist.github.com/1242279 |
| |
| Will have to put the answer at http://stackoverflow.com/questions/7556249. |
| :) |
| """ |
| if not pid in self._process_lookup: |
| # Another process tree, ignore. |
| return |
| match = self.RE_EXECVE.match(args) |
| if not match: |
| raise TracingFailure( |
| 'Failed to parse arguments: %r' % args, |
| None, None, None) |
| proc = self._process_lookup[pid] |
| proc.executable = match.group(1) |
| proc.command = self.process_escaped_arguments(match.group(3)) |
| if int(match.group(2)) != len(proc.command): |
| raise TracingFailure( |
| 'Failed to parse execve() arguments: %s' % args, |
| None, None, None) |
| |
| def handle_chdir(self, pid, args): |
| """Updates cwd.""" |
| if pid not in self._process_lookup: |
| # Another process tree, ignore. |
| return |
| cwd = self.RE_CHDIR.match(args).group(1) |
| if not cwd.startswith('/'): |
| cwd2 = os.path.join(self._process_lookup[pid].cwd, cwd) |
| logging.debug('handle_chdir(%d, %s) -> %s' % (pid, cwd, cwd2)) |
| else: |
| logging.debug('handle_chdir(%d, %s)' % (pid, cwd)) |
| cwd2 = cwd |
| self._process_lookup[pid].cwd = cwd2 |
| |
| def handle_open_nocancel(self, pid, args): |
| """Redirects to handle_open().""" |
| return self.handle_open(pid, args) |
| |
| def handle_open(self, pid, args): |
| if pid not in self._process_lookup: |
| # Another process tree, ignore. |
| return |
| match = self.RE_OPEN.match(args) |
| if not match: |
| raise TracingFailure( |
| 'Failed to parse arguments: %s' % args, |
| None, None, None) |
| flag = int(match.group(2), 16) |
| if self.O_DIRECTORY & flag == self.O_DIRECTORY: |
| # Ignore directories. |
| return |
| self._handle_file(pid, match.group(1)) |
| |
| def handle_rename(self, pid, args): |
| if pid not in self._process_lookup: |
| # Another process tree, ignore. |
| return |
| match = self.RE_RENAME.match(args) |
| if not match: |
| raise TracingFailure( |
| 'Failed to parse arguments: %s' % args, |
| None, None, None) |
| self._handle_file(pid, match.group(1)) |
| self._handle_file(pid, match.group(2)) |
| |
| def _handle_file(self, pid, filepath): |
| if not filepath.startswith('/'): |
| filepath = os.path.join(self._process_lookup[pid].cwd, filepath) |
| # We can get '..' in the path. |
| filepath = os.path.normpath(filepath) |
| # Sadly, still need to filter out directories here; |
| # saw open_nocancel(".", 0, 0) = 0 lines. |
| if os.path.isdir(filepath): |
| return |
| self._process_lookup[pid].add_file(filepath, False) |
| |
| def handle_ftruncate(self, pid, args): |
| """Just used as a signal to kill dtrace, ignoring.""" |
| pass |
| |
| @staticmethod |
| def _handle_ignored(pid, args): |
| """Is called for all the event traces that are not handled.""" |
| raise NotImplementedError('Please implement me') |
| |
| @staticmethod |
| def process_escaped_arguments(text): |
| """Extracts escaped arguments on a string and return the arguments as a |
| list. |
| |
| Implemented as an automaton. |
| |
| Example: |
| With text = '\\001python2.7\\001-c\\001print(\\"hi\\")\\0', the |
| function will return ['python2.7', '-c', 'print("hi")] |
| """ |
| if not text.endswith('\\0'): |
| raise ValueError('String is not null terminated: %r' % text, text) |
| text = text[:-2] |
| |
| def unescape(x): |
| """Replaces '\\' with '\' and '\?' (where ? is anything) with ?.""" |
| out = [] |
| escaped = False |
| for i in x: |
| if i == '\\' and not escaped: |
| escaped = True |
| continue |
| escaped = False |
| out.append(i) |
| return ''.join(out) |
| |
| return [unescape(i) for i in text.split('\\001')] |
| |
| class Tracer(ApiBase.Tracer): |
| # pylint: disable=C0301 |
| # |
| # To understand the following code, you'll want to take a look at: |
| # http://developers.sun.com/solaris/articles/dtrace_quickref/dtrace_quickref.html |
| # https://wikis.oracle.com/display/DTrace/Variables |
| # http://docs.oracle.com/cd/E19205-01/820-4221/ |
| # |
| # 0. Dump all the valid probes into a text file. It is important, you |
| # want to redirect into a file and you don't want to constantly 'sudo'. |
| # $ sudo dtrace -l > probes.txt |
| # |
| # 1. Count the number of probes: |
| # $ wc -l probes.txt |
| # 81823 # On OSX 10.7, including 1 header line. |
| # |
| # 2. List providers, intentionally skipping all the 'syspolicy10925' and the |
| # likes and skipping the header with NR>1: |
| # $ awk 'NR>1 { print $2 }' probes.txt | sort | uniq | grep -v '[[:digit:]]' |
| # dtrace |
| # fbt |
| # io |
| # ip |
| # lockstat |
| # mach_trap |
| # proc |
| # profile |
| # sched |
| # syscall |
| # tcp |
| # vminfo |
| # |
| # 3. List of valid probes: |
| # $ grep syscall probes.txt | less |
| # or use dtrace directly: |
| # $ sudo dtrace -l -P syscall | less |
| # |
| # trackedpid is an associative array where its value can be 0, 1 or 2. |
| # 0 is for untracked processes and is the default value for items not |
| # in the associative array. |
| # 1 is for tracked processes. |
| # 2 is for the script created by create_thunk() only. It is not tracked |
| # itself but all its decendants are. |
| # |
| # The script will kill itself only once waiting_to_die == 1 and |
| # current_processes == 0, so that both getlogin() was called and that |
| # all traced processes exited. |
| # |
| # TODO(maruel): Use cacheable predicates. See |
| # https://wikis.oracle.com/display/DTrace/Performance+Considerations |
| D_CODE = """ |
| dtrace:::BEGIN { |
| waiting_to_die = 0; |
| current_processes = 0; |
| logindex = 0; |
| printf("%d %d %s_%s(\\"%s\\")\\n", |
| logindex, PID, probeprov, probename, SCRIPT); |
| logindex++; |
| } |
| |
| proc:::start /trackedpid[ppid]/ { |
| trackedpid[pid] = 1; |
| current_processes += 1; |
| printf("%d %d %s_%s(%d, \\"%s\\", %d)\\n", |
| logindex, pid, probeprov, probename, |
| ppid, |
| execname, |
| current_processes); |
| logindex++; |
| } |
| /* Should use SCRIPT but there is no access to this variable at that |
| * point. */ |
| proc:::start /ppid == PID && execname == "Python"/ { |
| trackedpid[pid] = 2; |
| current_processes += 1; |
| printf("%d %d %s_%s(%d, \\"%s\\", %d)\\n", |
| logindex, pid, probeprov, probename, |
| ppid, |
| execname, |
| current_processes); |
| logindex++; |
| } |
| proc:::exit /trackedpid[pid] && |
| current_processes == 1 && |
| waiting_to_die == 1/ { |
| trackedpid[pid] = 0; |
| current_processes -= 1; |
| printf("%d %d %s_%s(%d)\\n", |
| logindex, pid, probeprov, probename, |
| current_processes); |
| logindex++; |
| exit(0); |
| } |
| proc:::exit /trackedpid[pid]/ { |
| trackedpid[pid] = 0; |
| current_processes -= 1; |
| printf("%d %d %s_%s(%d)\\n", |
| logindex, pid, probeprov, probename, |
| current_processes); |
| logindex++; |
| } |
| |
| /* Use an arcane function to detect when we need to die */ |
| syscall::ftruncate:entry /pid == PID && arg0 == FILE_ID/ { |
| waiting_to_die = 1; |
| printf("%d %d %s()\\n", logindex, pid, probefunc); |
| logindex++; |
| } |
| syscall::ftruncate:entry / |
| pid == PID && arg0 == FILE_ID && current_processes == 0/ { |
| exit(0); |
| } |
| |
| syscall::open*:entry /trackedpid[pid] == 1/ { |
| self->open_arg0 = arg0; |
| self->open_arg1 = arg1; |
| self->open_arg2 = arg2; |
| } |
| syscall::open*:return /trackedpid[pid] == 1 && errno == 0/ { |
| this->open_arg0 = copyinstr(self->open_arg0); |
| printf("%d %d %s(\\"%s\\", 0x%x, 0x%x)\\n", |
| logindex, pid, probefunc, |
| this->open_arg0, |
| self->open_arg1, |
| self->open_arg2); |
| logindex++; |
| this->open_arg0 = 0; |
| } |
| syscall::open*:return /trackedpid[pid] == 1/ { |
| self->open_arg0 = 0; |
| self->open_arg1 = 0; |
| self->open_arg2 = 0; |
| } |
| |
| syscall::rename:entry /trackedpid[pid] == 1/ { |
| self->rename_arg0 = arg0; |
| self->rename_arg1 = arg1; |
| } |
| syscall::rename:return /trackedpid[pid] == 1 && errno == 0/ { |
| this->rename_arg0 = copyinstr(self->rename_arg0); |
| this->rename_arg1 = copyinstr(self->rename_arg1); |
| printf("%d %d %s(\\"%s\\", \\"%s\\")\\n", |
| logindex, pid, probefunc, |
| this->rename_arg0, |
| this->rename_arg1); |
| logindex++; |
| this->rename_arg0 = 0; |
| this->rename_arg1 = 0; |
| } |
| syscall::rename:return /trackedpid[pid] == 1/ { |
| self->rename_arg0 = 0; |
| self->rename_arg1 = 0; |
| } |
| |
| /* Track chdir, it's painful because it is only receiving relative path. |
| */ |
| syscall::chdir:entry /trackedpid[pid] == 1/ { |
| self->chdir_arg0 = arg0; |
| } |
| syscall::chdir:return /trackedpid[pid] == 1 && errno == 0/ { |
| this->chdir_arg0 = copyinstr(self->chdir_arg0); |
| printf("%d %d %s(\\"%s\\")\\n", |
| logindex, pid, probefunc, |
| this->chdir_arg0); |
| logindex++; |
| this->chdir_arg0 = 0; |
| } |
| syscall::chdir:return /trackedpid[pid] == 1/ { |
| self->chdir_arg0 = 0; |
| } |
| """ |
| |
| # execve-specific code, tends to throw a lot of exceptions. |
| D_CODE_EXECVE = """ |
| /* Finally what we care about! */ |
| syscall::exec*:entry /trackedpid[pid]/ { |
| self->exec_arg0 = copyinstr(arg0); |
| /* Incrementally probe for a NULL in the argv parameter of execve() to |
| * figure out argc. */ |
| /* TODO(maruel): Skip the remaining copyin() when a NULL pointer was |
| * found. */ |
| self->exec_argc = 0; |
| /* Probe for argc==1 */ |
| this->exec_argv = (user_addr_t*)copyin( |
| arg1, sizeof(user_addr_t) * (self->exec_argc + 1)); |
| self->exec_argc = this->exec_argv[self->exec_argc] ? |
| (self->exec_argc + 1) : self->exec_argc; |
| |
| /* Probe for argc==2 */ |
| this->exec_argv = (user_addr_t*)copyin( |
| arg1, sizeof(user_addr_t) * (self->exec_argc + 1)); |
| self->exec_argc = this->exec_argv[self->exec_argc] ? |
| (self->exec_argc + 1) : self->exec_argc; |
| |
| /* Probe for argc==3 */ |
| this->exec_argv = (user_addr_t*)copyin( |
| arg1, sizeof(user_addr_t) * (self->exec_argc + 1)); |
| self->exec_argc = this->exec_argv[self->exec_argc] ? |
| (self->exec_argc + 1) : self->exec_argc; |
| |
| /* Probe for argc==4 */ |
| this->exec_argv = (user_addr_t*)copyin( |
| arg1, sizeof(user_addr_t) * (self->exec_argc + 1)); |
| self->exec_argc = this->exec_argv[self->exec_argc] ? |
| (self->exec_argc + 1) : self->exec_argc; |
| |
| /* Copy the inputs strings since there is no guarantee they'll be |
| * present after the call completed. */ |
| self->exec_argv0 = (self->exec_argc > 0) ? |
| copyinstr(this->exec_argv[0]) : ""; |
| self->exec_argv1 = (self->exec_argc > 1) ? |
| copyinstr(this->exec_argv[1]) : ""; |
| self->exec_argv2 = (self->exec_argc > 2) ? |
| copyinstr(this->exec_argv[2]) : ""; |
| self->exec_argv3 = (self->exec_argc > 3) ? |
| copyinstr(this->exec_argv[3]) : ""; |
| this->exec_argv = 0; |
| } |
| syscall::exec*:return /trackedpid[pid] && errno == 0/ { |
| /* We need to join strings here, as using multiple printf() would |
| * cause tearing when multiple threads/processes are traced. |
| * Since it is impossible to escape a string and join it to another one, |
| * like sprintf("%s%S", previous, more), use hackery. |
| * Each of the elements are split with a \\1. \\0 cannot be used because |
| * it is simply ignored. This will conflict with any program putting a |
| * \\1 in their execve() string but this should be "rare enough" */ |
| this->args = ""; |
| /* Process exec_argv[0] */ |
| this->args = strjoin( |
| this->args, (self->exec_argc > 0) ? self->exec_argv0 : ""); |
| |
| /* Process exec_argv[1] */ |
| this->args = strjoin( |
| this->args, (self->exec_argc > 1) ? "\\1" : ""); |
| this->args = strjoin( |
| this->args, (self->exec_argc > 1) ? self->exec_argv1 : ""); |
| |
| /* Process exec_argv[2] */ |
| this->args = strjoin( |
| this->args, (self->exec_argc > 2) ? "\\1" : ""); |
| this->args = strjoin( |
| this->args, (self->exec_argc > 2) ? self->exec_argv2 : ""); |
| |
| /* Process exec_argv[3] */ |
| this->args = strjoin( |
| this->args, (self->exec_argc > 3) ? "\\1" : ""); |
| this->args = strjoin( |
| this->args, (self->exec_argc > 3) ? self->exec_argv3 : ""); |
| |
| /* Prints self->exec_argc to permits verifying the internal |
| * consistency since this code is quite fishy. */ |
| printf("%d %d %s(\\"%s\\", [%d, %S])\\n", |
| logindex, pid, probefunc, |
| self->exec_arg0, |
| self->exec_argc, |
| this->args); |
| logindex++; |
| this->args = 0; |
| } |
| syscall::exec*:return /trackedpid[pid]/ { |
| self->exec_arg0 = 0; |
| self->exec_argc = 0; |
| self->exec_argv0 = 0; |
| self->exec_argv1 = 0; |
| self->exec_argv2 = 0; |
| self->exec_argv3 = 0; |
| } |
| """ |
| |
| # Code currently not used. |
| D_EXTRANEOUS = """ |
| /* This is a good learning experience, since it traces a lot of things |
| * related to the process and child processes. |
| * Warning: it generates a gigantic log. For example, tracing |
| * "data/trace_inputs/child1.py --child" generates a 2mb log and takes |
| * several minutes to execute. |
| */ |
| /* |
| mach_trap::: /trackedpid[pid] == 1 || trackedpid[ppid]/ { |
| printf("%d %d %s_%s() = %d\\n", |
| logindex, pid, probeprov, probefunc, errno); |
| logindex++; |
| } |
| proc::: /trackedpid[pid] == 1 || trackedpid[ppid]/ { |
| printf("%d %d %s_%s() = %d\\n", |
| logindex, pid, probeprov, probefunc, errno); |
| logindex++; |
| } |
| sched::: /trackedpid[pid] == 1 || trackedpid[ppid]/ { |
| printf("%d %d %s_%s() = %d\\n", |
| logindex, pid, probeprov, probefunc, errno); |
| logindex++; |
| } |
| syscall::: /trackedpid[pid] == 1 || trackedpid[ppid]/ { |
| printf("%d %d %s_%s() = %d\\n", |
| logindex, pid, probeprov, probefunc, errno); |
| logindex++; |
| } |
| vminfo::: /trackedpid[pid] == 1 || trackedpid[ppid]/ { |
| printf("%d %d %s_%s() = %d\\n", |
| logindex, pid, probeprov, probefunc, errno); |
| logindex++; |
| } |
| */ |
| /* TODO(maruel): *stat* functions and friends |
| syscall::access:return, |
| syscall::chdir:return, |
| syscall::chflags:return, |
| syscall::chown:return, |
| syscall::chroot:return, |
| syscall::getattrlist:return, |
| syscall::getxattr:return, |
| syscall::lchown:return, |
| syscall::lstat64:return, |
| syscall::lstat:return, |
| syscall::mkdir:return, |
| syscall::pathconf:return, |
| syscall::readlink:return, |
| syscall::removexattr:return, |
| syscall::setxattr:return, |
| syscall::stat64:return, |
| syscall::stat:return, |
| syscall::truncate:return, |
| syscall::unlink:return, |
| syscall::utimes:return, |
| */ |
| """ |
| |
| def __init__(self, logname): |
| """Starts the log collection with dtrace. |
| |
| Requires root access or chmod 4555 on dtrace. dtrace is asynchronous so |
| this needs to wait for dtrace to be "warmed up". |
| """ |
| super(Dtrace.Tracer, self).__init__(logname) |
| self._script = create_thunk() |
| # This unique dummy temp file is used to signal the dtrace script that it |
| # should stop as soon as all the child processes are done. A bit hackish |
| # but works fine enough. |
| self._dummy_file_id, self._dummy_file_name = tempfile.mkstemp( |
| prefix='trace_signal_file') |
| |
| # Note: do not use the -p flag. It's useless if the initial process quits |
| # too fast, resulting in missing traces from the grand-children. The D |
| # code manages the dtrace lifetime itself. |
| trace_cmd = [ |
| 'sudo', |
| 'dtrace', |
| # Use a larger buffer if getting 'out of scratch space' errors. |
| # Ref: https://wikis.oracle.com/display/DTrace/Options+and+Tunables |
| '-b', '10m', |
| '-x', 'dynvarsize=10m', |
| #'-x', 'dtrace_global_maxsize=1m', |
| '-x', 'evaltime=exec', |
| '-o', '/dev/stderr', |
| '-q', |
| '-n', self._get_dtrace_code(), |
| ] |
| with open(self._logname + '.log', 'wb') as logfile: |
| self._dtrace = subprocess.Popen( |
| trace_cmd, stdout=logfile, stderr=subprocess.STDOUT) |
| logging.debug('Started dtrace pid: %d' % self._dtrace.pid) |
| |
| # Reads until one line is printed, which signifies dtrace is up and ready. |
| with open(self._logname + '.log', 'rb') as logfile: |
| while 'dtrace_BEGIN' not in logfile.readline(): |
| if self._dtrace.poll() is not None: |
| # Do a busy wait. :/ |
| break |
| logging.debug('dtrace started') |
| |
| def _get_dtrace_code(self): |
| """Setups the D code to implement child process tracking. |
| |
| Injects the cookie in the script so it knows when to stop. |
| |
| The script will detect any instance of the script created with |
| create_thunk() and will start tracing it. |
| """ |
| return ( |
| 'inline int PID = %d;\n' |
| 'inline string SCRIPT = "%s";\n' |
| 'inline int FILE_ID = %d;\n' |
| '\n' |
| '%s') % ( |
| os.getpid(), |
| self._script, |
| self._dummy_file_id, |
| self.D_CODE) + self.D_CODE_EXECVE |
| |
| def trace(self, cmd, cwd, tracename, output): |
| """Runs dtrace on an executable. |
| |
| This dtruss is broken when it starts the process itself or when tracing |
| child processes, this code starts a wrapper process |
| generated with create_thunk() which starts the executable to trace. |
| """ |
| logging.info('trace(%s, %s, %s, %s)' % (cmd, cwd, tracename, output)) |
| assert os.path.isabs(cmd[0]), cmd[0] |
| assert os.path.isabs(cwd), cwd |
| assert os.path.normpath(cwd) == cwd, cwd |
| with self._lock: |
| if not self._initialized: |
| raise TracingFailure( |
| 'Called Tracer.trace() on an unitialized object', |
| None, None, None, tracename) |
| assert tracename not in (i['trace'] for i in self._traces) |
| |
| # Starts the script wrapper to start the child process. This signals the |
| # dtrace script that this process is to be traced. |
| stdout = stderr = None |
| if output: |
| stdout = subprocess.PIPE |
| stderr = subprocess.STDOUT |
| child_cmd = [ |
| sys.executable, |
| self._script, |
| tracename, |
| ] |
| # Call a dummy function so that dtrace knows I'm about to launch a process |
| # that needs to be traced. |
| # Yummy. |
| child = subprocess.Popen( |
| child_cmd + fix_python_path(cmd), |
| stdin=subprocess.PIPE, |
| stdout=stdout, |
| stderr=stderr, |
| cwd=cwd) |
| logging.debug('Started child pid: %d' % child.pid) |
| |
| out = child.communicate()[0] |
| # This doesn't mean tracing is done, one of the grand-child process may |
| # still be alive. It will be tracked with the dtrace script. |
| |
| with self._lock: |
| assert tracename not in (i['trace'] for i in self._traces) |
| self._traces.append( |
| { |
| 'cmd': cmd, |
| 'cwd': cwd, |
| # The pid of strace process, not very useful. |
| 'pid': child.pid, |
| 'trace': tracename, |
| 'output': out, |
| }) |
| return child.returncode, out |
| |
| def close(self, timeout=None): |
| """Terminates dtrace.""" |
| logging.debug('close(%s)' % timeout) |
| try: |
| try: |
| super(Dtrace.Tracer, self).close(timeout) |
| # Signal dtrace that it should stop now. |
| os.ftruncate(self._dummy_file_id, 0) |
| if timeout: |
| start = time.time() |
| # Use polling. :/ |
| while (self._dtrace.poll() is None and |
| (time.time() - start) < timeout): |
| time.sleep(0.1) |
| self._dtrace.kill() |
| self._dtrace.wait() |
| finally: |
| # Make sure to kill it in any case. |
| if self._dtrace.poll() is None: |
| try: |
| self._dtrace.kill() |
| self._dtrace.wait() |
| except OSError: |
| pass |
| |
| if self._dtrace.returncode != 0: |
| # Warn about any dtrace failure but basically ignore it. |
| print 'dtrace failure: %s' % self._dtrace.returncode |
| finally: |
| os.close(self._dummy_file_id) |
| os.remove(self._dummy_file_name) |
| os.remove(self._script) |
| |
| def post_process_log(self): |
| """Sorts the log back in order when each call occured. |
| |
| dtrace doesn't save the buffer in strict order since it keeps one buffer |
| per CPU. |
| """ |
| super(Dtrace.Tracer, self).post_process_log() |
| logname = self._logname + '.log' |
| with open(logname, 'rb') as logfile: |
| lines = [l for l in logfile if l.strip()] |
| errors = [l for l in lines if l.startswith('dtrace:')] |
| if errors: |
| raise TracingFailure( |
| 'Found errors in the trace: %s' % '\n'.join(errors), |
| None, None, None, logname) |
| try: |
| lines = sorted(lines, key=lambda l: int(l.split(' ', 1)[0])) |
| except ValueError: |
| raise TracingFailure( |
| 'Found errors in the trace: %s' % '\n'.join( |
| l for l in lines if l.split(' ', 1)[0].isdigit()), |
| None, None, None, logname) |
| with open(logname, 'wb') as logfile: |
| logfile.write(''.join(lines)) |
| |
| @staticmethod |
| def clean_trace(logname): |
| for ext in ('', '.log'): |
| if os.path.isfile(logname + ext): |
| os.remove(logname + ext) |
| |
| @classmethod |
| def parse_log(cls, logname, blacklist): |
| logging.info('parse_log(%s, ...)' % logname) |
| |
| def blacklist_more(filepath): |
| # All the HFS metadata is in the form /.vol/... |
| return blacklist(filepath) or re.match(r'^\/\.vol\/.+$', filepath) |
| |
| data = read_json(logname) |
| out = [] |
| for item in data['traces']: |
| context = cls.Context(blacklist_more, item['pid'], item['cwd']) |
| for line in open(logname + '.log', 'rb'): |
| context.on_line(line) |
| out.append( |
| { |
| 'results': context.to_results(), |
| 'trace': item['trace'], |
| 'output': item['output'], |
| }) |
| return out |
| |
| |
| class LogmanTrace(ApiBase): |
| """Uses the native Windows ETW based tracing functionality to trace a child |
| process. |
| |
| Caveat: this implementations doesn't track cwd or initial_cwd. It is because |
| the Windows Kernel doesn't have a concept of 'current working directory' at |
| all. A Win32 process has a map of current directories, one per drive letter |
| and it is managed by the user mode kernel32.dll. In kernel, a file is always |
| opened relative to another file_object or as an absolute path. All the current |
| working directory logic is done in user mode. |
| """ |
| class Context(ApiBase.Context): |
| """Processes a ETW log line and keeps the list of existent and non |
| existent files accessed. |
| |
| Ignores directories. |
| """ |
| # These indexes are for the stripped version in json. |
| EVENT_NAME = 0 |
| TYPE = 1 |
| PID = 2 |
| TID = 3 |
| PROCESSOR_ID = 4 |
| TIMESTAMP = 5 |
| USER_DATA = 6 |
| |
| class Process(ApiBase.Context.Process): |
| def __init__(self, *args): |
| super(LogmanTrace.Context.Process, self).__init__(*args) |
| # Handle file objects that succeeded. |
| self.file_objects = {} |
| |
| def __init__(self, blacklist, tracer_pid): |
| logging.info('%s(%d)' % (self.__class__.__name__, tracer_pid)) |
| super(LogmanTrace.Context, self).__init__(blacklist) |
| self._drive_map = DosDriveMap() |
| # Threads mapping to the corresponding process id. |
| self._threads_active = {} |
| # Process ID of the tracer, e.g. tracer_inputs.py |
| self._tracer_pid = tracer_pid |
| self._line_number = 0 |
| |
| def on_line(self, line): |
| """Processes a json Event line.""" |
| self._line_number += 1 |
| try: |
| # By Opcode |
| handler = getattr( |
| self, |
| 'handle_%s_%s' % (line[self.EVENT_NAME], line[self.TYPE]), |
| None) |
| if not handler: |
| raise TracingFailure( |
| 'Unexpected event %s_%s' % ( |
| line[self.EVENT_NAME], line[self.TYPE]), |
| None, None, None) |
| handler(line) |
| except TracingFailure, e: |
| # Hack in the values since the handler could be a static function. |
| e.pid = line[self.PID] |
| e.line = line |
| e.line_number = self._line_number |
| # Re-raise the modified exception. |
| raise |
| except (KeyError, NotImplementedError, ValueError), e: |
| raise TracingFailure( |
| 'Trace generated a %s exception: %s' % ( |
| e.__class__.__name__, str(e)), |
| line[self.PID], |
| self._line_number, |
| line, |
| e) |
| |
| def to_results(self): |
| if not self.root_process: |
| raise TracingFailure( |
| 'Failed to detect the initial process', |
| None, None, None) |
| process = self.root_process.to_results_process() |
| return Results(process) |
| |
| def _thread_to_process(self, tid): |
| """Finds the process from the thread id.""" |
| tid = int(tid, 16) |
| pid = self._threads_active.get(tid) |
| if not pid or not self._process_lookup.get(pid): |
| return |
| return self._process_lookup[pid] |
| |
| @classmethod |
| def handle_EventTrace_Header(cls, line): |
| """Verifies no event was dropped, e.g. no buffer overrun occured.""" |
| BUFFER_SIZE = cls.USER_DATA |
| #VERSION = cls.USER_DATA + 1 |
| #PROVIDER_VERSION = cls.USER_DATA + 2 |
| #NUMBER_OF_PROCESSORS = cls.USER_DATA + 3 |
| #END_TIME = cls.USER_DATA + 4 |
| #TIMER_RESOLUTION = cls.USER_DATA + 5 |
| #MAX_FILE_SIZE = cls.USER_DATA + 6 |
| #LOG_FILE_MODE = cls.USER_DATA + 7 |
| #BUFFERS_WRITTEN = cls.USER_DATA + 8 |
| #START_BUFFERS = cls.USER_DATA + 9 |
| #POINTER_SIZE = cls.USER_DATA + 10 |
| EVENTS_LOST = cls.USER_DATA + 11 |
| #CPU_SPEED = cls.USER_DATA + 12 |
| #LOGGER_NAME = cls.USER_DATA + 13 |
| #LOG_FILE_NAME = cls.USER_DATA + 14 |
| #BOOT_TIME = cls.USER_DATA + 15 |
| #PERF_FREQ = cls.USER_DATA + 16 |
| #START_TIME = cls.USER_DATA + 17 |
| #RESERVED_FLAGS = cls.USER_DATA + 18 |
| #BUFFERS_LOST = cls.USER_DATA + 19 |
| #SESSION_NAME_STRING = cls.USER_DATA + 20 |
| #LOG_FILE_NAME_STRING = cls.USER_DATA + 21 |
| if line[EVENTS_LOST] != '0': |
| raise TracingFailure( |
| ( '%s events were lost during trace, please increase the buffer ' |
| 'size from %s') % (line[EVENTS_LOST], line[BUFFER_SIZE]), |
| None, None, None) |
| |
| def handle_FileIo_Cleanup(self, line): |
| """General wisdom: if a file is closed, it's because it was opened. |
| |
| Note that FileIo_Close is not used since if a file was opened properly but |
| not closed before the process exits, only Cleanup will be logged. |
| """ |
| #IRP = self.USER_DATA |
| TTID = self.USER_DATA + 1 # Thread ID, that's what we want. |
| FILE_OBJECT = self.USER_DATA + 2 |
| #FILE_KEY = self.USER_DATA + 3 |
| proc = self._thread_to_process(line[TTID]) |
| if not proc: |
| # Not a process we care about. |
| return |
| file_object = line[FILE_OBJECT] |
| if file_object in proc.file_objects: |
| proc.add_file(proc.file_objects.pop(file_object), False) |
| |
| def handle_FileIo_Create(self, line): |
| """Handles a file open. |
| |
| All FileIo events are described at |
| http://msdn.microsoft.com/library/windows/desktop/aa363884.aspx |
| for some value of 'description'. |
| |
| " (..) process and thread id values of the IO events (..) are not valid " |
| http://msdn.microsoft.com/magazine/ee358703.aspx |
| |
| The FileIo.Create event doesn't return if the CreateFile() call |
| succeeded, so keep track of the file_object and check that it is |
| eventually closed with FileIo_Cleanup. |
| """ |
| #IRP = self.USER_DATA |
| TTID = self.USER_DATA + 1 # Thread ID, that's what we want. |
| FILE_OBJECT = self.USER_DATA + 2 |
| #CREATE_OPTIONS = self.USER_DATA + 3 |
| #FILE_ATTRIBUTES = self.USER_DATA + 4 |
| #self.USER_DATA + SHARE_ACCESS = 5 |
| OPEN_PATH = self.USER_DATA + 6 |
| |
| proc = self._thread_to_process(line[TTID]) |
| if not proc: |
| # Not a process we care about. |
| return |
| |
| match = re.match(r'^\"(.+)\"$', line[OPEN_PATH]) |
| raw_path = match.group(1) |
| # Ignore directories and bare drive right away. |
| if raw_path.endswith(os.path.sep): |
| return |
| filepath = self._drive_map.to_win32(raw_path) |
| # Ignore bare drive right away. Some may still fall through with format |
| # like '\\?\X:' |
| if len(filepath) == 2: |
| return |
| file_object = line[FILE_OBJECT] |
| if os.path.isdir(filepath): |
| # There is no O_DIRECTORY equivalent on Windows. The closed is |
| # FILE_FLAG_BACKUP_SEMANTICS but it's not exactly right either. So |
| # simply discard directories are they are found. |
| return |
| # Override any stale file object |
| proc.file_objects[file_object] = filepath |
| |
| def handle_FileIo_Rename(self, line): |
| # TODO(maruel): Handle? |
| pass |
| |
| def handle_Process_End(self, line): |
| pid = line[self.PID] |
| if self._process_lookup.get(pid): |
| logging.info('Terminated: %d' % pid) |
| self._process_lookup[pid] = None |
| else: |
| logging.debug('Terminated: %d' % pid) |
| |
| def handle_Process_Start(self, line): |
| """Handles a new child process started by PID.""" |
| #UNIQUE_PROCESS_KEY = self.USER_DATA |
| PROCESS_ID = self.USER_DATA + 1 |
| #PARENT_PID = self.USER_DATA + 2 |
| #SESSION_ID = self.USER_DATA + 3 |
| #EXIT_STATUS = self.USER_DATA + 4 |
| #DIRECTORY_TABLE_BASE = self.USER_DATA + 5 |
| #USER_SID = self.USER_DATA + 6 |
| IMAGE_FILE_NAME = self.USER_DATA + 7 |
| COMMAND_LINE = self.USER_DATA + 8 |
| |
| ppid = line[self.PID] |
| pid = int(line[PROCESS_ID], 16) |
| logging.debug( |
| 'New process %d->%d (%s) %s' % |
| (ppid, pid, line[IMAGE_FILE_NAME], line[COMMAND_LINE])) |
| |
| if ppid == self._tracer_pid: |
| # Need to ignore processes we don't know about because the log is |
| # system-wide. self._tracer_pid shall start only one process. |
| if self.root_process: |
| raise TracingFailure( |
| ( 'Parent process is _tracer_pid(%d) but root_process(%d) is ' |
| 'already set') % (self._tracer_pid, self.root_process.pid), |
| None, None, None) |
| proc = self.Process(self.blacklist, pid, None) |
| self.root_process = proc |
| ppid = None |
| elif self._process_lookup.get(ppid): |
| proc = self.Process(self.blacklist, pid, None) |
| self._process_lookup[ppid].children.append(proc) |
| else: |
| # Ignore |
| return |
| self._process_lookup[pid] = proc |
| |
| if (not line[IMAGE_FILE_NAME].startswith('"') or |
| not line[IMAGE_FILE_NAME].endswith('"')): |
| raise TracingFailure( |
| 'Command line is not properly quoted: %s' % line[IMAGE_FILE_NAME], |
| None, None, None) |
| |
| # TODO(maruel): Process escapes. |
| if (not line[COMMAND_LINE].startswith('"') or |
| not line[COMMAND_LINE].endswith('"')): |
| raise TracingFailure( |
| 'Command line is not properly quoted: %s' % line[COMMAND_LINE], |
| None, None, None) |
| proc.command = CommandLineToArgvW(line[COMMAND_LINE][1:-1]) |
| proc.executable = line[IMAGE_FILE_NAME][1:-1] |
| # proc.command[0] may be the absolute path of 'executable' but it may be |
| # anything else too. If it happens that command[0] ends with executable, |
| # use it, otherwise defaults to the base name. |
| cmd0 = proc.command[0].lower() |
| if not cmd0.endswith('.exe'): |
| # TODO(maruel): That's not strictly true either. |
| cmd0 += '.exe' |
| if cmd0.endswith(proc.executable) and os.path.isfile(cmd0): |
| # Fix the path. |
| cmd0 = cmd0.replace('/', os.path.sep) |
| cmd0 = os.path.normpath(cmd0) |
| proc.executable = get_native_path_case(cmd0) |
| logging.info( |
| 'New child: %s -> %d %s' % (ppid, pid, proc.executable)) |
| |
| def handle_Thread_End(self, line): |
| """Has the same parameters as Thread_Start.""" |
| tid = int(line[self.TID], 16) |
| self._threads_active.pop(tid, None) |
| |
| def handle_Thread_Start(self, line): |
| """Handles a new thread created. |
| |
| Do not use self.PID here since a process' initial thread is created by |
| the parent process. |
| """ |
| PROCESS_ID = self.USER_DATA |
| TTHREAD_ID = self.USER_DATA + 1 |
| #STACK_BASE = self.USER_DATA + 2 |
| #STACK_LIMIT = self.USER_DATA + 3 |
| #USER_STACK_BASE = self.USER_DATA + 4 |
| #USER_STACK_LIMIT = self.USER_DATA + 5 |
| #AFFINITY = self.USER_DATA + 6 |
| #WIN32_START_ADDR = self.USER_DATA + 7 |
| #TEB_BASE = self.USER_DATA + 8 |
| #SUB_PROCESS_TAG = self.USER_DATA + 9 |
| #BASE_PRIORITY = self.USER_DATA + 10 |
| #PAGE_PRIORITY = self.USER_DATA + 11 |
| #IO_PRIORITY = self.USER_DATA + 12 |
| #THREAD_FLAGS = self.USER_DATA + 13 |
| # Do not use self.PID here since a process' initial thread is created by |
| # the parent process. |
| pid = int(line[PROCESS_ID], 16) |
| tid = int(line[TTHREAD_ID], 16) |
| logging.debug('New thread pid:%d, tid:%d' % (pid, tid)) |
| self._threads_active[tid] = pid |
| |
| @classmethod |
| def supported_events(cls): |
| """Returns all the procesed events.""" |
| out = [] |
| for member in dir(cls): |
| match = re.match(r'^handle_([A-Za-z]+)_([A-Za-z]+)$', member) |
| if match: |
| out.append(match.groups()) |
| return out |
| |
| class Tracer(ApiBase.Tracer): |
| # The basic headers. |
| EXPECTED_HEADER = [ |
| u'Event Name', |
| u'Type', |
| u'Event ID', |
| u'Version', |
| u'Channel', |
| u'Level', # 5 |
| u'Opcode', |
| u'Task', |
| u'Keyword', |
| u'PID', |
| u'TID', # 10 |
| u'Processor Number', |
| u'Instance ID', |
| u'Parent Instance ID', |
| u'Activity ID', |
| u'Related Activity ID', # 15 |
| u'Clock-Time', |
| u'Kernel(ms)', # Both have a resolution of ~15ms which makes them |
| u'User(ms)', # pretty much useless. |
| u'User Data', # Extra arguments that are event-specific. |
| ] |
| # Only the useful headers common to all entries are listed there. Any column |
| # at 19 or higher is dependent on the specific event. |
| EVENT_NAME = 0 |
| TYPE = 1 |
| PID = 9 |
| TID = 10 |
| PROCESSOR_ID = 11 |
| TIMESTAMP = 16 |
| NULL_GUID = '{00000000-0000-0000-0000-000000000000}' |
| USER_DATA = 19 |
| |
| def __init__(self, logname): |
| """Starts the log collection. |
| |
| Requires administrative access. logman.exe is synchronous so no need for a |
| "warmup" call. 'Windows Kernel Trace' is *localized* so use its GUID |
| instead. The GUID constant name is SystemTraceControlGuid. Lovely. |
| |
| One can get the list of potentially interesting providers with: |
| "logman query providers | findstr /i file" |
| """ |
| super(LogmanTrace.Tracer, self).__init__(logname) |
| self._script = create_thunk() |
| cmd_start = [ |
| 'logman.exe', |
| 'start', |
| 'NT Kernel Logger', |
| '-p', '{9e814aad-3204-11d2-9a82-006008a86939}', |
| # splitio,fileiocompletion,syscall,file,cswitch,img |
| '(process,fileio,thread)', |
| '-o', self._logname + '.etl', |
| '-ets', # Send directly to kernel |
| # Values extracted out of thin air. |
| # Event Trace Session buffer size in kb. |
| '-bs', '10240', |
| # Number of Event Trace Session buffers. |
| '-nb', '16', '256', |
| ] |
| logging.debug('Running: %s' % cmd_start) |
| try: |
| subprocess.check_call( |
| cmd_start, |
| stdin=subprocess.PIPE, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError, e: |
| if e.returncode == -2147024891: |
| print >> sys.stderr, 'Please restart with an elevated admin prompt' |
| elif e.returncode == -2144337737: |
| print >> sys.stderr, ( |
| 'A kernel trace was already running, stop it and try again') |
| raise |
| |
| def trace(self, cmd, cwd, tracename, output): |
| logging.info('trace(%s, %s, %s, %s)' % (cmd, cwd, tracename, output)) |
| assert os.path.isabs(cmd[0]), cmd[0] |
| assert os.path.isabs(cwd), cwd |
| assert os.path.normpath(cwd) == cwd, cwd |
| with self._lock: |
| if not self._initialized: |
| raise TracingFailure( |
| 'Called Tracer.trace() on an unitialized object', |
| None, None, None, tracename) |
| assert tracename not in (i['trace'] for i in self._traces) |
| |
| # Use "logman -?" for help. |
| |
| stdout = stderr = None |
| if output: |
| stdout = subprocess.PIPE |
| stderr = subprocess.STDOUT |
| |
| # Run the child process. |
| logging.debug('Running: %s' % cmd) |
| # Use the temporary script generated with create_thunk() so we have a |
| # clear pid owner. Since trace_inputs.py can be used as a library and |
| # could trace multiple processes simultaneously, it makes it more complex |
| # if the executable to be traced is executed directly here. It also solves |
| # issues related to logman.exe that needs to be executed to control the |
| # kernel trace. |
| child_cmd = [ |
| sys.executable, |
| self._script, |
| tracename, |
| ] |
| child = subprocess.Popen( |
| child_cmd + fix_python_path(cmd), |
| cwd=cwd, |
| stdin=subprocess.PIPE, |
| stdout=stdout, |
| stderr=stderr) |
| logging.debug('Started child pid: %d' % child.pid) |
| out = child.communicate()[0] |
| # This doesn't mean all the grand-children are done. Sadly, we don't have |
| # a good way to determine that. |
| |
| with self._lock: |
| assert tracename not in (i['trace'] for i in self._traces) |
| self._traces.append({ |
| 'command': cmd, |
| 'cwd': cwd, |
| 'pid': child.pid, |
| 'trace': tracename, |
| 'output': out, |
| }) |
| |
| return child.returncode, out |
| |
| def close(self, _timeout=None): |
| """Stops the kernel log collection and converts the traces to text |
| representation. |
| """ |
| with self._lock: |
| if not self._initialized: |
| raise TracingFailure( |
| 'Called Tracer.close() on an unitialized object', |
| None, None, None) |
| os.remove(self._script) |
| # Save metadata, add 'format' key.. |
| data = { |
| 'format': 'csv', |
| 'traces': self._traces, |
| } |
| write_json(self._logname, data, False) |
| |
| cmd_stop = [ |
| 'logman.exe', |
| 'stop', |
| 'NT Kernel Logger', |
| '-ets', # Sends the command directly to the kernel. |
| ] |
| logging.debug('Running: %s' % cmd_stop) |
| subprocess.check_call( |
| cmd_stop, |
| stdin=subprocess.PIPE, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT) |
| self._initialized = False |
| |
| def post_process_log(self): |
| """Converts the .etl file into .csv then into .json.""" |
| super(LogmanTrace.Tracer, self).post_process_log() |
| logformat = 'csv' |
| self._convert_log(logformat) |
| |
| if logformat == 'csv_utf16': |
| def load_file(): |
| def utf_8_encoder(unicode_csv_data): |
| """Encodes the unicode object as utf-8 encoded str instance""" |
| for line in unicode_csv_data: |
| yield line.encode('utf-8') |
| |
| def unicode_csv_reader(unicode_csv_data, **kwargs): |
| """Encodes temporarily as UTF-8 since csv module doesn't do unicode. |
| """ |
| csv_reader = csv.reader(utf_8_encoder(unicode_csv_data), **kwargs) |
| for row in csv_reader: |
| # Decode str utf-8 instances back to unicode instances, cell by |
| # cell: |
| yield [cell.decode('utf-8') for cell in row] |
| |
| # The CSV file is UTF-16 so use codecs.open() to load the file into |
| # the python internal unicode format (utf-8). Then explicitly |
| # re-encode as utf8 as str instances so csv can parse it fine. Then |
| # decode the utf-8 str back into python unicode instances. This |
| # sounds about right. |
| for line in unicode_csv_reader( |
| codecs.open(self._logname + '.' + logformat, 'r', 'utf-16')): |
| # line is a list of unicode objects |
| # So much white space! |
| yield [i.strip() for i in line] |
| |
| elif logformat == 'csv': |
| def load_file(): |
| def ansi_csv_reader(ansi_csv_data, **kwargs): |
| """Loads an 'ANSI' code page and returns unicode() objects.""" |
| assert sys.getfilesystemencoding() == 'mbcs' |
| encoding = get_current_encoding() |
| for row in csv.reader(ansi_csv_data, **kwargs): |
| # Decode str 'ansi' instances to unicode instances, cell by cell: |
| yield [cell.decode(encoding) for cell in row] |
| |
| # The fastest and smallest format but only supports 'ANSI' file paths. |
| # E.g. the filenames are encoding in the 'current' encoding. |
| for line in ansi_csv_reader(open(self._logname + '.' + logformat)): |
| # line is a list of unicode objects. |
| yield [i.strip() for i in line] |
| |
| supported_events = LogmanTrace.Context.supported_events() |
| |
| def trim(generator): |
| for index, line in enumerate(generator): |
| if not index: |
| if line != self.EXPECTED_HEADER: |
| raise TracingFailure( |
| 'Found malformed header: %s' % ' '.join(line), |
| None, None, None) |
| continue |
| # As you can see, the CSV is full of useful non-redundant information: |
| if (line[2] != '0' or # Event ID |
| line[3] not in ('2', '3') or # Version |
| line[4] != '0' or # Channel |
| line[5] != '0' or # Level |
| line[7] != '0' or # Task |
| line[8] != '0x0000000000000000' or # Keyword |
| line[12] != '' or # Instance ID |
| line[13] != '' or # Parent Instance ID |
| line[14] != self.NULL_GUID or # Activity ID |
| line[15] != ''): # Related Activity ID |
| raise TracingFailure( |
| 'Found unexpected values in line: %s' % ' '.join(line), |
| None, None, None) |
| |
| if (line[self.EVENT_NAME], line[self.TYPE]) not in supported_events: |
| continue |
| |
| # Convert the PID in-place from hex. |
| line[self.PID] = int(line[self.PID], 16) |
| |
| yield [ |
| line[self.EVENT_NAME], |
| line[self.TYPE], |
| line[self.PID], |
| line[self.TID], |
| line[self.PROCESSOR_ID], |
| line[self.TIMESTAMP], |
| ] + line[self.USER_DATA:] |
| |
| write_json('%s.json' % self._logname, list(trim(load_file())), True) |
| |
| def _convert_log(self, logformat): |
| """Converts the ETL trace to text representation. |
| |
| Normally, 'csv' is sufficient. If complex scripts are used (like eastern |
| languages), use 'csv_utf16'. If localization gets in the way, use 'xml'. |
| |
| Arguments: |
| - logformat: Text format to be generated, csv, csv_utf16 or xml. |
| |
| Use "tracerpt -?" for help. |
| """ |
| LOCALE_INVARIANT = 0x7F |
| windll.kernel32.SetThreadLocale(LOCALE_INVARIANT) |
| cmd_convert = [ |
| 'tracerpt.exe', |
| '-l', self._logname + '.etl', |
| '-o', self._logname + '.' + logformat, |
| '-gmt', # Use UTC |
| '-y', # No prompt |
| # Use -of XML to get the header of each items after column 19, e.g. all |
| # the actual headers of 'User Data'. |
| ] |
| |
| if logformat == 'csv': |
| # tracerpt localizes the 'Type' column, for major brainfuck |
| # entertainment. I can't imagine any sane reason to do that. |
| cmd_convert.extend(['-of', 'CSV']) |
| elif logformat == 'csv_utf16': |
| # This causes it to use UTF-16, which doubles the log size but ensures |
| # the log is readable for non-ASCII characters. |
| cmd_convert.extend(['-of', 'CSV', '-en', 'Unicode']) |
| elif logformat == 'xml': |
| cmd_convert.extend(['-of', 'XML']) |
| else: |
| raise ValueError('Unexpected log format \'%s\'' % logformat) |
| logging.debug('Running: %s' % cmd_convert) |
| # This can takes tens of minutes for large logs. |
| # Redirects all output to stderr. |
| subprocess.check_call( |
| cmd_convert, |
| stdin=subprocess.PIPE, |
| stdout=sys.stderr, |
| stderr=sys.stderr) |
| |
| @staticmethod |
| def clean_trace(logname): |
| for ext in ('', '.csv', '.etl', '.json', '.xml'): |
| if os.path.isfile(logname + ext): |
| os.remove(logname + ext) |
| |
| @classmethod |
| def parse_log(cls, logname, blacklist): |
| logging.info('parse_log(%s, %s)' % (logname, blacklist)) |
| |
| def blacklist_more(filepath): |
| # All the NTFS metadata is in the form x:\$EXTEND or stuff like that. |
| return blacklist(filepath) or re.match(r'[A-Z]\:\\\$EXTEND', filepath) |
| |
| data = read_json(logname) |
| lines = read_json(logname + '.json') |
| out = [] |
| for item in data['traces']: |
| context = cls.Context(blacklist_more, item['pid']) |
| for line in lines: |
| context.on_line(line) |
| out.append( |
| { |
| 'results': context.to_results(), |
| 'trace': item['trace'], |
| 'output': item['output'], |
| }) |
| return out |
| |
| |
| def get_api(): |
| """Returns the correct implementation for the current OS.""" |
| if sys.platform == 'cygwin': |
| raise NotImplementedError( |
| 'Not implemented for cygwin, start the script from Win32 python') |
| flavors = { |
| 'win32': LogmanTrace, |
| 'darwin': Dtrace, |
| 'sunos5': Dtrace, |
| 'freebsd7': Dtrace, |
| 'freebsd8': Dtrace, |
| } |
| # Defaults to strace. |
| return flavors.get(sys.platform, Strace)() |
| |
| |
| def extract_directories(root_dir, files, blacklist): |
| """Detects if all the files in a directory are in |files| and if so, replace |
| the individual files by a Results.Directory instance. |
| |
| Takes a list of Results.File instances and returns a shorter list of |
| Results.File and Results.Directory instances. |
| |
| Arguments: |
| - root_dir: Optional base directory that shouldn't be search further. |
| - files: list of Results.File instances. |
| - blacklist: regexp of files to ignore, for example r'.+\.pyc'. |
| """ |
| logging.info( |
| 'extract_directories(%s, %d files, ...)' % (root_dir, len(files))) |
| assert not (root_dir or '').endswith(os.path.sep), root_dir |
| assert not root_dir or (get_native_path_case(root_dir) == root_dir) |
| assert not any(isinstance(f, Results.Directory) for f in files) |
| # Remove non existent files. |
| files = [f for f in files if f.existent] |
| if not files: |
| return files |
| # All files must share the same root, which can be None. |
| assert len(set(f.root for f in files)) == 1, set(f.root for f in files) |
| |
| # Creates a {directory: {filename: File}} mapping, up to root. |
| buckets = {} |
| if root_dir: |
| buckets[root_dir] = {} |
| for fileobj in files: |
| path = fileobj.full_path |
| directory = os.path.dirname(path) |
| assert directory |
| # Do not use os.path.basename() so trailing os.path.sep is kept. |
| basename = path[len(directory)+1:] |
| files_in_directory = buckets.setdefault(directory, {}) |
| files_in_directory[basename] = fileobj |
| # Add all the directories recursively up to root. |
| while True: |
| old_d = directory |
| directory = os.path.dirname(directory) |
| if directory + os.path.sep == root_dir or directory == old_d: |
| break |
| buckets.setdefault(directory, {}) |
| |
| root_prefix = len(root_dir) + 1 if root_dir else 0 |
| for directory in sorted(buckets, reverse=True): |
| actual = set(f for f in os.listdir(directory) if not blacklist(f)) |
| expected = set(buckets[directory]) |
| if not (actual - expected): |
| parent = os.path.dirname(directory) |
| buckets[parent][os.path.basename(directory)] = Results.Directory( |
| root_dir, |
| directory[root_prefix:], |
| False, |
| sum(f.size for f in buckets[directory].itervalues()), |
| sum(f.nb_files for f in buckets[directory].itervalues())) |
| # Remove the whole bucket. |
| del buckets[directory] |
| |
| # Reverse the mapping with what remains. The original instances are returned, |
| # so the cached meta data is kept. |
| files = sum((x.values() for x in buckets.itervalues()), []) |
| return sorted(files, key=lambda x: x.path) |
| |
| |
| def trace(logfile, cmd, cwd, api, output): |
| """Traces an executable. Returns (returncode, output) from api. |
| |
| Arguments: |
| - logfile: file to write to. |
| - cmd: command to run. |
| - cwd: current directory to start the process in. |
| - api: a tracing api instance. |
| - output: if True, returns output, otherwise prints it at the console. |
| """ |
| cmd = fix_python_path(cmd) |
| api.clean_trace(logfile) |
| with api.get_tracer(logfile) as tracer: |
| return tracer.trace(cmd, cwd, 'default', output) |
| |
| |
| def load_trace(logfile, root_dir, api, blacklist): |
| """Loads a trace file and returns the Results instance. |
| |
| Arguments: |
| - logfile: File to load. |
| - root_dir: Root directory to use to determine if a file is relevant to the |
| trace or not. |
| - api: A tracing api instance. |
| - blacklist: Optional blacklist function to filter out unimportant files. |
| """ |
| data = api.parse_log(logfile, (blacklist or (lambda _: False))) |
| assert len(data) == 1, 'More than one trace was detected!' |
| if 'exception' in data[0]: |
| # It got an exception, raise it. |
| raise data[0]['exception'] |
| results = data[0]['results'] |
| if root_dir: |
| results = results.strip_root(root_dir) |
| return results |
| |
| |
| def CMDclean(args): |
| """Cleans up traces.""" |
| parser = OptionParserTraceInputs(command='clean') |
| options, args = parser.parse_args(args) |
| api = get_api() |
| api.clean_trace(options.log) |
| return 0 |
| |
| |
| def CMDtrace(args): |
| """Traces an executable.""" |
| parser = OptionParserTraceInputs(command='trace') |
| parser.allow_interspersed_args = False |
| parser.add_option( |
| '-q', '--quiet', action='store_true', |
| help='Redirects traced executable output to /dev/null') |
| options, args = parser.parse_args(args) |
| |
| if not args: |
| parser.error('Please provide a command to run') |
| |
| if not os.path.isabs(args[0]) and os.access(args[0], os.X_OK): |
| args[0] = os.path.abspath(args[0]) |
| |
| api = get_api() |
| return trace(options.log, args, os.getcwd(), api, options.quiet)[0] |
| |
| |
| def CMDread(args): |
| """Reads the logs and prints the result.""" |
| parser = OptionParserTraceInputs(command='read') |
| parser.add_option( |
| '-V', '--variable', |
| nargs=2, |
| action='append', |
| dest='variables', |
| metavar='VAR_NAME directory', |
| default=[], |
| help=('Variables to replace relative directories against. Example: ' |
| '"-v \'$HOME\' \'/home/%s\'" will replace all occurence of your ' |
| 'home dir with $HOME') % getpass.getuser()) |
| parser.add_option( |
| '--root-dir', |
| help='Root directory to base everything off it. Anything outside of this ' |
| 'this directory will not be reported') |
| parser.add_option( |
| '-j', '--json', action='store_true', |
| help='Outputs raw result data as json') |
| parser.add_option( |
| '-b', '--blacklist', action='append', default=[], |
| help='List of regexp to use as blacklist filter') |
| options, args = parser.parse_args(args) |
| |
| if options.root_dir: |
| options.root_dir = os.path.abspath(options.root_dir) |
| |
| variables = dict(options.variables) |
| api = get_api() |
| def blacklist(f): |
| return any(re.match(b, f) for b in options.blacklist) |
| data = api.parse_log(options.log, blacklist) |
| # Process each trace. |
| output_as_json = [] |
| for item in data: |
| if 'exception' in item: |
| print >> sys.stderr, ( |
| 'Trace %s: Got an exception: %s' % (item['trace'], item['exception'])) |
| continue |
| results = item['results'] |
| if options.root_dir: |
| results = results.strip_root(options.root_dir) |
| |
| if options.json: |
| output_as_json.append(results.flatten()) |
| else: |
| simplified = extract_directories( |
| options.root_dir, results.files, blacklist) |
| simplified = [f.replace_variables(variables) for f in simplified] |
| if len(data) > 1: |
| print('Trace: %s' % item['trace']) |
| print('Total: %d' % len(results.files)) |
| print('Non existent: %d' % len(results.non_existent)) |
| for f in results.non_existent: |
| print(' %s' % f.path) |
| print( |
| 'Interesting: %d reduced to %d' % ( |
| len(results.existent), len(simplified))) |
| for f in simplified: |
| print(' %s' % f.path) |
| |
| if options.json: |
| write_json(sys.stdout, output_as_json, False) |
| return 0 |
| |
| |
| class OptionParserWithLogging(optparse.OptionParser): |
| """Adds --verbose option.""" |
| def __init__(self, verbose=0, **kwargs): |
| optparse.OptionParser.__init__(self, **kwargs) |
| self.add_option( |
| '-v', '--verbose', |
| action='count', |
| default=verbose, |
| help='Use multiple times to increase verbosity') |
| |
| def parse_args(self, *args, **kwargs): |
| options, args = optparse.OptionParser.parse_args(self, *args, **kwargs) |
| levels = [logging.ERROR, logging.INFO, logging.DEBUG] |
| logging.basicConfig( |
| level=levels[min(len(levels)-1, options.verbose)], |
| format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s') |
| return options, args |
| |
| |
| class OptionParserWithNiceDescription(OptionParserWithLogging): |
| """Generates the description with the command's docstring.""" |
| def __init__(self, **kwargs): |
| """Sets 'description' and 'usage' if not already specified.""" |
| command = kwargs.pop('command', 'help') |
| kwargs.setdefault( |
| 'description', |
| re.sub('[\r\n ]{2,}', ' ', get_command_handler(command).__doc__)) |
| kwargs.setdefault('usage', '%%prog %s [options]' % command) |
| OptionParserWithLogging.__init__(self, **kwargs) |
| |
| |
| class OptionParserTraceInputs(OptionParserWithNiceDescription): |
| """Adds automatic --log handling.""" |
| def __init__(self, **kwargs): |
| OptionParserWithNiceDescription.__init__(self, **kwargs) |
| self.add_option( |
| '-l', '--log', help='Log file to generate or read, required') |
| |
| def parse_args(self, *args, **kwargs): |
| """Makes sure the paths make sense. |
| |
| On Windows, / and \ are often mixed together in a path. |
| """ |
| options, args = OptionParserWithNiceDescription.parse_args( |
| self, *args, **kwargs) |
| if not options.log: |
| self.error('Must supply a log file with -l') |
| options.log = os.path.abspath(options.log) |
| return options, args |
| |
| |
| def extract_documentation(): |
| """Returns a dict {command: description} for each of documented command.""" |
| commands = ( |
| fn[3:] |
| for fn in dir(sys.modules['__main__']) |
| if fn.startswith('CMD') and get_command_handler(fn[3:]).__doc__) |
| return dict((fn, get_command_handler(fn).__doc__) for fn in commands) |
| |
| |
| def CMDhelp(args): |
| """Prints list of commands or help for a specific command.""" |
| doc = extract_documentation() |
| # Calculates the optimal offset. |
| offset = max(len(cmd) for cmd in doc) |
| format_str = ' %-' + str(offset + 2) + 's %s' |
| # Generate a one-liner documentation of each commands. |
| commands_description = '\n'.join( |
| format_str % (cmd, doc[cmd].split('\n')[0]) for cmd in sorted(doc)) |
| |
| parser = OptionParserWithNiceDescription( |
| usage='%prog <command> [options]', |
| description='Commands are:\n%s\n' % commands_description) |
| parser.format_description = lambda _: parser.description |
| |
| # Strip out any -h or --help argument. |
| _, args = parser.parse_args([i for i in args if not i in ('-h', '--help')]) |
| if len(args) == 1: |
| if not get_command_handler(args[0]): |
| parser.error('Unknown command %s' % args[0]) |
| # The command was "%prog help command", replaces ourself with |
| # "%prog command --help" so help is correctly printed out. |
| return main(args + ['--help']) |
| elif args: |
| parser.error('Unknown argument "%s"' % ' '.join(args)) |
| parser.print_help() |
| return 0 |
| |
| |
| def get_command_handler(name): |
| """Returns the command handler or CMDhelp if it doesn't exist.""" |
| return getattr(sys.modules['__main__'], 'CMD%s' % name, None) |
| |
| |
| def main_impl(argv): |
| command = get_command_handler(argv[0] if argv else 'help') |
| if not command: |
| return CMDhelp(argv) |
| return command(argv[1:]) |
| |
| def main(argv): |
| try: |
| main_impl(argv) |
| except TracingFailure, e: |
| sys.stderr.write('\nError: ') |
| sys.stderr.write(str(e)) |
| sys.stderr.write('\n') |
| return 1 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |