GoogleGit

blob: 8fcb5ebc02ba7e2aa3554c2c54cafb7ae9155f85 [file] [log] [blame]
  1. # Copyright (c) 2015 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Contains the parsing system of the Chromium Buildbot Annotator."""
  5. import os
  6. import sys
  7. import traceback
  8. # These are maps of annotation key -> number of expected arguments.
  9. STEP_ANNOTATIONS = {
  10. 'SET_BUILD_PROPERTY': 2,
  11. 'STEP_CLEAR': 0,
  12. 'STEP_EXCEPTION': 0,
  13. 'STEP_FAILURE': 0,
  14. 'STEP_LINK': 2,
  15. 'STEP_LOG_END': 1,
  16. 'STEP_LOG_END_PERF': 2,
  17. 'STEP_LOG_LINE': 2,
  18. 'STEP_SUMMARY_CLEAR': 0,
  19. 'STEP_SUMMARY_TEXT': 1,
  20. 'STEP_TEXT': 1,
  21. 'STEP_TRIGGER': 1,
  22. 'STEP_WARNINGS': 0,
  23. 'STEP_NEST_LEVEL': 1,
  24. }
  25. CONTROL_ANNOTATIONS = {
  26. 'STEP_CLOSED': 0,
  27. 'STEP_STARTED': 0,
  28. }
  29. STREAM_ANNOTATIONS = {
  30. 'HALT_ON_FAILURE': 0,
  31. 'HONOR_ZERO_RETURN_CODE': 0,
  32. 'SEED_STEP': 1,
  33. 'SEED_STEP_TEXT': 2,
  34. 'STEP_CURSOR': 1,
  35. }
  36. DEPRECATED_ANNOTATIONS = {
  37. 'BUILD_STEP': 1,
  38. }
  39. ALL_ANNOTATIONS = {}
  40. ALL_ANNOTATIONS.update(STEP_ANNOTATIONS)
  41. ALL_ANNOTATIONS.update(CONTROL_ANNOTATIONS)
  42. ALL_ANNOTATIONS.update(STREAM_ANNOTATIONS)
  43. ALL_ANNOTATIONS.update(DEPRECATED_ANNOTATIONS)
  44. # This is a mapping of old_annotation_name -> new_annotation_name.
  45. # Theoretically all annotator scripts should use the new names, but it's hard
  46. # to tell due to the decentralized nature of the annotator.
  47. DEPRECATED_ALIASES = {
  48. 'BUILD_FAILED': 'STEP_FAILURE',
  49. 'BUILD_WARNINGS': 'STEP_WARNINGS',
  50. 'BUILD_EXCEPTION': 'STEP_EXCEPTION',
  51. 'link': 'STEP_LINK',
  52. }
  53. # A couple of the annotations have the format:
  54. # @@@THING arg@@@
  55. # for reasons no one knows. We only need this case until all masters have been
  56. # restarted to pick up the new master-side parsing code.
  57. OLD_STYLE_ANNOTATIONS = set((
  58. 'SEED_STEP',
  59. 'STEP_CURSOR',
  60. ))
  61. def emit(line, stream, flush_before=None):
  62. if flush_before:
  63. flush_before.flush()
  64. print >> stream
  65. # WinDOS can only handle 64kb of output to the console at a time, per process.
  66. if sys.platform.startswith('win'):
  67. lim = 2**15
  68. while line:
  69. to_print, line = line[:lim], line[lim:]
  70. stream.write(to_print)
  71. stream.write('\n')
  72. else:
  73. print >> stream, line
  74. stream.flush()
  75. class MetaAnnotationPrinter(type):
  76. def __new__(mcs, name, bases, dct):
  77. annotation_map = dct.get('ANNOTATIONS')
  78. if annotation_map:
  79. for key, v in annotation_map.iteritems():
  80. key = key.lower()
  81. dct[key] = mcs.make_printer_fn(key, v)
  82. return type.__new__(mcs, name, bases, dct)
  83. @staticmethod
  84. def make_printer_fn(name, n_args):
  85. """Generates a method which emits an annotation to the log stream."""
  86. upname = name.upper()
  87. if upname in OLD_STYLE_ANNOTATIONS:
  88. assert n_args >= 1
  89. fmt = '@@@%s %%s%s@@@' % (upname, '@%s' * (n_args - 1))
  90. else:
  91. fmt = '@@@%s%s@@@' % (upname, '@%s' * n_args)
  92. inner_args = n_args + 1 # self counts
  93. infix = '1 argument' if inner_args == 1 else ('%d arguments' % inner_args)
  94. err = '%s() takes %s (%%d given)' % (name, infix)
  95. def printer(self, *args):
  96. if len(args) != n_args:
  97. raise TypeError(err % (len(args) + 1))
  98. self.emit(fmt % args)
  99. printer.__name__ = name
  100. printer.__doc__ = """Emits an annotation for %s.""" % name.upper()
  101. return printer
  102. class AnnotationPrinter(object):
  103. """A derivable class which will inject annotation-printing methods into the
  104. subclass.
  105. A subclass should define a class variable ANNOTATIONS equal to a
  106. dictionary of the form { '<ANNOTATION_NAME>': <# args> }. This class will
  107. then inject methods whose names are the undercased version of your
  108. annotation names, and which take the number of arguments specified in the
  109. dictionary.
  110. Example:
  111. >>> my_annotations = { 'STEP_LOG_LINE': 2 }
  112. >>> class MyObj(AnnotationPrinter):
  113. ... ANNOTATIONS = my_annotations
  114. ...
  115. >>> o = MyObj()
  116. >>> o.step_log_line('logname', 'here is a line to put in the log')
  117. @@@STEP_LOG_LINE@logname@here is a line to put in the log@@@
  118. >>> o.step_log_line()
  119. Traceback (most recent call last):
  120. File "<stdin>", line 1, in <module>
  121. TypeError: step_log_line() takes exactly 3 arguments (1 given)
  122. >>> o.setp_log_line.__doc__
  123. "Emits an annotation for STEP_LOG_LINE."
  124. >>>
  125. """
  126. __metaclass__ = MetaAnnotationPrinter
  127. def __init__(self, stream, flush_before):
  128. self.stream = stream
  129. self.flush_before = flush_before
  130. def emit(self, line):
  131. emit(line, self.stream, self.flush_before)
  132. class StepCommands(AnnotationPrinter):
  133. """Class holding step commands. Intended to be subclassed."""
  134. ANNOTATIONS = STEP_ANNOTATIONS
  135. def __init__(self, stream, flush_before):
  136. super(StepCommands, self).__init__(stream, flush_before)
  137. self.emitted_logs = set()
  138. def write_log_lines(self, logname, lines, perf=None):
  139. if logname in self.emitted_logs:
  140. raise ValueError('Log %s has been emitted multiple times.' % logname)
  141. self.emitted_logs.add(logname)
  142. logname = logname.replace('/', '&#x2f;')
  143. for line in lines:
  144. for actual_line in line.split('\n'):
  145. self.step_log_line(logname, actual_line)
  146. if perf:
  147. self.step_log_end_perf(logname, perf)
  148. else:
  149. self.step_log_end(logname)
  150. class StepControlCommands(AnnotationPrinter):
  151. """Subclass holding step control commands. Intended to be subclassed.
  152. This is subclassed out so callers in StructuredAnnotationStep can't call
  153. step_started() or step_closed().
  154. """
  155. ANNOTATIONS = CONTROL_ANNOTATIONS
  156. class StructuredAnnotationStep(StepCommands, StepControlCommands):
  157. """Helper class to provide context for a step."""
  158. def __init__(self, annotation_stream, *args, **kwargs):
  159. self.annotation_stream = annotation_stream
  160. super(StructuredAnnotationStep, self).__init__(*args, **kwargs)
  161. self.control = StepControlCommands(self.stream, self.flush_before)
  162. self.emitted_logs = set()
  163. def __enter__(self):
  164. return self.step_started()
  165. def step_started(self):
  166. self.control.step_started()
  167. return self
  168. def __exit__(self, exc_type, exc_value, tb):
  169. self.annotation_stream.step_cursor(self.annotation_stream.current_step)
  170. #TODO(martinis) combine this and step_ended
  171. if exc_type:
  172. self.step_exception_occured(exc_type, exc_value, tb)
  173. self.control.step_closed()
  174. self.annotation_stream.current_step = ''
  175. return not exc_type
  176. def step_exception_occured(self, exc_type, exc_value, tb):
  177. trace = traceback.format_exception(exc_type, exc_value, tb)
  178. trace_lines = ''.join(trace).split('\n')
  179. self.write_log_lines('exception', filter(None, trace_lines))
  180. self.step_exception()
  181. def step_ended(self):
  182. self.annotation_stream.step_cursor(self.annotation_stream.current_step)
  183. self.control.step_closed()
  184. self.annotation_stream.current_step = ''
  185. return True
  186. class StructuredAnnotationStream(AnnotationPrinter):
  187. """Provides an interface to handle an annotated build.
  188. StructuredAnnotationStream handles most of the step setup and closure calls
  189. for you. All you have to do is execute your code within the steps and set any
  190. failures or warnings that come up. You may optionally provide a list of steps
  191. to seed before execution.
  192. Usage:
  193. stream = StructuredAnnotationStream()
  194. with stream.step('compile') as s:
  195. # do something
  196. if error:
  197. s.step_failure()
  198. with stream.step('test') as s:
  199. # do something
  200. if warnings:
  201. s.step_warnings()
  202. """
  203. ANNOTATIONS = STREAM_ANNOTATIONS
  204. def __init__(self, stream=sys.stdout,
  205. flush_before=sys.stderr,
  206. seed_steps=None): # pylint: disable=W0613
  207. super(StructuredAnnotationStream, self).__init__(stream=stream,
  208. flush_before=flush_before)
  209. self.current_step = ''
  210. def step(self, name):
  211. """Provide a context with which to execute a step."""
  212. if self.current_step:
  213. raise Exception('Can\'t start step %s while in step %s.' % (
  214. name, self.current_step))
  215. self.seed_step(name)
  216. self.step_cursor(name)
  217. self.current_step = name
  218. return StructuredAnnotationStep(self, stream=self.stream,
  219. flush_before=self.flush_before)
  220. def MatchAnnotation(line, callback_implementor):
  221. """Call back into |callback_implementor| if line contains an annotation.
  222. Args:
  223. line (str) - The line to analyze
  224. callback_implementor (object) - An object which contains methods
  225. corresponding to all of the annotations in the |ALL_ANNOTATIONS|
  226. dictionary. For example, it should contain a method STEP_SUMMARY_TEXT
  227. taking a single argument.
  228. Parsing method:
  229. * if line doesn't match /^@@@.*@@@$/, return without calling back
  230. * Look for the first '@' or ' '
  231. """
  232. if not (line.startswith('@@@') and line.endswith('@@@') and len(line) > 6):
  233. return
  234. line = line[3:-3]
  235. # look until the first @ or ' '
  236. idx = min((x for x in (line.find('@'), line.find(' '), len(line)) if x > 0))
  237. cmd_text = line[:idx]
  238. cmd = DEPRECATED_ALIASES.get(cmd_text, cmd_text)
  239. field_count = ALL_ANNOTATIONS.get(cmd)
  240. if field_count is None:
  241. raise Exception('Unrecognized annotator command "%s"' % cmd_text)
  242. if field_count:
  243. if idx == len(line):
  244. raise Exception('Annotator command "%s" expects %d args, got 0.'
  245. % (cmd_text, field_count))
  246. line = line[idx+1:]
  247. args = line.split('@', field_count-1)
  248. if len(args) != field_count:
  249. raise Exception('Annotator command "%s" expects %d args, got %d.'
  250. % (cmd_text, field_count, len(args)))
  251. else:
  252. line = line[len(cmd_text):]
  253. if line:
  254. raise Exception('Annotator command "%s" expects no args, got cruft "%s".'
  255. % (cmd_text, line))
  256. args = []
  257. fn = getattr(callback_implementor, cmd, None)
  258. if fn is None:
  259. raise Exception('"%s" does not implement "%s"'
  260. % (callback_implementor, cmd))
  261. fn(*args)
  262. def print_step(step, env, stream):
  263. """Prints the step command and relevant metadata.
  264. Intended to be similar to the information that Buildbot prints at the
  265. beginning of each non-annotator step.
  266. """
  267. step_info_lines = []
  268. step_info_lines.append(' '.join(step['cmd']))
  269. step_info_lines.append('in dir %s:' % (step['cwd'] or os.getcwd()))
  270. for key, value in sorted(step.items()):
  271. if value is not None:
  272. if callable(value):
  273. # This prevents functions from showing up as:
  274. # '<function foo at 0x7f523ec7a410>'
  275. # which is tricky to test.
  276. value = value.__name__+'(...)'
  277. step_info_lines.append(' %s: %s' % (key, value))
  278. step_info_lines.append('full environment:')
  279. for key, value in sorted(env.items()):
  280. step_info_lines.append(' %s: %s' % (key, value))
  281. step_info_lines.append('')
  282. stream.emit('\n'.join(step_info_lines))