GoogleGit

blob: 95e01b516b1a95e8b206681ec0a694bb5083242b [file] [log] [blame]
  1. # Copyright (c) 2015 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Contains the parsing system of the Chromium Buildbot Annotator."""
  5. import os
  6. import sys
  7. import traceback
  8. # These are maps of annotation key -> number of expected arguments.
  9. STEP_ANNOTATIONS = {
  10. 'SET_BUILD_PROPERTY': 2,
  11. 'STEP_CLEAR': 0,
  12. 'STEP_EXCEPTION': 0,
  13. 'STEP_FAILURE': 0,
  14. 'STEP_LINK': 2,
  15. 'STEP_LOG_END': 1,
  16. 'STEP_LOG_END_PERF': 2,
  17. 'STEP_LOG_LINE': 2,
  18. 'STEP_SUMMARY_CLEAR': 0,
  19. 'STEP_SUMMARY_TEXT': 1,
  20. 'STEP_TEXT': 1,
  21. 'STEP_TRIGGER': 1,
  22. 'STEP_WARNINGS': 0,
  23. 'STEP_NEST_LEVEL': 1,
  24. }
  25. CONTROL_ANNOTATIONS = {
  26. 'STEP_CLOSED': 0,
  27. 'STEP_STARTED': 0,
  28. }
  29. STREAM_ANNOTATIONS = {
  30. 'HALT_ON_FAILURE': 0,
  31. 'HONOR_ZERO_RETURN_CODE': 0,
  32. 'SEED_STEP': 1,
  33. 'SEED_STEP_TEXT': 2,
  34. 'STEP_CURSOR': 1,
  35. }
  36. DEPRECATED_ANNOTATIONS = {
  37. 'BUILD_STEP': 1,
  38. }
  39. ALL_ANNOTATIONS = {}
  40. ALL_ANNOTATIONS.update(STEP_ANNOTATIONS)
  41. ALL_ANNOTATIONS.update(CONTROL_ANNOTATIONS)
  42. ALL_ANNOTATIONS.update(STREAM_ANNOTATIONS)
  43. ALL_ANNOTATIONS.update(DEPRECATED_ANNOTATIONS)
  44. # This is a mapping of old_annotation_name -> new_annotation_name.
  45. # Theoretically all annotator scripts should use the new names, but it's hard
  46. # to tell due to the decentralized nature of the annotator.
  47. DEPRECATED_ALIASES = {
  48. 'BUILD_FAILED': 'STEP_FAILURE',
  49. 'BUILD_WARNINGS': 'STEP_WARNINGS',
  50. 'BUILD_EXCEPTION': 'STEP_EXCEPTION',
  51. 'link': 'STEP_LINK',
  52. }
  53. # A couple of the annotations have the format:
  54. # @@@THING arg@@@
  55. # for reasons no one knows. We only need this case until all masters have been
  56. # restarted to pick up the new master-side parsing code.
  57. OLD_STYLE_ANNOTATIONS = set((
  58. 'SEED_STEP',
  59. 'STEP_CURSOR',
  60. ))
  61. def emit(line, stream, flush_before=None):
  62. if flush_before:
  63. flush_before.flush()
  64. print >> stream
  65. # WinDOS can only handle 64kb of output to the console at a time, per process.
  66. if sys.platform.startswith('win'):
  67. lim = 2**15
  68. while line:
  69. to_print, line = line[:lim], line[lim:]
  70. stream.write(to_print)
  71. stream.write('\n')
  72. else:
  73. print >> stream, line
  74. stream.flush()
  75. class MetaAnnotationPrinter(type):
  76. def __new__(mcs, name, bases, dct):
  77. annotation_map = dct.get('ANNOTATIONS')
  78. if annotation_map:
  79. for key, v in annotation_map.iteritems():
  80. key = key.lower()
  81. dct[key] = mcs.make_printer_fn(key, v)
  82. return type.__new__(mcs, name, bases, dct)
  83. @staticmethod
  84. def make_printer_fn(name, n_args):
  85. """Generates a method which emits an annotation to the log stream."""
  86. upname = name.upper()
  87. if upname in OLD_STYLE_ANNOTATIONS:
  88. assert n_args >= 1
  89. fmt = '@@@%s %%s%s@@@' % (upname, '@%s' * (n_args - 1))
  90. else:
  91. fmt = '@@@%s%s@@@' % (upname, '@%s' * n_args)
  92. inner_args = n_args + 1 # self counts
  93. infix = '1 argument' if inner_args == 1 else ('%d arguments' % inner_args)
  94. err = '%s() takes %s (%%d given)' % (name, infix)
  95. def printer(self, *args):
  96. if len(args) != n_args:
  97. raise TypeError(err % (len(args) + 1))
  98. self.emit(fmt % args)
  99. printer.__name__ = name
  100. printer.__doc__ = """Emits an annotation for %s.""" % name.upper()
  101. return printer
  102. class AnnotationPrinter(object):
  103. """A derivable class which will inject annotation-printing methods into the
  104. subclass.
  105. A subclass should define a class variable ANNOTATIONS equal to a
  106. dictionary of the form { '<ANNOTATION_NAME>': <# args> }. This class will
  107. then inject methods whose names are the undercased version of your
  108. annotation names, and which take the number of arguments specified in the
  109. dictionary.
  110. Example:
  111. >>> my_annotations = { 'STEP_LOG_LINE': 2 }
  112. >>> class MyObj(AnnotationPrinter):
  113. ... ANNOTATIONS = my_annotations
  114. ...
  115. >>> o = MyObj()
  116. >>> o.step_log_line('logname', 'here is a line to put in the log')
  117. @@@STEP_LOG_LINE@logname@here is a line to put in the log@@@
  118. >>> o.step_log_line()
  119. Traceback (most recent call last):
  120. File "<stdin>", line 1, in <module>
  121. TypeError: step_log_line() takes exactly 3 arguments (1 given)
  122. >>> o.setp_log_line.__doc__
  123. "Emits an annotation for STEP_LOG_LINE."
  124. >>>
  125. """
  126. __metaclass__ = MetaAnnotationPrinter
  127. def __init__(self, stream, flush_before):
  128. self.stream = stream
  129. self.flush_before = flush_before
  130. def emit(self, line):
  131. emit(line, self.stream, self.flush_before)
  132. class StepCommands(AnnotationPrinter):
  133. """Class holding step commands. Intended to be subclassed."""
  134. ANNOTATIONS = STEP_ANNOTATIONS
  135. def __init__(self, stream, flush_before):
  136. super(StepCommands, self).__init__(stream, flush_before)
  137. self.emitted_logs = set()
  138. def write_log_lines(self, logname, lines, perf=None):
  139. if logname in self.emitted_logs:
  140. raise ValueError('Log %s has been emitted multiple times.' % logname)
  141. self.emitted_logs.add(logname)
  142. logname = logname.replace('/', '&#x2f;')
  143. for line in lines:
  144. self.step_log_line(logname, line)
  145. if perf:
  146. self.step_log_end_perf(logname, perf)
  147. else:
  148. self.step_log_end(logname)
  149. class StepControlCommands(AnnotationPrinter):
  150. """Subclass holding step control commands. Intended to be subclassed.
  151. This is subclassed out so callers in StructuredAnnotationStep can't call
  152. step_started() or step_closed().
  153. """
  154. ANNOTATIONS = CONTROL_ANNOTATIONS
  155. class StructuredAnnotationStep(StepCommands, StepControlCommands):
  156. """Helper class to provide context for a step."""
  157. def __init__(self, annotation_stream, *args, **kwargs):
  158. self.annotation_stream = annotation_stream
  159. super(StructuredAnnotationStep, self).__init__(*args, **kwargs)
  160. self.control = StepControlCommands(self.stream, self.flush_before)
  161. self.emitted_logs = set()
  162. def __enter__(self):
  163. return self.step_started()
  164. def step_started(self):
  165. self.control.step_started()
  166. return self
  167. def __exit__(self, exc_type, exc_value, tb):
  168. self.annotation_stream.step_cursor(self.annotation_stream.current_step)
  169. #TODO(martinis) combine this and step_ended
  170. if exc_type:
  171. self.step_exception_occured(exc_type, exc_value, tb)
  172. self.control.step_closed()
  173. self.annotation_stream.current_step = ''
  174. return not exc_type
  175. def step_exception_occured(self, exc_type, exc_value, tb):
  176. trace = traceback.format_exception(exc_type, exc_value, tb)
  177. trace_lines = ''.join(trace).split('\n')
  178. self.write_log_lines('exception', filter(None, trace_lines))
  179. self.step_exception()
  180. def step_ended(self):
  181. self.annotation_stream.step_cursor(self.annotation_stream.current_step)
  182. self.control.step_closed()
  183. self.annotation_stream.current_step = ''
  184. return True
  185. class StructuredAnnotationStream(AnnotationPrinter):
  186. """Provides an interface to handle an annotated build.
  187. StructuredAnnotationStream handles most of the step setup and closure calls
  188. for you. All you have to do is execute your code within the steps and set any
  189. failures or warnings that come up. You may optionally provide a list of steps
  190. to seed before execution.
  191. Usage:
  192. stream = StructuredAnnotationStream()
  193. with stream.step('compile') as s:
  194. # do something
  195. if error:
  196. s.step_failure()
  197. with stream.step('test') as s:
  198. # do something
  199. if warnings:
  200. s.step_warnings()
  201. """
  202. ANNOTATIONS = STREAM_ANNOTATIONS
  203. def __init__(self, stream=sys.stdout,
  204. flush_before=sys.stderr,
  205. seed_steps=None): # pylint: disable=W0613
  206. super(StructuredAnnotationStream, self).__init__(stream=stream,
  207. flush_before=flush_before)
  208. self.current_step = ''
  209. def step(self, name):
  210. """Provide a context with which to execute a step."""
  211. if self.current_step:
  212. raise Exception('Can\'t start step %s while in step %s.' % (
  213. name, self.current_step))
  214. self.seed_step(name)
  215. self.step_cursor(name)
  216. self.current_step = name
  217. return StructuredAnnotationStep(self, stream=self.stream,
  218. flush_before=self.flush_before)
  219. def MatchAnnotation(line, callback_implementor):
  220. """Call back into |callback_implementor| if line contains an annotation.
  221. Args:
  222. line (str) - The line to analyze
  223. callback_implementor (object) - An object which contains methods
  224. corresponding to all of the annotations in the |ALL_ANNOTATIONS|
  225. dictionary. For example, it should contain a method STEP_SUMMARY_TEXT
  226. taking a single argument.
  227. Parsing method:
  228. * if line doesn't match /^@@@.*@@@$/, return without calling back
  229. * Look for the first '@' or ' '
  230. """
  231. if not (line.startswith('@@@') and line.endswith('@@@') and len(line) > 6):
  232. return
  233. line = line[3:-3]
  234. # look until the first @ or ' '
  235. idx = min((x for x in (line.find('@'), line.find(' '), len(line)) if x > 0))
  236. cmd_text = line[:idx]
  237. cmd = DEPRECATED_ALIASES.get(cmd_text, cmd_text)
  238. field_count = ALL_ANNOTATIONS.get(cmd)
  239. if field_count is None:
  240. raise Exception('Unrecognized annotator command "%s"' % cmd_text)
  241. if field_count:
  242. if idx == len(line):
  243. raise Exception('Annotator command "%s" expects %d args, got 0.'
  244. % (cmd_text, field_count))
  245. line = line[idx+1:]
  246. args = line.split('@', field_count-1)
  247. if len(args) != field_count:
  248. raise Exception('Annotator command "%s" expects %d args, got %d.'
  249. % (cmd_text, field_count, len(args)))
  250. else:
  251. line = line[len(cmd_text):]
  252. if line:
  253. raise Exception('Annotator command "%s" expects no args, got cruft "%s".'
  254. % (cmd_text, line))
  255. args = []
  256. fn = getattr(callback_implementor, cmd, None)
  257. if fn is None:
  258. raise Exception('"%s" does not implement "%s"'
  259. % (callback_implementor, cmd))
  260. fn(*args)
  261. def print_step(step, env, stream):
  262. """Prints the step command and relevant metadata.
  263. Intended to be similar to the information that Buildbot prints at the
  264. beginning of each non-annotator step.
  265. """
  266. step_info_lines = []
  267. step_info_lines.append(' '.join(step['cmd']))
  268. step_info_lines.append('in dir %s:' % (step['cwd'] or os.getcwd()))
  269. for key, value in sorted(step.items()):
  270. if value is not None:
  271. if callable(value):
  272. # This prevents functions from showing up as:
  273. # '<function foo at 0x7f523ec7a410>'
  274. # which is tricky to test.
  275. value = value.__name__+'(...)'
  276. step_info_lines.append(' %s: %s' % (key, value))
  277. step_info_lines.append('full environment:')
  278. for key, value in sorted(env.items()):
  279. step_info_lines.append(' %s: %s' % (key, value))
  280. step_info_lines.append('')
  281. stream.emit('\n'.join(step_info_lines))