chrome/browser/resources/hotword/nacl_manager.js

// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

cr.define('hotword', function() {
'use strict';

/**
 * Class used to manage the state of the NaCl recognizer plugin. Handles all
 * control of the NaCl plugin, including creation, start, stop, trigger, and
 * shutdown.
 *
 * @param {boolean} loggingEnabled Whether audio logging is enabled.
 * @param {boolean} hotwordStream Whether the audio input stream is from a
 *     hotword stream.
 * @constructor
 * @extends {cr.EventTarget}
 */
function NaClManager(loggingEnabled, hotwordStream) {
  /**
   * Current state of this manager.
   * @private {hotword.NaClManager.ManagerState_}
   */
  this.recognizerState_ = ManagerState_.UNINITIALIZED;

  /**
   * The window.timeout ID associated with a pending message.
   * @private {?number}
   */
  this.naclTimeoutId_ = null;

  /**
   * The expected message that will cancel the current timeout.
   * @private {?string}
   */
  this.expectingMessage_ = null;

  /**
   * Whether the plugin will be started as soon as it stops.
   * @private {boolean}
   */
  this.restartOnStop_ = false;

  /**
   * NaCl plugin element on extension background page.
   * @private {?HTMLEmbedElement}
   */
  this.plugin_ = null;

  /**
   * URL containing hotword-model data file.
   * @private {string}
   */
  this.modelUrl_ = '';

  /**
   * Media stream containing an audio input track.
   * @private {?MediaStream}
   */
  this.stream_ = null;

  /**
   * The mode to start the recognizer in.
   * @private {?chrome.hotwordPrivate.RecognizerStartMode}
   */
  this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;

  /**
   * Whether audio logging is enabled.
   * @private {boolean}
   */
  this.loggingEnabled_ = loggingEnabled;

  /**
   * Whether the audio input stream is from a hotword stream.
   * @private {boolean}
   */
  this.hotwordStream_ = hotwordStream;

  /**
   * Audio log of X seconds before hotword triggered.
   * @private {?Object}
   */
  this.preambleLog_ = null;
};

/**
 * States this manager can be in. Since messages to/from the plugin are
 * asynchronous (and potentially queued), it's not possible to know what state
 * the plugin is in. However, track a state machine for NaClManager based on
 * what messages are sent/received.
 * @enum {number}
 * @private
 */
NaClManager.ManagerState_ = {
  UNINITIALIZED: 0,
  LOADING: 1,
  STOPPING: 2,
  STOPPED: 3,
  STARTING: 4,
  RUNNING: 5,
  ERROR: 6,
  SHUTDOWN: 7,
};
var ManagerState_ = NaClManager.ManagerState_;
var Error_ = hotword.constants.Error;
var UmaNaClMessageTimeout_ = hotword.constants.UmaNaClMessageTimeout;
var UmaNaClPluginLoadResult_ = hotword.constants.UmaNaClPluginLoadResult;

NaClManager.prototype.__proto__ = cr.EventTarget.prototype;

/**
 * Called when an error occurs. Dispatches an event.
 * @param {!hotword.constants.Error} error
 * @private
 */
NaClManager.prototype.handleError_ = function(error) {
  var event = new Event(hotword.constants.Event.ERROR);
  event.data = error;
  this.dispatchEvent(event);
};

/**
 * Record the result of loading the NaCl plugin to UMA.
 * @param {!hotword.constants.UmaNaClPluginLoadResult} error
 * @private
 */
NaClManager.prototype.logPluginLoadResult_ = function(error) {
  hotword.metrics.recordEnum(
      hotword.constants.UmaMetrics.NACL_PLUGIN_LOAD_RESULT,
      error,
      UmaNaClPluginLoadResult_.MAX);
};

/**
 * Set a timeout. Only allow one timeout to exist at any given time.
 * @param {!function()} func
 * @param {number} timeout
 * @private
 */
NaClManager.prototype.setTimeout_ = function(func, timeout) {
  assert(!this.naclTimeoutId_, 'Timeout already exists');
  this.naclTimeoutId_ = window.setTimeout(
      function() {
        this.naclTimeoutId_ = null;
        func();
      }.bind(this), timeout);
};

/**
 * Clears the current timeout.
 * @private
 */
NaClManager.prototype.clearTimeout_ = function() {
  window.clearTimeout(this.naclTimeoutId_);
  this.naclTimeoutId_ = null;
};

/**
 * Starts a stopped or stopping hotword recognizer (NaCl plugin).
 * @param {hotword.constants.RecognizerStartMode} mode The mode to start the
 *     recognizer in.
 */
NaClManager.prototype.startRecognizer = function(mode) {
  this.startMode_ = mode;
  if (this.recognizerState_ == ManagerState_.STOPPED) {
    this.preambleLog_ = null;
    this.recognizerState_ = ManagerState_.STARTING;
    if (mode == hotword.constants.RecognizerStartMode.NEW_MODEL) {
      hotword.debug('Starting Recognizer in START training mode');
      this.sendDataToPlugin_(hotword.constants.NaClPlugin.BEGIN_SPEAKER_MODEL);
    } else if (mode == hotword.constants.RecognizerStartMode.ADAPT_MODEL) {
      hotword.debug('Starting Recognizer in ADAPT training mode');
      this.sendDataToPlugin_(hotword.constants.NaClPlugin.ADAPT_SPEAKER_MODEL);
    } else {
      hotword.debug('Starting Recognizer in NORMAL mode');
      this.sendDataToPlugin_(hotword.constants.NaClPlugin.RESTART);
    }
    // Normally, there would be a waitForMessage_(READY_FOR_AUDIO) here.
    // However, this message is sent the first time audio data is read and in
    // some cases (ie. using the hotword stream), this won't happen until a
    // potential hotword trigger is seen. Having a waitForMessage_() would time
    // out in this case, so just leave it out. This ends up sacrificing a bit of
    // error detection in the non-hotword-stream case, but I think we can live
    // with that.
  } else if (this.recognizerState_ == ManagerState_.STOPPING) {
    // Wait until the plugin is stopped before trying to start it.
    this.restartOnStop_ = true;
  } else {
    throw 'Attempting to start NaCl recogniser not in STOPPED or STOPPING ' +
        'state';
  }
};

/**
 * Stops the hotword recognizer.
 */
NaClManager.prototype.stopRecognizer = function() {
  if (this.recognizerState_ == ManagerState_.STARTING) {
    // If the recognizer is stopped before it finishes starting, it causes an
    // assertion to be raised in waitForMessage_() since we're waiting for the
    // READY_FOR_AUDIO message. Clear the current timeout and expecting message
    // since we no longer expect it and may never receive it.
    this.clearTimeout_();
    this.expectingMessage_ = null;
  }
  this.sendDataToPlugin_(hotword.constants.NaClPlugin.STOP);
  this.recognizerState_ = ManagerState_.STOPPING;
  this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
                       hotword.constants.NaClPlugin.STOPPED);
};

/**
 * Saves the speaker model.
 */
NaClManager.prototype.finalizeSpeakerModel = function() {
  if (this.recognizerState_ == ManagerState_.UNINITIALIZED ||
      this.recognizerState_ == ManagerState_.ERROR ||
      this.recognizerState_ == ManagerState_.SHUTDOWN ||
      this.recognizerState_ == ManagerState_.LOADING) {
    return;
  }
  this.sendDataToPlugin_(hotword.constants.NaClPlugin.FINISH_SPEAKER_MODEL);
};

/**
 * Checks whether the file at the given path exists.
 * @param {!string} path Path to a file. Can be any valid URL.
 * @return {boolean} True if the patch exists.
 * @private
 */
NaClManager.prototype.fileExists_ = function(path) {
  var xhr = new XMLHttpRequest();
  xhr.open('HEAD', path, false);
  try {
    xhr.send();
  } catch (err) {
    return false;
  }
  if (xhr.readyState != xhr.DONE || xhr.status != 200) {
    return false;
  }
  return true;
};

/**
 * Creates and returns a list of possible languages to check for hotword
 * support.
 * @return {!Array<string>} Array of languages.
 * @private
 */
NaClManager.prototype.getPossibleLanguages_ = function() {
  // Create array used to search first for language-country, if not found then
  // search for language, if not found then no language (empty string).
  // For example, search for 'en-us', then 'en', then ''.
  var langs = new Array();
  if (hotword.constants.UI_LANGUAGE) {
    // Chrome webstore doesn't support uppercase path: crbug.com/353407
    var language = hotword.constants.UI_LANGUAGE.toLowerCase();
    langs.push(language);  // Example: 'en-us'.
    // Remove country to add just the language to array.
    var hyphen = language.lastIndexOf('-');
    if (hyphen >= 0) {
      langs.push(language.substr(0, hyphen));  // Example: 'en'.
    }
  }
  langs.push('');
  return langs;
};

/**
 * Creates a NaCl plugin object and attaches it to the page.
 * @param {!string} src Location of the plugin.
 * @return {!HTMLEmbedElement} NaCl plugin DOM object.
 * @private
 */
NaClManager.prototype.createPlugin_ = function(src) {
  var plugin = /** @type {HTMLEmbedElement} */(document.createElement('embed'));
  plugin.src = src;
  plugin.type = 'application/x-nacl';
  document.body.appendChild(plugin);
  return plugin;
};

/**
 * Initializes the NaCl manager.
 * @param {!string} naclArch Either 'arm', 'x86-32' or 'x86-64'.
 * @param {!MediaStream} stream A stream containing an audio source track.
 * @return {boolean} True if the successful.
 */
NaClManager.prototype.initialize = function(naclArch, stream) {
  assert(this.recognizerState_ == ManagerState_.UNINITIALIZED,
         'Recognizer not in uninitialized state. State: ' +
         this.recognizerState_);
  assert(this.plugin_ == null);
  var langs = this.getPossibleLanguages_();
  var i, j;
  // For country-lang variations. For example, when combined with path it will
  // attempt to find: '/x86-32_en-gb/', else '/x86-32_en/', else '/x86-32_/'.
  for (i = 0; i < langs.length; i++) {
    var folder = hotword.constants.SHARED_MODULE_ROOT + '/_platform_specific/' +
        naclArch + '_' + langs[i] + '/';
    var dataSrc = folder + hotword.constants.File.RECOGNIZER_CONFIG;
    var pluginSrc = hotword.constants.SHARED_MODULE_ROOT + '/hotword_' +
        langs[i] + '.nmf';
    var dataExists = this.fileExists_(dataSrc) && this.fileExists_(pluginSrc);
    if (!dataExists) {
      continue;
    }

    var plugin = this.createPlugin_(pluginSrc);
    if (!plugin || !plugin.postMessage) {
      document.body.removeChild(plugin);
      this.recognizerState_ = ManagerState_.ERROR;
      return false;
    }
    this.plugin_ = plugin;
    this.modelUrl_ = chrome.extension.getURL(dataSrc);
    this.stream_ = stream;
    this.recognizerState_ = ManagerState_.LOADING;

    plugin.addEventListener('message',
                            this.handlePluginMessage_.bind(this),
                            false);

    plugin.addEventListener('crash',
                            function() {
                              this.handleError_(Error_.NACL_CRASH);
                              this.logPluginLoadResult_(
                                  UmaNaClPluginLoadResult_.CRASH);
                            }.bind(this),
                            false);
    return true;
  }
  this.recognizerState_ = ManagerState_.ERROR;
  this.logPluginLoadResult_(UmaNaClPluginLoadResult_.NO_MODULE_FOUND);
  return false;
};

/**
 * Shuts down the NaCl plugin and frees all resources.
 */
NaClManager.prototype.shutdown = function() {
  if (this.plugin_ != null) {
    document.body.removeChild(this.plugin_);
    this.plugin_ = null;
  }
  this.clearTimeout_();
  this.recognizerState_ = ManagerState_.SHUTDOWN;
  if (this.stream_)
    this.stream_.getAudioTracks()[0].stop();
  this.stream_ = null;
};

/**
 * Sends data to the NaCl plugin.
 * @param {!string|!MediaStreamTrack} data Command to be sent to NaCl plugin.
 * @private
 */
NaClManager.prototype.sendDataToPlugin_ = function(data) {
  assert(this.recognizerState_ != ManagerState_.UNINITIALIZED,
         'Recognizer in uninitialized state');
  this.plugin_.postMessage(data);
};

/**
 * Waits, with a timeout, for a message to be received from the plugin. If the
 * message is not seen within the timeout, dispatch an 'error' event and go into
 * the ERROR state.
 * @param {number} timeout Timeout, in milliseconds, to wait for the message.
 * @param {!string} message Message to wait for.
 * @private
 */
NaClManager.prototype.waitForMessage_ = function(timeout, message) {
  assert(this.expectingMessage_ == null, 'Cannot wait for message: ' +
      message + ', already waiting for message ' + this.expectingMessage_);
  this.setTimeout_(
      function() {
        this.recognizerState_ = ManagerState_.ERROR;
        this.handleError_(Error_.TIMEOUT);
        switch (this.expectingMessage_) {
          case hotword.constants.NaClPlugin.REQUEST_MODEL:
            var metricValue = UmaNaClMessageTimeout_.REQUEST_MODEL;
            break;
          case hotword.constants.NaClPlugin.MODEL_LOADED:
            var metricValue = UmaNaClMessageTimeout_.MODEL_LOADED;
            break;
          case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
            var metricValue = UmaNaClMessageTimeout_.READY_FOR_AUDIO;
            break;
          case hotword.constants.NaClPlugin.STOPPED:
            var metricValue = UmaNaClMessageTimeout_.STOPPED;
            break;
          case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
            var metricValue = UmaNaClMessageTimeout_.HOTWORD_DETECTED;
            break;
          case hotword.constants.NaClPlugin.MS_CONFIGURED:
            var metricValue = UmaNaClMessageTimeout_.MS_CONFIGURED;
            break;
        }
        hotword.metrics.recordEnum(
            hotword.constants.UmaMetrics.NACL_MESSAGE_TIMEOUT,
            metricValue,
            UmaNaClMessageTimeout_.MAX);
      }.bind(this), timeout);
  this.expectingMessage_ = message;
};

/**
 * Called when a message is received from the plugin. If we're waiting for that
 * message, cancel the pending timeout.
 * @param {string} message Message received.
 * @private
 */
NaClManager.prototype.receivedMessage_ = function(message) {
  if (message == this.expectingMessage_) {
    this.clearTimeout_();
    this.expectingMessage_ = null;
  }
};

/**
 * Handle a REQUEST_MODEL message from the plugin.
 * The plugin sends this message immediately after starting.
 * @private
 */
NaClManager.prototype.handleRequestModel_ = function() {
  if (this.recognizerState_ != ManagerState_.LOADING) {
    return;
  }
  this.logPluginLoadResult_(UmaNaClPluginLoadResult_.SUCCESS);
  this.sendDataToPlugin_(
      hotword.constants.NaClPlugin.MODEL_PREFIX + this.modelUrl_);
  this.waitForMessage_(hotword.constants.TimeoutMs.LONG,
                       hotword.constants.NaClPlugin.MODEL_LOADED);

  // Configure logging in the plugin. This can be configured any time before
  // starting the recognizer, and now is as good a time as any.
  if (this.loggingEnabled_) {
    this.sendDataToPlugin_(
        hotword.constants.NaClPlugin.LOG + ':' +
        hotword.constants.AUDIO_LOG_SECONDS);
  }

  // If the audio stream is from a hotword stream, tell the plugin.
  if (this.hotwordStream_) {
    this.sendDataToPlugin_(
        hotword.constants.NaClPlugin.DSP + ':' +
        hotword.constants.HOTWORD_STREAM_TIMEOUT_SECONDS);
  }
};

/**
 * Handle a MODEL_LOADED message from the plugin.
 * The plugin sends this message after successfully loading the language model.
 * @private
 */
NaClManager.prototype.handleModelLoaded_ = function() {
  if (this.recognizerState_ != ManagerState_.LOADING) {
    return;
  }
  this.sendDataToPlugin_(this.stream_.getAudioTracks()[0]);
  // The plugin will send a MS_CONFIGURED, but don't set a timeout waiting for
  // it. MediaStreamAudioTrack::Configure() will remain pending until the first
  // audio buffer is received. When the audio source is a DSP for always-on
  // detection, no audio sample is sent until the DSP detects a potential
  // hotword trigger. Thus, Configure would remain pending indefinitely if we
  // were to wait here. See https://crbug.com/616203
};

/**
 * Handle a MS_CONFIGURED message from the plugin.
 * The plugin sends this message after successfully configuring the audio input
 * stream.
 * @private
 */
NaClManager.prototype.handleMsConfigured_ = function() {
  if (this.recognizerState_ != ManagerState_.LOADING) {
    return;
  }
  this.recognizerState_ = ManagerState_.STOPPED;
  this.dispatchEvent(new Event(hotword.constants.Event.READY));
};

/**
 * Handle a READY_FOR_AUDIO message from the plugin.
 * The plugin sends this message after the recognizer is started and
 * successfully receives and processes audio data.
 * @private
 */
NaClManager.prototype.handleReadyForAudio_ = function() {
  if (this.recognizerState_ != ManagerState_.STARTING) {
    return;
  }
  this.recognizerState_ = ManagerState_.RUNNING;
};

/**
 * Handle a HOTWORD_DETECTED message from the plugin.
 * The plugin sends this message after detecting the hotword.
 * @private
 */
NaClManager.prototype.handleHotwordDetected_ = function() {
  if (this.recognizerState_ != ManagerState_.RUNNING) {
    return;
  }
  // We'll receive a STOPPED message very soon.
  this.recognizerState_ = ManagerState_.STOPPING;
  this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
                       hotword.constants.NaClPlugin.STOPPED);
  var event = new Event(hotword.constants.Event.TRIGGER);
  event.log = this.preambleLog_;
  this.dispatchEvent(event);
};

/**
 * Handle a STOPPED message from the plugin.
 * This plugin sends this message after stopping the recognizer. This can happen
 * either in response to a stop request, or after the hotword is detected.
 * @private
 */
NaClManager.prototype.handleStopped_ = function() {
  this.recognizerState_ = ManagerState_.STOPPED;
  if (this.restartOnStop_) {
    this.restartOnStop_ = false;
    this.startRecognizer(this.startMode_);
  }
};

/**
 * Handle a TIMEOUT message from the plugin.
 * The plugin sends this message when it thinks the stream is from a DSP and
 * a hotword wasn't detected within a timeout period after arrival of the first
 * audio samples.
 * @private
 */
NaClManager.prototype.handleTimeout_ = function() {
  if (this.recognizerState_ != ManagerState_.RUNNING) {
    return;
  }
  this.recognizerState_ = ManagerState_.STOPPED;
  this.dispatchEvent(new Event(hotword.constants.Event.TIMEOUT));
};

/**
 * Handle a SPEAKER_MODEL_SAVED message from the plugin.
 * The plugin sends this message after writing the model to a file.
 * @private
 */
NaClManager.prototype.handleSpeakerModelSaved_ = function() {
  this.dispatchEvent(new Event(hotword.constants.Event.SPEAKER_MODEL_SAVED));
};

/**
 * Handles a message from the NaCl plugin.
 * @param {!Event} msg Message from NaCl plugin.
 * @private
 */
NaClManager.prototype.handlePluginMessage_ = function(msg) {
  if (msg['data']) {
    if (typeof(msg['data']) == 'object') {
      // Save the preamble for delivery to the trigger handler when the trigger
      // message arrives.
      this.preambleLog_ = msg['data'];
      return;
    }
    this.receivedMessage_(msg['data']);
    switch (msg['data']) {
      case hotword.constants.NaClPlugin.REQUEST_MODEL:
        this.handleRequestModel_();
        break;
      case hotword.constants.NaClPlugin.MODEL_LOADED:
        this.handleModelLoaded_();
        break;
      case hotword.constants.NaClPlugin.MS_CONFIGURED:
        this.handleMsConfigured_();
        break;
      case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
        this.handleReadyForAudio_();
        break;
      case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
        this.handleHotwordDetected_();
        break;
      case hotword.constants.NaClPlugin.STOPPED:
        this.handleStopped_();
        break;
      case hotword.constants.NaClPlugin.TIMEOUT:
        this.handleTimeout_();
        break;
      case hotword.constants.NaClPlugin.SPEAKER_MODEL_SAVED:
        this.handleSpeakerModelSaved_();
        break;
    }
  }
};

return {
  NaClManager: NaClManager
};

});