blob: 33cb89a01a01473038ac349e130feb43fccd0773 [file] [log] [blame]
// Copyright 2016 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/common/input/actions_parser.h"
#include <utility>
#include "base/format_macros.h"
#include "base/strings/string_split.h"
#include "base/strings/stringprintf.h"
#include "base/values.h"
#include "components/viz/common/frame_sinks/begin_frame_args.h"
#include "content/common/input/input_injector.mojom.h"
#include "content/common/input/synthetic_smooth_scroll_gesture_params.h"
#include "ui/events/types/scroll_types.h"
using Button = content::SyntheticPointerActionParams::Button;
using PointerActionType =
content::SyntheticPointerActionParams::PointerActionType;
namespace content {
namespace {
PointerActionType ToSyntheticPointerActionType(const std::string& action_type) {
if (action_type == "pointerDown")
return PointerActionType::PRESS;
if (action_type == "pointerMove")
return PointerActionType::MOVE;
if (action_type == "pointerUp")
return PointerActionType::RELEASE;
if (action_type == "pointerLeave")
return PointerActionType::LEAVE;
if (action_type == "pause")
return PointerActionType::IDLE;
return PointerActionType::NOT_INITIALIZED;
}
content::mojom::GestureSourceType ToSyntheticGestureSourceType(
const std::string& pointer_type) {
if (pointer_type == "touch")
return content::mojom::GestureSourceType::kTouchInput;
else if (pointer_type == "mouse")
return content::mojom::GestureSourceType::kMouseInput;
else if (pointer_type == "pen")
return content::mojom::GestureSourceType::kPenInput;
return content::mojom::GestureSourceType::kDefaultInput;
}
Button ToSyntheticMouseButton(int button) {
if (button == 0)
return Button::LEFT;
if (button == 1)
return Button::MIDDLE;
if (button == 2)
return Button::RIGHT;
if (button == 3)
return Button::BACK;
if (button == 4)
return Button::FORWARD;
NOTREACHED() << "Unexpected button";
return Button();
}
int ToKeyModifiers(const std::string& key) {
if (key == "Alt")
return blink::WebInputEvent::kAltKey;
if (key == "Control")
return blink::WebInputEvent::kControlKey;
if (key == "Meta")
return blink::WebInputEvent::kMetaKey;
if (key == "Shift")
return blink::WebInputEvent::kShiftKey;
if (key == "CapsLock")
return blink::WebInputEvent::kCapsLockOn;
if (key == "NumLock")
return blink::WebInputEvent::kNumLockOn;
if (key == "AltGraph")
return blink::WebInputEvent::kAltGrKey;
return 0;
}
} // namespace
ActionsParser::ActionsParser(base::Value action_sequence_list)
: action_sequence_list_(std::move(action_sequence_list)) {
// We have two different JSON formats from testdriver Action API or
// gpuBenchmarking.pointerActionSequence. Below we are deciding where the
// action sequence list comes from.
if (action_sequence_list_.is_list() &&
action_sequence_list_.GetList().size() > 0 &&
action_sequence_list_.GetList()[0].is_dict()) {
use_testdriver_api_ = ActionsDictionaryUsesTestDriverApi(
action_sequence_list_.GetList()[0].GetDict());
}
}
ActionsParser::~ActionsParser() {}
bool ActionsParser::Parse() {
if (!action_sequence_list_.is_list() ||
action_sequence_list_.GetList().size() == 0) {
error_message_ =
std::string("provided action sequence list is not a list or is empty");
return false;
}
for (const auto& action_sequence : action_sequence_list_.GetList()) {
if (!action_sequence.is_dict()) {
error_message_ =
std::string("Expected ActionSequence is not a dictionary");
return false;
}
if (use_testdriver_api_) {
if (!ParseTestDriverActionSequence(action_sequence.GetDict())) {
return false;
}
} else {
if (!ParseGpuBenchmarkingActionSequence(action_sequence.GetDict())) {
return false;
}
}
}
if (source_type_ == "wheel")
return true;
gesture_params_ = std::make_unique<SyntheticPointerActionListParams>();
SyntheticPointerActionListParams* pointer_actions =
static_cast<SyntheticPointerActionListParams*>(gesture_params_.get());
pointer_actions->gesture_source_type =
ToSyntheticGestureSourceType(pointer_type_);
// Group a list of actions from all pointers into a
// SyntheticPointerActionListParams object, which is a list of actions, which
// will be dispatched together.
for (size_t index = 0; index < longest_action_sequence_; ++index) {
SyntheticPointerActionListParams::ParamList param_list;
size_t longest_pause_frame = 0;
for (const auto& pointer_action_list : pointer_actions_lists_) {
if (index < pointer_action_list.size()) {
param_list.push_back(pointer_action_list[index]);
if (pointer_action_list[index].pointer_action_type() ==
PointerActionType::IDLE) {
size_t num_pause_frame = static_cast<size_t>(std::ceil(
pointer_action_list[index].duration().InMilliseconds() /
viz::BeginFrameArgs::DefaultInterval().InMilliseconds()));
longest_pause_frame = std::max(longest_pause_frame, num_pause_frame);
}
}
}
pointer_actions->PushPointerActionParamsList(param_list);
for (size_t pause_index = 1; pause_index < longest_pause_frame;
++pause_index) {
SyntheticPointerActionListParams::ParamList pause_param_list;
SyntheticPointerActionParams pause_action_param(PointerActionType::IDLE);
for (size_t i = 0; i < param_list.size(); ++i) {
pause_param_list.push_back(pause_action_param);
}
pointer_actions->PushPointerActionParamsList(pause_param_list);
}
}
return true;
}
bool ActionsParser::ActionsDictionaryUsesTestDriverApi(
const base::Value::Dict& action_sequence) {
// If the JSON format of each action_sequence has "type" element, it is from
// the new Action API, otherwise it is from
// gpuBenchmarking.pointerActionSequence API. We have to keep both formats
// for now, but later on once we switch to the new Action API in all tests,
// we will remove the old format.
if (action_sequence.contains("type")) {
return true;
}
return false;
}
bool ActionsParser::ParseGpuBenchmarkingActionSequence(
const base::Value::Dict& action_sequence) {
// The GpuBenchmarking format is implicitly for pointers only and for
// historic reasons, the "source" key refers to what TestDriver calls the
// pointer_type_.
source_type_ = "pointer";
if (use_testdriver_api_ !=
ActionsDictionaryUsesTestDriverApi(action_sequence)) {
error_message_ = std::string(
"all action sequences must be of the same gpuBenchmarking format");
return false;
}
const std::string* pointer_type = action_sequence.FindString("source");
if (!pointer_type) {
error_message_ = std::string("source type is not defined or not a string");
return false;
}
if (*pointer_type != "touch" && *pointer_type != "mouse" &&
*pointer_type != "pen") {
error_message_ = base::StringPrintf(
"source type %s is an unsupported input type", (*pointer_type).c_str());
return false;
}
if (pointer_type_.empty()) {
pointer_type_ = *pointer_type;
} else if (pointer_type_ != *pointer_type) {
error_message_ =
std::string("currently multiple input types are not not supported");
return false;
}
if (*pointer_type != "touch" && input_source_count_ > 0) {
error_message_ = std::string(
"for source type of mouse and pen, we only support one device "
"in one sequence");
return false;
}
const base::Value::List* actions = action_sequence.FindList("actions");
if (!actions) {
error_message_ = base::StringPrintf(
"action_sequence[%zu].actions is not defined or not a list",
action_index_);
return false;
} else if (actions->size() == 0) {
error_message_ = base::StringPrintf(
"action_sequence[%zu].actions is an empty list", action_index_);
return false;
}
if (!ParseActionItemList(*actions, "pointer"))
return false;
input_source_count_++;
return true;
}
bool ActionsParser::ParseTestDriverActionSequence(
const base::Value::Dict& action_sequence) {
if (use_testdriver_api_ !=
ActionsDictionaryUsesTestDriverApi(action_sequence)) {
error_message_ = std::string(
"all action sequences must be of the same TestDriver format");
return false;
}
const std::string* source_type = action_sequence.FindString("type");
if (!source_type) {
error_message_ =
std::string("input source type is not defined or not a string");
return false;
}
if (*source_type != "none") {
if (source_type_.empty()) {
source_type_ = *source_type;
} else if (source_type_ != *source_type) {
error_message_ = std::string(
"currently multiple input source types are not supported");
return false;
}
}
if (*source_type == "pointer") {
if (!ParsePointerParameters(action_sequence))
return false;
} else if (*source_type == "key") {
error_message_ =
std::string("we do not support action sequence type of key");
return false;
} else if (*source_type == "wheel") {
// do nothing
} else if (*source_type == "none") {
// do nothing
} else {
error_message_ = base::StringPrintf("the input source type %s is invalid",
(*source_type).c_str());
return false;
}
const base::Value::List* actions = action_sequence.FindList("actions");
if (!actions) {
error_message_ = base::StringPrintf(
"action_sequence[%zu].actions is not defined or not a list",
action_index_);
return false;
} else if (actions->size() == 0) {
error_message_ = base::StringPrintf(
"action_sequence[%zu].actions is an empty list", action_index_);
return false;
} else if (*source_type == "wheel" && actions->size() > 1) {
error_message_ = base::StringPrintf(
"action_sequence[%zu].actions should only have one action for the "
"wheel input source",
action_index_);
return false;
}
if (!ParseActionItemList(*actions, *source_type))
return false;
if (*source_type != "none")
input_source_count_++;
return true;
}
bool ActionsParser::ParsePointerParameters(
const base::Value::Dict& action_sequence) {
const base::Value* parameters = action_sequence.Find("parameters");
// The default pointer type is mouse.
std::string pointer_type = "mouse";
if (parameters) {
if (!parameters->is_dict()) {
error_message_ =
std::string("action sequence parameters is not a dictionary");
return false;
}
const std::string* pointer_type_value =
parameters->GetDict().FindString("pointerType");
if (!pointer_type_value) {
error_message_ = std::string(
"action sequence pointer type is not defined or not a string");
return false;
}
if (*pointer_type_value != "touch" && *pointer_type_value != "mouse" &&
*pointer_type_value != "pen") {
error_message_ = base::StringPrintf(
"action sequence pointer type %s is an unsupported input type",
(*pointer_type_value).c_str());
return false;
}
pointer_type = *pointer_type_value;
}
if (pointer_type_.empty()) {
pointer_type_ = pointer_type;
} else if (pointer_type_ != pointer_type) {
error_message_ = std::string(
"currently multiple action sequence pointer type are not "
"supported");
return false;
}
if (pointer_type != "touch" && input_source_count_ > 0) {
error_message_ = std::string(
"for input type of mouse and pen, we only support one device");
return false;
}
// TODO(lanwei): according to the Webdriver spec, "Let id be the result of
// getting the property id from action sequence.", we should move "id" from
// parameters dictionary to action sequence.
const std::string* pointer_name = action_sequence.FindString("id");
if (!pointer_name) {
error_message_ = std::string("pointer name is not defined or not a string");
return false;
}
if (pointer_name_set_.find(*pointer_name) != pointer_name_set_.end()) {
error_message_ = std::string("pointer name already exists");
return false;
}
pointer_name_set_.insert(*pointer_name);
return true;
}
bool ActionsParser::ParseActionItemList(const base::Value::List& actions,
std::string source_type) {
DCHECK(source_type == "none" || source_type == source_type_);
SyntheticPointerActionListParams::ParamList param_list;
for (const auto& action : actions) {
if (!action.is_dict()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions is not defined or not a dictionary",
action_index_);
return false;
} else if (!ParseAction(action.GetDict(), param_list, source_type)) {
return false;
}
}
if (param_list.size() > longest_action_sequence_)
longest_action_sequence_ = param_list.size();
pointer_actions_lists_.push_back(param_list);
return true;
}
bool ActionsParser::ParseAction(
const base::Value::Dict& action,
SyntheticPointerActionListParams::ParamList& param_list,
std::string source_type) {
std::string subtype;
if (use_testdriver_api_) {
const std::string* type_value = action.FindString("type");
if (!type_value) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.type is not defined or not a string",
action_index_);
return false;
}
subtype = *type_value;
} else {
const std::string* name_value = action.FindString("name");
if (!name_value) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.name is not defined or not a string",
action_index_);
return false;
}
subtype = *name_value;
}
if (source_type == "wheel") {
return ParseWheelAction(action, subtype);
} else if (source_type == "pointer") {
return ParsePointerAction(action, subtype, param_list);
} else if (source_type == "none") {
return ParseNullAction(action, subtype, param_list);
} else {
NOTREACHED();
}
return false;
}
bool ActionsParser::ParseWheelAction(const base::Value::Dict& action,
std::string subtype) {
if (subtype == "pause") {
error_message_ = base::StringPrintf(
"actions[%zu].actions.type of pause is not supported now",
action_index_);
return false;
} else if (subtype != "scroll") {
error_message_ = base::StringPrintf(
"actions[%zu].actions.type is not scroll or pause when source type is "
"wheel",
action_index_);
return false;
}
double position_x = 0;
double position_y = 0;
if (!GetPosition(action, position_x, position_y))
return false;
int delta_x = 0;
int delta_y = 0;
if (!GetScrollDelta(action, delta_x, delta_y))
return false;
gesture_params_ = std::make_unique<SyntheticSmoothScrollGestureParams>();
SyntheticSmoothScrollGestureParams* scroll =
static_cast<SyntheticSmoothScrollGestureParams*>(gesture_params_.get());
scroll->gesture_source_type = content::mojom::GestureSourceType::kMouseInput;
scroll->speed_in_pixels_s = 8000;
scroll->prevent_fling = true;
scroll->granularity = ui::ScrollGranularity::kScrollByPrecisePixel;
scroll->anchor.SetPoint(position_x, position_y);
scroll->fling_velocity_x = 0;
scroll->fling_velocity_y = 0;
scroll->distances.push_back(-gfx::Vector2dF(delta_x, delta_y));
scroll->modifiers = 0;
return true;
}
bool ActionsParser::ParsePointerAction(
const base::Value::Dict& action,
std::string subtype,
SyntheticPointerActionListParams::ParamList& param_list) {
double position_x = 0;
double position_y = 0;
if ((subtype == "pointerDown" || subtype == "pointerMove") &&
!GetPosition(action, position_x, position_y)) {
return false;
}
PointerActionType pointer_action_type = PointerActionType::NOT_INITIALIZED;
pointer_action_type = ToSyntheticPointerActionType(subtype);
if (pointer_action_type == PointerActionType::NOT_INITIALIZED) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.name is an unsupported action name",
action_index_);
return false;
}
Button button = pointer_action_type == PointerActionType::MOVE
? Button::NO_BUTTON
: Button::LEFT;
const base::Value* button_id_value = action.Find("button");
if (button_id_value) {
if (!button_id_value->is_int()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.button is not an integer", action_index_);
return false;
}
int button_id = button_id_value->GetInt();
if (button_id < 0 || button_id > 4) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.button is an unsupported button",
action_index_);
return false;
}
button = ToSyntheticMouseButton(button_id);
}
std::string keys;
const base::Value* keys_value = action.Find("keys");
if (keys_value) {
if (!keys_value->is_string()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.key is not a string", action_index_);
return false;
}
keys = keys_value->GetString();
}
int key_modifiers = 0;
std::vector<std::string> key_list =
base::SplitString(keys, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
for (std::string& key : key_list) {
int key_modifier = ToKeyModifiers(key);
if (key_modifier == 0) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.key is not a valid key", action_index_);
return false;
}
key_modifiers |= key_modifier;
}
const absl::optional<double> width_optional = action.FindDouble("width");
double width = width_optional.value_or(40);
if (width < 0) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.width should not be negative", action_index_);
return false;
}
const absl::optional<double> height_optional = action.FindDouble("height");
double height = height_optional.value_or(40);
if (height < 0) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.height should not be negative", action_index_);
return false;
}
const absl::optional<double> pressure_optional =
action.FindDouble("pressure");
double pressure = pressure_optional.value_or(0.5);
if (pressure < 0 || pressure > 1) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.pressure must be a non-negative number in the "
"range of [0,1]",
action_index_);
return false;
}
const absl::optional<double> tangential_pressure_optional =
action.FindDouble("tangentialPressure");
double tangential_pressure = tangential_pressure_optional.value_or(0);
if (tangential_pressure < -1 || tangential_pressure > 1) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.tangentialPressure must be a non-negative "
"number in the range of [-1,1]",
action_index_);
return false;
}
int tilt_x = 0;
const base::Value* tilt_x_value = action.Find("tiltX");
if (tilt_x_value) {
if (!tilt_x_value->is_int()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.tiltX is not an integer", action_index_);
return false;
}
tilt_x = tilt_x_value->GetInt();
if (tilt_x < -90 || tilt_x > 90) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.tiltX must be an integer in the range of "
"[-90,90]",
action_index_);
return false;
}
}
int tilt_y = 0;
const base::Value* tilt_y_value = action.Find("tiltY");
if (tilt_y_value) {
if (!tilt_y_value->is_int()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.tiltY is not an integer", action_index_);
return false;
}
tilt_y = tilt_y_value->GetInt();
if (tilt_y < -90 || tilt_y > 90) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.tiltY must be an integer in the range of "
"[-90,90]",
action_index_);
return false;
}
}
int twist = 0;
const base::Value* twist_value = action.Find("twist");
if (twist_value) {
if (!twist_value->is_int()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.twist is not an integer", action_index_);
return false;
}
twist = twist_value->GetInt();
if (twist < 0 || twist > 359) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.twist must be an integer in the range of "
"[0,359]",
action_index_);
return false;
}
}
int duration = viz::BeginFrameArgs::DefaultInterval().InMilliseconds();
if (pointer_action_type == PointerActionType::IDLE &&
!GetPauseDuration(action, duration)) {
return false;
}
SyntheticPointerActionParams action_param(pointer_action_type);
action_param.set_pointer_id(input_source_count_);
switch (pointer_action_type) {
case PointerActionType::PRESS:
action_param.set_position(gfx::PointF(position_x, position_y));
action_param.set_button(button);
action_param.set_key_modifiers(key_modifiers);
action_param.set_width(width);
action_param.set_height(height);
action_param.set_force(pressure);
action_param.set_tangential_pressure(tangential_pressure);
action_param.set_tilt_x(tilt_x);
action_param.set_tilt_y(tilt_y);
action_param.set_rotation_angle(twist);
break;
case PointerActionType::MOVE:
action_param.set_position(gfx::PointF(position_x, position_y));
action_param.set_key_modifiers(key_modifiers);
action_param.set_width(width);
action_param.set_height(height);
action_param.set_force(pressure);
action_param.set_tangential_pressure(tangential_pressure);
action_param.set_tilt_x(tilt_x);
action_param.set_tilt_y(tilt_y);
action_param.set_rotation_angle(twist);
action_param.set_button(button);
break;
case PointerActionType::RELEASE:
action_param.set_button(button);
action_param.set_key_modifiers(key_modifiers);
break;
case PointerActionType::IDLE:
action_param.set_duration(base::Milliseconds(duration));
break;
case PointerActionType::CANCEL:
case PointerActionType::LEAVE:
case PointerActionType::NOT_INITIALIZED:
break;
}
param_list.push_back(action_param);
return true;
}
bool ActionsParser::ParseNullAction(
const base::Value::Dict& action,
std::string subtype,
SyntheticPointerActionListParams::ParamList& param_list) {
PointerActionType pointer_action_type = PointerActionType::NOT_INITIALIZED;
pointer_action_type = ToSyntheticPointerActionType(subtype);
if (pointer_action_type != PointerActionType::IDLE) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.name should only be pause", action_index_);
return false;
}
int duration = viz::BeginFrameArgs::DefaultInterval().InMilliseconds();
if (!GetPauseDuration(action, duration))
return false;
SyntheticPointerActionParams action_param(pointer_action_type);
action_param.set_pointer_id(0);
action_param.set_duration(base::Milliseconds(duration));
param_list.push_back(action_param);
return true;
}
bool ActionsParser::GetPosition(const base::Value::Dict& action,
double& position_x,
double& position_y) {
const absl::optional<double> position_x_optional = action.FindDouble("x");
const absl::optional<double> position_y_optional = action.FindDouble("y");
// TODO(lanwei): we should clarify the case when x or y is undefined in the
// WebDriver spec.
// https://www.w3.org/TR/webdriver/#dfn-process-a-pointer-move-action.
if (!position_x_optional) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.x is not defined or not a number", action_index_);
return false;
}
position_x = position_x_optional.value();
if (!position_y_optional) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.y is not defined or not a number", action_index_);
return false;
}
position_y = position_y_optional.value();
return true;
}
bool ActionsParser::GetScrollDelta(const base::Value::Dict& action,
int& delta_x,
int& delta_y) {
const absl::optional<int> delta_x_optional = action.FindInt("deltaX");
const absl::optional<int> delta_y_optional = action.FindInt("deltaY");
if (!delta_x_optional) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.delta_x is not defined or not an integer",
action_index_);
return false;
}
delta_x = delta_x_optional.value();
if (!delta_y_optional) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.delta_y is not defined or not an integer",
action_index_);
return false;
}
delta_y = delta_y_optional.value();
return true;
}
bool ActionsParser::GetPauseDuration(const base::Value::Dict& action,
int& duration) {
const base::Value* duration_value = action.Find("duration");
// TODO(lanwei): we should always have a duration value for pause action.
if (duration_value) {
if (!duration_value->is_double() && !duration_value->is_int()) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.duration is not a number", action_index_);
return false;
}
duration = duration_value->GetDouble();
}
if (duration < 0) {
error_message_ = base::StringPrintf(
"actions[%zu].actions.duration should not be negative", action_index_);
return false;
}
return true;
}
} // namespace content