| // Copyright 2015 The LUCI Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package datastore |
| |
| import ( |
| "bytes" |
| "fmt" |
| "sort" |
| "strings" |
| "sync" |
| |
| "go.chromium.org/luci/common/data/stringset" |
| "go.chromium.org/luci/common/errors" |
| ) |
| |
| var ( |
| // ErrMultipleInequalityFilter is returned from Query.Finalize if you build a |
| // query which has inequality filters on multiple fields. |
| ErrMultipleInequalityFilter = errors.New( |
| "inequality filters on multiple properties in the same Query is not allowed") |
| |
| // ErrNullQuery is returned from Query.Finalize if you build a query for which |
| // there cannot possibly be any results. |
| ErrNullQuery = errors.New( |
| "the query is overconstrained and can never have results") |
| ) |
| |
| // Query is a builder-object for building a datastore query. It may represent |
| // an invalid query, but the error will only be observable when you call |
| // Finalize. |
| // |
| // Fields like "$id" are technically usable at the datastore level, but using |
| // them through the non-raw interface is likely a mistake. |
| // |
| // For example, instead of using: |
| // > datastore.NewQuery(...).Lte("$id", ...) |
| // One should use: |
| // > datastore.NewQuery(...).Lte("__key__", ...) |
| type Query struct { |
| queryFields |
| |
| // These are set by Finalize as a way to cache the 1-1 correspondence of |
| // a Query to its FinalizedQuery form. err may also be set by intermediate |
| // Query functions if there's a problem before finalization. |
| // |
| // Query implements lazy finalization, meaning that it will happen at most |
| // once. This means that the finalization state and cached finalization must |
| // be locked around. |
| finalizeOnce sync.Once |
| finalized *FinalizedQuery |
| finalizeErr error |
| } |
| |
| // queryFields are the Query's read-only fields. |
| // |
| // All Property and PropertySlice inside must have comparable types. |
| type queryFields struct { |
| kind string |
| |
| // Indicate if the query is executed to a firestore (with datastore API) |
| firestoreMode bool |
| eventualConsistency bool |
| keysOnly bool |
| distinct bool |
| |
| limit *int32 |
| offset *int32 |
| |
| order []IndexColumn |
| project stringset.Set |
| |
| eqFilts map[string]PropertySlice |
| inFilts map[string][]PropertySlice |
| |
| ineqFiltProp string |
| ineqFiltLow Property |
| ineqFiltLowIncl bool |
| ineqFiltLowSet bool |
| ineqFiltHigh Property |
| ineqFiltHighIncl bool |
| ineqFiltHighSet bool |
| |
| start Cursor |
| end Cursor |
| |
| err error |
| } |
| |
| // NewQuery returns a new Query for the given kind. If kind may be empty to |
| // begin a kindless query. |
| func NewQuery(kind string) *Query { |
| return &Query{ |
| queryFields: queryFields{ |
| kind: kind, |
| }, |
| } |
| } |
| |
| func (q *Query) mod(cb func(*Query)) *Query { |
| if q.err != nil { |
| return q |
| } |
| |
| ret := Query{ |
| queryFields: q.queryFields, |
| } |
| if len(q.order) > 0 { |
| ret.order = make([]IndexColumn, len(q.order)) |
| copy(ret.order, q.order) |
| } |
| if q.project != nil { |
| ret.project = q.project.Dup() |
| } |
| if len(q.eqFilts) > 0 { |
| ret.eqFilts = make(map[string]PropertySlice, len(q.eqFilts)) |
| for k, v := range q.eqFilts { |
| newV := make(PropertySlice, len(v)) |
| copy(newV, v) |
| ret.eqFilts[k] = newV |
| } |
| } |
| if len(q.inFilts) > 0 { |
| ret.inFilts = make(map[string][]PropertySlice, len(q.inFilts)) |
| for k, v := range q.inFilts { |
| // Note that we never mutate individual `v` elements (which are property |
| // slices), so there's no need to deep-clone them. We only need to clone |
| // the top container slice (i.e. `v` itself), since we do mutate it in |
| // In(...) by appending more elements to it. |
| newV := make([]PropertySlice, len(v)) |
| copy(newV, v) |
| ret.inFilts[k] = newV |
| } |
| } |
| cb(&ret) |
| return &ret |
| } |
| |
| // Kind alters the kind of this query. |
| func (q *Query) Kind(kind string) *Query { |
| return q.mod(func(q *Query) { |
| q.kind = kind |
| }) |
| } |
| |
| // Ancestor sets the ancestor filter for this query. |
| // |
| // If ancestor is nil, then this removes the Ancestor restriction from the |
| // query. |
| func (q *Query) Ancestor(ancestor *Key) *Query { |
| return q.mod(func(q *Query) { |
| if q.eqFilts == nil { |
| q.eqFilts = map[string]PropertySlice{} |
| } |
| if ancestor == nil { |
| delete(q.eqFilts, "__ancestor__") |
| if len(q.eqFilts) == 0 { |
| q.eqFilts = nil |
| } |
| } else { |
| q.eqFilts["__ancestor__"] = PropertySlice{MkProperty(ancestor)} |
| } |
| }) |
| } |
| |
| // EventualConsistency changes the EventualConsistency setting for this query. |
| func (q *Query) EventualConsistency(on bool) *Query { |
| return q.mod(func(q *Query) { |
| q.eventualConsistency = on |
| }) |
| } |
| |
| // Limit sets the limit (max items to return) for this query. If limit < 0, this |
| // removes the limit from the query entirely. |
| func (q *Query) Limit(limit int32) *Query { |
| return q.mod(func(q *Query) { |
| if limit < 0 { |
| q.limit = nil |
| } else { |
| q.limit = &limit |
| } |
| }) |
| } |
| |
| // Offset sets the offset (number of items to skip) for this query. If |
| // offset < 0, this removes the offset from the query entirely. |
| func (q *Query) Offset(offset int32) *Query { |
| return q.mod(func(q *Query) { |
| if offset < 0 { |
| q.offset = nil |
| } else { |
| q.offset = &offset |
| } |
| }) |
| } |
| |
| // KeysOnly makes this into a query which only returns keys (but doesn't fetch |
| // values). It's incompatible with projection queries. |
| func (q *Query) KeysOnly(on bool) *Query { |
| return q.mod(func(q *Query) { |
| q.keysOnly = on |
| }) |
| } |
| |
| // Order sets one or more orders for this query. |
| func (q *Query) Order(fieldNames ...string) *Query { |
| if len(fieldNames) == 0 { |
| return q |
| } |
| return q.mod(func(q *Query) { |
| for _, fn := range fieldNames { |
| ic, err := ParseIndexColumn(fn) |
| if err != nil { |
| q.err = err |
| return |
| } |
| if q.reserved(ic.Property) { |
| return |
| } |
| q.order = append(q.order, ic) |
| } |
| }) |
| } |
| |
| // ClearOrder removes all orders from this Query. |
| func (q *Query) ClearOrder() *Query { |
| return q.mod(func(q *Query) { |
| q.order = nil |
| }) |
| } |
| |
| // Project lists one or more field names to project. |
| func (q *Query) Project(fieldNames ...string) *Query { |
| if len(fieldNames) == 0 { |
| return q |
| } |
| return q.mod(func(q *Query) { |
| for _, f := range fieldNames { |
| if q.reserved(f) { |
| return |
| } |
| if f == "__key__" { |
| q.err = fmt.Errorf("cannot project on %q", f) |
| return |
| } |
| if q.project == nil { |
| q.project = stringset.New(1) |
| } |
| q.project.Add(f) |
| } |
| }) |
| } |
| |
| // Distinct makes a projection query only return distinct values. This has |
| // no effect on non-projection queries. |
| func (q *Query) Distinct(on bool) *Query { |
| return q.mod(func(q *Query) { |
| q.distinct = on |
| }) |
| } |
| |
| // ClearProject removes all projected fields from this Query. |
| func (q *Query) ClearProject() *Query { |
| return q.mod(func(q *Query) { |
| q.project = nil |
| }) |
| } |
| |
| // Start sets a starting cursor. The cursor is implementation-defined by the |
| // particular 'impl' you have installed. |
| func (q *Query) Start(c Cursor) *Query { |
| return q.mod(func(q *Query) { |
| q.start = c |
| }) |
| } |
| |
| // End sets the ending cursor. The cursor is implementation-defined by the |
| // particular 'impl' you have installed. |
| func (q *Query) End(c Cursor) *Query { |
| return q.mod(func(q *Query) { |
| q.end = c |
| }) |
| } |
| |
| // Eq adds one or more equality restrictions to the query. |
| // |
| // Equality filters interact with multiply-defined properties by ensuring that |
| // the given field has /at least one/ value which is equal to the specified |
| // constraint. |
| // |
| // So a query with `.Eq("thing", 1, 2)` will only return entities where the |
| // field "thing" is multiply defined and contains both a value of 1 and a value |
| // of 2. If the field is singular, such check will never pass. To query for |
| // entities with a field matching any one of values use `.In("thing", 1, 2)` |
| // filter instead. |
| // |
| // `Eq("thing", 1).Eq("thing", 2)` and `.Eq("thing", 1, 2)` have identical |
| // meaning. |
| func (q *Query) Eq(field string, values ...any) *Query { |
| if len(values) == 0 { |
| return q |
| } |
| return q.mod(func(q *Query) { |
| if !q.reserved(field) { |
| if q.eqFilts == nil { |
| q.eqFilts = make(map[string]PropertySlice, 1) |
| } |
| s := q.eqFilts[field] |
| for _, value := range values { |
| p := Property{} |
| if q.err = p.SetValue(value, ShouldIndex); q.err != nil { |
| return |
| } |
| if q.err = checkComparable(field, p.Type()); q.err != nil { |
| return |
| } |
| idx := sort.Search(len(s), func(i int) bool { |
| // s[i] >= p is the same as: |
| return s[i].Equal(&p) || p.Less(&s[i]) |
| }) |
| if idx == len(s) || !s[idx].Equal(&p) { |
| s = append(s, Property{}) |
| copy(s[idx+1:], s[idx:]) |
| s[idx] = p |
| } |
| } |
| q.eqFilts[field] = s |
| } |
| }) |
| } |
| |
| // reserved checks whether a field is reserved. |
| // |
| // Set the q.err as a side-effect if field is invalid. |
| func (q *Query) reserved(field string) bool { |
| if field == "__key__" || field == "__scatter__" { |
| return false |
| } |
| if strings.HasPrefix(field, "$") { |
| q.err = fmt.Errorf( |
| `LUCI fields such as "$id" and "$kind" are not real fields: rejecting field %q`, field) |
| return true |
| } |
| if field == "" { |
| q.err = fmt.Errorf( |
| "cannot filter/project on: %q", field) |
| return true |
| } |
| if strings.HasPrefix(field, "__") && strings.HasSuffix(field, "__") { |
| q.err = fmt.Errorf( |
| "cannot filter/project on reserved property: %q", field) |
| return true |
| } |
| return false |
| } |
| |
| func (q *Query) ineqOK(field string, value Property) bool { |
| if q.reserved(field) { |
| return false |
| } |
| if field == "__key__" && value.Type() != PTKey { |
| q.err = fmt.Errorf( |
| "filters on %q must have type *Key (got %s)", field, value.Type()) |
| return false |
| } |
| if q.ineqFiltProp != "" && q.ineqFiltProp != field { |
| q.err = ErrMultipleInequalityFilter |
| return false |
| } |
| return true |
| } |
| |
| // Lt imposes a 'less-than' inequality restriction on the Query. |
| // |
| // Inequality filters interact with multiply-defined properties by ensuring that |
| // the given field has /exactly one/ value which matches /all/ of the inequality |
| // constraints. |
| // |
| // So a query with `.Gt("thing", 5).Lt("thing", 10)` will only return entities |
| // where the field "thing" has a single value where `5 < val < 10`. |
| func (q *Query) Lt(field string, value any) *Query { |
| p := Property{} |
| err := p.SetValue(value, ShouldIndex) |
| if err == nil { |
| err = checkComparable(field, p.Type()) |
| } |
| |
| if err == nil && q.ineqFiltHighSet { |
| if q.ineqFiltHigh.Less(&p) { |
| return q |
| } else if q.ineqFiltHigh.Equal(&p) && !q.ineqFiltHighIncl { |
| return q |
| } |
| } |
| |
| return q.mod(func(q *Query) { |
| if q.err = err; err != nil { |
| return |
| } |
| if q.ineqOK(field, p) { |
| q.ineqFiltProp = field |
| q.ineqFiltHighSet = true |
| q.ineqFiltHigh = p |
| q.ineqFiltHighIncl = false |
| } |
| }) |
| } |
| |
| // Lte imposes a 'less-than-or-equal' inequality restriction on the Query. |
| // |
| // Inequality filters interact with multiply-defined properties by ensuring that |
| // the given field has /exactly one/ value which matches /all/ of the inequality |
| // constraints. |
| // |
| // So a query with `.Gt("thing", 5).Lt("thing", 10)` will only return entities |
| // where the field "thing" has a single value where `5 < val < 10`. |
| func (q *Query) Lte(field string, value any) *Query { |
| p := Property{} |
| err := p.SetValue(value, ShouldIndex) |
| if err == nil { |
| err = checkComparable(field, p.Type()) |
| } |
| |
| if err == nil && q.ineqFiltHighSet { |
| if q.ineqFiltHigh.Less(&p) { |
| return q |
| } else if q.ineqFiltHigh.Equal(&p) { |
| return q |
| } |
| } |
| |
| return q.mod(func(q *Query) { |
| if q.err = err; err != nil { |
| return |
| } |
| if q.ineqOK(field, p) { |
| q.ineqFiltProp = field |
| q.ineqFiltHighSet = true |
| q.ineqFiltHigh = p |
| q.ineqFiltHighIncl = true |
| } |
| }) |
| } |
| |
| // Gt imposes a 'greater-than' inequality restriction on the Query. |
| // |
| // Inequality filters interact with multiply-defined properties by ensuring that |
| // the given field has /exactly one/ value which matches /all/ of the inequality |
| // constraints. |
| // |
| // So a query with `.Gt("thing", 5).Lt("thing", 10)` will only return entities |
| // where the field "thing" has a single value where `5 < val < 10`. |
| func (q *Query) Gt(field string, value any) *Query { |
| p := Property{} |
| err := p.SetValue(value, ShouldIndex) |
| if err == nil { |
| err = checkComparable(field, p.Type()) |
| } |
| |
| if err == nil && q.ineqFiltLowSet { |
| if p.Less(&q.ineqFiltLow) { |
| return q |
| } else if p.Equal(&q.ineqFiltLow) && !q.ineqFiltLowIncl { |
| return q |
| } |
| } |
| |
| return q.mod(func(q *Query) { |
| if q.err = err; err != nil { |
| return |
| } |
| if q.ineqOK(field, p) { |
| q.ineqFiltProp = field |
| q.ineqFiltLowSet = true |
| q.ineqFiltLow = p |
| q.ineqFiltLowIncl = false |
| } |
| }) |
| } |
| |
| // Gte imposes a 'greater-than-or-equal' inequality restriction on the Query. |
| // |
| // Inequality filters interact with multiply-defined properties by ensuring that |
| // the given field has /exactly one/ value which matches /all/ of the inequality |
| // constraints. |
| // |
| // So a query with `.Gt("thing", 5).Lt("thing", 10)` will only return entities |
| // where the field "thing" has a single value where `5 < val < 10`. |
| func (q *Query) Gte(field string, value any) *Query { |
| p := Property{} |
| err := p.SetValue(value, ShouldIndex) |
| if err == nil { |
| err = checkComparable(field, p.Type()) |
| } |
| |
| if err == nil && q.ineqFiltLowSet { |
| if p.Less(&q.ineqFiltLow) { |
| return q |
| } else if p.Equal(&q.ineqFiltLow) { |
| return q |
| } |
| } |
| |
| return q.mod(func(q *Query) { |
| if q.err = err; err != nil { |
| return |
| } |
| if q.ineqOK(field, p) { |
| q.ineqFiltProp = field |
| q.ineqFiltLowSet = true |
| q.ineqFiltLow = p |
| q.ineqFiltLowIncl = true |
| } |
| }) |
| } |
| |
| // In imposes a 'is-in-a-set' equality restriction on the Query. |
| // |
| // Equality filters interact with multiply-defined properties by ensuring that |
| // the given field has /at least one/ value which is equal to the specified |
| // constraint. So a query with `.In("thing", 1, 2)` will return entities |
| // where at least one value of the field "thing" is either 1 or 2. |
| // |
| // Multiple `In` filters on the same property are AND-ed together, e.g. |
| // `.In("thing", 1, 2).In("thing", 3, 4)` will return entities whose repeated |
| // "thing" field has a value equal to 1 or 2 AND another value equal to 3 or 4. |
| func (q *Query) In(field string, values ...any) *Query { |
| return q.mod(func(q *Query) { |
| if q.reserved(field) { |
| return |
| } |
| props := make(PropertySlice, len(values)) |
| for idx, value := range values { |
| if q.err = props[idx].SetValue(value, ShouldIndex); q.err != nil { |
| return |
| } |
| if q.err = checkComparable(field, props[idx].Type()); q.err != nil { |
| return |
| } |
| } |
| if q.inFilts == nil { |
| q.inFilts = make(map[string][]PropertySlice, 1) |
| } |
| q.inFilts[field] = append(q.inFilts[field], props) |
| }) |
| } |
| |
| // ClearFilters clears all equality and inequality filters from the Query. It |
| // does not clear the Ancestor filter if one is defined. |
| func (q *Query) ClearFilters() *Query { |
| return q.mod(func(q *Query) { |
| anc := q.eqFilts["__ancestor__"] |
| if anc != nil { |
| q.eqFilts = map[string]PropertySlice{"__ancestor__": anc} |
| } else { |
| q.eqFilts = nil |
| } |
| q.inFilts = nil |
| q.ineqFiltLowSet = false |
| q.ineqFiltHighSet = false |
| }) |
| } |
| |
| // Finalize converts this Query to a FinalizedQuery. If the Query has any |
| // inconsistencies or violates any of the query rules, that will be returned |
| // here. |
| func (q *Query) Finalize() (*FinalizedQuery, error) { |
| if q.err != nil { |
| return nil, q.err |
| } |
| |
| q.finalizeOnce.Do(func() { |
| q.finalized, q.finalizeErr = q.finalizeImpl() |
| }) |
| return q.finalized, q.finalizeErr |
| } |
| |
| func (q *Query) finalizeImpl() (*FinalizedQuery, error) { |
| ancestor := (*Key)(nil) |
| if slice, ok := q.eqFilts["__ancestor__"]; ok { |
| ancestor = slice[0].Value().(*Key) |
| } |
| |
| if q.kind == "" { // kindless query checks |
| if q.ineqFiltProp != "" && q.ineqFiltProp != "__key__" { |
| return nil, fmt.Errorf( |
| "kindless queries can only filter on __key__, got %q", q.ineqFiltProp) |
| } |
| allowedEqs := 0 |
| if ancestor != nil { |
| allowedEqs = 1 |
| } |
| if len(q.eqFilts) > allowedEqs || len(q.inFilts) > 0 { |
| return nil, fmt.Errorf("kindless queries may not have any equality filters") |
| } |
| for _, o := range q.order { |
| if o.Property != "__key__" || o.Descending { |
| return nil, fmt.Errorf("invalid order for kindless query: %#v", o) |
| } |
| } |
| } |
| |
| if q.keysOnly && q.project != nil && q.project.Len() > 0 { |
| return nil, errors.New("cannot project a keysOnly query") |
| } |
| |
| if q.ineqFiltProp != "" { |
| if len(q.order) > 0 && q.order[0].Property != q.ineqFiltProp { |
| return nil, fmt.Errorf( |
| "first sort order must match inequality filter: %q v %q", |
| q.order[0].Property, q.ineqFiltProp) |
| } |
| if q.ineqFiltLowSet && q.ineqFiltHighSet { |
| if q.ineqFiltHigh.Less(&q.ineqFiltLow) || |
| (q.ineqFiltHigh.Equal(&q.ineqFiltLow) && |
| (!q.ineqFiltLowIncl || !q.ineqFiltHighIncl)) { |
| return nil, ErrNullQuery |
| } |
| } |
| if q.ineqFiltProp == "__key__" { |
| if q.ineqFiltLowSet { |
| if ancestor != nil && !q.ineqFiltLow.Value().(*Key).HasAncestor(ancestor) { |
| return nil, fmt.Errorf( |
| "inequality filters on __key__ must be descendants of the __ancestor__") |
| } |
| } |
| if q.ineqFiltHighSet { |
| if ancestor != nil && !q.ineqFiltHigh.Value().(*Key).HasAncestor(ancestor) { |
| return nil, fmt.Errorf( |
| "inequality filters on __key__ must be descendants of the __ancestor__") |
| } |
| } |
| } |
| } |
| |
| if q.project != nil { |
| var err error |
| q.project.Iter(func(p string) bool { |
| _, iseq := q.eqFilts[p] |
| _, isin := q.inFilts[p] |
| if iseq || isin { |
| err = fmt.Errorf("cannot project on equality filter field: %s", p) |
| return false |
| } |
| return true |
| }) |
| if err != nil { |
| return nil, err |
| } |
| } |
| |
| for _, slices := range q.inFilts { |
| for _, set := range slices { |
| if len(set) == 0 { |
| return nil, ErrNullQuery |
| } |
| } |
| } |
| |
| ret := &FinalizedQuery{ |
| original: q, |
| kind: q.kind, |
| |
| keysOnly: q.keysOnly, |
| eventuallyConsistent: q.getEventualConsistency(ancestor), |
| limit: q.limit, |
| offset: q.offset, |
| start: q.start, |
| end: q.end, |
| |
| eqFilts: q.eqFilts, |
| inFilts: q.inFilts, |
| |
| ineqFiltProp: q.ineqFiltProp, |
| ineqFiltLow: q.ineqFiltLow, |
| ineqFiltLowIncl: q.ineqFiltLowIncl, |
| ineqFiltLowSet: q.ineqFiltLowSet, |
| ineqFiltHigh: q.ineqFiltHigh, |
| ineqFiltHighIncl: q.ineqFiltHighIncl, |
| ineqFiltHighSet: q.ineqFiltHighSet, |
| } |
| // If a starting cursor is provided, ignore the offset, as it would have been |
| // accounted for in the query that produced the cursor. |
| if ret.start != nil { |
| ret.offset = nil |
| } |
| |
| if q.project != nil { |
| ret.project = q.project.ToSlice() |
| ret.distinct = q.distinct && q.project.Len() > 0 |
| |
| // If we're DISTINCT && have an inequality filter, we must project that |
| // inequality property as well. |
| if ret.distinct && ret.ineqFiltProp != "" && !q.project.Has(ret.ineqFiltProp) { |
| ret.project = append([]string{ret.ineqFiltProp}, ret.project...) |
| } |
| } |
| |
| seenOrders := stringset.New(len(q.order)) |
| |
| // if len(q.order) > 0, we already enforce that the first order |
| // is the same as the inequality above. Otherwise we need to add it. |
| if len(q.order) == 0 && q.ineqFiltProp != "" { |
| ret.orders = []IndexColumn{{Property: q.ineqFiltProp}} |
| seenOrders.Add(q.ineqFiltProp) |
| } |
| |
| // drop orders where there's an equality filter |
| // https://cloud.google.com/appengine/docs/go/datastore/queries#sort_orders_are_ignored_on_properties_with_equality_filters |
| // Deduplicate orders |
| for _, o := range q.order { |
| if _, iseq := q.eqFilts[o.Property]; !iseq { |
| if seenOrders.Add(o.Property) { |
| ret.orders = append(ret.orders, o) |
| } |
| } |
| } |
| |
| // Add any projection columns not mentioned in the user-defined order as |
| // ASCENDING orders. Technically we could be smart and automatically use |
| // a DESCENDING ordered index, if it fit, but the logic gets insane, since all |
| // suffixes of all used indexes need to be PRECISELY equal (and so you'd have |
| // to hunt/invalidate/something to find the combination of indexes that are |
| // compatible with each other as well as the query). If you want to use |
| // a DESCENDING column, just add it to the user sort order, and this loop will |
| // not synthesize a new suffix entry for it. |
| // |
| // NOTE: if you want to use an index that sorts by -__key__, you MUST |
| // include all of the projected fields for that index in the order explicitly. |
| // Otherwise the generated orders will be wacky. So: |
| // Query("Foo").Project("A", "B").Order("A").Order("-__key__") |
| // |
| // will turn into a orders of: |
| // A, ASCENDING |
| // __key__, DESCENDING |
| // B, ASCENDING |
| // __key__, ASCENDING |
| // |
| // To prevent this, your query should have another Order("B") clause before |
| // the -__key__ clause. |
| if len(ret.project) > 0 { |
| sort.Strings(ret.project) |
| for _, p := range ret.project { |
| if !seenOrders.Has(p) { |
| ret.orders = append(ret.orders, IndexColumn{Property: p}) |
| } |
| } |
| } |
| |
| // If the suffix format ends with __key__ already (e.g. .Order("__key__")), |
| // then we're good to go. Otherwise we need to add it as the last bit of the |
| // suffix, since all indexes implicitly have it as the last column. |
| if len(ret.orders) == 0 || ret.orders[len(ret.orders)-1].Property != "__key__" { |
| ret.orders = append(ret.orders, IndexColumn{Property: "__key__"}) |
| } |
| |
| return ret, nil |
| } |
| |
| func (q *Query) String() string { |
| ret := &bytes.Buffer{} |
| needComma := false |
| p := func(fmtStr string, stuff ...any) { |
| if needComma { |
| if _, err := ret.WriteString(", "); err != nil { |
| panic(err) |
| } |
| } |
| needComma = true |
| fmt.Fprintf(ret, fmtStr, stuff...) |
| } |
| if _, err := ret.WriteString("Query("); err != nil { |
| panic(err) |
| } |
| if q.err != nil { |
| p("ERROR=%q", q.err.Error()) |
| } |
| |
| // Filters |
| if q.kind != "" { |
| p("Kind=%q", q.kind) |
| } |
| if q.eqFilts["__ancestor__"] != nil { |
| p("Ancestor=%s", q.eqFilts["__ancestor__"][0].Value().(*Key).String()) |
| } |
| for prop, vals := range q.eqFilts { |
| if prop == "__ancestor__" { |
| continue |
| } |
| for _, v := range vals { |
| p("Filter(%q == %s)", prop, v.GQL()) |
| } |
| } |
| for prop, slices := range q.inFilts { |
| for _, vals := range slices { |
| gql := make([]string, len(vals)) |
| for i, v := range vals { |
| gql[i] = v.GQL() |
| } |
| p("Filter(%q in [%s])", prop, strings.Join(gql, ", ")) |
| } |
| } |
| if q.ineqFiltProp != "" { |
| if q.ineqFiltLowSet { |
| op := ">" |
| if q.ineqFiltLowIncl { |
| op = ">=" |
| } |
| p("Filter(%q %s %s)", q.ineqFiltProp, op, q.ineqFiltLow.GQL()) |
| } |
| if q.ineqFiltHighSet { |
| op := "<" |
| if q.ineqFiltHighIncl { |
| op = "<=" |
| } |
| p("Filter(%q %s %s)", q.ineqFiltProp, op, q.ineqFiltHigh.GQL()) |
| } |
| } |
| |
| // Order |
| if len(q.order) > 0 { |
| orders := make([]string, len(q.order)) |
| for i, o := range q.order { |
| orders[i] = o.String() |
| } |
| p("Order(%s)", strings.Join(orders, ", ")) |
| } |
| |
| // Projection |
| if q.project != nil && q.project.Len() > 0 { |
| f := "Project(%s)" |
| if q.distinct { |
| f = "Project[DISTINCT](%s)" |
| } |
| p(f, strings.Join(q.project.ToSlice(), ", ")) |
| } |
| |
| // Cursors |
| if q.start != nil { |
| p("Start(%q)", q.start.String()) |
| } |
| if q.end != nil { |
| p("End(%q)", q.end.String()) |
| } |
| |
| // Modifiers |
| if q.limit != nil { |
| p("Limit=%d", *q.limit) |
| } |
| if q.offset != nil { |
| p("Offset=%d", *q.offset) |
| } |
| if q.eventualConsistency { |
| p("EventualConsistency") |
| } |
| if q.keysOnly { |
| p("KeysOnly") |
| } |
| |
| if _, err := ret.WriteRune(')'); err != nil { |
| panic(err) |
| } |
| |
| return ret.String() |
| } |
| |
| // FirestoreMode set the firestore mode. It removes internal checks for |
| // this Query which don't apply when using Firestore-in-Datastore mode. |
| // |
| // In firestore mode all Datastore queries become strongly consistent by |
| // default, but still can be made eventually consistent via a call to |
| // EventualConsistency(true). In particular this is useful for aggregation |
| // queries like Count(). |
| // |
| // Note that firestore mode allows non-ancestor queries within a transaction. |
| func (q *Query) FirestoreMode(on bool) *Query { |
| return q.mod(func(q *Query) { |
| q.firestoreMode = on |
| }) |
| } |
| |
| // GetFirestoreMode returns the firestore mode. |
| func (q *Query) GetFirestoreMode() bool { |
| return q.firestoreMode |
| } |
| |
| func (q *Query) getEventualConsistency(ancestor *Key) bool { |
| return q.eventualConsistency || (!q.firestoreMode && ancestor == nil) |
| } |
| |
| // checkComparable returns an error if this property type is not comparable. |
| func checkComparable(field string, pt PropertyType) error { |
| if !pt.Comparable() { |
| return fmt.Errorf("a non-comparable value in a filter on field %q", field) |
| } |
| return nil |
| } |
| |
| // All cmp* functions below return -1 if a < b, 0 if a == b, 1 if a > b. |
| |
| func cmpInteger(a, b int) int { |
| switch { |
| case a == b: |
| return 0 |
| case a < b: |
| return -1 |
| default: |
| return 1 |
| } |
| } |
| |
| func cmpStringSet(a, b stringset.Set) int { |
| // Quick check for the most common case to skip heavy calls below. |
| if a.Len() == 0 && b.Len() == 0 { |
| return 0 |
| } |
| |
| // Compare as math sets: discard common elements and then compare remainders |
| // lexicographically. |
| common := a.Intersect(b) |
| auniq := a.Difference(common).ToSortedSlice() |
| buniq := b.Difference(common).ToSortedSlice() |
| |
| for i := 0; i < len(auniq) && i < len(buniq); i++ { |
| switch { |
| case auniq[i] < buniq[i]: |
| return -1 |
| case auniq[i] > buniq[i]: |
| return 1 |
| } |
| } |
| |
| return cmpInteger(len(auniq), len(buniq)) |
| } |
| |
| func cmpStr(a, b string) int { |
| switch { |
| case a == b: |
| return 0 |
| case a < b: |
| return -1 |
| default: |
| return 1 |
| } |
| } |
| |
| func cmpBoolean(a, b bool) int { |
| switch { |
| case a == b: |
| return 0 |
| case !a: |
| return -1 |
| default: |
| return 1 |
| } |
| } |
| |
| func cmpEqFilters(a, b map[string]PropertySlice) int { |
| // Quick checks for common cases. |
| switch { |
| case len(a) == 0 && len(b) == 0: |
| return 0 |
| case len(a) == 0 && len(b) != 0: |
| return -1 |
| case len(a) != 0 && len(b) == 0: |
| return 1 |
| } |
| |
| // Compare maps in the sorted order of keys. |
| cap := len(a) |
| if len(b) > cap { |
| cap = len(b) |
| } |
| keys := stringset.New(cap) |
| for k := range a { |
| keys.Add(k) |
| } |
| for k := range b { |
| keys.Add(k) |
| } |
| for _, key := range keys.ToSortedSlice() { |
| if cmp := cmpPropertySliceSet(a[key], b[key]); cmp != 0 { |
| return cmp |
| } |
| } |
| return cmpInteger(len(a), len(b)) |
| } |
| |
| func cmpInFilters(a, b map[string][]PropertySlice) int { |
| // Quick checks for common cases. |
| switch { |
| case len(a) == 0 && len(b) == 0: |
| return 0 |
| case len(a) == 0 && len(b) != 0: |
| return -1 |
| case len(a) != 0 && len(b) == 0: |
| return 1 |
| } |
| |
| // Compare maps in the sorted order of keys. Compare values lexicographically. |
| cap := len(a) |
| if len(b) > cap { |
| cap = len(b) |
| } |
| keys := stringset.New(cap) |
| for k := range a { |
| keys.Add(k) |
| } |
| for k := range b { |
| keys.Add(k) |
| } |
| for _, key := range keys.ToSortedSlice() { |
| alist := a[key] |
| blist := b[key] |
| for i := 0; i < len(alist) && i < len(blist); i++ { |
| if cmp := cmpPropertySliceSet(alist[i], blist[i]); cmp != 0 { |
| return cmp |
| } |
| } |
| if cmp := cmpInteger(len(alist), len(blist)); cmp != 0 { |
| return cmp |
| } |
| } |
| return cmpInteger(len(a), len(b)) |
| } |
| |
| func cmpPropertySliceSet(a, b PropertySlice) int { |
| // Quick checks for common cases. |
| switch { |
| case len(a) == 0 && len(b) == 0: |
| return 0 |
| case len(a) == 0 && len(b) != 0: |
| return -1 |
| case len(a) != 0 && len(b) == 0: |
| return 1 |
| } |
| |
| asorted := a.Slice() |
| sort.Sort(asorted) |
| |
| bsorted := b.Slice() |
| sort.Sort(bsorted) |
| |
| for i := 0; i < len(asorted) && i < len(bsorted); i++ { |
| if cmp := asorted[i].Compare(&bsorted[i]); cmp != 0 { |
| return cmp |
| } |
| } |
| |
| return cmpInteger(len(asorted), len(bsorted)) |
| } |
| |
| func cmpIneqOp(a *Property, aincl, aset bool, b *Property, bincl, bset bool) int { |
| switch { |
| case !aset && !bset: |
| // If both are unset, don't bother comparing properties. They are null. |
| return 0 |
| case !aset && bset: |
| // If only 'b' set, then a < b. |
| return -1 |
| case aset && !bset: |
| // If only 'a' set, then a > b. |
| return 1 |
| default: |
| // If both are set, compare the rest of the fields. |
| if cmp := a.Compare(b); cmp != 0 { |
| return cmp |
| } |
| return cmpBoolean(aincl, bincl) |
| } |
| } |
| |
| func cmpIndexColumnList(a, b []IndexColumn) int { |
| for i := 0; i < len(a) && i < len(b); i++ { |
| if cmp := cmpStr(a[i].Property, b[i].Property); cmp != 0 { |
| return cmp |
| } |
| if cmp := cmpBoolean(a[i].Descending, b[i].Descending); cmp != 0 { |
| return cmp |
| } |
| } |
| return cmpInteger(len(a), len(b)) |
| } |
| |
| func cmpOptionalInt32(a, b *int32) int { |
| switch { |
| case a == nil && b == nil: |
| return 0 |
| case a == nil && b != nil: |
| return -1 |
| case a != nil && b == nil: |
| return 1 |
| default: |
| switch { |
| case *a == *b: |
| return 0 |
| case *a < *b: |
| return -1 |
| default: |
| return 1 |
| } |
| } |
| } |
| |
| func cmpCursor(a, b Cursor) int { |
| var astr string |
| if a != nil { |
| astr = a.String() |
| } |
| var bstr string |
| if b != nil { |
| bstr = b.String() |
| } |
| return cmpStr(astr, bstr) |
| } |
| |
| // Less returns true if a < b. It is used for local sorting of lists of queries, |
| // there is nothing datastore specific about this. |
| func (a *Query) Less(b *Query) bool { |
| // For concreteness compare query components in the same order they would |
| // appear in a GQL query string. Things that do not show up in GQL are |
| // compared last. Note this should cover every field in queryFields struct. |
| // |
| // See https://cloud.google.com/datastore/docs/reference/gql_reference |
| if cmp := cmpStringSet(a.project, b.project); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpStr(a.kind, b.kind); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpBoolean(a.distinct, b.distinct); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpEqFilters(a.eqFilts, b.eqFilts); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpInFilters(a.inFilts, b.inFilts); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpStr(a.ineqFiltProp, b.ineqFiltProp); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpIneqOp( |
| &a.ineqFiltLow, a.ineqFiltLowIncl, a.ineqFiltLowSet, |
| &b.ineqFiltLow, b.ineqFiltLowIncl, b.ineqFiltLowSet, |
| ); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpIneqOp( |
| &a.ineqFiltHigh, a.ineqFiltHighIncl, a.ineqFiltHighSet, |
| &b.ineqFiltHigh, b.ineqFiltHighIncl, b.ineqFiltHighSet, |
| ); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpIndexColumnList(a.order, b.order); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpOptionalInt32(a.limit, b.limit); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpOptionalInt32(a.offset, b.offset); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpCursor(a.start, b.start); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpCursor(a.end, b.end); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpBoolean(a.keysOnly, b.keysOnly); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpBoolean(a.firestoreMode, b.firestoreMode); cmp != 0 { |
| return cmp < 0 |
| } |
| if cmp := cmpBoolean(a.eventualConsistency, b.eventualConsistency); cmp != 0 { |
| return cmp < 0 |
| } |
| // They are equal, which means `a < b` is false. |
| return false |
| } |