blob: 47be5a4d7e396ff407558cf2b90c3a9d4f8d7428 [file] [log] [blame]
// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aggrmetrics
import (
"context"
"go.chromium.org/luci/common/data/stringset"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/common/retry/transient"
"go.chromium.org/luci/common/tsmon/field"
"go.chromium.org/luci/common/tsmon/metric"
"go.chromium.org/luci/common/tsmon/types"
"go.chromium.org/luci/gae/service/datastore"
"go.chromium.org/luci/cv/internal/prjmanager"
)
var (
metricPMStateCLs = metric.NewInt(
"cv/internal/pm/cls",
"Number of tracked CLs(# of vertices in the CL graph).",
nil,
field.String("project"),
)
metricPMStateDeps = metric.NewInt(
"cv/internal/pm/deps",
"Number of tracked CL dependencies (# of edges in the CL graph)",
nil,
field.String("project"),
)
metricPMStateComponents = metric.NewInt(
"cv/internal/pm/components",
"Number of tracked CL components (~ connected components in the CL graph)",
nil,
field.String("project"),
)
metricPMEntitySize = metric.NewInt(
"cv/internal/pm/entity_size",
"Approximate size of the Project entity in Datastore",
&types.MetricMetadata{Units: types.Bytes},
field.String("project"),
)
)
// maxProjects is the max number of projects to which the pmReporter will
// scale in its current implementation.
//
// If a bigger amount is desired, batching should be implemented to avoid
// high memory usage.
const maxProjects = 500
type pmReporter struct{}
// metrics implements aggregator interface.
func (*pmReporter) metrics() []types.Metric {
return []types.Metric{
metricPMStateCLs,
metricPMStateDeps,
metricPMStateComponents,
metricPMEntitySize,
}
}
// metrics implements aggregator interface.
func (*pmReporter) prepare(ctx context.Context, activeProjects stringset.Set) (reportFunc, error) {
if activeProjects.Len() > maxProjects {
logging.Errorf(ctx, "FIXME: too many active projects (>%d) to report metrics for", maxProjects)
return nil, errors.New("too many active projects")
}
// We need to load & decode Datastore entities as prjmanager.Project objects
// in order to compute metrics like the number of components.
// But we also need to figure out the Datastore entities' size in bytes, which
// is unfortunately not easy to compute off the prjmanager.Project objects.
//
// So, load each project entity as both prjmanager.Project and the
// projectEntitySizeEstimator objects. Note, that both objects have
// datastore's Kind is exactly the same (see `gae:"$kind..."` attribute in
// both structs.
//
// Finally, behind the scenes, datastore library should optimize this to load
// only N entities instead of 2*N.
eSizes := make([]projectEntitySizeEstimator, activeProjects.Len())
entities := make([]prjmanager.Project, activeProjects.Len())
i := 0
for p := range activeProjects {
eSizes[i].ID = p
entities[i].ID = p
i++
}
err := datastore.Get(ctx, eSizes, entities)
if err != nil {
return nil, errors.Annotate(err, "failed to fetch all active projects").Tag(transient.Tag).Err()
}
stats := make([]struct {
cls, deps, components int64
}, len(entities))
for i, e := range entities {
deps := 0
for _, pcl := range e.State.GetPcls() {
deps += len(pcl.GetDeps())
}
stats[i].cls = int64(len(e.State.GetPcls()))
stats[i].deps = int64(deps)
stats[i].components = int64(len(e.State.GetComponents()))
}
// Free memory, since decoded Project entities can use be 1MB or more.
entities = nil
return func(ctx context.Context) {
for i, stat := range stats {
project := eSizes[i].ID
metricPMEntitySize.Set(ctx, eSizes[i].bytes, project)
metricPMStateCLs.Set(ctx, stat.cls, project)
metricPMStateDeps.Set(ctx, stat.deps, project)
metricPMStateComponents.Set(ctx, stat.components, project)
}
}, nil
}
type projectEntitySizeEstimator struct {
// Kind must match the kind of the prjmanager.Project entity.
_kind string `gae:"$kind,Project"`
ID string `gae:"$id"`
bytes int64
}
// Load implements datastore.PropertyLoadSaver.
func (e *projectEntitySizeEstimator) Load(m datastore.PropertyMap) error {
e.bytes = m.EstimateSize()
return nil
}
// Save implements datastore.PropertyLoadSaver.
func (e *projectEntitySizeEstimator) Save(withMeta bool) (datastore.PropertyMap, error) {
panic("not needed")
}