Milo: Store console defs as their own entities
BUG=468053
Review-Url: https://codereview.chromium.org/2982183002
diff --git a/milo/api/config/project.pb.go b/milo/api/config/project.pb.go
index 4498728..8361154 100644
--- a/milo/api/config/project.pb.go
+++ b/milo/api/config/project.pb.go
@@ -68,8 +68,9 @@
Name string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
// RepoURL is the URL of the git repository to display as the rows of the console.
RepoURL string `protobuf:"bytes,3,opt,name=RepoURL" json:"RepoURL,omitempty"`
- // Branch is the branch to pull commits from when displaying the console.
- Branch string `protobuf:"bytes,4,opt,name=Branch" json:"Branch,omitempty"`
+ // Ref is the ref to pull commits from when displaying the console.
+ // Eg. refs/heads/master
+ Ref string `protobuf:"bytes,4,opt,name=Ref" json:"Ref,omitempty"`
// ManifestName is the name of the manifest the waterfall looks at.
// By convention, Manifest Names can be:
// * UNPATCHED - For non patched builds, such as continuous builds
@@ -105,9 +106,9 @@
return ""
}
-func (m *Console) GetBranch() string {
+func (m *Console) GetRef() string {
if m != nil {
- return m.Branch
+ return m.Ref
}
return ""
}
@@ -177,22 +178,22 @@
}
var fileDescriptor0 = []byte{
- // 263 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xd1, 0x4a, 0xc3, 0x30,
- 0x14, 0x86, 0x69, 0x37, 0xdb, 0xed, 0x28, 0x0a, 0xb9, 0x90, 0x20, 0x5e, 0x8c, 0x5e, 0x0d, 0xc4,
- 0x06, 0x14, 0x5f, 0x60, 0x1b, 0xc2, 0x40, 0x45, 0x22, 0xe2, 0x75, 0x16, 0xb3, 0x36, 0xd2, 0xf6,
- 0x94, 0x34, 0xbd, 0xf0, 0xad, 0x7c, 0x44, 0x31, 0x89, 0xd5, 0xb2, 0x9b, 0x72, 0xfe, 0xef, 0x3f,
- 0x3d, 0x7c, 0x04, 0xee, 0x0a, 0x6d, 0xcb, 0x7e, 0x97, 0x4b, 0xac, 0x59, 0xd5, 0x4b, 0xed, 0x3e,
- 0xd7, 0x05, 0xb2, 0x5a, 0x57, 0xc8, 0x44, 0xab, 0x99, 0xc4, 0x66, 0xaf, 0x0b, 0xd6, 0x1a, 0xfc,
- 0x50, 0xd2, 0xe6, 0xad, 0x41, 0x8b, 0x24, 0xf1, 0x34, 0xbb, 0x87, 0xf4, 0xd9, 0x17, 0xe4, 0x14,
- 0xe2, 0xed, 0x86, 0x46, 0x8b, 0x68, 0x39, 0xe7, 0xf1, 0x76, 0x43, 0xae, 0x60, 0xb6, 0xc6, 0xa6,
- 0xc3, 0x4a, 0x75, 0x34, 0x5e, 0x4c, 0x96, 0xc7, 0x37, 0x67, 0xb9, 0xff, 0x2b, 0x0f, 0x9c, 0x0f,
- 0x0b, 0xd9, 0x57, 0x04, 0x69, 0x08, 0x07, 0x87, 0x08, 0x4c, 0x9f, 0x44, 0xad, 0x68, 0xec, 0x88,
- 0x9b, 0x09, 0x85, 0x94, 0xab, 0x16, 0x5f, 0xf9, 0x03, 0x9d, 0x38, 0xfc, 0x1b, 0xc9, 0x39, 0x24,
- 0x2b, 0x23, 0x1a, 0x59, 0xd2, 0xa9, 0x2b, 0x42, 0x22, 0x19, 0x9c, 0x3c, 0x8a, 0x46, 0xef, 0x55,
- 0x67, 0xdd, 0xb5, 0x23, 0xd7, 0x8e, 0xd8, 0x8f, 0xf2, 0xaa, 0xd7, 0xd5, 0xbb, 0x32, 0x1d, 0x4d,
- 0xc6, 0xca, 0x81, 0xf3, 0x61, 0x21, 0x7b, 0x83, 0x34, 0xcc, 0x83, 0x61, 0xf4, 0xcf, 0xf0, 0x02,
- 0x66, 0x6b, 0x61, 0x55, 0x81, 0xe6, 0x33, 0x98, 0x0f, 0x99, 0x5c, 0xc2, 0xfc, 0xa5, 0x44, 0xe3,
- 0x45, 0xbc, 0xff, 0x1f, 0xd8, 0x25, 0xee, 0x89, 0x6f, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xfb,
- 0x59, 0x08, 0x73, 0x9b, 0x01, 0x00, 0x00,
+ // 260 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xc1, 0x4a, 0xc3, 0x40,
+ 0x10, 0x86, 0x49, 0x52, 0x93, 0x76, 0x14, 0x95, 0x3d, 0x2d, 0xe2, 0xa1, 0xe4, 0x54, 0x10, 0xb3,
+ 0xa0, 0xf8, 0x02, 0xb6, 0x08, 0x05, 0x15, 0x59, 0x11, 0xcf, 0x69, 0xdc, 0xa4, 0x2b, 0x49, 0x26,
+ 0x6c, 0x36, 0x07, 0x5f, 0xc9, 0xa7, 0x94, 0xce, 0xae, 0xd1, 0xd2, 0x4b, 0xf8, 0xff, 0x6f, 0x26,
+ 0xc3, 0xc7, 0xc2, 0x5d, 0xa5, 0xed, 0x76, 0xd8, 0x64, 0x05, 0x36, 0xa2, 0x1e, 0x0a, 0x4d, 0x9f,
+ 0xeb, 0x0a, 0x45, 0xa3, 0x6b, 0x14, 0x79, 0xa7, 0x45, 0x81, 0x6d, 0xa9, 0x2b, 0xd1, 0x19, 0xfc,
+ 0x54, 0x85, 0xcd, 0x3a, 0x83, 0x16, 0x59, 0xec, 0x68, 0xfa, 0x00, 0xc9, 0x8b, 0x1b, 0xb0, 0x53,
+ 0x08, 0xd7, 0x2b, 0x1e, 0xcc, 0x83, 0xc5, 0x4c, 0x86, 0xeb, 0x15, 0xbb, 0x82, 0xe9, 0x12, 0xdb,
+ 0x1e, 0x6b, 0xd5, 0xf3, 0x70, 0x1e, 0x2d, 0x8e, 0x6f, 0xce, 0x32, 0xf7, 0x57, 0xe6, 0xb9, 0x1c,
+ 0x17, 0xd2, 0xef, 0x00, 0x12, 0x5f, 0x0e, 0x0e, 0x31, 0x98, 0x3c, 0xe7, 0x8d, 0xe2, 0x21, 0x11,
+ 0xca, 0x8c, 0x43, 0x22, 0x55, 0x87, 0x6f, 0xf2, 0x91, 0x47, 0x84, 0x7f, 0x2b, 0x3b, 0x87, 0x48,
+ 0xaa, 0x92, 0x4f, 0x88, 0xee, 0x22, 0x4b, 0xe1, 0xe4, 0x29, 0x6f, 0x75, 0xa9, 0x7a, 0x4b, 0x77,
+ 0x8e, 0x68, 0xb4, 0xc7, 0x76, 0xb2, 0xf7, 0x83, 0xae, 0x3f, 0x94, 0xe9, 0x79, 0xbc, 0x2f, 0xeb,
+ 0xb9, 0x1c, 0x17, 0xd2, 0x77, 0x48, 0x7c, 0x1e, 0xdd, 0x82, 0x7f, 0x6e, 0x17, 0x30, 0x5d, 0xe6,
+ 0x56, 0x55, 0x68, 0xbe, 0xbc, 0xf3, 0xd8, 0xd9, 0x25, 0xcc, 0x5e, 0xb7, 0x68, 0x9c, 0x88, 0x33,
+ 0xff, 0x03, 0x9b, 0x98, 0x1e, 0xf7, 0xf6, 0x27, 0x00, 0x00, 0xff, 0xff, 0x83, 0x36, 0x82, 0xe0,
+ 0x95, 0x01, 0x00, 0x00,
}
diff --git a/milo/api/config/project.proto b/milo/api/config/project.proto
index 163feff..aa8ad6a 100644
--- a/milo/api/config/project.proto
+++ b/milo/api/config/project.proto
@@ -28,8 +28,9 @@
// RepoURL is the URL of the git repository to display as the rows of the console.
string RepoURL = 3;
- // Branch is the branch to pull commits from when displaying the console.
- string Branch = 4;
+ // Ref is the ref to pull commits from when displaying the console.
+ // Eg. refs/heads/master
+ string Ref = 4;
// ManifestName is the name of the manifest the waterfall looks at.
// By convention, Manifest Names can be:
diff --git a/milo/api/resp/build.go b/milo/api/resp/build.go
index b316f18..174f328 100644
--- a/milo/api/resp/build.go
+++ b/milo/api/resp/build.go
@@ -315,7 +315,7 @@
Name: "REVISION",
ID: []byte(rb.SourceStamp.Revision.Label),
})
- consoles, err := common.GetConsolesForBuilder(c, bs.BuilderID)
+ consoles, err := common.GetAllConsoles(c, bs.BuilderID)
if err != nil {
return err
}
@@ -324,7 +324,7 @@
// definitions will specify their manifest as "REVISION", and we'll do
// lookups with null URL fields.
bs.AddManifestKey(
- con.ProjectID, con.Console.Name, "REVISION", "", revisionBytes)
+ con.GetProjectName(), con.ID, "REVISION", "", revisionBytes)
}
}
}
diff --git a/milo/buildsource/buildbot/pubsub_test.go b/milo/buildsource/buildbot/pubsub_test.go
index 3952b09..f02b6f0 100644
--- a/milo/buildsource/buildbot/pubsub_test.go
+++ b/milo/buildsource/buildbot/pubsub_test.go
@@ -472,8 +472,6 @@
})
So(h.Code, ShouldEqual, 200)
Convey("And stores correctly", func() {
- err := common.UpdateProjectConfigs(c)
- So(err, ShouldBeNil)
c = auth.WithState(c, &authtest.FakeState{
Identity: "user:alicebob@google.com",
IdentityGroups: []string{"googlers", "all"},
diff --git a/milo/buildsource/console.go b/milo/buildsource/console.go
index e8840c3..f38abdb 100644
--- a/milo/buildsource/console.go
+++ b/milo/buildsource/console.go
@@ -25,7 +25,7 @@
"github.com/luci/luci-go/common/errors"
"github.com/luci/luci-go/common/sync/parallel"
- "github.com/luci/luci-go/milo/api/config"
+ "github.com/luci/luci-go/milo/common"
"github.com/luci/luci-go/milo/common/model"
)
@@ -41,7 +41,7 @@
// GetConsoleRows returns a row-oriented collection of BuildSummary
// objects. Each row corresponds to the similarly-indexed commit in the
// `commits` slice.
-func GetConsoleRows(c context.Context, project string, console *config.Console, commits, builders []string) ([]*ConsoleRow, error) {
+func GetConsoleRows(c context.Context, project string, console *common.Console, commits, builders []string) ([]*ConsoleRow, error) {
rawCommits := make([][]byte, len(commits))
for i, c := range commits {
var err error
@@ -60,7 +60,7 @@
if console.ManifestName == "REVISION" {
url = ""
}
- partialKey := model.NewPartialManifestKey(project, console.Name, console.ManifestName, url)
+ partialKey := model.NewPartialManifestKey(project, console.ID, console.ManifestName, url)
q := datastore.NewQuery("BuildSummary").KeysOnly(true)
err := parallel.WorkPool(4, func(ch chan<- func() error) {
for i := range rawCommits {
diff --git a/milo/common/acl_test.go b/milo/common/acl_test.go
index 770e8a0..bf8e5ca 100644
--- a/milo/common/acl_test.go
+++ b/milo/common/acl_test.go
@@ -38,7 +38,7 @@
Convey("Set up projects", func() {
c = testconfig.WithCommonClient(c, memcfg.New(aclConfgs))
- err := UpdateProjectConfigs(c)
+ err := UpdateConsoles(c)
So(err, ShouldBeNil)
Convey("Anon wants to...", func() {
diff --git a/milo/common/config.go b/milo/common/config.go
index 42097a9..dc0ef8d 100644
--- a/milo/common/config.go
+++ b/milo/common/config.go
@@ -16,6 +16,7 @@
import (
"fmt"
+ "strings"
"time"
"github.com/golang/protobuf/proto"
@@ -23,24 +24,81 @@
"github.com/luci/gae/service/datastore"
"github.com/luci/gae/service/info"
+ configInterface "github.com/luci/luci-go/common/config"
"github.com/luci/luci-go/common/data/caching/proccache"
+ "github.com/luci/luci-go/common/data/stringset"
"github.com/luci/luci-go/common/errors"
"github.com/luci/luci-go/common/logging"
"github.com/luci/luci-go/luci_config/server/cfgclient"
"github.com/luci/luci-go/luci_config/server/cfgclient/backend"
- "github.com/luci/luci-go/luci_config/server/cfgclient/textproto"
"github.com/luci/luci-go/milo/api/config"
)
-// Project is a LUCI project.
-type Project struct {
- // ID of the project, as per self defined. This is the luci-config name.
+// Console is af datastore entity representing a single console.
+type Console struct {
+ // Parent is a key to the parent Project entity where this console was
+ // defined in.
+ Parent *datastore.Key `gae:"$parent"`
+ // ID is the ID of the console.
ID string `gae:"$id"`
- // Data is the Project data in protobuf binary format.
- Data []byte `gae:",noindex"`
- // Revision is the latest revision we have of this project's config.
+ // RepoURL and Ref combined defines the commits the show up on the left
+ // hand side of a Console.
+ RepoURL string
+ // RepoURL and Ref combined defines the commits the show up on the left
+ // hand side of a Console.
+ Ref string
+ // ManifestName is the name of the manifest to look for when querying for
+ // builds under this console.
+ ManifestName string
+ // URL is the URL to the luci-config definition of this console.
+ URL string
+ // Revision is the luci-config reivision from when this Console was retrieved.
Revision string
+ // Builders is a list of universal builder IDs.
+ Builders []string
+}
+
+// GetProjectName retrieves the project name of the console out of the Console's
+// parent key.
+func (con *Console) GetProjectName() string {
+ return con.Parent.StringID()
+}
+
+// NewConsole creates a fully populated console out of the luci-config proto
+// definition of a console.
+func NewConsole(project *datastore.Key, URL, revision string, con *config.Console) *Console {
+ return &Console{
+ Parent: project,
+ ID: con.ID,
+ RepoURL: con.RepoURL,
+ Ref: con.Ref,
+ ManifestName: con.ManifestName,
+ Revision: revision,
+ URL: URL,
+ Builders: BuilderFromProto(con.Builders),
+ }
+}
+
+// BuilderFromProto tranforms a luci-config proto builder format into the datastore
+// format.
+func BuilderFromProto(cb []*config.Builder) []string {
+ builders := make([]string, len(cb))
+ for i, b := range cb {
+ builders[i] = b.Name
+ }
+ return builders
+}
+
+// LuciConfigURL returns a user friendly URL that specifies where to view
+// this console definition.
+func LuciConfigURL(c context.Context, configSet, path, revision string) string {
+ // TODO(hinoka): This shouldn't be hardcoded, instead we should get the
+ // luci-config instance from the context. But we only use this instance at
+ // the moment so it is okay for now.
+ // TODO(hinoka): The UI doesn't allow specifying paths and revision yet. Add
+ // that in when it is supported.
+ return fmt.Sprintf("https://luci-config.appspot.com/newui#/%s", configSet)
}
// The key for the service config entity in datastore.
@@ -178,149 +236,144 @@
return settings, nil
}
-// UpdateProjectConfigs internal project configuration based off luci-config.
-// update updates Milo's configuration based off luci config. This includes
-// scanning through all project and extract all console configs.
-func UpdateProjectConfigs(c context.Context) error {
+// updateProjectConsoles updates all of the consoles for a given project,
+// and then returns a set of known console names.
+func updateProjectConsoles(c context.Context, projectName string, cfg *configInterface.Config) (stringset.Set, error) {
+ proj := config.Project{}
+ if err := proto.UnmarshalText(cfg.Content, &proj); err != nil {
+ return nil, errors.Annotate(err, "unmarshalling proto").Err()
+ }
+
+ // Keep a list of known consoles so we can prune deleted ones later.
+ knownConsoles := stringset.New(len(proj.Consoles))
+ // Iterate through all the proto consoles, adding and replacing the
+ // known ones if needed.
+ parentKey := datastore.MakeKey(c, "Project", projectName)
+ for _, pc := range proj.Consoles {
+ knownConsoles.Add(pc.ID)
+ con, err := GetConsole(c, projectName, pc.ID)
+ switch err {
+ case datastore.ErrNoSuchEntity:
+ // continue
+ case nil:
+ // Check if revisions match, if so just skip it.
+ if con.Revision == cfg.Revision {
+ continue
+ }
+ default:
+ return nil, errors.Annotate(err, "checking %s", pc.ID).Err()
+ }
+ URL := LuciConfigURL(c, cfg.ConfigSet, cfg.Path, cfg.Revision)
+ con = NewConsole(parentKey, URL, cfg.Revision, pc)
+ if err = datastore.Put(c, con); err != nil {
+ return nil, errors.Annotate(err, "saving %s", pc.ID).Err()
+ } else {
+ logging.Infof(c, "saved a new %s / %s (revision %s)", projectName, con.ID, cfg.Revision)
+ }
+ }
+ return knownConsoles, nil
+}
+
+// UpdateConsoles updates internal console definitions entities based off luci-config.
+func UpdateConsoles(c context.Context) error {
cfgName := info.AppID(c) + ".cfg"
- var (
- configs []*config.Project
- metas []*cfgclient.Meta
- merr errors.MultiError
- )
-
logging.Debugf(c, "fetching configs for %s", cfgName)
- if err := cfgclient.Projects(c, cfgclient.AsService, cfgName, textproto.Slice(&configs), &metas); err != nil {
- merr = err.(errors.MultiError)
- // Some configs errored out, but we can continue with the ones that work still.
+ // Acquire the raw config client.
+ lucicfg := backend.Get(c).GetConfigInterface(c, backend.AsService)
+ // Project configs for Milo contains console definitions.
+ configs, err := lucicfg.GetProjectConfigs(c, cfgName, false)
+ if err != nil {
+ return errors.Annotate(err, "while fetching project configs").Err()
}
+ logging.Infof(c, "got %d project configs", len(configs))
- // A map of project ID to project.
- projects := map[string]*Project{}
- for i, proj := range configs {
- meta := metas[i]
- projectName, _, _ := meta.ConfigSet.SplitProject()
- name := string(projectName)
- projects[name] = nil
- if merr != nil && merr[i] != nil {
- logging.WithError(merr[i]).Warningf(c, "skipping %s due to error", name)
- continue
+ merr := errors.MultiError{}
+ knownProjects := map[string]stringset.Set{}
+ // Iterate through each project config, extracting the console definition.
+ for _, cfg := range configs {
+ // This looks like "projects/<project name>"
+ splitPath := strings.SplitN(cfg.ConfigSet, "/", 2)
+ if len(splitPath) != 2 {
+ return fmt.Errorf("Invalid config set path %s", cfg.ConfigSet)
}
-
- p := &Project{
- ID: name,
- Revision: meta.Revision,
- }
-
- logging.Infof(c, "prossing %s", name)
-
- var err error
- p.Data, err = proto.Marshal(proj)
- if err != nil {
- logging.WithError(err).Errorf(c, "Encountered error while processing project %s", name)
- // Set this to nil to signal this project exists but we don't want to update it.
- projects[name] = nil
- continue
- }
- projects[name] = p
- }
-
- // Now load all the data into the datastore.
- projs := make([]*Project, 0, len(projects))
- for _, proj := range projects {
- if proj != nil {
- projs = append(projs, proj)
+ projectName := splitPath[1]
+ knownProjects[projectName] = nil
+ if kp, err := updateProjectConsoles(c, projectName, &cfg); err != nil {
+ err = errors.Annotate(err, "processing project %s", cfg.ConfigSet).Err()
+ merr = append(merr, err)
+ } else {
+ knownProjects[projectName] = kp
}
}
- if err := datastore.Put(c, projs); err != nil {
- return err
+
+ // Delete all the consoles that no longer exists or are part of deleted projects.
+ toDelete := []*datastore.Key{}
+ err = datastore.Run(c, datastore.NewQuery("Console"), func(key *datastore.Key) error {
+ proj := key.Parent().StringID()
+ id := key.StringID()
+ // If this console is either:
+ // 1. In a project that no longer exists, or
+ // 2. Not in the project, then delete it.
+ knownConsoles, ok := knownProjects[proj]
+ if !ok {
+ logging.Infof(
+ c, "deleting %s/%s because the project no longer exists", proj, id)
+ toDelete = append(toDelete, key)
+ return nil
+ }
+ if knownConsoles == nil {
+ // The project exists but we couldn't check it this time. Skip it and
+ // try again the next cron cycle.
+ return nil
+ }
+ if !knownConsoles.Has(id) {
+ logging.Infof(
+ c, "deleting %s/%s because the console no longer exists", proj, id)
+ toDelete = append(toDelete, key)
+ }
+ return nil
+ })
+ if err != nil {
+ merr = append(merr, err)
+ } else if err := datastore.Delete(c, toDelete); err != nil {
+ merr = append(merr, err)
}
- // Delete entries that no longer exist.
- q := datastore.NewQuery("Project").KeysOnly(true)
- allProjs := []Project{}
- datastore.GetAll(c, q, &allProjs)
- toDelete := []Project{}
- for _, proj := range allProjs {
- if _, ok := projects[proj.ID]; !ok {
- toDelete = append(toDelete, proj)
- }
+ // Print some stats.
+ processedConsoles := 0
+ for _, cons := range knownProjects {
+ processedConsoles += cons.Len()
}
- return datastore.Delete(c, toDelete)
+ logging.Infof(
+ c, "processed %d consoles over %d projects", len(knownProjects), processedConsoles)
+
+ if len(merr) == 0 {
+ return nil
+ }
+ return merr
}
-// GetAllProjects returns all registered projects.
-func GetAllProjects(c context.Context) ([]*config.Project, error) {
- q := datastore.NewQuery("Project")
+// GetAllConsoles returns all registered projects with the builder name.
+// If builderName is empty, then this retrieves all Consoles.
+func GetAllConsoles(c context.Context, builderName string) ([]*Console, error) {
+ q := datastore.NewQuery("Console")
+ if builderName != "" {
+ q = q.Eq("Builders", builderName)
+ }
q.Order("ID")
-
- ps := []*Project{}
- err := datastore.GetAll(c, q, &ps)
- if err != nil {
- return nil, err
- }
- results := make([]*config.Project, len(ps))
- for i, p := range ps {
- results[i] = &config.Project{}
- if err := proto.Unmarshal(p.Data, results[i]); err != nil {
- return nil, err
- }
- }
- return results, nil
+ con := []*Console{}
+ err := datastore.GetAll(c, q, &con)
+ return con, err
}
-// GetProject returns the requested project.
-func GetProject(c context.Context, projName string) (*config.Project, error) {
- // Next, Try datastore
- p := Project{ID: projName}
- if err := datastore.Get(c, &p); err != nil {
- return nil, err
+// GetConsole returns the requested console.
+func GetConsole(c context.Context, proj, id string) (*Console, error) {
+ // TODO(hinoka): Memcache this.
+ con := Console{
+ Parent: datastore.MakeKey(c, "Project", proj),
+ ID: id,
}
- mp := config.Project{}
- if err := proto.Unmarshal(p.Data, &mp); err != nil {
- return nil, err
- }
-
- return &mp, nil
-}
-
-// GetConsole returns the requested console instance.
-func GetConsole(c context.Context, projName, consoleName string) (*config.Console, error) {
- p, err := GetProject(c, projName)
- if err != nil {
- return nil, err
- }
- for _, cs := range p.Consoles {
- if cs.Name == consoleName {
- return cs, nil
- }
- }
- return nil, fmt.Errorf("Console %s not found in project %s", consoleName, projName)
-}
-
-// ProjectConsole is a simple tuple type for GetConsolesForBuilder.
-type ProjectConsole struct {
- ProjectID string
- Console *config.Console
-}
-
-// GetConsolesForBuilder retrieves all the console definitions that this builder
-// belongs to.
-func GetConsolesForBuilder(c context.Context, builderName string) ([]*ProjectConsole, error) {
- projs, err := GetAllProjects(c)
- if err != nil {
- return nil, err
- }
- ret := []*ProjectConsole{}
- for _, p := range projs {
- for _, con := range p.Consoles {
- for _, b := range con.Builders {
- if b.Name == builderName {
- ret = append(ret, &ProjectConsole{p.ID, con})
- }
- }
- }
- }
- return ret, nil
+ err := datastore.Get(c, &con)
+ return &con, err
}
diff --git a/milo/common/config_test.go b/milo/common/config_test.go
index ee1863b..89260f3 100644
--- a/milo/common/config_test.go
+++ b/milo/common/config_test.go
@@ -61,19 +61,12 @@
_, err := UpdateServiceConfig(c)
So(err, ShouldBeNil)
// Send update here
- err = UpdateProjectConfigs(c)
- So(err, ShouldBeNil)
-
- Convey("Check Project config updated", func() {
- p, err := GetProject(c, "foo")
- So(err, ShouldBeNil)
- So(p.ID, ShouldEqual, "foo")
- })
+ So(UpdateConsoles(c), ShouldBeNil)
Convey("Check Console config updated", func() {
cs, err := GetConsole(c, "foo", "default")
So(err, ShouldBeNil)
- So(cs.Name, ShouldEqual, "default")
+ So(cs.ID, ShouldEqual, "default")
So(cs.RepoURL, ShouldEqual, "https://chromium.googlesource.com/foo/bar")
})
})
@@ -83,9 +76,9 @@
var fooCfg = `
ID: "foo"
Consoles: {
- Name: "default"
+ ID: "default"
RepoURL: "https://chromium.googlesource.com/foo/bar"
- Branch: "master"
+ Ref: "master"
Builders: {
Name: "buildbucket/luci.foo.something/bar"
Category: "main|something"
diff --git a/milo/frontend/appengine/templates/pages/configs.html b/milo/frontend/appengine/templates/pages/configs.html
index 237ec9c..6592983 100644
--- a/milo/frontend/appengine/templates/pages/configs.html
+++ b/milo/frontend/appengine/templates/pages/configs.html
@@ -7,31 +7,17 @@
<a href="{{ .Navi.SiteTitle.URL }}">{{ .Navi.SiteTitle.Label }}</a>
</div>
<div class="content">
- <h1>All Projects</h1>
- <h2>These are all of the projects and their settings as defined by luci config</h2>
- {{ range .Projects }}
- <h3> {{ .ID }} </h3>
- <h4> Readers </h4>
- <ul>
- {{ range .Readers }}
- <li> {{ . }} </li>
- {{ end }}
- </ul>
- {{ range .Consoles }}
- <h4> Console: {{ .Name }} </h4>
- <p> <b>ID</b>: {{ .ID }} </p>
- <p> <b>Repo</b>: {{ .RepoURL }} </p>
- <p> <b>Branch</b>: {{ .Branch}} </p>
- <ul>
- {{ range .Builders }}
- <li> <b>{{ .Module }}/{{ .Name }}</b>: {{ .Category }} - {{ .ShortName }} </li>
- {{ end }}
- </ul>
- {{ else }}
- <h4> No consoles defined </h4>
- {{ end }}
+ <h1>All Consoles</h1>
+ {{ range .Consoles }}
+ <h4> {{ .GetProjectName }} / {{ .ID }} </h4>
+ <p> <b>Repo</b>: {{ .RepoURL }} </p>
+ <p> <b>Ref</b>: {{ .Ref }} </p>
+ <p> <b>Manifest Name</b>: {{ .ManifestName }} </p>
+ <p> <b>URL</b>: {{ .URL }} </p>
+ <p> <b>Revision</b>: {{ .Revision }} </p>
+ <ul>{{ range .Builders }}<li>{{ . }}</li>{{ end }}</ul>
{{ else }}
- <h3> No projects defined </h3>
+ <h4> No consoles defined </h4>
{{ end }}
<h1>Service Configs</h1>
diff --git a/milo/frontend/view_config.go b/milo/frontend/view_config.go
index fe6c391..9797016 100644
--- a/milo/frontend/view_config.go
+++ b/milo/frontend/view_config.go
@@ -29,7 +29,7 @@
// ConfigsHandler renders the page showing the currently loaded set of luci-configs.
func ConfigsHandler(c *router.Context) {
- projects, err := common.GetAllProjects(c.Context)
+ consoles, err := common.GetAllConsoles(c.Context, "")
if err != nil {
ErrorHandler(c, errors.Annotate(err, "Error while getting projects").Err())
return
@@ -41,7 +41,7 @@
}
templates.MustRender(c.Context, c.Writer, "pages/configs.html", templates.Args{
- "Projects": projects,
+ "Consoles": consoles,
"ServiceConfig": sc,
})
}
@@ -51,7 +51,7 @@
c, h := ctx.Context, ctx.Writer
// Needed to access the PubSub API
c = appengine.WithContext(c, ctx.Request)
- projErr := common.UpdateProjectConfigs(c)
+ projErr := common.UpdateConsoles(c)
if projErr != nil {
logging.WithError(projErr).Errorf(c, "project update handler encountered error")
}
diff --git a/milo/frontend/view_console.go b/milo/frontend/view_console.go
index a5cb312..1150d21 100644
--- a/milo/frontend/view_console.go
+++ b/milo/frontend/view_console.go
@@ -19,7 +19,6 @@
"fmt"
"html/template"
"net/http"
- "strings"
"golang.org/x/net/context"
@@ -30,7 +29,6 @@
"github.com/luci/luci-go/server/router"
"github.com/luci/luci-go/server/templates"
- "github.com/luci/luci-go/milo/api/config"
"github.com/luci/luci-go/milo/api/resp"
"github.com/luci/luci-go/milo/buildsource"
"github.com/luci/luci-go/milo/common"
@@ -42,7 +40,7 @@
// If the user is not a reader of the project, this will return a 404.
// TODO(hinoka): If the user is not a reader of any of of the builders returned,
// that builder will be removed from list of results.
-func getConsoleDef(c context.Context, project, name string) (*config.Console, error) {
+func getConsoleDef(c context.Context, project, name string) (*common.Console, error) {
cs, err := common.GetConsole(c, project, name)
if err != nil {
return nil, err
@@ -57,7 +55,7 @@
if err != nil {
return nil, err
}
- commitInfo, err := git.GetHistory(c, def.RepoURL, def.Branch, 25)
+ commitInfo, err := git.GetHistory(c, def.RepoURL, def.Ref, 25)
if err != nil {
return nil, err
}
@@ -67,10 +65,9 @@
builderNames := make([]string, len(def.Builders))
builders := make([]resp.BuilderRef, len(def.Builders))
for i, b := range def.Builders {
- builderNames[i] = b.Name
- builders[i].Name = b.Name
- builders[i].Category = strings.Split(b.Category, "|")
- builders[i].ShortName = b.ShortName
+ builderNames[i] = b
+ builders[i].Name = b
+ // TODO(hinoka): Add Categories and ShortNames back in.
}
commitNames := make([]string, len(commitInfo.Commits))
@@ -92,7 +89,7 @@
AuthorEmail: commit.AuthorEmail,
CommitTime: google.TimeFromProto(commit.CommitTime),
Repo: def.RepoURL,
- Branch: def.Branch,
+ Branch: def.Ref, // TODO(hinoka): Actually this doesn't match, change branch to ref.
Description: commit.Msg,
Revision: resp.NewLink(commitNames[row], def.RepoURL+"/+/"+commitNames[row]),
}
@@ -106,7 +103,7 @@
}
return &resp.Console{
- Name: def.Name,
+ Name: def.ID,
Commit: ccb,
BuilderRef: builders,
}, nil
