| // Copyright 2016 The LUCI Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package buildbot |
| |
| import ( |
| "bytes" |
| "compress/gzip" |
| "encoding/json" |
| "fmt" |
| "io/ioutil" |
| "strconv" |
| |
| "github.com/luci/gae/service/datastore" |
| "github.com/luci/luci-go/milo/api/resp" |
| "github.com/luci/luci-go/milo/common/model" |
| ) |
| |
| // This file contains all of the structs that define buildbot json endpoints. |
| // This is primarily used for unmarshalling buildbot master and build json. |
| // json.UnmarshalJSON can directly unmarshal buildbot jsons into these structs. |
| // Many of the structs were initially built using https://mholt.github.io/json-to-go/ |
| |
| // buildbotStep represents a single step in a buildbot build. |
| type buildbotStep struct { |
| // We actually don't care about ETA. This is represented as a string if |
| // it's fetched from a build json, but a float if it's dug out of the |
| // slave portion of a master json. We'll just set it to interface and |
| // ignore it. |
| Eta interface{} `json:"eta"` |
| Expectations [][]interface{} `json:"expectations"` |
| Hidden bool `json:"hidden"` |
| IsFinished bool `json:"isFinished"` |
| IsStarted bool `json:"isStarted"` |
| Logs [][]string `json:"logs"` |
| Name string `json:"name"` |
| Results []interface{} `json:"results"` |
| Statistics struct { |
| } `json:"statistics"` |
| StepNumber int `json:"step_number"` |
| Text []string `json:"text"` |
| Times []*float64 `json:"times"` |
| Urls map[string]string `json:"urls"` |
| |
| // Log link aliases. The key is a log name that is being aliases. It should, |
| // generally, exist within the Logs. The value is the set of aliases attached |
| // to that key. |
| Aliases map[string][]*buildbotLinkAlias `json:"aliases"` |
| } |
| |
| // buildbotSourceStamp is a list of changes (commits) tagged with where the changes |
| // came from, ie. the project/repository. Also includes a "main" revision." |
| type buildbotSourceStamp struct { |
| Branch *string `json:"branch"` |
| Changes []buildbotChange `json:"changes"` |
| Haspatch bool `json:"hasPatch"` |
| Project string `json:"project"` |
| Repository string `json:"repository"` |
| Revision string `json:"revision"` |
| } |
| |
| type buildbotLinkAlias struct { |
| URL string `json:"url"` |
| Text string `json:"text"` |
| } |
| |
| func (a *buildbotLinkAlias) toLink() *resp.Link { |
| return resp.NewLink(a.Text, a.URL) |
| } |
| |
| type buildbotProperty struct { |
| Name string |
| Value interface{} |
| Source string |
| } |
| |
| func (p *buildbotProperty) MarshalJSON() ([]byte, error) { |
| return json.Marshal([]interface{}{p.Name, p.Value, p.Source}) |
| } |
| |
| func (p *buildbotProperty) UnmarshalJSON(d []byte) error { |
| // The raw BuildBot representation is a slice of interfaces. |
| var raw []interface{} |
| if err := json.Unmarshal(d, &raw); err != nil { |
| return err |
| } |
| |
| switch len(raw) { |
| case 3: |
| if s, ok := raw[2].(string); ok { |
| p.Source = s |
| } |
| fallthrough |
| |
| case 2: |
| p.Value = raw[1] |
| fallthrough |
| |
| case 1: |
| if s, ok := raw[0].(string); ok { |
| p.Name = s |
| } |
| } |
| return nil |
| } |
| |
| // buildbotBuild is a single build json on buildbot. |
| type buildbotBuild struct { |
| Master string `gae:"$master"` |
| Blame []string `json:"blame" gae:"-"` |
| Buildername string `json:"builderName"` |
| // This needs to be reflected. This can be either a String or a buildbotStep. |
| Currentstep interface{} `json:"currentStep" gae:"-"` |
| // We don't care about this one. |
| Eta interface{} `json:"eta" gae:"-"` |
| Logs [][]string `json:"logs" gae:"-"` |
| Number int `json:"number"` |
| // This is a slice of tri-tuples of [property name, value, source]. |
| // property name is always a string |
| // value can be a string or float |
| // source is optional, but is always a string if present |
| Properties []*buildbotProperty `json:"properties" gae:"-"` |
| Reason string `json:"reason"` |
| Results *int `json:"results" gae:"-"` |
| Slave string `json:"slave"` |
| Sourcestamp *buildbotSourceStamp `json:"sourceStamp" gae:"-"` |
| Steps []buildbotStep `json:"steps" gae:"-"` |
| Text []string `json:"text" gae:"-"` |
| Times []*float64 `json:"times" gae:"-"` |
| // This one is injected by Milo. Does not exist in a normal json query. |
| TimeStamp *int `json:"timeStamp" gae:"-"` |
| // This one is marked by Milo, denotes whether or not the build is internal. |
| Internal bool `json:"internal" gae:"-"` |
| // This one is computed by Milo for indexing purposes. It does so by |
| // checking to see if times[1] is null or not. |
| Finished bool `json:"finished"` |
| // OS is a string representation of the OS the build ran on. This is |
| // derived best-effort from the slave information in the master JSON. |
| // This information is injected into the buildbot builds via puppet, and |
| // comes as Family + Version. Family is (windows, Darwin, Debian), while |
| // Version is the version of the OS, such as (XP, 7, 10) for windows. |
| OSFamily string `json:"osFamily"` |
| OSVersion string `json:"osVersion"` |
| } |
| |
| func (b *buildbotBuild) toStatus() model.Status { |
| var result model.Status |
| if b.Currentstep != nil { |
| result = model.Running |
| } else { |
| result = result2Status(b.Results) |
| } |
| return result |
| } |
| |
| var _ datastore.PropertyLoadSaver = (*buildbotBuild)(nil) |
| var _ datastore.MetaGetterSetter = (*buildbotBuild)(nil) |
| |
| // getID is a helper function that returns the datastore key for a given |
| // build. |
| func (b *buildbotBuild) getID() string { |
| s := []string{b.Master, b.Buildername, strconv.Itoa(b.Number)} |
| id, err := json.Marshal(s) |
| if err != nil { |
| panic(err) // This really shouldn't fail. |
| } |
| return string(id) |
| } |
| |
| // setKeys is the inverse of getID(). |
| func (b *buildbotBuild) setKeys(id string) error { |
| s := []string{} |
| err := json.Unmarshal([]byte(id), &s) |
| if err != nil { |
| return err |
| } |
| if len(s) != 3 { |
| return fmt.Errorf("%s does not have 3 items", id) |
| } |
| b.Master = s[0] |
| b.Buildername = s[1] |
| b.Number, err = strconv.Atoi(s[2]) |
| return err // or nil. |
| } |
| |
| // GetMeta is overridden so that a query for "id" calls getID() instead of |
| // the superclass method. |
| func (b *buildbotBuild) GetMeta(key string) (interface{}, bool) { |
| if key == "id" { |
| if b.Master == "" || b.Buildername == "" { |
| panic(fmt.Errorf("No Master or Builder found")) |
| } |
| return b.getID(), true |
| } |
| return datastore.GetPLS(b).GetMeta(key) |
| } |
| |
| // GetAllMeta is overridden for the same reason GetMeta() is. |
| func (b *buildbotBuild) GetAllMeta() datastore.PropertyMap { |
| p := datastore.GetPLS(b).GetAllMeta() |
| p.SetMeta("id", b.getID()) |
| return p |
| } |
| |
| // SetMeta is the inverse of GetMeta(). |
| func (b *buildbotBuild) SetMeta(key string, val interface{}) bool { |
| if key == "id" { |
| err := b.setKeys(val.(string)) |
| if err != nil { |
| panic(err) |
| } |
| } |
| return datastore.GetPLS(b).SetMeta(key, val) |
| } |
| |
| // Load translates a propertymap into the struct and loads the data into |
| // the struct. |
| func (b *buildbotBuild) Load(p datastore.PropertyMap) error { |
| if _, ok := p["data"]; !ok { |
| // This is probably from a keys-only query. No need to load the rest. |
| return datastore.GetPLS(b).Load(p) |
| } |
| gz, err := p.Slice("data")[0].Project(datastore.PTBytes) |
| if err != nil { |
| return err |
| } |
| reader, err := gzip.NewReader(bytes.NewReader(gz.([]byte))) |
| if err != nil { |
| return err |
| } |
| bs, err := ioutil.ReadAll(reader) |
| if err != nil { |
| return err |
| } |
| return json.Unmarshal(bs, b) |
| } |
| |
| func (b *buildbotBuild) getPropertyValue(name string) interface{} { |
| for _, prop := range b.Properties { |
| if prop.Name == name { |
| return prop.Value |
| } |
| } |
| return "" |
| } |
| |
| type errTooBig struct { |
| error |
| } |
| |
| // Save returns a property map of the struct to save in the datastore. It |
| // contains two fields, the ID which is the key, and a data field which is a |
| // serialized and gzipped representation of the entire struct. |
| func (b *buildbotBuild) Save(withMeta bool) (datastore.PropertyMap, error) { |
| bs, err := json.Marshal(b) |
| if err != nil { |
| return nil, err |
| } |
| gzbs := bytes.Buffer{} |
| gsw := gzip.NewWriter(&gzbs) |
| _, err = gsw.Write(bs) |
| if err != nil { |
| return nil, err |
| } |
| err = gsw.Close() |
| if err != nil { |
| return nil, err |
| } |
| blob := gzbs.Bytes() |
| // Datastore has a max size of 1MB. If the blob is over 9.5MB, it probably |
| // won't fit after accounting for overhead. |
| if len(blob) > 950000 { |
| return nil, errTooBig{ |
| fmt.Errorf("buildbotBuild: Build too big to store (%d bytes)", len(blob))} |
| } |
| p := datastore.PropertyMap{ |
| "data": datastore.MkPropertyNI(blob), |
| } |
| if withMeta { |
| p["id"] = datastore.MkPropertyNI(b.getID()) |
| p["master"] = datastore.MkProperty(b.Master) |
| p["builder"] = datastore.MkProperty(b.Buildername) |
| p["number"] = datastore.MkProperty(b.Number) |
| p["finished"] = datastore.MkProperty(b.Finished) |
| } |
| return p, nil |
| } |
| |
| type buildbotPending struct { |
| Source buildbotSourceStamp `json:"source"` |
| Reason string `json:"reason"` |
| SubmittedAt int `json:"submittedAt"` |
| BuilderName string `json:"builderName"` |
| } |
| |
| // buildbotBuilder is a builder struct from the master json, _not_ the builder json. |
| type buildbotBuilder struct { |
| Basedir string `json:"basedir"` |
| CachedBuilds []int `json:"cachedBuilds"` |
| PendingBuilds int `json:"pendingBuilds"` |
| // This one is specific to the pubsub interface. This is limited to 75, |
| // so it could differ from PendingBuilds |
| PendingBuildStates []*buildbotPending `json:"pendingBuildStates"` |
| Category string `json:"category"` |
| CurrentBuilds []int `json:"currentBuilds"` |
| Slaves []string `json:"slaves"` |
| State string `json:"state"` |
| } |
| |
| // buildbotChangeSource is a changesource (ie polling source) usually tied to a master's scheduler. |
| type buildbotChangeSource struct { |
| Description string `json:"description"` |
| } |
| |
| // buildbotChange describes a commit in a repository as part of a changesource of blamelist. |
| type buildbotChange struct { |
| At string `json:"at"` |
| Branch *string `json:"branch"` |
| Category string `json:"category"` |
| Comments string `json:"comments"` |
| // This could be a list of strings or list of struct { Name string } . |
| Files []interface{} `json:"files"` |
| Number int `json:"number"` |
| Project string `json:"project"` |
| Properties [][]interface{} `json:"properties"` |
| Repository string `json:"repository"` |
| Rev string `json:"rev"` |
| Revision string `json:"revision"` |
| Revlink string `json:"revlink"` |
| When int `json:"when"` |
| Who string `json:"who"` |
| } |
| |
| func (bc *buildbotChange) GetFiles() []string { |
| files := make([]string, 0, len(bc.Files)) |
| for _, f := range bc.Files { |
| // Buildbot stores files both as a string, or as a dict with a single entry |
| // named "name". It doesn't matter to us what the type is, but we need |
| // to reflect on the type anyways. |
| switch fn := f.(type) { |
| case string: |
| files = append(files, fn) |
| case map[string]interface{}: |
| if name, ok := fn["name"]; ok { |
| files = append(files, fmt.Sprintf("%s", name)) |
| } |
| } |
| } |
| return files |
| } |
| |
| // buildbotSlave describes a slave on a master from a master json, and also includes the |
| // full builds of any currently running builds. |
| type buildbotSlave struct { |
| // RecentBuilds is a map of builder name to a list of recent finished build |
| // numbers on that builder. |
| RecentBuilds map[string][]int `json:"builders"` |
| Connected bool `json:"connected"` |
| Host string `json:"host"` |
| Name string `json:"name"` |
| Runningbuilds []*buildbotBuild `json:"runningBuilds"` |
| Version string `json:"version"` |
| // This is like runningbuilds, but instead of storing the full build, |
| // just reference the build by builder: build num. |
| RunningbuildsMap map[string][]int `json:"runningBuildsMap"` |
| } |
| |
| type buildbotProject struct { |
| BuildbotURL string `json:"buildbotURL"` |
| Title string `json:"title"` |
| Titleurl string `json:"titleURL"` |
| } |
| |
| // buildbotMaster This is json definition for https://build.chromium.org/p/<master>/json |
| // endpoints. |
| type buildbotMaster struct { |
| AcceptingBuilds struct { |
| AcceptingBuilds bool `json:"accepting_builds"` |
| } `json:"accepting_builds"` |
| |
| Builders map[string]*buildbotBuilder `json:"builders"` |
| |
| Buildstate struct { |
| AcceptingBuilds bool `json:"accepting_builds"` |
| Builders []struct { |
| Basedir string `json:"basedir"` |
| Buildername string `json:"builderName"` |
| Cachedbuilds []int `json:"cachedBuilds"` |
| Category string `json:"category"` |
| Currentbuilds []int `json:"currentBuilds"` |
| Slaves []string `json:"slaves"` |
| State string `json:"state"` |
| } `json:"builders"` |
| Project struct { |
| BuildbotURL string `json:"buildbotURL"` |
| Title string `json:"title"` |
| Titleurl string `json:"titleURL"` |
| } `json:"project"` |
| Timestamp float64 `json:"timestamp"` |
| } `json:"buildstate"` |
| |
| ChangeSources map[string]buildbotChangeSource `json:"change_sources"` |
| |
| Changes map[string]buildbotChange `json:"changes"` |
| |
| Clock struct { |
| Current struct { |
| Local string `json:"local"` |
| Utc string `json:"utc"` |
| UtcTs float64 `json:"utc_ts"` |
| } `json:"current"` |
| ServerStarted struct { |
| Local string `json:"local"` |
| Utc string `json:"utc"` |
| UtcTs float64 `json:"utc_ts"` |
| } `json:"server_started"` |
| ServerUptime float64 `json:"server_uptime"` |
| } `json:"clock"` |
| |
| Project buildbotProject `json:"project"` |
| |
| Slaves map[string]*buildbotSlave `json:"slaves"` |
| |
| Varz struct { |
| AcceptingBuilds bool `json:"accepting_builds"` |
| Builders map[string]struct { |
| ConnectedSlaves int `json:"connected_slaves"` |
| CurrentBuilds int `json:"current_builds"` |
| PendingBuilds int `json:"pending_builds"` |
| State string `json:"state"` |
| TotalSlaves int `json:"total_slaves"` |
| } `json:"builders"` |
| ServerUptime float64 `json:"server_uptime"` |
| } `json:"varz"` |
| |
| // This is injected by the pubsub publisher on the buildbot side. |
| Name string `json:"name"` |
| } |
