blob: 7881f6165f14d71911fd840f8e73bcaeb6c3c4bb [file] [log] [blame]
// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package notify
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"go.chromium.org/luci/appengine/gaetesting"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/common/logging/memlogger"
"go.chromium.org/luci/gae/service/datastore"
notifypb "go.chromium.org/luci/luci_notify/api/config"
"go.chromium.org/luci/luci_notify/config"
. "github.com/smartystreets/goconvey/convey"
)
// fakeTreeStatusClient simulates the behaviour of a real tree status instance,
// but locally, in-memory.
type fakeTreeStatusClient struct {
statusForHosts map[string]treeStatus
nextKey int64
mu sync.Mutex
}
func (ts *fakeTreeStatusClient) getStatus(c context.Context, host string) (*treeStatus, error) {
ts.mu.Lock()
defer ts.mu.Unlock()
status, exists := ts.statusForHosts[host]
if exists {
return &status, nil
}
return nil, errors.New(fmt.Sprintf("No status for host %s", host))
}
func (ts *fakeTreeStatusClient) postStatus(c context.Context, host, message string, prevKey int64) error {
ts.mu.Lock()
defer ts.mu.Unlock()
currStatus, exists := ts.statusForHosts[host]
if exists && currStatus.key != prevKey {
return errors.New(fmt.Sprintf(
"prevKey %q passed to postStatus doesn't match previously stored key %q",
prevKey, currStatus.key))
}
key := ts.nextKey
ts.nextKey++
var status config.TreeCloserStatus
if strings.Contains(message, "close") {
status = config.Closed
} else {
status = config.Open
}
ts.statusForHosts[host] = treeStatus{
"buildbot@chromium.org", message, key, status, time.Now(),
}
return nil
}
func TestUpdateTrees(t *testing.T) {
Convey("Test environment", t, func() {
c := gaetesting.TestingContextWithAppID("luci-notify-test")
datastore.GetTestable(c).Consistent(true)
c = memlogger.Use(c)
log := logging.Get(c).(*memlogger.MemLogger)
project1 := &config.Project{Name: "chromium", TreeClosingEnabled: true}
project1Key := datastore.KeyForObj(c, project1)
builder1 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder1"}
builder2 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder2"}
builder3 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder3"}
builder4 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder4"}
project2 := &config.Project{Name: "infra", TreeClosingEnabled: false}
project2Key := datastore.KeyForObj(c, project2)
builder5 := &config.Builder{ProjectKey: project2Key, ID: "ci/builder5"}
builder6 := &config.Builder{ProjectKey: project2Key, ID: "ci/builder6"}
So(datastore.Put(c, project1, builder1, builder2, builder3, builder4, project2, builder5, builder6), ShouldBeNil)
earlierTime := time.Now().AddDate(-1, 0, 0).UTC()
evenEarlierTime := time.Now().AddDate(-2, 0, 0).UTC()
cleanup := func() {
var treeClosers []*config.TreeCloser
So(datastore.GetAll(c, datastore.NewQuery("TreeClosers"), &treeClosers), ShouldBeNil)
datastore.Delete(c, treeClosers)
}
// Helper function for basic tests. Sets an initial tree state, adds two tree closers
// for the tree, and checks that updateTrees sets the tree to the correct state.
testUpdateTrees := func(initialTreeStatus, builder1Status, builder2Status, expectedStatus config.TreeCloserStatus) {
var statusMessage string
if initialTreeStatus == config.Open {
statusMessage = "Open for business"
} else {
statusMessage = "Closed up"
}
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: statusMessage,
key: -1,
status: initialTreeStatus,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: builder1Status,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: builder2Status,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, expectedStatus)
}
Convey("Open, both TCs failing, closes", func() {
testUpdateTrees(config.Open, config.Closed, config.Closed, config.Closed)
})
Convey("Open, 1 failing & 1 passing TC, closes", func() {
testUpdateTrees(config.Open, config.Closed, config.Open, config.Closed)
})
Convey("Open, both TCs passing, stays open", func() {
testUpdateTrees(config.Open, config.Open, config.Open, config.Open)
})
Convey("Closed, both TCs failing, stays closed", func() {
testUpdateTrees(config.Closed, config.Closed, config.Closed, config.Closed)
})
Convey("Closed, 1 failing & 1 passing TC, stays closed", func() {
testUpdateTrees(config.Closed, config.Closed, config.Open, config.Closed)
})
Convey("Closed, both TCs, stays closed", func() {
testUpdateTrees(config.Closed, config.Closed, config.Open, config.Closed)
})
Convey("Closed manually, doesn't re-open", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: "somedev@chromium.org",
message: "Closed because of reasons",
key: -1,
status: config.Closed,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Closed)
})
Convey("Opened manually, stays open with no new failures", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: "somedev@chromium.org",
message: "Opened, because I feel like it",
key: -1,
status: config.Open,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: evenEarlierTime,
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldEqual, "Opened, because I feel like it")
})
Convey("Opened manually, closes on new failure", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: "somedev@chromium.org",
message: "Opened, because I feel like it",
key: -1,
status: config.Open,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Closed)
})
Convey("Multiple trees", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: "Closed up",
key: -1,
status: config.Closed,
timestamp: evenEarlierTime,
},
"v8-status.appspot.com": {
username: botUsername,
message: "Open for business",
key: -1,
status: config.Open,
timestamp: evenEarlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder3),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "v8-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder3),
TreeStatusHost: "v8-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder4),
TreeStatusHost: "v8-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: earlierTime,
Message: "Correct message",
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
status, err = ts.getStatus(c, "v8-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Closed)
So(status.message, ShouldEqual, "Tree is closed (Automatic: Correct message)")
})
Convey("Doesn't close when build is older than last status update", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: "somedev@chromium.org",
message: "Opened, because I feel like it",
key: -1,
status: config.Open,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: evenEarlierTime,
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldEqual, "Opened, because I feel like it")
})
Convey("Doesn't open when build is older than last status update", func() {
// This test replicates the likely state just after we've
// automatically closed the tree: the tree is closed with
// our username, and there is some failing TreeCloser older
// than the status update.
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: "Tree is closed (Automatic: some builder failed)",
key: -1,
status: config.Closed,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: evenEarlierTime,
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: evenEarlierTime,
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Closed)
So(status.message, ShouldEqual, "Tree is closed (Automatic: some builder failed)")
})
Convey("Doesn't open when a builder is still failing", func() {
// This test replicates the likely state after we've automatically
// closed the tree, but some other builder has had a successful
// build.
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: "Tree is closed (Automatic: some builder failed)",
key: -1,
status: config.Closed,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: evenEarlierTime,
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Closed)
So(status.message, ShouldEqual, "Tree is closed (Automatic: some builder failed)")
})
Convey("Multiple projects", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: "Tree is closed (Automatic: some builder failed)",
key: -1,
status: config.Closed,
timestamp: earlierTime,
},
"infra-status.appspot.com": {
username: botUsername,
message: "Tree is open (Automatic: Yes!)",
key: -1,
status: config.Open,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder5),
TreeStatusHost: "infra-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: time.Now().UTC(),
Message: "Close it up!",
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
status, err = ts.getStatus(c, "infra-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
hasExpectedLog := false
for _, log := range log.Messages() {
if log.Level == logging.Info {
hasExpectedLog = true
So(log.Msg, ShouldEqual, `Would update status for infra-status.appspot.com to "Tree is closed (Automatic: Close it up!)"`)
}
}
So(hasExpectedLog, ShouldBeTrue)
})
Convey("Multiple projects, overlapping tree status hosts", func() {
ts := fakeTreeStatusClient{
statusForHosts: map[string]treeStatus{
"chromium-status.appspot.com": {
username: botUsername,
message: "Tree is open (Flake)",
key: -1,
status: config.Open,
timestamp: earlierTime,
},
"infra-status.appspot.com": {
username: botUsername,
message: "Tree is closed (Automatic: Some builder failed)",
key: -1,
status: config.Closed,
timestamp: earlierTime,
},
},
}
So(datastore.Put(c, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder1),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder5),
TreeStatusHost: "chromium-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: time.Now().UTC(),
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder2),
TreeStatusHost: "infra-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Open,
Timestamp: time.Now().UTC(),
}, &config.TreeCloser{
BuilderKey: datastore.KeyForObj(c, builder6),
TreeStatusHost: "infra-status.appspot.com",
TreeCloser: notifypb.TreeCloser{},
Status: config.Closed,
Timestamp: time.Now().UTC(),
}), ShouldBeNil)
defer cleanup()
So(updateTrees(c, &ts), ShouldBeNil)
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldEqual, "Tree is open (Flake)")
status, err = ts.getStatus(c, "infra-status.appspot.com")
So(err, ShouldBeNil)
So(status.status, ShouldEqual, config.Open)
So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
})
})
}
func TestHttpTreeStatusClient(t *testing.T) {
Convey("Test environment for httpTreeStatusClient", t, func() {
c := gaetesting.TestingContextWithAppID("luci-notify-test")
// Real responses, with usernames redacted and readable formatting applied.
responses := map[string]string{
"https://chromium-status.appspot.com/current?format=json": `{
"username": "someone@google.com",
"can_commit_freely": false,
"general_state": "throttled",
"key": 5656890264518656,
"date": "2020-03-31 05:33:52.682351",
"message": "Tree is throttled (win rel 32 appears to be a goma flake. the other builds seem to be charging ahead OK. will fully open / fully close if win32 does/doesn't improve)"
}`,
"https://v8-status.appspot.com/current?format=json": `{
"username": "someone-else@google.com",
"can_commit_freely": true,
"general_state": "open",
"key": 5739466035560448,
"date": "2020-04-02 15:21:39.981072",
"message": "open (flake?)"
}`,
}
get := func(_ context.Context, url string) ([]byte, error) {
if s, e := responses[url]; e {
return []byte(s), nil
} else {
return nil, fmt.Errorf("Key not present: %q", url)
}
}
var postUrls []string
post := func(_ context.Context, url string) error {
postUrls = append(postUrls, url)
return nil
}
ts := httpTreeStatusClient{get, post}
Convey("getStatus, open tree", func() {
status, err := ts.getStatus(c, "chromium-status.appspot.com")
So(err, ShouldBeNil)
expectedTime := time.Date(2020, time.March, 31, 5, 33, 52, 682351000, time.UTC)
So(status, ShouldResemble, &treeStatus{
username: "someone@google.com",
message: "Tree is throttled (win rel 32 appears to be a goma flake. the other builds seem to be charging ahead OK. will fully open / fully close if win32 does/doesn't improve)",
key: 5656890264518656,
status: config.Closed,
timestamp: expectedTime,
})
})
Convey("getStatus, closed tree", func() {
status, err := ts.getStatus(c, "v8-status.appspot.com")
So(err, ShouldBeNil)
expectedTime := time.Date(2020, time.April, 2, 15, 21, 39, 981072000, time.UTC)
So(status, ShouldResemble, &treeStatus{
username: "someone-else@google.com",
message: "open (flake?)",
key: 5739466035560448,
status: config.Open,
timestamp: expectedTime,
})
})
Convey("postStatus", func() {
err := ts.postStatus(c, "dart-status.appspot.com", "open for business", 1234)
So(err, ShouldBeNil)
So(postUrls, ShouldHaveLength, 1)
So(postUrls[0], ShouldEqual, "https://dart-status.appspot.com/?last_status_key=1234&message=open+for+business")
})
})
}