| // Copyright 2023 The LUCI Authors. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| package server | |
| import ( | |
| "context" | |
| "fmt" | |
| "google.golang.org/grpc/codes" | |
| "google.golang.org/grpc/status" | |
| "google.golang.org/protobuf/proto" | |
| bbpb "go.chromium.org/luci/buildbucket/proto" | |
| "go.chromium.org/luci/common/clock" | |
| "go.chromium.org/luci/common/errors" | |
| "go.chromium.org/luci/common/logging" | |
| "go.chromium.org/luci/gae/service/datastore" | |
| "go.chromium.org/luci/server/tq" | |
| "go.chromium.org/luci/bisection/compilefailureanalysis/nthsection" | |
| "go.chromium.org/luci/bisection/compilefailureanalysis/statusupdater" | |
| "go.chromium.org/luci/bisection/model" | |
| "go.chromium.org/luci/bisection/nthsectionsnapshot" | |
| pb "go.chromium.org/luci/bisection/proto/v1" | |
| "go.chromium.org/luci/bisection/server/updatetestrerun" | |
| taskpb "go.chromium.org/luci/bisection/task/proto" | |
| "go.chromium.org/luci/bisection/util/datastoreutil" | |
| "go.chromium.org/luci/bisection/util/loggingutil" | |
| ) | |
| // BotUpdatesServer implements the LUCI Bisection proto service for BotUpdates. | |
| type BotUpdatesServer struct { | |
| pb.UnimplementedBotUpdatesServer | |
| ACL func(context.Context, string, proto.Message) (context.Context, error) | |
| } | |
| // UpdateAnalysisProgress is an RPC endpoints used by the recipes to update | |
| // analysis progress. | |
| func (server *BotUpdatesServer) UpdateAnalysisProgress(c context.Context, req *pb.UpdateAnalysisProgressRequest) (*pb.UpdateAnalysisProgressResponse, error) { | |
| c, err := server.ACL(c, "UpdateAnalysisProgress", req) | |
| if err != nil { | |
| return nil, err | |
| } | |
| err = verifyUpdateAnalysisProgressRequest(c, req) | |
| if err != nil { | |
| return nil, status.Errorf(codes.InvalidArgument, "Invalid request: %s", err) | |
| } | |
| c = loggingutil.SetAnalysisID(c, req.AnalysisId) | |
| c = loggingutil.SetRerunBBID(c, req.Bbid) | |
| logging.Infof(c, "Update analysis with rerun_build_id = %d analysis_id = %d gitiles_commit=%v ", req.Bbid, req.AnalysisId, req.GitilesCommit) | |
| cfa, err := datastoreutil.GetCompileFailureAnalysis(c, req.AnalysisId) | |
| if err != nil { | |
| err = errors.Fmt("failed GetCompileFailureAnalysis ID: %d: %w", req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error GetCompileFailureAnalysis") | |
| } | |
| if cfa.CompileFailure != nil && cfa.CompileFailure.Parent() != nil { | |
| c = loggingutil.SetAnalyzedBBID(c, cfa.CompileFailure.Parent().IntID()) | |
| } | |
| // Get rerun model | |
| rerunModel := &model.CompileRerunBuild{ | |
| Id: req.Bbid, | |
| } | |
| switch err := datastore.Get(c, rerunModel); { | |
| case err == datastore.ErrNoSuchEntity: | |
| return nil, status.Errorf(codes.NotFound, "could not find rerun build with id %d", req.Bbid) | |
| case err != nil: | |
| return nil, status.Errorf(codes.Internal, "error finding rerun build") | |
| default: | |
| //continue | |
| } | |
| lastRerun, err := datastoreutil.GetLastRerunForRerunBuild(c, rerunModel) | |
| if err != nil { | |
| err = errors.Fmt("failed getting last rerun for build %d. Analysis ID: %d: %w", rerunModel.Id, req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error getting last rerun build") | |
| } | |
| // Update rerun model | |
| err = updateRerun(c, req, lastRerun) | |
| if err != nil { | |
| err = errors.Fmt("failed updating rerun for build %d. Analysis ID: %d: %w", rerunModel.Id, req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error updating rerun build") | |
| } | |
| // Safeguard, we really don't expect any other type | |
| if lastRerun.Type != model.RerunBuildType_CulpritVerification && lastRerun.Type != model.RerunBuildType_NthSection { | |
| logging.Errorf(c, "Invalid type %v for analysis %d", lastRerun.Type, req.AnalysisId) | |
| return nil, status.Errorf(codes.Internal, "Invalid type %v", lastRerun.Type) | |
| } | |
| // Culprit verification | |
| if lastRerun.Type == model.RerunBuildType_CulpritVerification { | |
| err := updateSuspectWithRerunData(c, lastRerun) | |
| if err != nil { | |
| err = errors.Fmt("updateSuspectWithRerunData for build id %d. Analysis ID: %d: %w", rerunModel.Id, req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error updating suspect") | |
| } | |
| // Update analysis status | |
| err = statusupdater.UpdateAnalysisStatus(c, cfa) | |
| if err != nil { | |
| err = errors.Fmt("statusupdater.UpdateAnalysisStatus. Analysis ID: %d: %w", req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error UpdateAnalysisStatus") | |
| } | |
| // TODO (nqmtuan): It is possible that we schedule an nth-section run right after | |
| // a culprit verification run within the same build. We will do this later, for | |
| // safety, after we verify nth-section analysis is running fine. | |
| return &pb.UpdateAnalysisProgressResponse{}, nil | |
| } | |
| // Nth section | |
| if lastRerun.Type == model.RerunBuildType_NthSection { | |
| nsa, err := processNthSectionUpdate(c, req) | |
| if err != nil { | |
| err = errors.Fmt("processNthSectionUpdate. Analysis ID: %d: %w", req.AnalysisId, err) | |
| logging.Errorf(c, err.Error()) | |
| // If there is an error, then nthsection analysis may ended | |
| // if there is no unfinised nthsection runs | |
| e := setNthSectionError(c, nsa) | |
| if e != nil { | |
| e = errors.Fmt("setNthSectionError. Analysis ID: %d: %w", req.AnalysisId, e) | |
| logging.Errorf(c, e.Error()) | |
| } | |
| // Also the main analysis status may need to change as well | |
| e = statusupdater.UpdateAnalysisStatus(c, cfa) | |
| if e != nil { | |
| e = errors.Fmt("UpdateAnalysisStatus. Analysis ID: %d: %w", req.AnalysisId, e) | |
| logging.Errorf(c, e.Error()) | |
| } | |
| return nil, status.Error(codes.Internal, err.Error()) | |
| } | |
| // Update analysis status | |
| err = statusupdater.UpdateAnalysisStatus(c, cfa) | |
| if err != nil { | |
| err = errors.Fmt("statusupdater.UpdateAnalysisStatus. Analysis ID: %d: %w", req.AnalysisId, err) | |
| errors.Log(c, err) | |
| return nil, status.Errorf(codes.Internal, "error UpdateAnalysisStatus") | |
| } | |
| return &pb.UpdateAnalysisProgressResponse{}, nil | |
| } | |
| return nil, status.Errorf(codes.Internal, "unknown error") | |
| } | |
| func (server *BotUpdatesServer) UpdateTestAnalysisProgress(c context.Context, req *pb.UpdateTestAnalysisProgressRequest) (*pb.UpdateTestAnalysisProgressResponse, error) { | |
| c, err := server.ACL(c, "UpdateTestAnalysisProgress", req) | |
| if err != nil { | |
| return nil, err | |
| } | |
| err = updatetestrerun.Update(c, req) | |
| if err != nil { | |
| return nil, err | |
| } | |
| return &pb.UpdateTestAnalysisProgressResponse{}, nil | |
| } | |
| func setNthSectionError(c context.Context, nsa *model.CompileNthSectionAnalysis) error { | |
| if nsa == nil { | |
| return nil | |
| } | |
| reruns, err := datastoreutil.GetRerunsForNthSectionAnalysis(c, nsa) | |
| if err != nil { | |
| return errors.Fmt("GetRerunsForNthSectionAnalysis: %w", err) | |
| } | |
| for _, rerun := range reruns { | |
| // There are some rerun running, so do not mark this as error yet | |
| if rerun.Status == pb.RerunStatus_RERUN_STATUS_IN_PROGRESS { | |
| return nil | |
| } | |
| } | |
| return datastore.RunInTransaction(c, func(c context.Context) error { | |
| e := datastore.Get(c, nsa) | |
| if e != nil { | |
| return e | |
| } | |
| nsa.Status = pb.AnalysisStatus_ERROR | |
| nsa.RunStatus = pb.AnalysisRunStatus_ENDED | |
| nsa.EndTime = clock.Now(c) | |
| return datastore.Put(c, nsa) | |
| }, nil) | |
| } | |
| // processNthSectionUpdate processes the bot update for nthsection analysis run | |
| // It will schedule the next run for nthsection analysis targeting the same bot | |
| func processNthSectionUpdate(c context.Context, req *pb.UpdateAnalysisProgressRequest) (*model.CompileNthSectionAnalysis, error) { | |
| cfa, err := datastoreutil.GetCompileFailureAnalysis(c, req.AnalysisId) | |
| if err != nil { | |
| return nil, err | |
| } | |
| // We should not schedule any more run for this analysis | |
| if cfa.ShouldCancel { | |
| return nil, nil | |
| } | |
| nsa, err := datastoreutil.GetNthSectionAnalysis(c, cfa) | |
| if err != nil { | |
| return nil, err | |
| } | |
| // There is no nthsection analysis for this analysis | |
| if nsa == nil { | |
| return nil, nil | |
| } | |
| snapshot, err := nthsection.CreateSnapshot(c, nsa) | |
| if err != nil { | |
| return nsa, errors.Fmt("couldn't create snapshot: %w", err) | |
| } | |
| // Check if we already found the culprit or not | |
| ok, cul := snapshot.GetCulprit() | |
| // Found culprit -> Update the nthsection analysis | |
| if ok { | |
| err := nthsection.SaveSuspectAndTriggerCulpritVerification(c, nsa, cfa, snapshot.BlameList.Commits[cul]) | |
| if err != nil { | |
| return nsa, errors.Fmt("save suspect and trigger culprit verification: %w", err) | |
| } | |
| return nsa, nil | |
| } | |
| shouldRunNthSection, err := nthsection.ShouldRunNthSectionAnalysis(c, cfa) | |
| if err != nil { | |
| return nsa, errors.Fmt("couldn't fetch config for nthsection: %w", err) | |
| } | |
| if !shouldRunNthSection { | |
| return nsa, nil | |
| } | |
| commit, err := snapshot.FindNextSingleCommitToRun() | |
| var badRangeError *nthsectionsnapshot.BadRangeError | |
| if err != nil { | |
| if !errors.As(err, &badRangeError) { | |
| return nsa, errors.Fmt("find next single commit to run: %w", err) | |
| } | |
| // BadRangeError suggests the regression range is invalid. | |
| // This is not really an error, but more of a indication of no suspect can be found | |
| // in this regression range. | |
| logging.Warningf(c, "find next single commit to run %s", err.Error()) | |
| } | |
| if commit == "" || errors.As(err, &badRangeError) { | |
| // We don't have more run to wait -> we've failed to find the suspect | |
| if snapshot.NumInProgress == 0 { | |
| return nsa, updateNthSectionModelNotFound(c, nsa) | |
| } | |
| return nsa, nil | |
| } | |
| // We got the next commit to run. We will schedule a rerun targetting the same bot | |
| gitilesCommit := &bbpb.GitilesCommit{ | |
| Host: req.GitilesCommit.Host, | |
| Project: req.GitilesCommit.Project, | |
| Ref: req.GitilesCommit.Ref, | |
| Id: commit, | |
| } | |
| dims := map[string]string{ | |
| "id": req.BotId, | |
| } | |
| err = nthsection.RerunCommit(c, nsa, gitilesCommit, cfa.FirstFailedBuildId, dims) | |
| if err != nil { | |
| return nsa, errors.Fmt("rerun commit for %s: %w", commit, err) | |
| } | |
| return nsa, nil | |
| } | |
| func updateNthSectionModelNotFound(c context.Context, nsa *model.CompileNthSectionAnalysis) error { | |
| err := datastore.RunInTransaction(c, func(c context.Context) error { | |
| e := datastore.Get(c, nsa) | |
| if e != nil { | |
| return e | |
| } | |
| nsa.EndTime = clock.Now(c) | |
| nsa.Status = pb.AnalysisStatus_NOTFOUND | |
| nsa.RunStatus = pb.AnalysisRunStatus_ENDED | |
| return datastore.Put(c, nsa) | |
| }, nil) | |
| if err != nil { | |
| return errors.Fmt("failed updating nthsectionModel: %w", err) | |
| } | |
| return nil | |
| } | |
| func updateSuspectWithRerunData(c context.Context, rerun *model.SingleRerun) error { | |
| // Get the suspect for the rerun build | |
| if rerun.Suspect == nil { | |
| return fmt.Errorf("no suspect for rerun %d", rerun.Id) | |
| } | |
| suspect := &model.Suspect{ | |
| Id: rerun.Suspect.IntID(), | |
| ParentAnalysis: rerun.Suspect.Parent(), | |
| } | |
| err := datastore.Get(c, suspect) | |
| if err != nil { | |
| return errors.Fmt("couldn't find suspect for rerun %d: %w", rerun.Id, err) | |
| } | |
| err = updateSuspect(c, suspect) | |
| if err != nil { | |
| return errors.Fmt("error updating suspect for rerun %d: %w", rerun.Id, err) | |
| } | |
| if suspect.VerificationStatus == model.SuspectVerificationStatus_ConfirmedCulprit { | |
| err = updateSuspectAsConfirmedCulprit(c, suspect) | |
| if err != nil { | |
| return errors.Fmt("error updateSuspectAsConfirmedCulprit for rerun %d: %w", rerun.Id, err) | |
| } | |
| // Cancel all remaining runs | |
| analysisID := suspect.ParentAnalysis.Parent().IntID() | |
| err = tq.AddTask(c, &tq.Task{ | |
| Title: fmt.Sprintf("cancel_analysis_%d", analysisID), | |
| Payload: &taskpb.CancelAnalysisTask{ | |
| AnalysisId: analysisID, | |
| }, | |
| }) | |
| if err != nil { | |
| // Non-critical, just log the error | |
| err := errors.Fmt("schedule canceling analysis %d: %w", analysisID, err) | |
| logging.Errorf(c, err.Error()) | |
| } | |
| // Add task to revert the confirmed culprit | |
| // TODO(@beining): Schedule this task when suspect is VerificationError too. | |
| // According to go/luci-bisection-integrating-gerrit, | |
| // we want to also perform gerrit action when suspect is VerificationError. | |
| err = tq.AddTask(c, &tq.Task{ | |
| Title: fmt.Sprintf("revert_culprit_%d_%d", suspect.Id, analysisID), | |
| Payload: &taskpb.RevertCulpritTask{ | |
| AnalysisId: analysisID, | |
| CulpritId: suspect.Id, | |
| }, | |
| }) | |
| if err != nil { | |
| return errors.Fmt("error creating task in task queue to revert culprit (analysis ID=%d, suspect ID=%d): %w", | |
| analysisID, suspect.Id, err) | |
| } | |
| } | |
| return nil | |
| } | |
| func verifyUpdateAnalysisProgressRequest(_ context.Context, req *pb.UpdateAnalysisProgressRequest) error { | |
| if req.AnalysisId == 0 { | |
| return fmt.Errorf("analysis_id is required") | |
| } | |
| if req.Bbid == 0 { | |
| return fmt.Errorf("build bucket id is required") | |
| } | |
| if req.GitilesCommit == nil { | |
| return fmt.Errorf("gitiles commit is required") | |
| } | |
| if req.RerunResult == nil { | |
| return fmt.Errorf("rerun result is required") | |
| } | |
| if req.BotId == "" { | |
| return fmt.Errorf("bot_id is required") | |
| } | |
| return nil | |
| } | |
| // updateSuspect looks at rerun and set the suspect status | |
| func updateSuspect(c context.Context, suspect *model.Suspect) error { | |
| rerunStatus, err := getSingleRerunStatus(c, suspect.SuspectRerunBuild.IntID()) | |
| if err != nil { | |
| return err | |
| } | |
| parentRerunStatus, err := getSingleRerunStatus(c, suspect.ParentRerunBuild.IntID()) | |
| if err != nil { | |
| return err | |
| } | |
| // Update suspect based on rerunStatus and parentRerunStatus | |
| suspectStatus := model.SuspectStatus(rerunStatus, parentRerunStatus) | |
| return datastore.RunInTransaction(c, func(ctx context.Context) error { | |
| e := datastore.Get(ctx, suspect) | |
| if e != nil { | |
| return e | |
| } | |
| if suspect.VerificationStatus == model.SuspectVerificationStatus_ConfirmedCulprit || | |
| suspect.VerificationStatus == model.SuspectVerificationStatus_Vindicated || | |
| suspect.VerificationStatus == model.SuspectVerificationStatus_Canceled { | |
| logging.Infof(ctx, "Suspect %d is already in terminal state %s, skipping update", suspect.Id, suspect.VerificationStatus) | |
| return nil | |
| } | |
| suspect.VerificationStatus = suspectStatus | |
| return datastore.Put(ctx, suspect) | |
| }, nil) | |
| } | |
| // updateSuspectAsConfirmedCulprit update the suspect as the confirmed culprit of analysis | |
| func updateSuspectAsConfirmedCulprit(c context.Context, suspect *model.Suspect) error { | |
| analysisKey := suspect.ParentAnalysis.Parent() | |
| analysis := &model.CompileFailureAnalysis{ | |
| Id: analysisKey.IntID(), | |
| } | |
| err := datastore.Get(c, analysis) | |
| if err != nil { | |
| return err | |
| } | |
| verifiedCulprits := analysis.VerifiedCulprits | |
| verifiedCulprits = append(verifiedCulprits, datastore.KeyForObj(c, suspect)) | |
| if len(verifiedCulprits) > 1 { | |
| // Just log the warning here, as it is a rare case | |
| logging.Warningf(c, "found more than 2 suspects for analysis %d", analysis.Id) | |
| } | |
| err = datastore.RunInTransaction(c, func(ctx context.Context) error { | |
| e := datastore.Get(c, analysis) | |
| if e != nil { | |
| return e | |
| } | |
| analysis.VerifiedCulprits = verifiedCulprits | |
| return datastore.Put(c, analysis) | |
| }, nil) | |
| if err != nil { | |
| return err | |
| } | |
| return statusupdater.UpdateAnalysisStatus(c, analysis) | |
| } | |
| // updateRerun updates the last SingleRerun for rerunModel with the information from req. | |
| // Returns the last SingleRerun and error (if it occur). | |
| func updateRerun(c context.Context, req *pb.UpdateAnalysisProgressRequest, rerun *model.SingleRerun) error { | |
| // Verify the gitiles commit, making sure it was the right rerun we are updating | |
| if !sameGitilesCommit(req.GitilesCommit, &rerun.GitilesCommit) { | |
| logging.Errorf(c, "Got different Gitles commit for rerun build %d", req.Bbid) | |
| return fmt.Errorf("different gitiles commit for rerun") | |
| } | |
| err := datastore.RunInTransaction(c, func(ctx context.Context) error { | |
| e := datastore.Get(c, rerun) | |
| if e != nil { | |
| return e | |
| } | |
| rerun.EndTime = clock.Now(c) | |
| rerun.Status = req.RerunResult.RerunStatus | |
| return datastore.Put(c, rerun) | |
| }, nil) | |
| if err != nil { | |
| logging.Errorf(c, "Error updating SingleRerun for build %d: %s", req.Bbid, rerun) | |
| return errors.Fmt("saving SingleRerun: %w", err) | |
| } | |
| return nil | |
| } | |
| func getSingleRerunStatus(c context.Context, rerunId int64) (pb.RerunStatus, error) { | |
| rerunBuild := &model.CompileRerunBuild{ | |
| Id: rerunId, | |
| } | |
| err := datastore.Get(c, rerunBuild) | |
| if err != nil { | |
| return pb.RerunStatus_RERUN_STATUS_UNSPECIFIED, err | |
| } | |
| // Get SingleRerun | |
| singleRerun, err := datastoreutil.GetLastRerunForRerunBuild(c, rerunBuild) | |
| if err != nil { | |
| return pb.RerunStatus_RERUN_STATUS_UNSPECIFIED, err | |
| } | |
| return singleRerun.Status, nil | |
| } | |
| func sameGitilesCommit(g1 *bbpb.GitilesCommit, g2 *bbpb.GitilesCommit) bool { | |
| return g1.Host == g2.Host && g1.Project == g2.Project && g1.Id == g2.Id && g1.Ref == g2.Ref | |
| } |