| // Copyright 2019 The Chromium OS Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| package repo |
| |
| import ( |
| "context" |
| "encoding/xml" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "net/http" |
| "path/filepath" |
| "regexp" |
| "strings" |
| |
| "go.chromium.org/chromiumos/infra/go/internal/gerrit" |
| bbproto "go.chromium.org/luci/buildbucket/proto" |
| "go.chromium.org/luci/common/errors" |
| ) |
| |
| var ( |
| // Name of the root XML file to seek in manifest-internal. |
| rootXml = "default.xml" |
| ) |
| |
| // Manifest is a top-level Repo definition file. |
| type Manifest struct { |
| XMLName xml.Name `xml:"manifest"` |
| Includes []Include `xml:"include"` |
| Remotes []Remote `xml:"remote"` |
| Default Default `xml:"default"` |
| Notice string `xml:"notice,omitempty"` |
| RepoHooks []RepoHooks `xml:"repo-hooks"` |
| Projects []Project `xml:"project"` |
| } |
| |
| // Project is an element of a manifest containing a Gerrit project to source path definition. |
| type Project struct { |
| Path string `xml:"path,attr,omitempty"` |
| Name string `xml:"name,attr,omitempty"` |
| Revision string `xml:"revision,attr,omitempty"` |
| Upstream string `xml:"upstream,attr,omitempty"` |
| RemoteName string `xml:"remote,attr,omitempty"` |
| Annotations []Annotation `xml:"annotation"` |
| Groups string `xml:"groups,attr,omitempty"` |
| SyncC string `xml:"sync-c,attr,omitempty"` |
| CopyFiles []CopyFile `xml:"copyfile"` |
| } |
| |
| // Annotation is an element of a manifest annotating the parent element. |
| type Annotation struct { |
| Name string `xml:"name,attr,omitempty"` |
| Value string `xml:"value,attr,omitempty"` |
| } |
| |
| // Include is a manifest element that imports another manifest file. |
| type Include struct { |
| Name string `xml:"name,attr,omitempty"` |
| } |
| |
| // Remote is a manifest element that lists a remote. |
| type Remote struct { |
| Fetch string `xml:"fetch,attr,omitempty"` |
| Name string `xml:"name,attr,omitempty"` |
| Revision string `xml:"revision,attr,omitempty"` |
| Alias string `xml:"alias,attr,omitempty"` |
| } |
| |
| // Default is a manifest element that lists the default. |
| type Default struct { |
| RemoteName string `xml:"remote,attr,omitempty"` |
| Revision string `xml:"revision,attr,omitempty"` |
| SyncJ string `xml:"sync-j,attr,omitempty"` |
| } |
| |
| type CopyFile struct { |
| Dest string `xml:"dest,attr,omitempty"` |
| Src string `xml:"src,attr,omitempty"` |
| } |
| |
| type RepoHooks struct { |
| EnabledList string `xml:"enabled-list,attr,omitempty"` |
| InProject string `xml:"in-project,attr,omitempty"` |
| } |
| |
| // GitName returns the git name of the remote, which |
| // is Alias if it is set, and Name otherwise. |
| func (r *Remote) GitName() string { |
| if r.Alias != "" { |
| return r.Alias |
| } |
| return r.Name |
| } |
| |
| // GetRemoteByName returns a pointer to the remote with |
| // the given name/alias in the given manifest. |
| func (m *Manifest) GetRemoteByName(name string) *Remote { |
| for i, remote := range m.Remotes { |
| if remote.Name == name { |
| return &m.Remotes[i] |
| } |
| } |
| return nil |
| } |
| |
| // GetProjectByName returns a pointer to the remote with |
| // the given path in the given manifest. |
| func (m *Manifest) GetProjectByName(name string) (*Project, error) { |
| for i, project := range m.Projects { |
| if project.Name == name { |
| return &m.Projects[i], nil |
| } |
| } |
| return nil, fmt.Errorf("project %s does not exist in manifest", name) |
| } |
| |
| // GetProjectByPath returns a pointer to the remote with |
| // the given path in the given manifest. |
| func (m *Manifest) GetProjectByPath(path string) (*Project, error) { |
| for i, project := range m.Projects { |
| if project.Path == path { |
| return &m.Projects[i], nil |
| } |
| } |
| return nil, fmt.Errorf("project %s does not exist in manifest", path) |
| } |
| |
| type projectType string |
| |
| const ( |
| singleCheckout projectType = "single" |
| multiCheckout projectType = "multi" |
| pinned projectType = "pinned" |
| tot projectType = "tot" |
| ) |
| |
| func (m *Manifest) getProjects(ptype projectType) []*Project { |
| projectCount := make(map[string]int) |
| |
| for _, project := range m.Projects { |
| projectCount[project.Name] += 1 |
| } |
| |
| projects := []*Project{} |
| for i, project := range m.Projects { |
| includeProject := false |
| projectMode := m.ProjectBranchMode(project) |
| if projectMode == Pinned { |
| includeProject = ptype == pinned |
| } else if projectMode == Tot { |
| includeProject = ptype == tot |
| } else if projectCount[project.Name] == 1 { |
| includeProject = ptype == singleCheckout |
| } |
| // Restart the if/else if block here because it is possible |
| // to have a project with multiple checkouts, some of which |
| // are pinned/tot and some of which are not. |
| if projectCount[project.Name] > 1 { |
| includeProject = includeProject || ptype == multiCheckout |
| } |
| if includeProject { |
| projects = append(projects, &m.Projects[i]) |
| } |
| } |
| return projects |
| } |
| |
| // GetSingleCheckoutProjects returns all projects in the manifest that have a |
| // single checkout and are not pinned/tot. |
| func (m *Manifest) GetSingleCheckoutProjects() []*Project { |
| return m.getProjects(singleCheckout) |
| } |
| |
| // GetMultiCheckoutProjects returns all projects in the manifest that have a |
| // multiple checkouts and are not pinned/tot. |
| func (m *Manifest) GetMultiCheckoutProjects() []*Project { |
| return m.getProjects(multiCheckout) |
| } |
| |
| // GetPinnedProjects returns all projects in the manifest that are |
| // pinned. |
| func (m *Manifest) GetPinnedProjects() []*Project { |
| return m.getProjects(pinned) |
| } |
| |
| // GetTotProjects returns all projects in the manifest that are |
| // tot. |
| func (m *Manifest) GetTotProjects() []*Project { |
| return m.getProjects(tot) |
| } |
| |
| var ( |
| GOB_HOST = "%s.googlesource.com" |
| EXTERNAL_GOB_INSTANCE = "chromium" |
| EXTERNAL_GOB_HOST = fmt.Sprintf(GOB_HOST, EXTERNAL_GOB_INSTANCE) |
| EXTERNAL_GOB_URL = fmt.Sprintf("https://%s", EXTERNAL_GOB_HOST) |
| |
| INTERNAL_GOB_INSTANCE = "chrome-internal" |
| INTERNAL_GOB_HOST = fmt.Sprintf(GOB_HOST, INTERNAL_GOB_INSTANCE) |
| INTERNAL_GOB_URL = fmt.Sprintf("https://%s", INTERNAL_GOB_HOST) |
| |
| AOSP_GOB_INSTANCE = "android" |
| AOSP_GOB_HOST = fmt.Sprintf(GOB_HOST, AOSP_GOB_INSTANCE) |
| AOSP_GOB_URL = fmt.Sprintf("https://%s", AOSP_GOB_HOST) |
| |
| WEAVE_GOB_INSTANCE = "weave" |
| WEAVE_GOB_HOST = fmt.Sprintf(GOB_HOST, WEAVE_GOB_INSTANCE) |
| WEAVE_GOB_URL = fmt.Sprintf("https://%s", WEAVE_GOB_HOST) |
| |
| external_remote = "cros" |
| internal_remote = "cros-internal" |
| |
| CROS_REMOTES = map[string]string{ |
| external_remote: EXTERNAL_GOB_URL, |
| internal_remote: INTERNAL_GOB_URL, |
| "aosp": AOSP_GOB_URL, |
| "weave": WEAVE_GOB_URL, |
| } |
| |
| // Mapping 'remote name' -> regexp that matches names of repositories on |
| // that remote that can be branched when creating CrOS branch. |
| // Branching script will actually create a new git ref when branching |
| // these projects. It won't attempt to create a git ref for other projects |
| // that may be mentioned in a manifest. If a remote is missing from this |
| // dictionary, all projects on that remote are considered to not be |
| // branchable. |
| BRANCHABLE_PROJECTS = map[string]*regexp.Regexp{ |
| external_remote: regexp.MustCompile("(chromiumos|aosp)/(.+)"), |
| internal_remote: regexp.MustCompile("chromeos/(.+)"), |
| } |
| |
| MANIFEST_ATTR_BRANCHING_CREATE = "create" |
| MANIFEST_ATTR_BRANCHING_PIN = "pin" |
| MANIFEST_ATTR_BRANCHING_TOT = "tot" |
| ) |
| |
| type BranchMode string |
| |
| const ( |
| UnspecifiedMode BranchMode = "unspecified" |
| Pinned BranchMode = "pinned" |
| Tot BranchMode = "tot" |
| Create BranchMode = "create" |
| ) |
| |
| // ProjectBranchMode returns the branch mode (create, pinned, tot) of a project. |
| func (m *Manifest) ProjectBranchMode(project Project) BranchMode { |
| // Anotation is set. |
| explicitMode, _ := project.GetAnnotation("branch-mode") |
| if explicitMode != "" { |
| switch explicitMode { |
| case MANIFEST_ATTR_BRANCHING_CREATE: |
| return Create |
| case MANIFEST_ATTR_BRANCHING_PIN: |
| return Pinned |
| case MANIFEST_ATTR_BRANCHING_TOT: |
| return Tot |
| default: |
| return UnspecifiedMode |
| } |
| } |
| |
| // Othwerise, peek at remote. |
| remote := m.GetRemoteByName(project.RemoteName) |
| if remote == nil { |
| return UnspecifiedMode |
| } |
| remoteName := remote.GitName() |
| _, inCrosRemote := CROS_REMOTES[remoteName] |
| projectRegexp, inBranchableProjects := BRANCHABLE_PROJECTS[remoteName] |
| |
| if inCrosRemote && inBranchableProjects && projectRegexp.MatchString(project.Name) { |
| return Create |
| } else { |
| return Pinned |
| } |
| } |
| |
| // GetAnnotation returns the value of the annotation with the |
| // given name, if it exists. It also returns a bool indicating |
| // whether or not the annotation exists. |
| func (p *Project) GetAnnotation(name string) (string, bool) { |
| for _, annotation := range p.Annotations { |
| if annotation.Name == name { |
| return annotation.Value, true |
| } |
| } |
| return "", false |
| } |
| |
| // LoadManifestFromFile loads the manifest at the given file into a |
| // Manifest struct. |
| func LoadManifestFromFile(file string) (Manifest, error) { |
| manifestMap, err := LoadManifestTree(file) |
| if err != nil { |
| return Manifest{}, err |
| } |
| manifest, exists := manifestMap[filepath.Base(file)] |
| if !exists { |
| return Manifest{}, fmt.Errorf("failed to read %s", file) |
| } |
| return *manifest, nil |
| } |
| |
| // LoadManifestFromFileRaw loads the manifest at the given file and returns |
| // the file contents as a byte array. |
| func LoadManifestFromFileRaw(file string) ([]byte, error) { |
| data, err := ioutil.ReadFile(file) |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to open and read %s", file).Err() |
| } |
| // We don't ResolveImplicitLinks or otherwise do anything else in this function (as it returns "raw" data). |
| return data, nil |
| } |
| |
| // LoadManifestFromFileWithIncludes loads the manifest at the given files but also |
| // calls MergeManifests to resolve includes. |
| func LoadManifestFromFileWithIncludes(file string) (*Manifest, error) { |
| manifestMap, err := LoadManifestTree(file) |
| if err != nil { |
| return nil, err |
| } |
| manifest, err := MergeManifests(filepath.Base(file), &manifestMap) |
| return manifest, err |
| } |
| |
| // ResolveImplicitLinks explicitly sets remote/revision information |
| // for each project in the manifest. |
| func (m *Manifest) ResolveImplicitLinks() *Manifest { |
| newManifest := *m |
| for i, project := range m.Projects { |
| // Set default remote on projects without an explicit remote |
| if project.RemoteName == "" { |
| project.RemoteName = m.Default.RemoteName |
| } |
| // Set default revision on projects without an explicit revision |
| if project.Revision == "" { |
| remote := m.GetRemoteByName(project.RemoteName) |
| if remote == nil || remote.Revision == "" { |
| project.Revision = m.Default.Revision |
| } else { |
| project.Revision = remote.Revision |
| } |
| } |
| // Path defaults to name. |
| if project.Path == "" { |
| project.Path = project.Name |
| } |
| newManifest.Projects[i] = project |
| } |
| return &newManifest |
| } |
| |
| // LoadManifestTree loads the manifest at the given file path into |
| // a Manifest struct. It also loads all included manifests. |
| // Returns a map mapping manifest filenames to file contents. |
| func LoadManifestTree(file string) (map[string]*Manifest, error) { |
| results := make(map[string]*Manifest) |
| |
| data, err := ioutil.ReadFile(file) |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to open and read %s", file).Err() |
| } |
| manifest := &Manifest{} |
| if err = xml.Unmarshal(data, manifest); err != nil { |
| return nil, errors.Annotate(err, "failed to unmarshal %s", file).Err() |
| } |
| manifest.XMLName = xml.Name{} |
| manifest = manifest.ResolveImplicitLinks() |
| results[filepath.Base(file)] = manifest |
| |
| // Recursively fetch manifests listed in "include" elements. |
| for _, incl := range manifest.Includes { |
| // Include paths are relative to the manifest location. |
| inclPath := filepath.Join(filepath.Dir(file), incl.Name) |
| subResults, err := LoadManifestTree(inclPath) |
| if err != nil { |
| return nil, err |
| } |
| for k, v := range subResults { |
| results[filepath.Join(filepath.Dir(incl.Name), k)] = v |
| } |
| } |
| return results, nil |
| } |
| |
| func fetchManifestRecursive(authedClient *http.Client, ctx context.Context, gc *bbproto.GitilesCommit, file string) (map[string]*Manifest, error) { |
| results := make(map[string]*Manifest) |
| log.Printf("Fetching manifest file %s at revision '%s'", file, gc.Id) |
| files, err := gerrit.FetchFilesFromGitiles( |
| authedClient, |
| ctx, |
| gc.Host, |
| gc.Project, |
| gc.Id, |
| []string{file}) |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to fetch %s", file).Err() |
| } |
| manifest := &Manifest{} |
| if err = xml.Unmarshal([]byte((*files)[file]), manifest); err != nil { |
| return nil, errors.Annotate(err, "failed to unmarshal %s", file).Err() |
| } |
| manifest.XMLName = xml.Name{} |
| results[file] = manifest |
| // Recursively fetch manifests listed in "include" elements. |
| for _, incl := range manifest.Includes { |
| subResults, err := fetchManifestRecursive(authedClient, ctx, gc, incl.Name) |
| if err != nil { |
| return nil, err |
| } |
| for k, v := range subResults { |
| results[k] = v |
| } |
| } |
| return results, nil |
| } |
| |
| // GetRepoToSourceRootFromManifests constructs a Gerrit project to path mapping by fetching manifest |
| // XML files from Gitiles. |
| func GetRepoToRemoteBranchToSourceRootFromManifests(authedClient *http.Client, ctx context.Context, gc *bbproto.GitilesCommit) (map[string]map[string]string, error) { |
| manifests, err := fetchManifestRecursive(authedClient, ctx, gc, rootXml) |
| if err != nil { |
| return nil, err |
| } |
| repoToSourceRoot := getRepoToRemoteBranchToSourceRootFromLoadedManifests(manifests) |
| log.Printf("Found %d repo to source root mappings from manifest files", len(repoToSourceRoot)) |
| return repoToSourceRoot, nil |
| } |
| |
| func GetRepoToRemoteBranchToSourceRootFromManifestFile(file string) (map[string]map[string]string, error) { |
| manifests, err := LoadManifestTree(file) |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to load local manifest %s", file).Err() |
| } |
| repoToSourceRoot := getRepoToRemoteBranchToSourceRootFromLoadedManifests(manifests) |
| log.Printf("Found %d repo to source root mappings from manifest files", len(repoToSourceRoot)) |
| return repoToSourceRoot, nil |
| } |
| |
| func getRepoToRemoteBranchToSourceRootFromLoadedManifests(manifests map[string]*Manifest) map[string]map[string]string { |
| repoToSourceRoot := make(map[string]map[string]string) |
| for _, m := range manifests { |
| for _, p := range m.Projects { |
| if _, found := repoToSourceRoot[p.Name]; !found { |
| repoToSourceRoot[p.Name] = make(map[string]string) |
| } |
| branch := p.Upstream |
| if branch == "" { |
| branch = "refs/heads/master" |
| } |
| if !strings.HasPrefix(branch, "refs/heads/") { |
| branch = "refs/heads/" + branch |
| } |
| |
| if oldPath, found := repoToSourceRoot[p.Name][branch]; found { |
| log.Printf("Source root for (%s, %s) is currently %s, overwriting with %s", p.Name, branch, oldPath, p.Path) |
| } |
| |
| repoToSourceRoot[p.Name][branch] = p.Path |
| } |
| } |
| return repoToSourceRoot |
| } |
| |
| // GetUnique Project returns the unique project with the given name |
| // (nil if the project DNE). It returns an error if multiple projects with the |
| // given name exist. |
| func (m *Manifest) GetUniqueProject(name string) (Project, error) { |
| var project Project |
| matchingProjects := 0 |
| for _, p := range m.Projects { |
| if p.Name == name { |
| matchingProjects++ |
| if matchingProjects > 1 { |
| return Project{}, fmt.Errorf("multiple projects named %s", name) |
| } |
| project = p |
| } |
| } |
| if matchingProjects == 0 { |
| return Project{}, fmt.Errorf("no project named %s", name) |
| } |
| return project, nil |
| } |
| |
| // Write writes the manifest to the given path. |
| func (m *Manifest) Write(path string) error { |
| data, err := xml.MarshalIndent(m, "", " ") |
| if err != nil { |
| return errors.Annotate(err, "failed to write manifest").Err() |
| } |
| err = ioutil.WriteFile(path, []byte(xml.Header+string(data)), 0644) |
| if err != nil { |
| return errors.Annotate(err, "failed to write manifest").Err() |
| } |
| return nil |
| } |
| |
| func projectInArr(project Project, projects []Project) bool { |
| for _, proj := range projects { |
| // Path is a unniue identifier for projects. |
| if project.Path == proj.Path { |
| return true |
| } |
| } |
| return false |
| } |
| |
| func remoteInArr(remote Remote, remotes []Remote) bool { |
| for _, rem := range remotes { |
| if remote.Name == rem.Name { |
| return true |
| } |
| } |
| return false |
| } |
| |
| // MergeManifests will merge the given manifests based on includes, taking |
| // manifests[path] to be the top-level manifest. |
| // manifests maps manifest filenames to the Manifest structs themselves. |
| // This basically re-implements `repo manifest` but is necessary because we can't run |
| // `repo manifest` on a singular git repository. |
| func MergeManifests(root string, manifests *map[string]*Manifest) (*Manifest, error) { |
| baseManifest, ok := (*manifests)[root] |
| if !ok { |
| return nil, fmt.Errorf("manifest %s does not exist", root) |
| } |
| |
| // Merge each included manifest. Then merge into baseManifest. |
| for _, includedManifest := range baseManifest.Includes { |
| mergedManifest, err := MergeManifests(includedManifest.Name, manifests) |
| if err != nil { |
| return nil, err |
| } |
| // Merge projects. |
| for _, project := range mergedManifest.Projects { |
| if !projectInArr(project, baseManifest.Projects) { |
| baseManifest.Projects = append(baseManifest.Projects, project) |
| } |
| } |
| // Merge remotes. |
| for _, remote := range mergedManifest.Remotes { |
| if !remoteInArr(remote, baseManifest.Remotes) { |
| baseManifest.Remotes = append(baseManifest.Remotes, remote) |
| } |
| } |
| // Keep the base manifest's default. |
| } |
| |
| // Clear includes. |
| baseManifest.Includes = []Include{} |
| baseManifest = baseManifest.ResolveImplicitLinks() |
| return baseManifest, nil |
| } |