package util
import (
"go/build"
"go/parser"
"go/token"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
)
// IsStandardImportPath reports whether $GOROOT/src/path should be considered
// part of the standard distribution. For historical reasons we allow people to add
// their own code to $GOROOT instead of using $GOPATH, but we assume that
// code will start with a domain name (dot in the first element).
//
// Note that this function is meant to evaluate whether a directory found in GOROOT
// should be treated as part of the standard library. It should not be used to decide
// that a directory found in GOPATH should be rejected: directories in GOPATH
// need not have dots in the first element, and they just take their chances
// with future collisions in the standard library.
//
// This function was copied from src/cmd/go/internal/search/search.go in Go's code.
func isStandardImportPath(path string) bool {
i := strings.Index(path, "/")
if i < 0 {
i = len(path)
}
elem := path[:i]
return !strings.Contains(elem, ".")
}
// Ignore specifies a set of resources to ignore
type Ignore struct {
Dirs []string
Trees []string
Regexes []*regexp.Regexp
}
func (ignore *Ignore) ignore(path string) bool {
if ignore == nil {
return false
}
for _, re := range ignore.Regexes {
if re.MatchString(path) {
return true
}
}
for _, dir := range ignore.Trees {
if strings.HasPrefix(path+"/", dir+"/") {
return true
}
}
for _, dir := range ignore.Dirs {
if path == dir {
return true
}
}
return false
}
type PackageInfoCollector struct {
packageInfos map[string]*build.Package
mainFiles map[string][]string
packagePath string
ignore *Ignore
includeMD bool
extensions []string
otherResources *OtherResources
isStdPackages map[string]bool
}
type OtherResources struct {
ProtoFiles []string
TmplFiles []string
MDFiles []string
SFiles []string
Other []string
}
func NewPackageInfoCollector(ignore *Ignore, includeMD bool, extensions []string) *PackageInfoCollector {
return &PackageInfoCollector{
packageInfos: make(map[string]*build.Package),
mainFiles: make(map[string][]string),
isStdPackages: make(map[string]bool),
ignore: ignore,
includeMD: includeMD,
extensions: extensions,
}
}
func (p *PackageInfoCollector) isStandardPackage(pkg *build.Package) bool {
return pkg.Goroot && pkg.ImportPath != "" && isStandardImportPath(pkg.ImportPath)
}
func (p *PackageInfoCollector) isStandard(pkg string) (bool, error) {
if is, exists := p.isStdPackages[pkg]; exists {
return is, nil
}
pkgInfo, err := build.Import(pkg, "", build.IgnoreVendor)
if err != nil {
return false, err
}
p.isStdPackages[pkg] = p.isStandardPackage(pkgInfo)
return p.isStdPackages[pkg], nil
}
func (p *PackageInfoCollector) resolveImports(baseDir string, imports []string) []string {
var paths []string
for _, importPath := range imports {
importedPackage, err := build.Import(importPath, baseDir, 0)
if err == nil {
paths = append(paths, importedPackage.ImportPath)
} else {
// Just assume we haven't 'go get'd it yet.
paths = append(paths, importPath)
}
}
return paths
}
func (p *PackageInfoCollector) CollectPackageInfos(packagePath string) error {
pkg, err := build.Import(packagePath, "", build.IgnoreVendor)
if err != nil {
// Don't worry about lack of Go files; there may be Go files in
// subdirectories. We just want to find the absolute path in
// GOPATH/GOROOT, but return error if not found at all.
if _, ok := err.(*build.NoGoError); !ok {
return err
}
}
p.otherResources = &OtherResources{}
if err := filepath.Walk(pkg.Dir+"/", func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
// Cited from https://golang.org/cmd/go/#hdr-Package_lists:
// Directory and file names that begin with "." or "_" are ignored
// by the go tool, as are directories named "testdata".
if base := filepath.Base(path); base[0] == '.' || base[0] == '_' {
return nil
}
for _, ext := range p.extensions {
if ext[0] != '.' {
ext = "." + ext
}
if strings.HasSuffix(path, ext) {
switch ext {
case ".proto":
p.otherResources.ProtoFiles = append(p.otherResources.ProtoFiles, path)
case ".md":
p.otherResources.MDFiles = append(p.otherResources.MDFiles, path)
case ".s":
p.otherResources.SFiles = append(p.otherResources.SFiles, path)
default:
p.otherResources.Other = append(p.otherResources.Other, path)
}
}
}
return nil
}
relPath := path[len(pkg.SrcRoot)+1:]
if strings.HasSuffix(relPath, "/") {
relPath = relPath[:len(relPath)-1]
}
// Cited from https://golang.org/cmd/go/#hdr-Package_lists:
// Directory and file names that begin with "." or "_" are ignored
// by the go tool, as are directories named "testdata".
if base := filepath.Base(relPath); base[0] == '.' || base[0] == '_' || base == "testdata" || base == "vendor" {
return filepath.SkipDir
}
if p.ignore.ignore(relPath) {
return nil
}
pkgInfo, err := build.ImportDir(path, build.IgnoreVendor)
if err != nil {
if _, ok := err.(*build.NoGoError); ok {
return nil
}
panic(err)
}
if len(pkgInfo.GoFiles) > 0 || len(pkgInfo.CgoFiles) > 0 {
// Show vendor-expanded paths in listing
pkgInfo.TestImports = p.resolveImports(pkg.Dir, pkgInfo.TestImports)
pkgInfo.XTestImports = p.resolveImports(pkg.Dir, pkgInfo.XTestImports)
p.packageInfos[relPath] = pkgInfo
}
return nil
}); err != nil {
return err
}
p.packagePath = packagePath
return nil
}
func (p *PackageInfoCollector) BuildArtifact() (*ProjectData, error) {
data := &ProjectData{}
var err error
// Get provided packages
if data.Packages, err = p.BuildPackageTree(false, false, false); err != nil {
return nil, err
}
sort.Strings(data.Packages)
// Get imported packages
data.Dependencies = make(map[string][]string)
for pkgName, info := range p.packageInfos {
data.Dependencies[pkgName] = []string{}
for _, item := range info.Imports {
if pos := strings.LastIndex(item, "/vendor/"); pos != -1 {
item = item[pos+8:]
}
if is, _ := p.isStandard(item); !is {
data.Dependencies[pkgName] = append(data.Dependencies[pkgName], item)
}
}
}
// Get tests
data.Tests = make(map[string][]string)
for pkgName, pkgInfo := range p.packageInfos {
if len(pkgInfo.TestGoFiles) > 0 {
data.Tests[pkgName] = []string{}
for _, item := range pkgInfo.TestImports {
if is, _ := p.isStandard(item); !is {
data.Tests[pkgName] = append(data.Tests[pkgName], item)
}
}
for _, item := range pkgInfo.XTestImports {
if is, _ := p.isStandard(item); !is {
data.Tests[pkgName] = append(data.Tests[pkgName], item)
}
}
}
}
// Get main files
data.MainFiles = make(map[string][]string)
for pkgName, item := range p.mainFiles {
data.MainFiles[pkgName] = []string{}
for _, dep := range item {
if is, _ := p.isStandard(dep); !is {
data.MainFiles[pkgName] = append(data.MainFiles[pkgName], dep)
}
}
}
return data, nil
}
func (p *PackageInfoCollector) CollectInstalledResources() ([]string, error) {
var resources []string
for _, info := range p.packageInfos {
resources = append(resources, info.Dir)
allFileLists := [][]string{
info.GoFiles, info.IgnoredGoFiles, info.CgoFiles,
info.CFiles, info.CXXFiles, info.MFiles, info.HFiles, info.FFiles,
info.SFiles, info.SwigFiles, info.SwigCXXFiles, info.SysoFiles,
info.TestGoFiles, info.XTestGoFiles}
for _, fileList := range allFileLists {
for _, file := range fileList {
resources = append(resources, path.Join(info.Dir, file))
}
}
}
if p.otherResources.ProtoFiles != nil {
resources = append(resources, p.otherResources.ProtoFiles...)
}
if p.includeMD && p.otherResources.MDFiles != nil {
resources = append(resources, p.otherResources.MDFiles...)
}
if p.otherResources.SFiles != nil {
resources = append(resources, p.otherResources.SFiles...)
}
if p.otherResources.Other != nil {
resources = append(resources, p.otherResources.Other...)
}
return resources, nil
}
func (p *PackageInfoCollector) CollectProjectDeps(standard, skipSelf, withTests, testsOnly bool) ([]string, error) {
imports := make(map[string]struct{})
for _, info := range p.packageInfos {
var pkgImports []string
if testsOnly {
pkgImports = append(info.TestImports, info.XTestImports...)
} else if withTests {
pkgImports = append(info.Imports, info.TestImports...)
pkgImports = append(pkgImports, info.XTestImports...)
} else {
pkgImports = info.Imports
}
for _, item := range pkgImports {
if item == "C" {
continue
}
if pos := strings.LastIndex(item, "/vendor/"); pos != -1 {
item = item[pos+8:]
}
if _, ok := imports[item]; !ok {
imports[item] = struct{}{}
}
}
}
var pkgs []string
for relPath := range imports {
if skipSelf && strings.HasPrefix(relPath, p.packagePath) {
continue
}
if p.ignore.ignore(relPath) {
continue
}
pkgInfo, err := build.Import(relPath, "", build.IgnoreVendor)
// assuming the stdlib is always processed properly
if !standard && err == nil && p.isStandardPackage(pkgInfo) {
continue
}
pkgs = append(pkgs, relPath)
}
return pkgs, nil
}
func (p *PackageInfoCollector) BuildPackageTree(includeMain, withTests, testsOnly bool) ([]string, error) {
// TODO(jchaloup): strip all main package unless explicitely requested
var entryPoints []string
if testsOnly || withTests {
for p, pkgInfo := range p.packageInfos {
if len(pkgInfo.TestGoFiles) > 0 || len(pkgInfo.XTestGoFiles) > 0 {
entryPoints = append(entryPoints, p)
}
}
if testsOnly {
return entryPoints, nil
}
}
for pkgName, pkgInfo := range p.packageInfos {
// check package name of each file
var nonMainFiles []string
files := pkgInfo.GoFiles
files = append(files, pkgInfo.CgoFiles...)
for _, file := range files {
f, err := parser.ParseFile(token.NewFileSet(), path.Join(pkgInfo.Dir, file), nil, 0)
if err != nil {
return nil, err
}
if f.Name.Name == "main" && file != "doc.go" {
importsList := make([]string, 0)
for _, i := range f.Imports {
importsList = append(importsList, i.Path.Value[1:len(i.Path.Value)-1])
}
p.mainFiles[path.Join(pkgInfo.ImportPath, file)] = importsList
}
if !includeMain && f.Name.Name == "main" {
continue
}
nonMainFiles = append(nonMainFiles, file)
}
if len(nonMainFiles) > 0 {
entryPoints = append(entryPoints, pkgName)
}
}
return entryPoints, nil
}
type ProjectData struct {
Packages []string `json:"packages"`
Dependencies map[string][]string `json:"dependencies"`
Tests map[string][]string `json:"tests"`
MainFiles map[string][]string `json:"main"`
}