From 832905804f0bcc1bb5468b8d5b1a4b5ba4f274cb Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Thu, 21 Feb 2019 15:44:43 +0100 Subject: [PATCH 1/6] Install contributions with a version (flogo install contrib@vX.Y.Z) --- api/project.go | 79 +++++++++++++++++++++++++++++++++++++++++++++----- go.mod | 1 + util/mod.go | 35 ++++++++++++++-------- 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/api/project.go b/api/project.go index f513a45..3d73017 100644 --- a/api/project.go +++ b/api/project.go @@ -1,12 +1,17 @@ package api import ( + "encoding/json" "fmt" + "github.com/project-flogo/core/app" // dependency to core ensures the CLI always uses an up-to-date struct for JSON manipulation (this dependency already exists implicitly in the "flogo create" command) "go/parser" "go/printer" "go/token" + "io/ioutil" "os" + "os/exec" "path/filepath" + "regexp" "runtime" "github.com/project-flogo/cli/common" @@ -104,8 +109,7 @@ func (p *appProjectImpl) GetPath(pkg string) (string, error) { return p.dm.GetPath(pkg) } -func (p *appProjectImpl) AddImports(ignoreError bool, imports ...string) error { - +func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...string) error { importsFile := filepath.Join(p.SrcDir(), fileImportsGo) fset := token.NewFileSet() @@ -114,17 +118,16 @@ func (p *appProjectImpl) AddImports(ignoreError bool, imports ...string) error { return err } - for _, impPath := range imports { - path, version := util.ParseImportPath(impPath) - err := p.DepManager().AddDependency(path, version) + for _, importPath := range imports { + importPath, err := p.DepManager().AddDependency(importPath, "", true) if err != nil { if ignoreError { - fmt.Printf("Warning: unable to install %s\n", impPath) + fmt.Printf("Warning: unable to install %s\n", importPath) continue } return err } - util.AddImport(fset, file, path) + util.AddImport(fset, file, importPath) } f, err := os.Create(importsFile) @@ -135,9 +138,71 @@ func (p *appProjectImpl) AddImports(ignoreError bool, imports ...string) error { //p.dm.Finalize() + // using "go mod verify" can solve some dependencies conflicts (when contributions are dependent of others) + err = util.ExecCmd(exec.Command("go", "mod", "verify"), p.srcDir) + if err != nil { + return err + } + return nil } +func (p *appProjectImpl) addImportsInJson(ignoreError bool, imports ...string) error { + appDescriptorFile := filepath.Join(p.appDir, fileFlogoJson) + appDescriptorJsonFile, err := os.Open(appDescriptorFile) + if err != nil { + return err + } + defer appDescriptorJsonFile.Close() + + appDescriptorData, err := ioutil.ReadAll(appDescriptorJsonFile) + if err != nil { + return err + } + + var appDescriptor app.Config + json.Unmarshal([]byte(appDescriptorData), &appDescriptor) + + importPattern := regexp.MustCompile(`^([^ ]* )?([^@]*)@?(.*)?$`) // extract import path even if there is an alias and/or a version + + // list existing imports in JSON to avoid duplicates + existingImports := make(map[string]bool) + for _, e := range appDescriptor.Imports { + importPath := importPattern.FindStringSubmatch(e)[2] + existingImports[importPath] = true + } + + for _, i := range imports { + if _, ok := existingImports[i]; !ok { + appDescriptor.Imports = append(appDescriptor.Imports, importPattern.FindStringSubmatch(i)[2]) + } + } + + appDescriptorUpdated, err := json.MarshalIndent(appDescriptor, "", " ") + if err != nil { + return err + } + + appDescriptorUpdatedJson := string(appDescriptorUpdated) + + err = ioutil.WriteFile(appDescriptorFile, []byte(appDescriptorUpdatedJson), 0644) + if err != nil { + return err + } + + return nil +} + +func (p *appProjectImpl) AddImports(ignoreError bool, imports ...string) error { + err := p.addImportsInGo(ignoreError, imports...) // begin with Go imports as they are more likely to fail + if err != nil { + return err + } + err = p.addImportsInJson(ignoreError, imports...) // adding imports in JSON after Go imports ensure the flogo.json is self-sufficient + + return err +} + func (p *appProjectImpl) RemoveImports(imports ...string) error { importsFile := filepath.Join(p.SrcDir(), fileImportsGo) diff --git a/go.mod b/go.mod index 1e22454..5d0b41f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/project-flogo/cli require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/project-flogo/core v0.9.0-alpha.4.0.20190220191401-07116138c345 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.2.2 diff --git a/util/mod.go b/util/mod.go index f9c7401..e7e42fc 100644 --- a/util/mod.go +++ b/util/mod.go @@ -16,7 +16,7 @@ import ( type DepManager interface { Init() error - AddDependency(path, version string) error + AddDependency(path, version string, fetch bool) (importPath string, err error) GetPath(pkg string) (string, error) AddLocalContribForBuild() error InstallLocalPkg(string, string) @@ -41,20 +41,27 @@ func (m *ModDepManager) Init() error { return nil } -func (m *ModDepManager) AddDependency(path, version string) error { +func (m *ModDepManager) AddDependency(path, version string, fetch bool) (string, error) { - var dep string - if version != "" { - dep = path + "@" + version - } else { - dep = path + "@latest" + depVersion := version + if strings.Contains(path, "@v") { + depVersion = strings.Split(path, "@")[1] + path = strings.Split(path, "@")[0] + } + + if len(depVersion) == 0 { + //Latest changed to master. Need to clear out in future. + //Changed to master due to Issue in flogo-contrib/legacy-support + depVersion = "master" + } else if depVersion != "master" && depVersion[0] != 'v' { + depVersion = "v" + version } //note: hack, because go get doesn't add core to go.mod if path == "github.com/project-flogo/core" { err := ExecCmd(exec.Command("go", "mod", "edit", "-require", dep), m.srcDir) if err != nil { - return err + return "", err } } @@ -63,17 +70,21 @@ func (m *ModDepManager) AddDependency(path, version string) error { version = getLatestVersion("github.com/TIBCOSoftware/flogo-contrib") err := ExecCmd(exec.Command("go", "mod", "edit", "-require", "github.com/TIBCOSoftware/flogo-contrib@"+version), m.srcDir) if err != nil { - return err + return "", err } } - err := ExecCmd(exec.Command("go", "get", "-u", dep), m.srcDir) + // use "go mod edit" instead of "go get -u", "go mod verify" will ensure dependencies at the end of imports + err := ExecCmd(exec.Command("go", "mod", "edit", "-require", dep), m.srcDir) if err != nil { fmt.Println("Error in installing", dep) - return err + return "", err } - return nil + return path, nil // return import path without the version + // note for the future: based on resolved depVersion, the import path could be updated by adding + // the according vX suffix in import path, for instance: github.com/corp/contrib/v2 + // this import path will then be used in imports.go & flogo.json files } // GetPath gets the path of where the From 7da3c00ad1ff680b87a99b272d37b00b49a6f2ea Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Thu, 21 Feb 2019 17:02:53 +0100 Subject: [PATCH 2/6] Avoid mutiple "go mod edit -require " commands --- util/mod.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/mod.go b/util/mod.go index e7e42fc..b0a21ae 100644 --- a/util/mod.go +++ b/util/mod.go @@ -63,6 +63,7 @@ func (m *ModDepManager) AddDependency(path, version string, fetch bool) (string, if err != nil { return "", err } + return path, nil } //note: hack, because go get isn't picking up latest @@ -72,6 +73,7 @@ func (m *ModDepManager) AddDependency(path, version string, fetch bool) (string, if err != nil { return "", err } + return path, nil } // use "go mod edit" instead of "go get -u", "go mod verify" will ensure dependencies at the end of imports From 7c48949b831a4b9eaa6dd0a8191ad41c7d5927ff Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Fri, 22 Feb 2019 16:25:22 +0100 Subject: [PATCH 3/6] Introduce a new Flogo import syntax --- api/build.go | 16 +++++- api/create.go | 21 ++++++-- api/imports.go | 7 ++- api/install.go | 33 ++++++------ api/legacy.go | 6 ++- api/project.go | 32 +++++------- common/project.go | 4 +- go.mod | 2 +- util/flogo.go | 13 ++++- util/imports.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++ util/mod.go | 65 +++++++++-------------- 11 files changed, 238 insertions(+), 90 deletions(-) create mode 100644 util/imports.go diff --git a/api/build.go b/api/build.go index 8f46401..28f52ad 100644 --- a/api/build.go +++ b/api/build.go @@ -81,6 +81,8 @@ func BuildProject(project common.AppProject, options BuildOptions) error { fmt.Println("Path to exe is ", exePath) } if _, err := os.Stat(exePath); err == nil { + finalExePath := project.Executable() + os.MkdirAll(filepath.Dir(finalExePath), os.ModePerm) err = os.Rename(exePath, project.Executable()) if err != nil { @@ -125,7 +127,12 @@ func prepareShim(project common.AppProject, shim string) (bool, error) { } } - path, err := project.GetPath(ref) + refImport, err := util.NewFlogoImportFromPath(ref) + if err != nil { + return false, err + } + + path, err := project.GetPath(refImport) if err != nil { return false, err } @@ -205,7 +212,12 @@ func createShimSupportGoFile(project common.AppProject, create bool) error { return nil } - corePath, err := project.GetPath(flogoCoreRepo) + flogoCoreImport, err := util.NewFlogoImportFromPath(flogoCoreRepo) + if err != nil { + return err + } + + corePath, err := project.GetPath(flogoCoreImport) if err != nil { return err } diff --git a/api/create.go b/api/create.go index ee4faa1..0ecd42b 100644 --- a/api/create.go +++ b/api/create.go @@ -146,9 +146,10 @@ func setupAppDirectory(dm util.DepManager, appPath, coreVersion string) error { fmt.Println("installing core") - // add & fetch the core library + flogoCoreImport := util.NewFlogoImport(flogoCoreRepo, "", coreVersion, "") - dm.AddDependency(flogoCoreRepo, coreVersion) + // add & fetch the core library + dm.AddDependency(flogoCoreImport, true) return nil } @@ -193,7 +194,7 @@ func importDependencies(project common.AppProject) error { fmt.Printf("%-20s %s\n", instStr, imp) } - legacy, err := IsLegacySupportRequired(desc, path, imp, true) + legacy, err := IsLegacySupportRequired(desc, path, imp.ImportPath(), true) if err != nil { return err } @@ -211,7 +212,12 @@ func importDependencies(project common.AppProject) error { func createMain(dm util.DepManager, appDir string) error { - corePath, err := dm.GetPath(flogoCoreRepo) + flogoCoreImport, err := util.NewFlogoImportFromPath(flogoCoreRepo) + if err != nil { + return err + } + + corePath, err := dm.GetPath(flogoCoreImport) if err != nil { return err } @@ -234,7 +240,12 @@ func getAndUpdateAppJson(dm util.DepManager, appName, appJson string) (string, e if len(appJson) == 0 { // appJson wasn't provided, so lets grab the example - corePath, err := dm.GetPath(flogoCoreRepo) + flogoCoreImport, err := util.NewFlogoImportFromPath(flogoCoreRepo) + if err != nil { + return "", err + } + + corePath, err := dm.GetPath(flogoCoreImport) if err != nil { return "", err } diff --git a/api/imports.go b/api/imports.go index 81df20a..d2bee8c 100644 --- a/api/imports.go +++ b/api/imports.go @@ -59,7 +59,12 @@ func registerImport(project common.AppProject, anImport string) error { func getContribType(project common.AppProject, ref string) (string, error) { - path, err := project.GetPath(ref) + refAsFlogoImport, err := util.NewFlogoImportFromPath(ref) + if err != nil { + return "", err + } + + path, err := project.GetPath(refAsFlogoImport) if err != nil { return "", err } diff --git a/api/install.go b/api/install.go index 1829cef..8ea7bf4 100644 --- a/api/install.go +++ b/api/install.go @@ -15,12 +15,17 @@ import ( func InstallPackage(project common.AppProject, pkg string) error { - err := project.AddImports(false, pkg) + flogoImport, err := util.ParseImport(pkg) if err != nil { return err } - path, err := project.GetPath(pkg) + err = project.AddImports(false, flogoImport) + if err != nil { + return err + } + + path, err := project.GetPath(flogoImport) if Verbose() { fmt.Println("Installed path", path) } @@ -72,6 +77,7 @@ func InstallPalette(project common.AppProject, path string) error { return nil } + func ListPackages(project common.AppProject, format bool, all bool) error { err := util.ExecCmd(exec.Command("go", "mod", "tidy"), project.SrcDir()) @@ -80,21 +86,21 @@ func ListPackages(project common.AppProject, format bool, all bool) error { return err } - var contribs []string + var contribs util.Imports if all { - contribs, _ = util.GetAllImports(filepath.Join(project.SrcDir(), fileImportsGo)) // Get Imports from imports.go - + imports, _ := util.GetAllImports(filepath.Join(project.SrcDir(), fileImportsGo)) // Get Imports from imports.go + for _, i := range imports { + flogoImport, _ := util.ParseImport(i) + contribs = append(contribs, flogoImport) + } } else { contribs, _ = util.GetImports(filepath.Join(project.Dir(), fileFlogoJson)) // Get Imports from flogo.json - } var result []interface{} for _, contrib := range contribs { - contrib = clearVersion(contrib) - path, err := project.GetPath(contrib) if Verbose() { fmt.Println("Path of contrib", path, "for contrib", contrib) @@ -127,7 +133,7 @@ func ListPackages(project common.AppProject, format bool, all bool) error { desc.Type, desc.Description, desc.Homepage, - contrib, + contrib.ModulePath(), getDescriptorFile(path), } @@ -157,12 +163,3 @@ func getDescriptorFile(path string) string { } return "" } -func clearVersion(pkg string) string { - - if strings.Contains(pkg, "@") { - - return strings.Split(pkg, "@")[0] - - } - return pkg -} diff --git a/api/legacy.go b/api/legacy.go index 6fbafc8..a97e927 100644 --- a/api/legacy.go +++ b/api/legacy.go @@ -35,7 +35,11 @@ func IsLegacySupportRequired(desc *util.FlogoContribDescriptor, path, pkg string } func InstallLegacySupport(project common.AppProject) error { - err := project.AddImports(false, pkgLegacySupport) + pkgLegacySupportImport, err := util.NewFlogoImportFromPath(pkgLegacySupport) + if err != nil { + return err + } + err = project.AddImports(false, pkgLegacySupportImport) if err == nil { fmt.Println("Installed Legacy Support") } diff --git a/api/project.go b/api/project.go index 3d73017..ceb4698 100644 --- a/api/project.go +++ b/api/project.go @@ -11,7 +11,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "runtime" "github.com/project-flogo/cli/common" @@ -104,12 +103,11 @@ func (p *appProjectImpl) Executable() string { return execPath } -func (p *appProjectImpl) GetPath(pkg string) (string, error) { - - return p.dm.GetPath(pkg) +func (p *appProjectImpl) GetPath(flogoImport util.Import) (string, error) { + return p.dm.GetPath(flogoImport) } -func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...string) error { +func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...util.Import) error { importsFile := filepath.Join(p.SrcDir(), fileImportsGo) fset := token.NewFileSet() @@ -118,16 +116,16 @@ func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...string) err return err } - for _, importPath := range imports { - importPath, err := p.DepManager().AddDependency(importPath, "", true) + for _, i := range imports { + err := p.DepManager().AddDependency(i, true) if err != nil { if ignoreError { - fmt.Printf("Warning: unable to install %s\n", importPath) + fmt.Printf("Warning: unable to install %s\n", i) continue } return err } - util.AddImport(fset, file, importPath) + util.AddImport(fset, file, i.ImportPath()) } f, err := os.Create(importsFile) @@ -147,7 +145,7 @@ func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...string) err return nil } -func (p *appProjectImpl) addImportsInJson(ignoreError bool, imports ...string) error { +func (p *appProjectImpl) addImportsInJson(ignoreError bool, imports ...util.Import) error { appDescriptorFile := filepath.Join(p.appDir, fileFlogoJson) appDescriptorJsonFile, err := os.Open(appDescriptorFile) if err != nil { @@ -163,18 +161,16 @@ func (p *appProjectImpl) addImportsInJson(ignoreError bool, imports ...string) e var appDescriptor app.Config json.Unmarshal([]byte(appDescriptorData), &appDescriptor) - importPattern := regexp.MustCompile(`^([^ ]* )?([^@]*)@?(.*)?$`) // extract import path even if there is an alias and/or a version - // list existing imports in JSON to avoid duplicates existingImports := make(map[string]bool) - for _, e := range appDescriptor.Imports { - importPath := importPattern.FindStringSubmatch(e)[2] - existingImports[importPath] = true + jsonImports, _ := util.ParseImports(appDescriptor.Imports) + for _, e := range jsonImports { + existingImports[e.CanonicalImport()] = true } for _, i := range imports { - if _, ok := existingImports[i]; !ok { - appDescriptor.Imports = append(appDescriptor.Imports, importPattern.FindStringSubmatch(i)[2]) + if _, ok := existingImports[i.CanonicalImport()]; !ok { + appDescriptor.Imports = append(appDescriptor.Imports, i.CanonicalImport()) } } @@ -193,7 +189,7 @@ func (p *appProjectImpl) addImportsInJson(ignoreError bool, imports ...string) e return nil } -func (p *appProjectImpl) AddImports(ignoreError bool, imports ...string) error { +func (p *appProjectImpl) AddImports(ignoreError bool, imports ...util.Import) error { err := p.addImportsInGo(ignoreError, imports...) // begin with Go imports as they are more likely to fail if err != nil { return err diff --git a/common/project.go b/common/project.go index 35e8bc8..01f7041 100644 --- a/common/project.go +++ b/common/project.go @@ -9,8 +9,8 @@ type AppProject interface { BinDir() string SrcDir() string Executable() string - AddImports(ignoreError bool, imports ...string) error + AddImports(ignoreError bool, imports ...util.Import) error RemoveImports(imports ...string) error - GetPath(pkg string) (string, error) + GetPath(flogoImport util.Import) (string, error) DepManager() util.DepManager } diff --git a/go.mod b/go.mod index 5d0b41f..8a65928 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/project-flogo/cli require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/project-flogo/core v0.9.0-alpha.4.0.20190220191401-07116138c345 + github.com/project-flogo/core v0.9.0-alpha.4.0.20190222151024-3eb86689b764 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.2.2 diff --git a/util/flogo.go b/util/flogo.go index b0b401d..623c3e2 100644 --- a/util/flogo.go +++ b/util/flogo.go @@ -87,7 +87,7 @@ func GetContribDescriptor(path string) (*FlogoContribDescriptor, error) { } // ParseAppDescriptor parse the application descriptor -func GetImports(appJsonPath string) ([]string, error) { +func GetImports(appJsonPath string) (Imports, error) { importSet := make(map[string]struct{}) @@ -116,7 +116,16 @@ func GetImports(appJsonPath string) ([]string, error) { allImports = append(allImports, key) } - return allImports, nil + var result Imports + for _, i := range allImports { + parsedImport, err := ParseImport(i) + if err != nil { + return nil, err + } + result = append(result, parsedImport) + } + + return result, nil } func getImports(appJsonPath string) ([]string, error) { diff --git a/util/imports.go b/util/imports.go new file mode 100644 index 0000000..f496807 --- /dev/null +++ b/util/imports.go @@ -0,0 +1,129 @@ +package util + +import ( + "errors" + "fmt" + "regexp" +) + +/* util.Import struct defines the different fields which can be extracted from a Flogo import +these imports are stored in flogo.json in the "imports" array, for instance: + + "imports": [ + "github.com/project-flogo/contrib@v0.9.0-alpha.4:/activity/log", + "rest_activity github.com/project-flogo/contrib@v0.9.0:/activity/rest", + "rest_trigger github.com/project-flogo/contrib:/trigger/rest", + "github.com/project-flogo/flow" + ] + +*/ +type FlogoImport struct { + modulePath string + relativeImportPath string + version string + alias string +} + +func NewFlogoImportFromPath(flogoImportPath string) (Import, error) { + flogoImport, err := ParseImport(flogoImportPath) + if err != nil { + return nil, err + } + return flogoImport, nil +} + +func NewFlogoImport(modulePath, relativeImportPath, version, alias string) Import { + return &FlogoImport{modulePath: modulePath, relativeImportPath: relativeImportPath, version: version, alias: alias} +} + +type Import interface { + fmt.Stringer + + ModulePath() string + RelativeImportPath() string + Version() string + Alias() string + + ImportPath() string + CanonicalImport() string + ModulePathWithVersion() string +} + +type Imports []Import + +func (flogoImport *FlogoImport) ModulePath() string { + return flogoImport.modulePath +} +func (flogoImport *FlogoImport) RelativeImportPath() string { + return flogoImport.relativeImportPath +} +func (flogoImport *FlogoImport) Version() string { + return flogoImport.version +} +func (flogoImport *FlogoImport) Alias() string { + return flogoImport.alias +} +func (flogoImport *FlogoImport) ImportPath() string { + return flogoImport.modulePath + flogoImport.relativeImportPath +} +func (flogoImport *FlogoImport) CanonicalImport() string { + alias := "" + if flogoImport.alias != "" { + alias = flogoImport.alias + " " + } + version := "" + if flogoImport.version != "" { + version = "@" + flogoImport.version + } + relativeImportPath := "" + if flogoImport.relativeImportPath != "" { + relativeImportPath = ":" + flogoImport.relativeImportPath + } + + return alias + flogoImport.modulePath + version + relativeImportPath +} +func (flogoImport *FlogoImport) ModulePathWithVersion() string { + version := "@master" + if flogoImport.version != "" { + version = "@" + flogoImport.version + } + return flogoImport.modulePath + version +} +func (flogoImport *FlogoImport) String() string { + version := "" + if flogoImport.version != "" { + version = " " + flogoImport.version + } + relativeImportPath := "" + if flogoImport.relativeImportPath != "" { + relativeImportPath = flogoImport.relativeImportPath + } + + return flogoImport.modulePath + relativeImportPath + version +} + +var flogoImportPattern = regexp.MustCompile(`^(([^ ]*)[ ]+)?([^@:]*)@?([^:]*)?:?(.*)?$`) // extract import path even if there is an alias and/or a version + +func ParseImports(flogoImports []string) (Imports, error) { + var result Imports + + for _, flogoImportPath := range flogoImports { + flogoImport, err := ParseImport(flogoImportPath) + if err != nil { + return nil, err + } + result = append(result, flogoImport) + } + + return result, nil +} + +func ParseImport(flogoImport string) (Import, error) { + if !flogoImportPattern.MatchString(flogoImport) { + return nil, errors.New(fmt.Sprintf("The Flogo import '%s' cannot be parsed.", flogoImport)) + } + + matches := flogoImportPattern.FindStringSubmatch(flogoImport) + result := &FlogoImport{modulePath: matches[3], relativeImportPath: matches[5], version: matches[4], alias: matches[2]} + return result, nil +} diff --git a/util/mod.go b/util/mod.go index b0a21ae..fea0373 100644 --- a/util/mod.go +++ b/util/mod.go @@ -16,8 +16,8 @@ import ( type DepManager interface { Init() error - AddDependency(path, version string, fetch bool) (importPath string, err error) - GetPath(pkg string) (string, error) + AddDependency(flogoImport Import, fetch bool) error + GetPath(flogoImport Import) (string, error) AddLocalContribForBuild() error InstallLocalPkg(string, string) } @@ -41,62 +41,47 @@ func (m *ModDepManager) Init() error { return nil } -func (m *ModDepManager) AddDependency(path, version string, fetch bool) (string, error) { - - depVersion := version - if strings.Contains(path, "@v") { - depVersion = strings.Split(path, "@")[1] - path = strings.Split(path, "@")[0] - } - - if len(depVersion) == 0 { - //Latest changed to master. Need to clear out in future. - //Changed to master due to Issue in flogo-contrib/legacy-support - depVersion = "master" - } else if depVersion != "master" && depVersion[0] != 'v' { - depVersion = "v" + version +func (m *ModDepManager) addDependency(path string, fetch bool) error { + err := ExecCmd(exec.Command("go", "mod", "edit", "-require", path), m.srcDir) + if err != nil { + return err } - //note: hack, because go get doesn't add core to go.mod - if path == "github.com/project-flogo/core" { - err := ExecCmd(exec.Command("go", "mod", "edit", "-require", dep), m.srcDir) + // force resolution + if fetch { + err = ExecCmd(exec.Command("go", "mod", "verify"), m.srcDir) if err != nil { - return "", err - } - return path, nil - } + return err - //note: hack, because go get isn't picking up latest - if strings.HasPrefix(path, "github.com/TIBCOSoftware/flogo-contrib") { - version = getLatestVersion("github.com/TIBCOSoftware/flogo-contrib") - err := ExecCmd(exec.Command("go", "mod", "edit", "-require", "github.com/TIBCOSoftware/flogo-contrib@"+version), m.srcDir) - if err != nil { - return "", err } - return path, nil } - // use "go mod edit" instead of "go get -u", "go mod verify" will ensure dependencies at the end of imports - err := ExecCmd(exec.Command("go", "mod", "edit", "-require", dep), m.srcDir) + return nil +} + +func (m *ModDepManager) AddDependency(flogoImport Import, fetch bool) error { + + // use "go mod edit" instead of "go get -u" + err := m.addDependency(flogoImport.ModulePathWithVersion(), fetch) + if err != nil { - fmt.Println("Error in installing", dep) - return "", err + fmt.Printf("Error in installing '%s'", flogoImport.String()) + return err } - return path, nil // return import path without the version - // note for the future: based on resolved depVersion, the import path could be updated by adding - // the according vX suffix in import path, for instance: github.com/corp/contrib/v2 - // this import path will then be used in imports.go & flogo.json files + return nil } // GetPath gets the path of where the -func (m *ModDepManager) GetPath(pkg string) (string, error) { +func (m *ModDepManager) GetPath(flogoImport Import) (string, error) { currentDir, err := os.Getwd() if err != nil { return "", err } + pkg := flogoImport.ModulePath() + path, ok := m.localMods[pkg] if ok && path != "" { @@ -149,7 +134,7 @@ func (m *ModDepManager) GetPath(pkg string) (string, error) { pathForPartial = filepath.Join(os.Getenv("GOPATH"), "pkg", "mod", pkgPath, remainingPath) } else { - return filepath.Join(os.Getenv("GOPATH"), "pkg", "mod", pkgPath), nil + return filepath.Join(os.Getenv("GOPATH"), "pkg", "mod", pkgPath, flogoImport.RelativeImportPath()), nil } } } From 78b3606b7f15f311425e18b0a819f952505d462d Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Wed, 27 Feb 2019 11:00:15 +0100 Subject: [PATCH 4/6] Support both syntaxes & methods, go get & go mod edit --- api/create.go | 4 +-- api/project.go | 16 ++++------ go.mod | 5 ++- util/imports.go | 29 +++++++++++++---- util/mod.go | 85 +++++++++++++++++++++++++++++++++++++++---------- 5 files changed, 101 insertions(+), 38 deletions(-) diff --git a/api/create.go b/api/create.go index 0ecd42b..286abc9 100644 --- a/api/create.go +++ b/api/create.go @@ -149,7 +149,7 @@ func setupAppDirectory(dm util.DepManager, appPath, coreVersion string) error { flogoCoreImport := util.NewFlogoImport(flogoCoreRepo, "", coreVersion, "") // add & fetch the core library - dm.AddDependency(flogoCoreImport, true) + dm.AddDependency(flogoCoreImport) return nil } @@ -194,7 +194,7 @@ func importDependencies(project common.AppProject) error { fmt.Printf("%-20s %s\n", instStr, imp) } - legacy, err := IsLegacySupportRequired(desc, path, imp.ImportPath(), true) + legacy, err := IsLegacySupportRequired(desc, path, imp.GoImportPath(), true) if err != nil { return err } diff --git a/api/project.go b/api/project.go index ceb4698..c3e5102 100644 --- a/api/project.go +++ b/api/project.go @@ -9,7 +9,6 @@ import ( "go/token" "io/ioutil" "os" - "os/exec" "path/filepath" "runtime" @@ -117,15 +116,18 @@ func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...util.Import } for _, i := range imports { - err := p.DepManager().AddDependency(i, true) + err := p.DepManager().AddDependency(i) if err != nil { if ignoreError { - fmt.Printf("Warning: unable to install %s\n", i) + fmt.Printf("Warning: unable to install '%s'\n", i) continue } + + fmt.Errorf("Error in installing '%s'\n", i) + return err } - util.AddImport(fset, file, i.ImportPath()) + util.AddImport(fset, file, i.GoImportPath()) } f, err := os.Create(importsFile) @@ -136,12 +138,6 @@ func (p *appProjectImpl) addImportsInGo(ignoreError bool, imports ...util.Import //p.dm.Finalize() - // using "go mod verify" can solve some dependencies conflicts (when contributions are dependent of others) - err = util.ExecCmd(exec.Command("go", "mod", "verify"), p.srcDir) - if err != nil { - return err - } - return nil } diff --git a/go.mod b/go.mod index 8a65928..9443dd6 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,8 @@ module github.com/project-flogo/cli require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/project-flogo/core v0.9.0-alpha.4.0.20190222151024-3eb86689b764 + github.com/msoap/byline v1.1.1 + github.com/project-flogo/core v0.9.0-alpha.4.0.20190225212330-f76237454d56 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.2.2 diff --git a/util/imports.go b/util/imports.go index f496807..d70893a 100644 --- a/util/imports.go +++ b/util/imports.go @@ -11,6 +11,7 @@ these imports are stored in flogo.json in the "imports" array, for instance: "imports": [ "github.com/project-flogo/contrib@v0.9.0-alpha.4:/activity/log", + "github.com/project-flogo/contrib/activity/rest@v0.9.0" "rest_activity github.com/project-flogo/contrib@v0.9.0:/activity/rest", "rest_trigger github.com/project-flogo/contrib:/trigger/rest", "github.com/project-flogo/flow" @@ -44,9 +45,11 @@ type Import interface { Version() string Alias() string - ImportPath() string - CanonicalImport() string - ModulePathWithVersion() string + CanonicalImport() string // canonical import is used in flogo.json imports array and to check for equality of imports + GoImportPath() string // the import path used in .go files + GoGetImportPath() string // the import path used by "go get" command + GoModImportPath() string // the import path used by "go mod edit" command + IsLegacy() bool // an import is "legacy" if it does not have a relative import path } type Imports []Import @@ -63,9 +66,7 @@ func (flogoImport *FlogoImport) Version() string { func (flogoImport *FlogoImport) Alias() string { return flogoImport.alias } -func (flogoImport *FlogoImport) ImportPath() string { - return flogoImport.modulePath + flogoImport.relativeImportPath -} + func (flogoImport *FlogoImport) CanonicalImport() string { alias := "" if flogoImport.alias != "" { @@ -82,13 +83,27 @@ func (flogoImport *FlogoImport) CanonicalImport() string { return alias + flogoImport.modulePath + version + relativeImportPath } -func (flogoImport *FlogoImport) ModulePathWithVersion() string { +func (flogoImport *FlogoImport) GoImportPath() string { + return flogoImport.modulePath + flogoImport.relativeImportPath +} +func (flogoImport *FlogoImport) GoGetImportPath() string { + version := "@master" + if flogoImport.version != "" { + version = "@" + flogoImport.version + } + return flogoImport.modulePath + flogoImport.relativeImportPath + version +} +func (flogoImport *FlogoImport) GoModImportPath() string { version := "@master" if flogoImport.version != "" { version = "@" + flogoImport.version } return flogoImport.modulePath + version } +func (flogoImport *FlogoImport) IsLegacy() bool { + return flogoImport.relativeImportPath == "" +} + func (flogoImport *FlogoImport) String() string { version := "" if flogoImport.version != "" { diff --git a/util/mod.go b/util/mod.go index fea0373..0a108a1 100644 --- a/util/mod.go +++ b/util/mod.go @@ -5,18 +5,20 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/msoap/byline" "io/ioutil" "log" "net/http" "os" "os/exec" "path/filepath" + "regexp" "strings" ) type DepManager interface { Init() error - AddDependency(flogoImport Import, fetch bool) error + AddDependency(flogoImport Import) error GetPath(flogoImport Import) (string, error) AddLocalContribForBuild() error InstallLocalPkg(string, string) @@ -41,31 +43,33 @@ func (m *ModDepManager) Init() error { return nil } -func (m *ModDepManager) addDependency(path string, fetch bool) error { - err := ExecCmd(exec.Command("go", "mod", "edit", "-require", path), m.srcDir) +func (m *ModDepManager) AddDependency(flogoImport Import) error { + + // use "go mod edit" (instead of "go get") as first method + err := ExecCmd(exec.Command("go", "mod", "edit", "-require", flogoImport.GoModImportPath()), m.srcDir) if err != nil { return err } // force resolution - if fetch { - err = ExecCmd(exec.Command("go", "mod", "verify"), m.srcDir) - if err != nil { - return err + // TODO: add a flag to skip download and perform download later (useful in 'flogo create' command for instance) + err = ExecCmd(exec.Command("go", "mod", "verify"), m.srcDir) + if err != nil { + // if the resolution fails and the Flogo import is "legacy" + // (meaning it does not separate module path from Go import path): + // 1. remove the import manually ("go mod edit -droprequire") would fail + // 2. try with "go get" instead + if flogoImport.IsLegacy() { + m.RemoveImport(flogoImport) + + err = ExecCmd(exec.Command("go", "get", "-u", flogoImport.GoGetImportPath()), m.srcDir) } + } else { + err = ExecCmd(exec.Command("go", "mod", "download", flogoImport.ModulePath()), m.srcDir) } - return nil -} - -func (m *ModDepManager) AddDependency(flogoImport Import, fetch bool) error { - - // use "go mod edit" instead of "go get -u" - err := m.addDependency(flogoImport.ModulePathWithVersion(), fetch) - if err != nil { - fmt.Printf("Error in installing '%s'", flogoImport.String()) return err } @@ -141,6 +145,55 @@ func (m *ModDepManager) GetPath(flogoImport Import) (string, error) { return pathForPartial, nil } +func (m *ModDepManager) RemoveImport(flogoImport Import) error { + + currentDir, err := os.Getwd() + if err != nil { + return err + } + + modulePath := flogoImport.ModulePath() + + defer os.Chdir(currentDir) + + os.Chdir(m.srcDir) + + file, err := os.Open(filepath.Join(m.srcDir, "go.mod")) + if err != nil { + return err + } + defer file.Close() + + modulePath = strings.Replace(modulePath, "/", "\\/", -1) + modulePath = strings.Replace(modulePath, ".", "\\.", -1) + importRegex := regexp.MustCompile(`\s*` + modulePath + `\s+` + flogoImport.Version() + `.*`) + + lr := byline.NewReader(file) + + lr.MapString(func(line string) string { + if importRegex.MatchString(line) { + return "" + } else { + return line + } + }) + + updatedGoMod, err := lr.ReadAll() + if err != nil { + return err + } + + file, err = os.Create(filepath.Join(m.srcDir, "go.mod")) + if err != nil { + return err + } + defer file.Close() + + file.Write(updatedGoMod) + + return nil +} + //This function converts capotal letters in package name // to !(smallercase). Eg C => !c . As this is the way // go.mod saves every repository in the $GOPATH/pkg/mod. From 4f64254c777023f9a18ca4e31b9df571741eb8ff Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Thu, 28 Feb 2019 13:18:22 +0100 Subject: [PATCH 5/6] Use @latest version as default --- util/imports.go | 4 ++-- util/mod.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/util/imports.go b/util/imports.go index d70893a..df36272 100644 --- a/util/imports.go +++ b/util/imports.go @@ -87,14 +87,14 @@ func (flogoImport *FlogoImport) GoImportPath() string { return flogoImport.modulePath + flogoImport.relativeImportPath } func (flogoImport *FlogoImport) GoGetImportPath() string { - version := "@master" + version := "@latest" if flogoImport.version != "" { version = "@" + flogoImport.version } return flogoImport.modulePath + flogoImport.relativeImportPath + version } func (flogoImport *FlogoImport) GoModImportPath() string { - version := "@master" + version := "@latest" if flogoImport.version != "" { version = "@" + flogoImport.version } diff --git a/util/mod.go b/util/mod.go index 0a108a1..1b3520d 100644 --- a/util/mod.go +++ b/util/mod.go @@ -53,7 +53,7 @@ func (m *ModDepManager) AddDependency(flogoImport Import) error { // force resolution // TODO: add a flag to skip download and perform download later (useful in 'flogo create' command for instance) - err = ExecCmd(exec.Command("go", "mod", "verify"), m.srcDir) + err = ExecCmd(exec.Command("go", "mod", "download", flogoImport.ModulePath()), m.srcDir) if err != nil { // if the resolution fails and the Flogo import is "legacy" @@ -65,8 +65,6 @@ func (m *ModDepManager) AddDependency(flogoImport Import) error { err = ExecCmd(exec.Command("go", "get", "-u", flogoImport.GoGetImportPath()), m.srcDir) } - } else { - err = ExecCmd(exec.Command("go", "mod", "download", flogoImport.ModulePath()), m.srcDir) } if err != nil { From 7936447b894b9fb1b46a59dbea93c09e7ddc4c68 Mon Sep 17 00:00:00 2001 From: Mathieu Debove Date: Thu, 28 Feb 2019 13:46:20 +0100 Subject: [PATCH 6/6] Rename 'legacy' imports to 'classic' imports --- util/imports.go | 4 ++-- util/mod.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/imports.go b/util/imports.go index df36272..d501362 100644 --- a/util/imports.go +++ b/util/imports.go @@ -49,7 +49,7 @@ type Import interface { GoImportPath() string // the import path used in .go files GoGetImportPath() string // the import path used by "go get" command GoModImportPath() string // the import path used by "go mod edit" command - IsLegacy() bool // an import is "legacy" if it does not have a relative import path + IsClassic() bool // an import is "classic" if it has no : character separator, hence no relative import path } type Imports []Import @@ -100,7 +100,7 @@ func (flogoImport *FlogoImport) GoModImportPath() string { } return flogoImport.modulePath + version } -func (flogoImport *FlogoImport) IsLegacy() bool { +func (flogoImport *FlogoImport) IsClassic() bool { return flogoImport.relativeImportPath == "" } diff --git a/util/mod.go b/util/mod.go index 1b3520d..f725de0 100644 --- a/util/mod.go +++ b/util/mod.go @@ -56,11 +56,11 @@ func (m *ModDepManager) AddDependency(flogoImport Import) error { err = ExecCmd(exec.Command("go", "mod", "download", flogoImport.ModulePath()), m.srcDir) if err != nil { - // if the resolution fails and the Flogo import is "legacy" + // if the resolution fails and the Flogo import is "classic" // (meaning it does not separate module path from Go import path): // 1. remove the import manually ("go mod edit -droprequire") would fail // 2. try with "go get" instead - if flogoImport.IsLegacy() { + if flogoImport.IsClassic() { m.RemoveImport(flogoImport) err = ExecCmd(exec.Command("go", "get", "-u", flogoImport.GoGetImportPath()), m.srcDir)