diff --git a/README.md b/README.md index 91f6ab7c..e16e3221 100644 --- a/README.md +++ b/README.md @@ -740,9 +740,25 @@ artifacts: ... ``` -The provided directory or file paths are relative to the task directory, and -they are only considered when Spread is run with the `-artifacts` flag pointing -to the target directory where content will be downloaded into. +Content generated by the project may be retrieved after the project completes +by registering the desired content under the same `artifacts` field: + +_$PROJECT/spread.yaml_ +``` +project: my-project + +artifacts: + - some/file + - some/dir/ +... +``` + +### Task-level artifacts + +The provided directory or file paths are relative to the task directory for +task-level artifacts, and they are only considered when Spread is run with +the `-artifacts` flag pointing to the target directory where content will +be downloaded into. For example, consider the following command: ``` @@ -756,6 +772,42 @@ hold it after the job is executed. Residual content is fetched whether the job finishes successfully or not, and even if some of the provided paths are missing. +### Suite-level artifacts + +The provided directory or file paths are relative to the suite directory for +suite-level artifacts, and they are only considered when Spread is run with +the `-artifacts` flag pointing to the target directory where content will +be downloaded into. + +For example, consider the following command: +``` +$ spread -artifacts=./artifacts lxd:ubuntu-16.04:mysuite/task-one:variant-a +``` + +Assuming the given task has residual content registered, the directory +`./artifacts/lxd:ubuntu-16.04:mysuite/` would be created to +hold it after the job is executed. + +Residual content is fetched whether the job finishes successfully or not, +and even if some of the provided paths are missing. + +### Project-level artifacts + +The provided directory or file paths are relative to the project directory +(contained in the environment variable `$SPREAD_PATH`) for project-level +artifacts, and they are only considered when Spread is run with the +`-artifacts` flag pointing to the target directory where content will be +downloaded into. + +For example, consider the following command: +``` +$ spread -artifacts=./artifacts lxd:ubuntu-16.04 +``` + +Assuming the given project has residual content registered, the directory +`./artifacts` would be created and contain the following: +- `./artifacts/some/file` +- `./artifacts/some/dir` diff --git a/spread/project.go b/spread/project.go index 7b730e79..78b20987 100644 --- a/spread/project.go +++ b/spread/project.go @@ -23,6 +23,8 @@ type Project struct { Environment *Environment + Artifacts []string + Repack string Reroot string `yaml:"reroot"` Prepare string @@ -312,6 +314,8 @@ type Suite struct { Systems []string Backends []string + Artifacts []string + Variants []string Environment *Environment @@ -470,6 +474,10 @@ func (jobs jobsByName) Less(i, j int) bool { return ji.Name < jj.Name } +func isPathRelativeAndInsideWorkingDir(path string) bool { + return !filepath.IsAbs(path) && path == filepath.Clean(path) && !strings.HasPrefix(path, "../") +} + func SplitVariants(s string) (prefix string, variants []string) { if i := strings.LastIndex(s, "/"); i >= 0 { return s[:i], strings.Split(s[i+1:], ",") @@ -517,6 +525,12 @@ func Load(path string) (*Project, error) { return nil, err } + for _, fname := range project.Artifacts { + if !isPathRelativeAndInsideWorkingDir(fname) { + return nil, fmt.Errorf("the project has an invalid artifact path: %s", fname) + } + } + for bname, backend := range project.Backends { if !validName.MatchString(bname) { return nil, fmt.Errorf("invalid backend name: %q", bname) @@ -616,6 +630,12 @@ func Load(path string) (*Project, error) { return nil, fmt.Errorf("%s is missing a summary", suite) } + for _, fname := range suite.Artifacts { + if !isPathRelativeAndInsideWorkingDir(fname) { + return nil, fmt.Errorf("the suite has an invalid artifact path: %s", fname) + } + } + if err := checkEnv(suite, &suite.Environment); err != nil { return nil, err } @@ -679,7 +699,7 @@ func Load(path string) (*Project, error) { } for _, fname := range task.Artifacts { - if filepath.IsAbs(fname) || fname != filepath.Clean(fname) || strings.HasPrefix(fname, "../") { + if !isPathRelativeAndInsideWorkingDir(fname) { return nil, fmt.Errorf("%s has improper artifact path: %s", task.Name, fname) } } diff --git a/spread/runner.go b/spread/runner.go index c84ff63d..dac76c78 100644 --- a/spread/runner.go +++ b/spread/runner.go @@ -633,7 +633,7 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { repeat = -1 } if !abend && !r.options.Restore && repeat <= 0 { - if err := r.fetchArtifacts(client, job); err != nil { + if err := r.fetchJobArtifacts(client, job); err != nil { printf("Cannot fetch artifacts of %s: %v", job, err) r.tomb.Killf("cannot fetch artifacts of %s: %v", job, err) } @@ -647,6 +647,10 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { } if !abend && insideSuite != nil { + if err := r.fetchSuiteArtifacts(client, insideSuite, last); err != nil { + printf("Cannot copy contents %v", err) + r.tomb.Killf("cannot copy contents: %v", err) + } if !r.run(client, last, restoring, insideSuite, insideSuite.Restore, insideSuite.Debug, &abend) { r.add(&stats.SuiteRestoreError, last) } @@ -659,6 +663,10 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { insideBackend = false } if !abend && insideProject { + if err := r.fetchProjectArtifacts(client); err != nil { + printf("Cannot copy contents %v", err) + r.tomb.Killf("cannot copy contents: %v", err) + } if !r.run(client, last, restoring, r.project, r.project.Restore, r.project.Debug, &abend) { r.add(&stats.ProjectRestoreError, last) } @@ -815,12 +823,11 @@ func (r *Runner) client(backend *Backend, system *System) *Client { return nil } -func (r *Runner) fetchArtifacts(client *Client, job *Job) error { - if r.options.Artifacts == "" || len(job.Task.Artifacts) == 0 { +func (r *Runner) fetchArtifacts(client *Client, localDir string, remoteDir string, artifactDeclaration []string) error { + if r.options.Artifacts == "" || len(artifactDeclaration) == 0 { return nil } - localDir := filepath.Join(r.options.Artifacts, job.Name) if err := os.MkdirAll(localDir, 0755); err != nil { return fmt.Errorf("cannot create artifacts directory: %v", err) } @@ -837,16 +844,34 @@ func (r *Runner) fetchArtifacts(client *Client, job *Job) error { return fmt.Errorf("cannot start unpacking tar: %v", err) } - printf("Fetching artifacts of %s...", job) + printf("Fetching artifacts...") - remoteDir := filepath.Join(r.project.RemotePath, job.Task.Name) - err = client.RecvTar(remoteDir, job.Task.Artifacts, tarw) + err = client.RecvTar(remoteDir, artifactDeclaration, tarw) tarw.Close() terr := cmd.Wait() return firstErr(err, terr) } +func (r *Runner) fetchSuiteArtifacts(client *Client, suite *Suite, lastJob *Job) error { + suiteFolder := fmt.Sprintf("%s:%s:%s", lastJob.Backend.Name, lastJob.System.Name, suite.Name) + localDir := filepath.Join(r.options.Artifacts, suiteFolder) + remoteDir := filepath.Join(r.project.RemotePath, suite.Name) + return r.fetchArtifacts(client, localDir, remoteDir, suite.Artifacts) +} + +func (r *Runner) fetchProjectArtifacts(client *Client) error { + localDir := filepath.Join(r.options.Artifacts) + remoteDir := filepath.Join(r.project.RemotePath) + return r.fetchArtifacts(client, localDir, remoteDir, r.project.Artifacts) +} + +func (r *Runner) fetchJobArtifacts(client *Client, job *Job) error { + localDir := filepath.Join(r.options.Artifacts, job.Name) + remoteDir := filepath.Join(r.project.RemotePath, job.Task.Name) + return r.fetchArtifacts(client, localDir, remoteDir, job.Task.Artifacts) +} + func (r *Runner) discardServer(server Server) { if err := server.Discard(r.tomb.Context(nil)); err != nil { printf("Error discarding %s: %v", server, err) diff --git a/tests/project-artifacts/checks/main/task.yaml b/tests/project-artifacts/checks/main/task.yaml new file mode 100644 index 00000000..f532f4e8 --- /dev/null +++ b/tests/project-artifacts/checks/main/task.yaml @@ -0,0 +1,16 @@ +summary: Ensure it works. + +environment: + FAIL_CHECK: $(HOST:echo $FAIL_CHECK) + +restore: | + rm -f *.txt + +execute: | + mkdir -p artifacts + touch artifacts/one.txt + if [ "$FAIL_CHECK" = 1 ]; then + echo "Failing as requested." + exit 1 + fi + touch artifacts/two.txt diff --git a/tests/project-artifacts/spread.yaml b/tests/project-artifacts/spread.yaml new file mode 100644 index 00000000..32804509 --- /dev/null +++ b/tests/project-artifacts/spread.yaml @@ -0,0 +1,23 @@ +project: spread + +backends: + lxd: + systems: + - ubuntu-16.04 + +path: /home/test + + +artifacts: + - my-artifacts + +suites: + checks/: + summary: Verification tasks. + restore-each: | + mkdir -p "${SPREAD_PATH}/my-artifacts" + if [ -d "artifacts" ]; then + cp artifacts/* "${SPREAD_PATH}/my-artifacts" + fi + +# vim:ts=4:sw=4:et diff --git a/tests/project-artifacts/task.yaml b/tests/project-artifacts/task.yaml new file mode 100644 index 00000000..188e11f7 --- /dev/null +++ b/tests/project-artifacts/task.yaml @@ -0,0 +1,44 @@ +summary: Test downloading of project artifacts. + +prepare: | + if [ ! -f .spread-reuse.yaml ]; then + touch /run/spread-reuse.yaml + ln -s /run/spread-reuse.yaml .spread-reuse.yaml + fi + +restore: + rm -rf artifacts + +execute: | + # Works even with different variants producing different data. + FAIL_CHECK=0 spread -v -reuse -resend -artifacts=./artifacts + + test -f artifacts/my-artifacts/one.txt + test -f artifacts/my-artifacts/two.txt + + ls artifacts/ | wc -l | grep 1 + ls artifacts/my-artifacts | wc -l | grep 2 + + rm -rf artifacts/ + + # On failures, downloads whatever partial data is available. + if FAIL_CHECK=1 spread -v -reuse -resend -artifacts=./artifacts; then + echo 'Expected spread to fail with $FAIL_CHECK set to 1.' + exit 1 + fi + + test -f artifacts/my-artifacts/one.txt + test -f artifacts/my-artifacts/one.txt + + test ! -f artifacts/my-artifacts/two.txt + test ! -f artifacts/my-artifacts/two.txt + + rm -rf artifacts/ + + # Without -residual, nothing happens. + FAIL_CHECK=0 spread -v -reuse -resend + + test ! -e artifacts/ + +debug: | + find artifacts/ diff --git a/tests/suite-artifacts/checks/main/task.yaml b/tests/suite-artifacts/checks/main/task.yaml new file mode 100644 index 00000000..f532f4e8 --- /dev/null +++ b/tests/suite-artifacts/checks/main/task.yaml @@ -0,0 +1,16 @@ +summary: Ensure it works. + +environment: + FAIL_CHECK: $(HOST:echo $FAIL_CHECK) + +restore: | + rm -f *.txt + +execute: | + mkdir -p artifacts + touch artifacts/one.txt + if [ "$FAIL_CHECK" = 1 ]; then + echo "Failing as requested." + exit 1 + fi + touch artifacts/two.txt diff --git a/tests/suite-artifacts/spread.yaml b/tests/suite-artifacts/spread.yaml new file mode 100644 index 00000000..a1acc9ba --- /dev/null +++ b/tests/suite-artifacts/spread.yaml @@ -0,0 +1,21 @@ +project: spread + +backends: + lxd: + systems: + - ubuntu-16.04 + +path: /home/test + +suites: + checks/: + summary: Verification tasks. + artifacts: + - my-artifacts + restore-each: | + mkdir -p "${SPREAD_PATH}/${SPREAD_SUITE}/my-artifacts" + if [ -d "artifacts" ]; then + cp artifacts/* "${SPREAD_PATH}/${SPREAD_SUITE}/my-artifacts" + fi + +# vim:ts=4:sw=4:et diff --git a/tests/suite-artifacts/task.yaml b/tests/suite-artifacts/task.yaml new file mode 100644 index 00000000..3e57d795 --- /dev/null +++ b/tests/suite-artifacts/task.yaml @@ -0,0 +1,45 @@ +summary: Test downloading of project artifacts. + +prepare: | + if [ ! -f .spread-reuse.yaml ]; then + touch /run/spread-reuse.yaml + ln -s /run/spread-reuse.yaml .spread-reuse.yaml + fi + +restore: + rm -rf artifacts + +execute: | + # Works even with different variants producing different data. + FAIL_CHECK=0 spread -v -reuse -resend -artifacts=./artifacts + + test -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/one.txt + test -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/two.txt + + ls artifacts/ | wc -l | grep 1 + ls artifacts/lxd:ubuntu-16.04:checks | wc -l | grep 1 + ls artifacts/lxd:ubuntu-16.04:checks/my-artifacts | wc -l | grep 2 + + rm -rf artifacts/ + + # On failures, downloads whatever partial data is available. + if FAIL_CHECK=1 spread -v -reuse -resend -artifacts=./artifacts; then + echo 'Expected spread to fail with $FAIL_CHECK set to 1.' + exit 1 + fi + + test -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/one.txt + test -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/one.txt + + test ! -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/two.txt + test ! -f artifacts/lxd:ubuntu-16.04:checks/my-artifacts/two.txt + + rm -rf artifacts/ + + # Without -residual, nothing happens. + FAIL_CHECK=0 spread -v -reuse -resend + + test ! -e artifacts/ + +debug: | + find artifacts/ diff --git a/tests/artifacts/checks/main/task.yaml b/tests/task-artifacts/checks/main/task.yaml similarity index 100% rename from tests/artifacts/checks/main/task.yaml rename to tests/task-artifacts/checks/main/task.yaml diff --git a/tests/artifacts/spread.yaml b/tests/task-artifacts/spread.yaml similarity index 100% rename from tests/artifacts/spread.yaml rename to tests/task-artifacts/spread.yaml diff --git a/tests/artifacts/task.yaml b/tests/task-artifacts/task.yaml similarity index 100% rename from tests/artifacts/task.yaml rename to tests/task-artifacts/task.yaml