diff --git a/config/config.go b/config/config.go index 79c5931..5f3fc48 100644 --- a/config/config.go +++ b/config/config.go @@ -23,7 +23,8 @@ func (in *Param) IsFile() bool { } func (in *Param) IsDir() bool { - return strings.ToLower(in.Type) == "path" + t := strings.ToLower(in.Type) + return t == "path" || t == "dir" } func TrimPrefix(s string) string { @@ -38,5 +39,6 @@ func (in *ParamRequest) IsFile() bool { } func (in *ParamRequest) IsDir() bool { - return strings.ToLower(in.Type) == "path" + t := strings.ToLower(in.Type) + return t == "path" || t == "dir" } diff --git a/docs/docs/config.md b/docs/docs/config.md index 38ab63d..391e21c 100644 --- a/docs/docs/config.md +++ b/docs/docs/config.md @@ -1,8 +1,8 @@ --- -title: Paramaters +title: Parameters --- -## Paramaters Variables +## Parameters Variables Playbooks can be parameterized. They are defined in the `params` section of the playbook YAML file. diff --git a/docs/docs/inputs/index.md b/docs/docs/inputs/index.md index 76c1817..f858b3c 100644 --- a/docs/docs/inputs/index.md +++ b/docs/docs/inputs/index.md @@ -23,7 +23,7 @@ The `table` loader is a good starting point because it demonstrates the typical params: gafFile: type: File - value: ../../source/go/goa_human.gaf.gz + default: ../../source/go/goa_human.gaf.gz inputs: gafLoad: diff --git a/docs/docs/schema.md b/docs/docs/schema.md index b4951ba..6faf8bd 100644 --- a/docs/docs/schema.md +++ b/docs/docs/schema.md @@ -87,7 +87,7 @@ Loads data from a SQLite database. ### `transpose` Loads a TSV and transposes it (making rows from columns). -- `input`: Path to the file. +- `path`: Path to the file. - `rowSkip`: Rows to skip. - `sep`: Separator. - `useDB`: Use a temporary disk database for large transpositions. diff --git a/extractors/sqldump_step.go b/extractors/sqldump_step.go index 256638f..cd346b0 100644 --- a/extractors/sqldump_step.go +++ b/extractors/sqldump_step.go @@ -15,7 +15,7 @@ import ( ) type SQLDumpStep struct { - Path string `json:"Path" jsonschema_description:"Path to the SQL dump file"` + Path string `json:"path" jsonschema_description:"Path to the SQL dump file"` Tables []string `json:"tables" jsonschema_description:"Array of transforms for the different tables in the SQL dump"` } diff --git a/extractors/tabular_load.go b/extractors/tabular_load.go index 002503b..d24bbb3 100644 --- a/extractors/tabular_load.go +++ b/extractors/tabular_load.go @@ -16,7 +16,7 @@ import ( ) type TableLoadStep struct { - Path string `json:"Path" jsonschema_description:"TSV to be transformed"` + Path string `json:"path" jsonschema_description:"TSV to be transformed"` RowSkip int `json:"rowSkip" jsonschema_description:"Number of header rows to skip"` Columns []string `json:"columns" jsonschema_description:"Manually set names of columns"` ExtraColumns string `json:"extraColumns" jsonschema_description:"Columns beyond originally declared columns will be placed in this array"` diff --git a/extractors/transpose_load.go b/extractors/transpose_load.go index 326914d..d282c62 100644 --- a/extractors/transpose_load.go +++ b/extractors/transpose_load.go @@ -271,7 +271,7 @@ func transposeInDB(workdir string, c csvReader, out chan map[string]any) error { } func transposeInTable(workdir string, fieldSize int, c csvReader, out chan map[string]any) error { - + defer close(out) table, err := os.Create(filepath.Join(workdir, "transpose")) if err != nil { return err @@ -316,12 +316,14 @@ func transposeInTable(workdir string, fieldSize int, c csvReader, out chan map[s columns := []string{} for row := int64(0); row < rowCount; row++ { buf := make([]byte, fieldSize) - _, err := table.ReadAt(buf, row*stepSize) - tmp := bytes.Split(buf, []byte{0}) - if err == nil { - columns = append(columns, string(tmp[0])) - } else { + n, err := table.ReadAt(buf, row*stepSize) + if err != nil && err != io.EOF { logger.Error("Column error", "message", err) + continue + } + if n > 0 { + tmp := bytes.Split(buf[:n], []byte{0}) + columns = append(columns, string(tmp[0])) } } @@ -341,7 +343,6 @@ func transposeInTable(workdir string, fieldSize int, c csvReader, out chan map[s out <- record } table.Close() - close(out) os.RemoveAll(workdir) return nil } diff --git a/go.sum b/go.sum index d45911b..60b9b20 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bmeg/flame v0.0.0-20231228021014-450efb0021a6 h1:7cacs+zs6d9oP0ylXMrmb6PlH8773WEnj/yohQRZkiA= -github.com/bmeg/flame v0.0.0-20231228021014-450efb0021a6/go.mod h1:XA9wVP0ORyATjjwdbwkjq/yaimsnreixGUR62sGMWsY= github.com/bmeg/flame v0.0.0-20250730184845-2dc57651053c h1:ktjBmJUTaHjRoabJio6+P/GPGNjm640gQi3OfcSlnBI= github.com/bmeg/flame v0.0.0-20250730184845-2dc57651053c/go.mod h1:telO7T1SSr9Ejv9nm//xcSMCQWynGFll08bnGMwBO/o= github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc h1:/0v/ZcXYjGs44InmjECrls31onIbVKVu1Q/E2cmnCEU= diff --git a/test/command_line_test.go b/test/command_line_test.go index 69da65f..119244c 100644 --- a/test/command_line_test.go +++ b/test/command_line_test.go @@ -60,7 +60,7 @@ func TestCommandLines(t *testing.T) { // read in conf, ie config.yaml in this case for _, c := range conf { cmd := exec.Command("../sifter", "run", c.Playbook) - fmt.Printf("Running: %s\n", c.Playbook) + t.Logf("Running: %s\n", c.Playbook) err = cmd.Run() if err != nil { t.Errorf("Failed running %s: %s", c.Playbook, err) @@ -68,7 +68,7 @@ func TestCommandLines(t *testing.T) { for i, chk := range c.Outputs { // iterate through expected output paths path := filepath.Join(filepath.Dir(c.Playbook), chk) - fmt.Printf("Checking %s \n", path) + t.Logf("Checking %s \n", path) if stat, err := os.Stat(path); err == nil { if stat.Size() > 0 { file, err := os.Open(path) diff --git a/test/resources/transpose_test.tsv b/test/resources/transpose_test.tsv new file mode 100644 index 0000000..521d356 --- /dev/null +++ b/test/resources/transpose_test.tsv @@ -0,0 +1,4 @@ +gene sample1 sample2 sample3 +EGFR 5.2 3.1 7.4 +KRAS 2.5 1.8 3.2 +TP53 6.1 4.5 5.8 diff --git a/test/unittests/playbook_examples_test.go b/test/unittests/playbook_examples_test.go deleted file mode 100644 index 605363d..0000000 --- a/test/unittests/playbook_examples_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package test - -import ( - "fmt" - "os" - "testing" - - "github.com/bmeg/sifter/cmd/run" - "github.com/bmeg/sifter/loader" - - "path/filepath" - - "github.com/ghodss/yaml" -) - -type PlaybookExampleConfig struct { - Playbook string `json:"playbook"` - Inputs map[string]string `json:"inputs"` - Outputs []string `json:"outputs"` -} - -func runPlaybook(playbook string, inputs map[string]string, outdir string) error { - workDir := "./" - os.Mkdir(outdir, 0700) - driver := "dir://" + outdir - ld, err := loader.NewLoader(driver) - if err != nil { - return err - } - defer ld.Close() - - dir, _ := os.MkdirTemp(workDir, "sifterwork_") - defer os.RemoveAll(dir) - - err = run.ExecuteFile(playbook, dir, outdir, inputs) - if err != nil { - return err - } - os.RemoveAll(dir) - return nil -} - -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func TestPlaybookExamples(t *testing.T) { - tests, err := filepath.Glob("test-playbook-*.yaml") - if err != nil { - t.Error(err) - } - for _, tPath := range tests { - raw, err := os.ReadFile(tPath) - if err != nil { - t.Error(fmt.Errorf("failed to read config %s", tPath)) - } - conf := PlaybookExampleConfig{} - if err := yaml.Unmarshal(raw, &conf); err != nil { - t.Error(fmt.Errorf("failed to read config %s", tPath)) - } - inputs := map[string]string{} - for k, v := range conf.Inputs { - inputs[k] = v - } - fmt.Printf("%s\n", conf) - outDir, _ := os.MkdirTemp("./", "testout_") - runPlaybook(conf.Playbook, inputs, outDir) - - for _, out := range conf.Outputs { - base := filepath.Base(out) - dst := filepath.Join(outDir, base) - if !fileExists(dst) { - t.Errorf("Output %s not produced", base) - } - } - os.RemoveAll(outDir) - } -} diff --git a/test/unittests/project_test.go b/test/unittests/project_test.go deleted file mode 100644 index 1c95be3..0000000 --- a/test/unittests/project_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/bmeg/sifter/cmd/run" - "github.com/bmeg/sifter/loader" -) - -func TestProject(t *testing.T) { - workDir := "./" - driver := "dir://." - ld, err := loader.NewLoader(driver) - if err != nil { - t.Error(err) - } - defer ld.Close() - - dir, err := ioutil.TempDir(workDir, "sifterwork_") - defer os.RemoveAll(dir) - - if err != nil { - t.Error(err) - } - inputs := map[string]string{} - - err = run.ExecuteFile("../resources/project.yaml", dir, "./", inputs) - if err != nil { - t.Error(err) - } - os.RemoveAll(dir) -}