diff --git a/README.md b/README.md index 13cba8f..df83c47 100644 --- a/README.md +++ b/README.md @@ -50,15 +50,15 @@ Written using [Pebble](https://github.com/cockroachdb/) |type|t|<[]byte> | |Desc|prefix|user ID| -The user ID is provided by the user, but should be checked to ensure it is unique. +The user ID is provided by the user, but should be checked to ensure it is unique. **Value** |bytes|0:4|4:...| |-|-|-------| |type|[]byte| -|Desc|BSON formatted Column definitions| +|Desc|Json formatted Column definitions| -First is the Table system ID, which is used as a prefix during key lookup. Then rest +First is the Table system ID, which is used as a prefix during key lookup. Then rest of the bytes describe a list of columns and their data types. #### Table ID @@ -68,7 +68,7 @@ of the bytes describe a list of columns and their data types. |type|T|uint32| |Desc|prefix|system table ID| -The generated ID for a table. +The generated ID for a table. **Value** |bytes|0:4|4:...| @@ -94,4 +94,4 @@ These map the user specified ID to a data block specified with offset and size. ### Data file format -Sequentially written [BSON](https://bsonspec.org/) entries. \ No newline at end of file +Sequentially written [JSON](https://www.json.org/json-en.html/) entries. diff --git a/bsontable/cache.go b/bsontable/cache.go deleted file mode 100644 index acd2284..0000000 --- a/bsontable/cache.go +++ /dev/null @@ -1,82 +0,0 @@ -package bsontable - -import ( - "bytes" - "time" - "context" - - "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/pebblebulk" - "github.com/bmeg/grip/log" - "github.com/maypok86/otter/v2" -) - - -func (dr *BSONDriver) PreloadCache() error { - var keys []string - prefix := []byte{benchtop.PosPrefix} - L_Start := time.Now() - - err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { - for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { - _, id := benchtop.ParsePosKey(it.Key()) - keys = append(keys, string(id)) - } - return nil - }) - if err != nil { - return err - } - - bulkLoader := otter.BulkLoaderFunc[string, benchtop.RowLoc](func(ctx context.Context, keys []string) (map[string]benchtop.RowLoc, error) { - result := make(map[string]benchtop.RowLoc, len(keys)) - err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { - for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { - tableId, id := benchtop.ParsePosKey(it.Key()) - val, err := it.Value() - if err != nil { - log.Errorf("Err on it.Value() in bulkLoader: %v", err) - continue - } - offset, size := benchtop.ParsePosValue(val) - result[string(id)] = benchtop.RowLoc{Offset: offset, Size: size, Label: tableId} - - } - return nil - }) - if err != nil { - return nil, err - } - return result, nil - }) - - _, err = dr.PageCache.BulkGet(context.Background(), keys, bulkLoader) - if err == nil { - log.Debugf("Successfully loaded %d keys in RowLoc cache in %s", len(keys), (time.Now().Sub(L_Start).String())) - } - return err -} - - -/* - * Old slow Cache Loading function. Will keep this here until it is clear that new cache loading function works as expected. - func (dr *BSONDriver) PreloadCache() error { - L_Start := time.Now() - err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { - prefix := []byte{benchtop.PosPrefix} - for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { - tableId, id := benchtop.ParsePosKey(it.Key()) - val, err := it.Value() - if err != nil { - log.Errorf("Err on it.Value() in PreloadCache") - } - offset, size := benchtop.ParsePosValue(val) - dr.PageCache.Set(string(id), benchtop.RowLoc{Offset: offset, Size: size, Label: tableId}) - } - return nil - }) - if err == nil { - log.Debugf("Successfully loaded RowLoc cache in %d seconds", (time.Now().Second() - L_Start.Second())) - } - return err -}*/ diff --git a/cmdline/benchtop/cmds/get/main.go b/cmdline/benchtop/cmds/get/main.go index 7b9db12..e6401bf 100644 --- a/cmdline/benchtop/cmds/get/main.go +++ b/cmdline/benchtop/cmds/get/main.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/grip/log" "github.com/cockroachdb/pebble" "github.com/spf13/cobra" @@ -22,7 +22,7 @@ var Cmd = &cobra.Command{ tableName := args[1] keys := args[2:] - driver, err := bsontable.NewBSONDriver(dbPath) + driver, err := jsontable.NewJSONDriver(dbPath) if err != nil { return err } @@ -32,7 +32,7 @@ var Cmd = &cobra.Command{ return err } - TS, _ := driver.(*bsontable.BSONDriver) + TS, _ := driver.(*jsontable.JSONDriver) for _, key := range keys { val, closer, err := TS.Pb.Db.Get([]byte(key)) if err != nil { diff --git a/cmdline/benchtop/cmds/keys/main.go b/cmdline/benchtop/cmds/keys/main.go index 05b116d..1cf7d05 100644 --- a/cmdline/benchtop/cmds/keys/main.go +++ b/cmdline/benchtop/cmds/keys/main.go @@ -3,7 +3,7 @@ package keys import ( "fmt" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/spf13/cobra" ) @@ -17,7 +17,7 @@ var Cmd = &cobra.Command{ dbPath := args[0] tableName := args[1] - driver, err := bsontable.NewBSONDriver(dbPath) + driver, err := jsontable.NewJSONDriver(dbPath) if err != nil { return err } diff --git a/cmdline/benchtop/cmds/load/main.go b/cmdline/benchtop/cmds/load/main.go index a340dac..e95541f 100644 --- a/cmdline/benchtop/cmds/load/main.go +++ b/cmdline/benchtop/cmds/load/main.go @@ -5,7 +5,7 @@ import ( "log" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/util" "github.com/schollz/progressbar/v3" "github.com/spf13/cobra" @@ -24,7 +24,7 @@ var Cmd = &cobra.Command{ tableName := args[1] filePath := args[2] - driver, err := bsontable.NewBSONDriver(dbPath) + driver, err := jsontable.NewJSONDriver(dbPath) if err != nil { return err } diff --git a/cmdline/benchtop/cmds/tables/main.go b/cmdline/benchtop/cmds/tables/main.go index 60a3670..9910f2a 100644 --- a/cmdline/benchtop/cmds/tables/main.go +++ b/cmdline/benchtop/cmds/tables/main.go @@ -3,7 +3,7 @@ package tables import ( "fmt" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/spf13/cobra" ) @@ -16,7 +16,7 @@ var Cmd = &cobra.Command{ dbPath := args[0] - driver, err := bsontable.NewBSONDriver(dbPath) + driver, err := jsontable.NewJSONDriver(dbPath) if err != nil { return err } diff --git a/examples/vecload.go b/examples/vecload.go index 01750f9..36f6450 100644 --- a/examples/vecload.go +++ b/examples/vecload.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/util" "github.com/schollz/progressbar/v3" @@ -19,7 +19,7 @@ func main() { file := flag.Arg(0) dbPath := flag.Arg(1) - db, err := bsontable.NewBSONDriver(dbPath) + db, err := jsontable.NewJSONDriver(dbPath) if err != nil { fmt.Printf("Error: %s", err) return diff --git a/interface.go b/interface.go index ec10f07..2ab9c86 100644 --- a/interface.go +++ b/interface.go @@ -1,29 +1,5 @@ package benchtop -import ( - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/bsontype" -) - -type OperatorType string - -const ( - OP_EQ OperatorType = "==" - OP_NEQ OperatorType = "!=" - OP_GT OperatorType = ">" - OP_LT OperatorType = "<" - OP_GTE OperatorType = ">=" - OP_LTE OperatorType = "<=" - OP_INSIDE OperatorType = "INSIDE" - OP_OUTSIDE OperatorType = "OUTSIDE" - OP_BETWEEN OperatorType = "BETWEEN" - OP_WITHIN OperatorType = "WITHIN" - OP_WITHOUT OperatorType = "WITHOUT" - OP_CONTAINS OperatorType = "CONTAINS" - OP_STARTSWITH OperatorType = "STARTSWITH" - OP_ENDSWITH OperatorType = "ENDSWITH" -) - type TableInfo struct { FileName string `json:"fileName"` Columns []ColumnDef `json:"columns"` @@ -33,10 +9,23 @@ type TableInfo struct { } type ColumnDef struct { - Key string `json:"key"` - Type FieldType `json:"type"` + Key string `json:"key"` + // Type FieldType `json:"type"` Remove this for now since not using bson anymore } +/* + Keep this code as a reminder for what the table field type architecture when bson was used + type FieldType bsontype.Type + + const ( + Double FieldType = FieldType(bson.TypeDouble) + Int64 FieldType = FieldType(bson.TypeInt64) + String FieldType = FieldType(bson.TypeString) + Bytes FieldType = FieldType(bson.TypeBinary) + VectorArray FieldType = FieldType(bson.TypeArray) + ) +*/ + type TableDriver interface { New(name string, columns []ColumnDef) (TableStore, error) Get(name string) (TableStore, error) @@ -93,13 +82,3 @@ type TableStore interface { Compact() error Close() } - -type FieldType bsontype.Type - -const ( - Double FieldType = FieldType(bson.TypeDouble) - Int64 FieldType = FieldType(bson.TypeInt64) - String FieldType = FieldType(bson.TypeString) - Bytes FieldType = FieldType(bson.TypeBinary) - VectorArray FieldType = FieldType(bson.TypeArray) -) diff --git a/jsontable/cache.go b/jsontable/cache.go new file mode 100644 index 0000000..7ead620 --- /dev/null +++ b/jsontable/cache.go @@ -0,0 +1,57 @@ +package jsontable + +import ( + "bytes" + "context" + "time" + + "github.com/bmeg/benchtop" + "github.com/bmeg/benchtop/pebblebulk" + "github.com/bmeg/grip/log" + "github.com/maypok86/otter/v2" +) + +func (dr *JSONDriver) PreloadCache() error { + var keys []string + prefix := []byte{benchtop.PosPrefix} + L_Start := time.Now() + + err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { + _, id := benchtop.ParsePosKey(it.Key()) + keys = append(keys, string(id)) + } + return nil + }) + if err != nil { + return err + } + + bulkLoader := otter.BulkLoaderFunc[string, benchtop.RowLoc](func(ctx context.Context, keys []string) (map[string]benchtop.RowLoc, error) { + result := make(map[string]benchtop.RowLoc, len(keys)) + err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { + tableId, id := benchtop.ParsePosKey(it.Key()) + val, err := it.Value() + if err != nil { + log.Errorf("Err on it.Value() in bulkLoader: %v", err) + continue + } + offset, size := benchtop.ParsePosValue(val) + result[string(id)] = benchtop.RowLoc{Offset: offset, Size: size, Label: tableId} + + } + return nil + }) + if err != nil { + return nil, err + } + return result, nil + }) + + _, err = dr.PageCache.BulkGet(context.Background(), keys, bulkLoader) + if err == nil { + log.Debugf("Successfully loaded %d keys in RowLoc cache in %s", len(keys), (time.Now().Sub(L_Start).String())) + } + return err +} diff --git a/bsontable/driver.go b/jsontable/driver.go similarity index 87% rename from bsontable/driver.go rename to jsontable/driver.go index 15db4ab..a127a66 100644 --- a/bsontable/driver.go +++ b/jsontable/driver.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "bytes" @@ -15,33 +15,33 @@ import ( "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/benchtop/util" "github.com/bmeg/grip/log" + "github.com/bytedance/sonic" "github.com/cockroachdb/pebble" multierror "github.com/hashicorp/go-multierror" "github.com/maypok86/otter/v2" - "github.com/bytedance/sonic" ) const BATCH_SIZE = 1000 const ROW_HSIZE = 12 const ROW_OFFSET_HSIZE = 8 -type BSONDriver struct { +type JSONDriver struct { base string Lock sync.RWMutex PebbleLock sync.Mutex db *pebble.DB Pb *pebblebulk.PebbleKV - PageCache *otter.Cache[string, benchtop.RowLoc] - PageLoader otter.LoaderFunc[string, benchtop.RowLoc] + PageCache *otter.Cache[string, benchtop.RowLoc] + PageLoader otter.LoaderFunc[string, benchtop.RowLoc] - Tables map[string]*BSONTable + Tables map[string]*JSONTable LabelLookup map[uint16]string // Fields is defined like label, field Fields map[string]map[string]struct{} } -func NewBSONDriver(path string) (benchtop.TableDriver, error) { +func NewJSONDriver(path string) (benchtop.TableDriver, error) { db, err := pebble.Open(path, &pebble.Options{}) if err != nil { return nil, err @@ -51,10 +51,10 @@ func NewBSONDriver(path string) (benchtop.TableDriver, error) { os.Mkdir(tableDir, 0700) } - driver := &BSONDriver{ + driver := &JSONDriver{ base: path, db: db, - Tables: map[string]*BSONTable{}, + Tables: map[string]*JSONTable{}, Pb: &pebblebulk.PebbleKV{ Db: db, InsertCount: 0, @@ -63,9 +63,9 @@ func NewBSONDriver(path string) (benchtop.TableDriver, error) { PageCache: otter.Must(&otter.Options[string, benchtop.RowLoc]{ MaximumSize: 10_000_000, }), - Fields: map[string]map[string]struct{}{}, - Lock: sync.RWMutex{}, - PebbleLock: sync.Mutex{}, + Fields: map[string]map[string]struct{}{}, + Lock: sync.RWMutex{}, + PebbleLock: sync.Mutex{}, LabelLookup: map[uint16]string{}, } @@ -85,8 +85,7 @@ func NewBSONDriver(path string) (benchtop.TableDriver, error) { return driver, nil } - -func LoadBSONDriver(path string) (benchtop.TableDriver, error) { +func LoadJSONDriver(path string) (benchtop.TableDriver, error) { db, err := pebble.Open(path, &pebble.Options{}) if err != nil { return nil, fmt.Errorf("failed to open database: %v", err) @@ -97,10 +96,10 @@ func LoadBSONDriver(path string) (benchtop.TableDriver, error) { return nil, fmt.Errorf("TABLES directory not found at %s", tableDir) } - driver := &BSONDriver{ + driver := &JSONDriver{ base: path, db: db, - Tables: map[string]*BSONTable{}, + Tables: map[string]*JSONTable{}, Pb: &pebblebulk.PebbleKV{ Db: db, InsertCount: 0, @@ -114,12 +113,11 @@ func LoadBSONDriver(path string) (benchtop.TableDriver, error) { }), LabelLookup: map[uint16]string{}, } - + err = driver.LoadFields() if err != nil { return nil, err } - for _, tableName := range driver.List() { table, err := driver.Get(tableName) @@ -127,21 +125,21 @@ func LoadBSONDriver(path string) (benchtop.TableDriver, error) { driver.Close() return nil, fmt.Errorf("failed to load table %s: %v", tableName, err) } - bsonTable, ok := table.(*BSONTable) + jsonTable, ok := table.(*JSONTable) if !ok { driver.Close() log.Errorf("invalid table type for %s", tableName) return nil, fmt.Errorf("invalid table type for %s", tableName) } // Pb is already set in Get, but ensure consistency if needed - bsonTable.Pb = &pebblebulk.PebbleKV{ + jsonTable.Pb = &pebblebulk.PebbleKV{ Db: db, InsertCount: 0, CompactLimit: uint32(1000), } driver.Lock.Lock() - driver.LabelLookup[bsonTable.TableId] = tableName[2:] - driver.Tables[tableName] = bsonTable + driver.LabelLookup[jsonTable.TableId] = tableName[2:] + driver.Tables[tableName] = jsonTable driver.Lock.Unlock() } @@ -169,7 +167,7 @@ func LoadBSONDriver(path string) (benchtop.TableDriver, error) { return driver, nil } -func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.TableStore, error) { +func (dr *JSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.TableStore, error) { dr.Lock.RLock() if p, ok := dr.Tables[name]; ok { dr.Lock.RUnlock() @@ -192,7 +190,7 @@ func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.T return nil, fmt.Errorf("failed to create table %s: %v", tPath, err) } - out := &BSONTable{ + out := &JSONTable{ columns: columns, handleLock: sync.RWMutex{}, columnMap: map[string]int{}, @@ -213,7 +211,7 @@ func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.T } dr.LabelLookup[newId] = name[2:] - + // Create TableInfo for serialization tinfo := &benchtop.TableInfo{ Columns: columns, @@ -228,7 +226,7 @@ func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.T f.Close() return nil, fmt.Errorf("failed to marshal table info: %v", err) } - + if err := dr.addTable(tinfo.Name, outData); err != nil { f.Close() log.Errorf("Error adding table: %s", err) @@ -236,7 +234,7 @@ func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.T } buffer := make([]byte, 12) - binary.LittleEndian.PutUint64(buffer[:8], uint64(0) + uint64(len(outData))+12) + binary.LittleEndian.PutUint64(buffer[:8], uint64(0)+uint64(len(outData))+12) binary.LittleEndian.PutUint32(buffer[8:12], uint32(len(outData))) if _, err := out.handle.Write(buffer); err != nil { @@ -259,7 +257,7 @@ func (dr *BSONDriver) New(name string, columns []benchtop.ColumnDef) (benchtop.T return out, nil } -func (dr *BSONDriver) List() []string { +func (dr *JSONDriver) List() []string { out := []string{} prefix := []byte{benchtop.TablePrefix} dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { @@ -272,11 +270,11 @@ func (dr *BSONDriver) List() []string { return out } -func (dr *BSONDriver) Close() { +func (dr *JSONDriver) Close() { dr.Lock.Lock() defer dr.Lock.Unlock() - log.Infoln("Closing BSONDriver...") + log.Infoln("Closing JSONDriver...") for tableName, table := range dr.Tables { table.handleLock.Lock() if table.handle != nil { @@ -293,7 +291,7 @@ func (dr *BSONDriver) Close() { table.handleLock.Unlock() table.Pb = nil } - dr.Tables = make(map[string]*BSONTable) + dr.Tables = make(map[string]*JSONTable) if dr.db != nil { if closeErr := dr.db.Close(); closeErr != nil { log.Errorf("Error closing Pebble database: %v", closeErr) @@ -303,11 +301,11 @@ func (dr *BSONDriver) Close() { } dr.Pb = nil dr.Fields = make(map[string]map[string]struct{}) - log.Infof("Successfully closed BSONDriver for path %s", dr.base) + log.Infof("Successfully closed JSONDriver for path %s", dr.base) return } -func (dr *BSONDriver) Get(name string) (benchtop.TableStore, error) { +func (dr *JSONDriver) Get(name string) (benchtop.TableStore, error) { dr.Lock.RLock() if x, ok := dr.Tables[name]; ok { dr.Lock.RUnlock() @@ -325,7 +323,7 @@ func (dr *BSONDriver) Get(name string) (benchtop.TableStore, error) { nkey := benchtop.NewTableKey([]byte(name)) value, closer, err := dr.db.Get(nkey) if err != nil { - log.Errorln("BSONDriver Get: ", err) + log.Errorln("JSONDriver Get: ", err) return nil, err } defer closer.Close() @@ -339,7 +337,7 @@ func (dr *BSONDriver) Get(name string) (benchtop.TableStore, error) { return nil, fmt.Errorf("failed to open table %s: %v", tPath, err) } - out := &BSONTable{ + out := &JSONTable{ columns: tinfo.Columns, db: dr.db, columnMap: map[string]int{}, @@ -370,7 +368,7 @@ func (dr *BSONDriver) Get(name string) (benchtop.TableStore, error) { } // Currently not used -func (dr *BSONDriver) Delete(name string) error { +func (dr *JSONDriver) Delete(name string) error { dr.Lock.Lock() defer dr.Lock.Unlock() @@ -400,21 +398,20 @@ func (dr *BSONDriver) Delete(name string) error { // BulkLoad // tx: set null to initialize pebble bulk write context -func (dr *BSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleBulk) error { - +func (dr *JSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleBulk) error { + if dr.Pb == nil || dr.Pb.Db == nil { - return fmt.Errorf("pebble database instance is nil") - } + return fmt.Errorf("pebble database instance is nil") + } var wg sync.WaitGroup tableChannels := make(map[string]chan *benchtop.Row) metadataChan := make(chan struct { - table *BSONTable + table *JSONTable fieldIndexKeys [][]byte metadata map[string]benchtop.RowLoc err error }, 100) - startTableGoroutine := func(tableName string) { snapshot := dr.Pb.Db.NewSnapshot() @@ -422,11 +419,11 @@ func (dr *BSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleB tableChannels[tableName] = ch wg.Add(1) go func() { - defer func() { - snapshot.Close() - wg.Done() - }() - var fieldIndexKeys [][]byte + defer func() { + snapshot.Close() + wg.Done() + }() + var fieldIndexKeys [][]byte metadata := make(map[string]benchtop.RowLoc) var localErr *multierror.Error @@ -438,14 +435,14 @@ func (dr *BSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleB if err != nil { localErr = multierror.Append(localErr, fmt.Errorf("failed to create table %s: %v", tableName, err)) metadataChan <- struct { - table *BSONTable + table *JSONTable fieldIndexKeys [][]byte metadata map[string]benchtop.RowLoc err error }{nil, nil, nil, localErr.ErrorOrNil()} return } - table = newTable.(*BSONTable) + table = newTable.(*JSONTable) dr.Lock.Lock() dr.Tables[tableName] = table dr.Lock.Unlock() @@ -517,10 +514,10 @@ func (dr *BSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleB batchData := make([]byte, totalLen) pos := 0 for i, bData := range bDatas { - binary.LittleEndian.PutUint64(batchData[pos:pos + ROW_OFFSET_HSIZE], offsets[i+1]) - binary.LittleEndian.PutUint32(batchData[pos + ROW_OFFSET_HSIZE: pos + ROW_HSIZE], uint32(len(bData))) + binary.LittleEndian.PutUint64(batchData[pos:pos+ROW_OFFSET_HSIZE], offsets[i+1]) + binary.LittleEndian.PutUint32(batchData[pos+ROW_OFFSET_HSIZE:pos+ROW_HSIZE], uint32(len(bData))) pos += ROW_HSIZE + len(bData) - copy(batchData[pos - len(bData):pos], bData) + copy(batchData[pos-len(bData):pos], bData) } _, err = table.handle.Write(batchData) @@ -538,7 +535,7 @@ func (dr *BSONDriver) BulkLoad(inputs chan *benchtop.Row, tx *pebblebulk.PebbleB } metadataChan <- struct { - table *BSONTable + table *JSONTable fieldIndexKeys [][]byte metadata map[string]benchtop.RowLoc err error diff --git a/bsontable/driverhelpers.go b/jsontable/driverhelpers.go similarity index 75% rename from bsontable/driverhelpers.go rename to jsontable/driverhelpers.go index b23031a..c487c5d 100644 --- a/bsontable/driverhelpers.go +++ b/jsontable/driverhelpers.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "bytes" @@ -6,14 +6,14 @@ import ( "github.com/bmeg/benchtop" "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/grip/log" - "go.mongodb.org/mongo-driver/bson" + "github.com/bytedance/sonic" ) // Specify a table type prefix to differentiate between edge tables and vertex tables -func (dr *BSONDriver) getMaxTablePrefix() uint16 { +func (dr *JSONDriver) getMaxTablePrefix() uint16 { // get the max table uint32. Useful for fetching keys. prefix := []byte{benchtop.TablePrefix} - + maxID := uint16(0) dr.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { @@ -24,29 +24,29 @@ func (dr *BSONDriver) getMaxTablePrefix() uint16 { maxID++ } return nil - }) - + }) + return maxID } -func (dr *BSONDriver) addTable(Name string, TinfoMarshal []byte) error { +func (dr *JSONDriver) addTable(Name string, TinfoMarshal []byte) error { nkey := benchtop.NewTableKey([]byte(Name)) return dr.db.Set(nkey, TinfoMarshal, nil) } -func (dr *BSONDriver) dropTable(name string) error { +func (dr *JSONDriver) dropTable(name string) error { nkey := benchtop.NewTableKey([]byte(name)) return dr.db.Delete(nkey, nil) } -func (dr *BSONDriver) getTableInfo(name string) (benchtop.TableInfo, error) { +func (dr *JSONDriver) getTableInfo(name string) (benchtop.TableInfo, error) { value, closer, err := dr.db.Get([]byte(name)) if err != nil { return benchtop.TableInfo{}, err } tinfo := benchtop.TableInfo{} - bson.Unmarshal(value, &tinfo) + sonic.ConfigFastest.Unmarshal(value, &tinfo) closer.Close() return tinfo, nil } diff --git a/bsontable/fields.go b/jsontable/fields.go similarity index 92% rename from bsontable/fields.go rename to jsontable/fields.go index f3f5b4f..c89c882 100644 --- a/bsontable/fields.go +++ b/jsontable/fields.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "bytes" @@ -12,7 +12,7 @@ import ( "github.com/bmeg/grip/gripql" ) -func (dr *BSONDriver) AddField(label, field string) error { +func (dr *JSONDriver) AddField(label, field string) error { dr.Lock.Lock() defer dr.Lock.Unlock() @@ -70,7 +70,7 @@ func (dr *BSONDriver) AddField(label, field string) error { return nil } -func (dr *BSONDriver) RemoveField(label string, field string) error { +func (dr *JSONDriver) RemoveField(label string, field string) error { dr.Lock.Lock() defer dr.Lock.Unlock() @@ -94,7 +94,7 @@ func (dr *BSONDriver) RemoveField(label string, field string) error { return nil } -func (dr *BSONDriver) LoadFields() error { +func (dr *JSONDriver) LoadFields() error { /* * Not sure wether to use a cache here as well or keep it how it is. */ @@ -128,7 +128,7 @@ type FieldInfo struct { Field string } -func (dr *BSONDriver) ListFields() []FieldInfo { +func (dr *JSONDriver) ListFields() []FieldInfo { /* Lists loaded fields. * Since fields on disk are loaded on startup this should be all that is needed */ @@ -148,7 +148,7 @@ func (dr *BSONDriver) ListFields() []FieldInfo { return out } -func (dr *BSONDriver) RowIdsByHas(fltField string, fltValue any, fltOp gripql.Condition) chan string { +func (dr *JSONDriver) RowIdsByHas(fltField string, fltValue any, fltOp gripql.Condition) chan string { dr.Lock.RLock() defer dr.Lock.RUnlock() @@ -181,7 +181,7 @@ func (dr *BSONDriver) RowIdsByHas(fltField string, fltValue any, fltOp gripql.Co return out } -func (dr *BSONDriver) RowIdsByLabelFieldValue(fltLabel string, fltField string, fltValue any, fltOp gripql.Condition) chan string { +func (dr *JSONDriver) RowIdsByLabelFieldValue(fltLabel string, fltField string, fltValue any, fltOp gripql.Condition) chan string { log.WithFields(log.Fields{"label": fltLabel, "field": fltField, "value": fltValue}).Debug("Running RowIdsByLabelFieldValue") dr.Lock.RLock() defer dr.Lock.RUnlock() @@ -212,7 +212,7 @@ func (dr *BSONDriver) RowIdsByLabelFieldValue(fltLabel string, fltField string, return out } -func (dr *BSONDriver) GetIDsForLabel(label string) chan string { +func (dr *JSONDriver) GetIDsForLabel(label string) chan string { dr.Lock.RLock() defer dr.Lock.RUnlock() diff --git a/bsontable/index.go b/jsontable/index.go similarity index 89% rename from bsontable/index.go rename to jsontable/index.go index acf421c..e6bb614 100644 --- a/bsontable/index.go +++ b/jsontable/index.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "bytes" @@ -10,7 +10,7 @@ import ( const bufferSize = 100 // List all unique col names held by all tables -func (dr *BSONDriver) GetAllColNames() chan string { +func (dr *JSONDriver) GetAllColNames() chan string { dr.Lock.RLock() defer dr.Lock.RUnlock() @@ -34,7 +34,7 @@ func (dr *BSONDriver) GetAllColNames() chan string { return out } -func (dr *BSONDriver) GetLabels(edges bool, removePrefix bool) chan string { +func (dr *JSONDriver) GetLabels(edges bool, removePrefix bool) chan string { dr.Lock.RLock() defer dr.Lock.RUnlock() @@ -48,7 +48,7 @@ func (dr *BSONDriver) GetLabels(edges bool, removePrefix bool) chan string { if (edges && strKey[:2] == "e_") || (!edges && strKey[:2] == "v_") { if removePrefix { out <- strKey[2:] - }else { + } else { out <- strKey } } diff --git a/bsontable/table.go b/jsontable/table.go similarity index 89% rename from bsontable/table.go rename to jsontable/table.go index 707d82d..bab88b8 100644 --- a/bsontable/table.go +++ b/jsontable/table.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "bufio" @@ -23,7 +23,7 @@ import ( "github.com/cockroachdb/pebble" ) -type BSONTable struct { +type JSONTable struct { Pb *pebblebulk.PebbleKV db *pebble.DB columns []benchtop.ColumnDef @@ -39,7 +39,7 @@ type BSONTable struct { FileName string } -func (b *BSONTable) Init(poolSize int) error { +func (b *JSONTable) Init(poolSize int) error { b.FilePool = make(chan *os.File, poolSize) for i := range poolSize { file, err := os.Open(b.Path) @@ -57,11 +57,11 @@ func (b *BSONTable) Init(poolSize int) error { return nil } -func (b *BSONTable) GetColumnDefs() []benchtop.ColumnDef { +func (b *JSONTable) GetColumnDefs() []benchtop.ColumnDef { return b.columns } -func (b *BSONTable) Close() { +func (b *JSONTable) Close() { if b.FilePool != nil { for len(b.FilePool) > 0 { if file, ok := <-b.FilePool; ok { @@ -77,7 +77,7 @@ func (b *BSONTable) Close() { //////////////////////////////////////////////////////////////// Unary single effect operations */ -func (b *BSONTable) AddRow(elem benchtop.Row) (*benchtop.RowLoc, error) { +func (b *JSONTable) AddRow(elem benchtop.Row) (*benchtop.RowLoc, error) { bData, err := sonic.ConfigFastest.Marshal( b.packData(elem.Data, string(elem.Id)), @@ -95,7 +95,7 @@ func (b *BSONTable) AddRow(elem benchtop.Row) (*benchtop.RowLoc, error) { } log.Debugln("WRITE ENTRY: ", offset, len(bData)) - writesize, err := b.writeBsonEntry(offset, bData) + writesize, err := b.writeJsonEntry(offset, bData) if err != nil { log.Errorf("write handler err in Load: bulkSet: %s", err) return nil, err @@ -108,7 +108,7 @@ func (b *BSONTable) AddRow(elem benchtop.Row) (*benchtop.RowLoc, error) { }, nil } -func (b *BSONTable) GetRow(loc benchtop.RowLoc) (map[string]any, error) { +func (b *JSONTable) GetRow(loc benchtop.RowLoc) (map[string]any, error) { file := <-b.FilePool defer func() { @@ -137,7 +137,7 @@ func (b *BSONTable) GetRow(loc benchtop.RowLoc) (map[string]any, error) { return out.(map[string]any), nil } -func (b *BSONTable) DeleteRow(name []byte) error { +func (b *JSONTable) DeleteRow(name []byte) error { offset, _, err := b.GetBlockPos(name) if err != nil { return err @@ -155,7 +155,7 @@ func (b *BSONTable) DeleteRow(name []byte) error { //////////////////////////////////////////////////////////////// Start of bulk, chan based functions */ -func (b *BSONTable) Keys() (chan benchtop.Index, error) { +func (b *JSONTable) Keys() (chan benchtop.Index, error) { out := make(chan benchtop.Index, 10) go func() { defer close(out) @@ -171,14 +171,14 @@ func (b *BSONTable) Keys() (chan benchtop.Index, error) { return out, nil } -func (b *BSONTable) Scan(loadData bool, filter benchtop.RowFilter) chan any { +func (b *JSONTable) Scan(loadData bool, filter benchtop.RowFilter) chan any { outChan := make(chan any, 100) go func() { defer close(outChan) handle := <-b.FilePool _, err := handle.Seek(0, io.SeekStart) if err != nil { - log.Errorln("Error in bsontable scan func", err) + log.Errorln("Error in jsontable scan func", err) return } @@ -205,16 +205,16 @@ func (b *BSONTable) Scan(loadData bool, filter benchtop.RowFilter) chan any { continue } - bsonStart := offset + ROW_HSIZE - bsonEnd := bsonStart + int(bSize) - if bsonEnd > len(m) { + jsonStart := offset + ROW_HSIZE + jsonEnd := jsonStart + int(bSize) + if jsonEnd > len(m) { log.Debugf("Incomplete record at end of file at offset %d", offset) break } - rowData := m[bsonStart:bsonEnd] + rowData := m[jsonStart:jsonEnd] - err = b.processBSONRowData(rowData, loadData, filter, outChan) + err = b.processJSONRowData(rowData, loadData, filter, outChan) if err != nil { log.Debugf("Skipping malformed row at offset %d: %v", offset, err) } @@ -228,7 +228,7 @@ func (b *BSONTable) Scan(loadData bool, filter benchtop.RowFilter) chan any { // processBSONRowData handles the parsing of row bytes, // applying filters, and sending the result to the output channel. // It returns an error if the row is malformed or cannot be processed. -func (b *BSONTable) processBSONRowData( +func (b *JSONTable) processJSONRowData( rowData []byte, loadData bool, filter benchtop.RowFilter, @@ -271,7 +271,7 @@ func (b *BSONTable) processBSONRowData( // Compact, Fetch, Load, And Remove methods are not currently being used in grip. // Compact should be introduced into grip in a future PR since the heavy load and delete design approach that we are taking -func (b *BSONTable) Compact() error { +func (b *JSONTable) Compact() error { const flushThreshold = 1000 flushCounter := 0 b.handleLock.Lock() @@ -319,20 +319,20 @@ func (b *BSONTable) Compact() error { continue } - bsonStart := offset + 12 - bsonEnd := bsonStart + int(bSize) - if bsonEnd > len(m) { - return fmt.Errorf("incomplete BSON data at offset %d, size %d", offset, bSize) + jsonStart := offset + 12 + jsonEnd := jsonStart + int(bSize) + if jsonEnd > len(m) { + return fmt.Errorf("incomplete JSON data at offset %d, size %d", offset, bSize) } - rowData := m[bsonStart:bsonEnd] + rowData := m[jsonStart:jsonEnd] var mRow RowData err = sonic.ConfigFastest.Unmarshal(rowData, &mRow) if err != nil { if err == io.EOF { - return fmt.Errorf("BSON data for row at offset %d, size %d was incomplete: %w", offset, bSize, err) + return fmt.Errorf("JSON data for row at offset %d, size %d was incomplete: %w", offset, bSize, err) } - return fmt.Errorf("failed to decode BSON row at offset %d, size %d: %w", offset, bSize, err) + return fmt.Errorf("failed to decode JSON row at offset %d, size %d: %w", offset, bSize, err) } node, err := sonic.Get(rowData, "1") @@ -354,7 +354,7 @@ func (b *BSONTable) Compact() error { } _, err = writer.Write(rowData) if err != nil { - return fmt.Errorf("failed writing BSON row at offset %d: %w", newOffset, err) + return fmt.Errorf("failed writing JSON row at offset %d: %w", newOffset, err) } flushCounter++ @@ -413,7 +413,7 @@ func (b *BSONTable) Compact() error { return nil } -func (b *BSONTable) Fetch(inputs chan benchtop.Index, workers int) <-chan benchtop.BulkResponse { +func (b *JSONTable) Fetch(inputs chan benchtop.Index, workers int) <-chan benchtop.BulkResponse { results := make(chan benchtop.BulkResponse, workers) var wg sync.WaitGroup go func() { @@ -453,7 +453,7 @@ func (b *BSONTable) Fetch(inputs chan benchtop.Index, workers int) <-chan bencht return results } -func (b *BSONTable) Load(inputs chan benchtop.Row) error { +func (b *JSONTable) Load(inputs chan benchtop.Row) error { var errs *multierror.Error b.handleLock.Lock() defer b.handleLock.Unlock() @@ -470,11 +470,11 @@ func (b *BSONTable) Load(inputs chan benchtop.Row) error { ) if err != nil { errs = multierror.Append(errs, err) - log.Errorf("bson Marshall err in Load: bulkSet: %s", err) + log.Errorf("json Marshall err in Load: bulkSet: %s", err) } // make Next offset equal to existing offset + length of data - writeSize, err := b.writeBsonEntry(offset, bData) + writeSize, err := b.writeJsonEntry(offset, bData) if err != nil { errs = multierror.Append(errs, err) log.Errorf("write handler err in Load: bulkSet: %s", err) @@ -492,7 +492,7 @@ func (b *BSONTable) Load(inputs chan benchtop.Row) error { } -func (b *BSONTable) Remove(inputs chan benchtop.Index, workers int) <-chan benchtop.BulkResponse { +func (b *JSONTable) Remove(inputs chan benchtop.Index, workers int) <-chan benchtop.BulkResponse { results := make(chan benchtop.BulkResponse, workers) batchDeletes := make(chan benchtop.Index, workers) diff --git a/bsontable/tablehelpers.go b/jsontable/tablehelpers.go similarity index 81% rename from bsontable/tablehelpers.go rename to jsontable/tablehelpers.go index da2d77d..198d3f3 100644 --- a/bsontable/tablehelpers.go +++ b/jsontable/tablehelpers.go @@ -1,4 +1,4 @@ -package bsontable +package jsontable import ( "encoding/binary" @@ -7,12 +7,12 @@ import ( "os" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable/tpath" + "github.com/bmeg/benchtop/jsontable/tpath" "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/grip/log" "github.com/bmeg/jsonpath" + "github.com/bytedance/sonic" "github.com/cockroachdb/pebble" - "go.mongodb.org/mongo-driver/bson" ) type RowData struct { @@ -20,14 +20,14 @@ type RowData struct { Key string `json:"1"` } -func (b *BSONTable) packData(entry map[string]any, key string) *RowData { +func (b *JSONTable) packData(entry map[string]any, key string) *RowData { return &RowData{ Data: entry, Key: key, } } -func (b *BSONTable) AddTableEntryInfo(tx *pebblebulk.PebbleBulk, rowId []byte, rowLoc benchtop.RowLoc) error { +func (b *JSONTable) AddTableEntryInfo(tx *pebblebulk.PebbleBulk, rowId []byte, rowLoc benchtop.RowLoc) error { value := benchtop.NewPosValue(rowLoc.Offset, rowLoc.Size) posKey := benchtop.NewPosKey(b.TableId, rowId) if tx != nil { @@ -57,7 +57,7 @@ func PathLookup(v map[string]any, path string) any { return res } -func (b *BSONTable) getTableEntryInfo(snap *pebble.Snapshot, id []byte) (*benchtop.RowLoc, error) { +func (b *JSONTable) getTableEntryInfo(snap *pebble.Snapshot, id []byte) (*benchtop.RowLoc, error) { // Really only want to see if anything was returned or not _, closer, err := snap.Get(benchtop.NewPosKey(b.TableId, id)) if err == pebble.ErrNotFound { @@ -70,7 +70,7 @@ func (b *BSONTable) getTableEntryInfo(snap *pebble.Snapshot, id []byte) (*bencht return &benchtop.RowLoc{}, nil } -func (b *BSONTable) unpackData(loadData bool, retId bool, doc *RowData) (any, error) { +func (b *JSONTable) unpackData(loadData bool, retId bool, doc *RowData) (any, error) { if doc == nil { return nil, fmt.Errorf("Doc is nil nothing to unpack") } @@ -84,7 +84,7 @@ func (b *BSONTable) unpackData(loadData bool, retId bool, doc *RowData) (any, er } -func (b *BSONTable) GetBlockPos(id []byte) (offset uint64, size uint64, err error) { +func (b *JSONTable) GetBlockPos(id []byte) (offset uint64, size uint64, err error) { log.Debugln("TABLE ID: ", b.TableId, "ID: ", string(id)) val, closer, err := b.db.Get(benchtop.NewPosKey(b.TableId, id)) if err != nil { @@ -99,7 +99,7 @@ func (b *BSONTable) GetBlockPos(id []byte) (offset uint64, size uint64, err erro return offset, size, nil } -func (b *BSONTable) setDataIndices(inputs chan benchtop.Index) { +func (b *JSONTable) setDataIndices(inputs chan benchtop.Index) { for index := range inputs { b.AddTableEntryInfo( nil, @@ -112,7 +112,7 @@ func (b *BSONTable) setDataIndices(inputs chan benchtop.Index) { } } -func (b *BSONTable) markDelete(offset uint64) error { +func (b *JSONTable) markDelete(offset uint64) error { file, err := os.OpenFile(b.Path, os.O_RDWR, 0644) if err != nil { return err @@ -135,7 +135,7 @@ func (b *BSONTable) markDelete(offset uint64) error { return nil } -func (b *BSONTable) readFromFile(offset uint64) (map[string]any, error) { +func (b *JSONTable) readFromFile(offset uint64) (map[string]any, error) { file, err := os.Open(b.Path) if err != nil { return nil, err @@ -147,7 +147,7 @@ func (b *BSONTable) readFromFile(offset uint64) (map[string]any, error) { return nil, err } - // Read BSON block size + // Read JSON block size sizeBytes := []byte{0x00, 0x00, 0x00, 0x00} _, err = file.Read(sizeBytes) if err != nil { @@ -162,7 +162,7 @@ func (b *BSONTable) readFromFile(offset uint64) (map[string]any, error) { return nil, err } var m *RowData = nil - bson.Unmarshal(rowData, m) + sonic.ConfigFastest.Unmarshal(rowData, m) out, err := b.unpackData(true, false, m) if err != nil { return nil, err @@ -170,7 +170,7 @@ func (b *BSONTable) readFromFile(offset uint64) (map[string]any, error) { return out.(map[string]any), nil } -func (b *BSONTable) writeBsonEntry(offset int64, bData []byte) (int, error) { +func (b *JSONTable) writeJsonEntry(offset int64, bData []byte) (int, error) { // make next offset equal to existing offset + length of data buffer := make([]byte, 12) binary.LittleEndian.PutUint64(buffer[:8], uint64(offset)+uint64(len(bData))+12) @@ -183,7 +183,7 @@ func (b *BSONTable) writeBsonEntry(offset int64, bData []byte) (int, error) { n, err := b.handle.Write(bData) if err != nil { - return 0, fmt.Errorf("write BSON error: %v", err) + return 0, fmt.Errorf("write JSON error: %v", err) } return n, nil } diff --git a/bsontable/tpath/tpath.go b/jsontable/tpath/tpath.go similarity index 100% rename from bsontable/tpath/tpath.go rename to jsontable/tpath/tpath.go diff --git a/keys.go b/keys.go index 2dde608..2bb2bc1 100644 --- a/keys.go +++ b/keys.go @@ -26,7 +26,6 @@ var FieldPrefix = []byte{'F'} // The '0x1F' invisible character unit seperator not supposed to appear in ASCII text var FieldSep = []byte{0x1F} - func FieldKey(field string, label string, value any, rowID []byte) []byte { /* creates a full field key for optimizing the beginning of a query */ valueBytes, err := json.Marshal(value) @@ -38,7 +37,7 @@ func FieldKey(field string, label string, value any, rowID []byte) []byte { FieldPrefix, // Static prefix []byte(field), // table field []byte(label), // label - valueBytes, // BSON-encoded value + valueBytes, // JSON-encoded value rowID, }, FieldSep, diff --git a/pybenchtop/Makefile b/pybenchtop/Makefile deleted file mode 100644 index 8626241..0000000 --- a/pybenchtop/Makefile +++ /dev/null @@ -1,3 +0,0 @@ - -pybenchtop.so: wrapper.go pybenchtop.c shim.c shim.h - go build -buildmode=c-shared -o pybenchtop.so \ No newline at end of file diff --git a/pybenchtop/pybenchtop.c b/pybenchtop/pybenchtop.c deleted file mode 100644 index 8f3f64e..0000000 --- a/pybenchtop/pybenchtop.c +++ /dev/null @@ -1,248 +0,0 @@ -//#define Py_LIMITED_API -#define PY_SSIZE_T_CLEAN - -// I leave this here to comment out the code. cgo seems not to recompile -// pybenchtop.h unless pybenchtop.c compiles correctly. So I set this to -// 0 and recompile to get an updated header file. -#if 1 - -#include -#include "structmember.h" -#include "pybenchtop.h" - - -//Header stuff - -typedef struct { - PyObject_HEAD - //driver here - uintptr_t driver; -} Driver; - -typedef struct { - PyObject_HEAD - //table here - uintptr_t table; -} Table; - -static PyTypeObject TableType; -static int Table_init(Table *self, PyObject *args, PyObject *kwds); - -// Benchtop Driver class - -static PyObject * Driver_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - Driver *self; - self = (Driver *)type->tp_alloc(type, 0); - self->driver = 0; - return (PyObject *)self; -} - -static void Driver_dealloc(Driver* self){ - if (self->driver != 0) { - DriverClose(self->driver); - } - self->driver = 0; - //self->ob_type->tp_free((PyObject*)self); -} - -static int Driver_init(Driver *self, PyObject *args, PyObject *kwds) { - char *base; - if (! PyArg_ParseTuple(args, "s", &base)) - return -1; - - uintptr_t dr = NewDriver(base) ; - self->driver = dr; - return 0; -} - -static PyObject * Driver_newtable(Driver *self, PyObject *args, PyObject *kwds) { - char *tableName; - PyObject *columnDef; - if (! PyArg_ParseTuple(args, "sO", &tableName, &columnDef)) - return NULL; - - printf("Adding table: %s\n", tableName); - - //TODO: should we release this table? - uintptr_t table = NewTable(self->driver, tableName, columnDef); - - PyObject *argList = Py_BuildValue("(Os)", self, tableName); - printf("Calling Object!\n"); - //PyObject *obj = PyObject_CallObject(&TableType, argList); - - PyObject *obj = PyObject_New(Table, &TableType); - if (Table_init(obj, argList, NULL) != 0) { - printf("table init error\n"); - } - - Py_DECREF(argList); - printf("Returning objct\n"); - return obj; -} - - -static PyObject * Driver_gettable(Driver *self, PyObject *args, PyObject *kwds) { - char *tableName; - if (! PyArg_ParseTuple(args, "s", &tableName)) - return NULL; - PyObject *argList = Py_BuildValue("(Os)", self, tableName); - PyObject *obj = PyObject_New(Table, &TableType); - if (Table_init(obj, argList, NULL) != 0) { - printf("table init error\n"); - } - Py_DECREF(argList); - return obj; -} - -static PyObject * Driver_close(Driver *self, PyObject *args, PyObject *kwds) { - if (self->driver != 0) { - DriverClose(self->driver); - } - self->driver = 0; - Py_RETURN_NONE; -} - -static PyMemberDef Driver_members[] = { - {NULL} /* Sentinel */ -}; - -static PyMethodDef Driver_methods[] = { - {"new", (PyCFunction)Driver_newtable, METH_VARARGS, "Generate a new table",}, - {"get", (PyCFunction)Driver_gettable, METH_VARARGS, "Get an existing table",}, - {"close", (PyCFunction)Driver_close, METH_VARARGS, "Close database",}, - {NULL} /* Sentinel */ -}; - - -static PyTypeObject DriverType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "pybenchtop.Driver", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(Driver), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE , - .tp_new = Driver_new, - .tp_init = (initproc) Driver_init, - .tp_dealloc = (destructor) Driver_dealloc, - .tp_members = Driver_members, - .tp_methods = Driver_methods, -}; - -// Table interface - - - -static PyObject * Table_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - printf("Calling table new\n"); - Table *self; - self = (Table *)type->tp_alloc(type, 0); - self->table = 0; - return (PyObject *)self; -} - -static void Table_dealloc(Table* self){ - if (self->table != 0) { - CloseTable(self->table); - } - //self->ob_type->tp_free((PyObject*)self); -} - -static int Table_init(Table *self, PyObject *args, PyObject *kwds) { - printf("Calling table init\n"); - char *name; - PyObject *pyObj; - - if (! PyArg_ParseTuple(args, "Os", &pyObj, &name)) - return -1; - - //check pyobject to ensure it is a driver - Driver *dr = (Driver *)pyObj; - - uintptr_t tb = GetTable(dr->driver, name); - if (tb == 0) { - printf("Table not found\n"); - PyErr_SetString(PyExc_TypeError, "table not found"); - return -1; - } - printf("Returning Table\n"); - self->table = tb; - return 0; -} - -static PyObject * Table_add(Table *self, PyObject *args, PyObject *kwds) { - char *key; - PyObject *data; - - if (! PyArg_ParseTuple(args, "sO", &key, &data)) - Py_RETURN_NONE; - - AddDataTable(self->table, key, data); - return PyUnicode_FromFormat("Running table add"); -} - -static PyObject * Table_get(Table *self, PyObject *args, PyObject *kwds) { - char *key; - - if (! PyArg_ParseTuple(args, "s", &key)) - Py_RETURN_NONE; - - PyObject *data = GetDataTable(self->table, key); - if (data == NULL) { - PyErr_SetString(PyExc_TypeError, "data not found"); - return NULL; - } - return data; -} - -static PyMemberDef Table_members[] = { - {NULL} /* Sentinel */ -}; - -static PyMethodDef Table_methods[] = { - {"add", (PyCFunction)Table_add, METH_VARARGS, "Add data to table",}, - {"get", (PyCFunction)Table_get, METH_VARARGS, "Get data from table",}, - {NULL} /* Sentinel */ -}; - - -static PyTypeObject TableType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "pybenchtop.Table", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(Table), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE , - .tp_new = Table_new, - .tp_init = (initproc) Table_init, - .tp_dealloc = (destructor) Table_dealloc, - .tp_members = Table_members, - .tp_methods = Table_methods, -}; - - -// Add methods to the class here -static PyMethodDef BenchMethods[] = { - {NULL, NULL, 0, NULL} // Sentinel -}; - -static struct PyModuleDef btmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "benchtop", // name of module - .m_size = -1, - .m_methods = BenchMethods -}; - -PyMODINIT_FUNC -PyInit_pybenchtop(void) { - PyObject *m = PyModule_Create(&btmodule); - - if (PyType_Ready(&DriverType) < 0) - return NULL; - - Py_INCREF(&DriverType); - PyModule_AddObject(m, "Driver", (PyObject *)&DriverType); - - return m; -} - -#endif \ No newline at end of file diff --git a/pybenchtop/pybenchtop.h b/pybenchtop/pybenchtop.h deleted file mode 100644 index 8b6b730..0000000 --- a/pybenchtop/pybenchtop.h +++ /dev/null @@ -1,95 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package github.com/bmeg/benchtop/pybenchtop */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "wrapper.go" - - #define Py_LIMITED_API - #include - #include // for uintptr_t - #include "shim.h" - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - -extern GoUintptr NewDriver(char* base); -extern void DriverClose(GoUintptr d); -extern GoUintptr NewTable(GoUintptr d, char* name, PyObject* def); -extern GoUintptr GetTable(GoUintptr d, char* name); -extern void CloseTable(GoUintptr tb); -extern void AddDataTable(GoUintptr tb, char* name, PyObject* obj); -extern PyObject* GetDataTable(GoUintptr tb, char* name); - -#ifdef __cplusplus -} -#endif diff --git a/pybenchtop/shim.c b/pybenchtop/shim.c deleted file mode 100644 index 3b5dce1..0000000 --- a/pybenchtop/shim.c +++ /dev/null @@ -1,36 +0,0 @@ - -#include - -// I have no idea why this is needed, but it works. -// Trying to call it directly gets the error: 'could not determine kind of name for C.PyDict_Check' -int _go_PyDict_Check(PyObject *p) { - return PyDict_Check(p); -} - -int _go_PyType_Check(PyObject *p) { - return PyType_Check(p); -} - -int _go_PyUnicode_Check(PyObject *p) { - return PyUnicode_Check(p); -} - -int _go_PyFloat_Check(PyObject *p) { - return PyFloat_Check(p); -} - -int _go_PyLong_Check(PyObject *p) { - return PyLong_Check(p); -} - -int _go_PyList_Check(PyObject *p) { - return PyList_Check(p); -} - -char * _go_PyUnicode_AsUTF8(PyObject *p) { - return (char *)PyUnicode_AsUTF8(p); -} - -PyObject* _go_PyList_GetItem(PyObject *obj, int i) { - return PyList_GetItem(obj, i); -} \ No newline at end of file diff --git a/pybenchtop/shim.h b/pybenchtop/shim.h deleted file mode 100644 index 3718408..0000000 --- a/pybenchtop/shim.h +++ /dev/null @@ -1,14 +0,0 @@ - -#include - -int _go_PyType_Check(PyObject *p); -int _go_PyDict_Check(PyObject *p); -int _go_PyUnicode_Check(PyObject *p); -int _go_PyFloat_Check(PyObject *p); -int _go_PyLong_Check(PyObject *p); -int _go_PyList_Check(PyObject *p); - - - -char * _go_PyUnicode_AsUTF8(PyObject *p); -PyObject * _go_PyList_GetItem(PyObject *d, int i); \ No newline at end of file diff --git a/pybenchtop/test.py b/pybenchtop/test.py deleted file mode 100644 index df6d1fb..0000000 --- a/pybenchtop/test.py +++ /dev/null @@ -1,17 +0,0 @@ - - -import pybenchtop - - -d = pybenchtop.Driver("test.data") -print(d) - -t = d.new("table_1", {"column_1":float}) - -print(t) - -print(t.add("key1", {"name": "Bob", "column_1": 0.9, "column_2": 1.2, "values" : [1,2.0,3.14]})) - -print(t.get("key1")) - -d.close() \ No newline at end of file diff --git a/pybenchtop/wrapper.go b/pybenchtop/wrapper.go deleted file mode 100644 index 2b59b3f..0000000 --- a/pybenchtop/wrapper.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -// #cgo pkg-config: python3-embed -// #define Py_LIMITED_API -// #include -// #include // for uintptr_t -// #include "shim.h" -import "C" - -import ( - "fmt" - "runtime/cgo" - "unsafe" - - "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -//export NewDriver -func NewDriver(base *C.char) uintptr { - fmt.Printf("Creating a driver\n") - s := C.GoString(base) - o, err := bsontable.NewBSONDriver(s) - if err != nil { - //TODO: clean this up - fmt.Printf("Error!!!: %s\n", err) - } - out := uintptr(cgo.NewHandle(o)) - return out -} - -//export DriverClose -func DriverClose(d uintptr) { - fmt.Printf("Calling db close\n") - dr := cgo.Handle(d).Value().(benchtop.TableDriver) - dr.Close() -} - -//export NewTable -func NewTable(d uintptr, name *C.char, def *C.PyObject) uintptr { - - nameField := C.CString("__name__") - defer C.free(unsafe.Pointer(nameField)) - - gname := C.GoString(name) - fmt.Printf("Building Table: %s\n", gname) - cdef := []benchtop.ColumnDef{} - if C._go_PyDict_Check(def) != 0 { - items := C.PyDict_Items(def) - itemCount := C.PyList_Size(items) - fmt.Printf("Dict with items: %#v (%d)\n", items, itemCount) - for i := 0; i < int(itemCount); i++ { - it := C.PyList_GetItem(items, C.Py_ssize_t(i)) - fmt.Printf("\tItem %#v\n", it) - key := C.PyTuple_GetItem(it, 0) - var keyBytes *C.char = C._go_PyUnicode_AsUTF8(key) - keyStr := C.GoString(keyBytes) - fmt.Printf("Key: %s\n", keyStr) - - value := C.PyTuple_GetItem(it, 1) - if C._go_PyType_Check(value) != 0 { - // typeName := C.PyType_GetName(value) // added in 3.12 - valueName := C.PyObject_GetAttrString(value, nameField) - valueNameCStr := C._go_PyUnicode_AsUTF8((*C.PyObject)(valueName)) - valueNameStr := C.GoString(valueNameCStr) - if valueNameStr == "float" { - fmt.Printf("Type float\n") - cdef = append(cdef, benchtop.ColumnDef{Key: keyStr, Type: benchtop.Double}) - } else { - fmt.Printf("Type Value: %s\n", valueNameStr) - } - } - } - } - dr := cgo.Handle(d).Value().(benchtop.TableDriver) - - table, err := dr.New(gname, cdef) - if err != nil { - return 0 - } - out := uintptr(cgo.NewHandle(table)) - return out -} - -//export GetTable -func GetTable(d uintptr, name *C.char) uintptr { - dr := cgo.Handle(d).Value().(benchtop.TableDriver) - table, err := dr.Get(C.GoString(name)) - if err != nil { - fmt.Printf("Error: %s\n", err) - return 0 - } - return uintptr(cgo.NewHandle(table)) -} - -//export CloseTable -func CloseTable(tb uintptr) { - table := cgo.Handle(tb).Value().(benchtop.TableStore) - table.Close() -} - -//export AddDataTable -func AddDataTable(tb uintptr, name *C.char, obj *C.PyObject) { - data := PyDict2Go(obj) - table := cgo.Handle(tb).Value().(benchtop.TableStore) - table.AddRow(benchtop.Row{Id: []byte(C.GoString(name)), Data: data}) -} - -//export GetDataTable -func GetDataTable(tb uintptr, name *C.char) *C.PyObject { - table := cgo.Handle(tb).Value().(benchtop.TableStore) - data, err := table.GetRow([]byte(C.GoString(name))) - if err != nil { - return nil - } - return Go2PyObject(data) -} - -func PyDict2Go(obj *C.PyObject) map[string]any { - out := map[string]any{} - items := C.PyDict_Items(obj) - itemCount := C.PyList_Size(items) - for i := 0; i < int(itemCount); i++ { - it := C.PyList_GetItem(items, C.Py_ssize_t(i)) - key := C.PyTuple_GetItem(it, 0) - var keyBytes *C.char = C._go_PyUnicode_AsUTF8(key) - keyStr := C.GoString(keyBytes) - value := C.PyTuple_GetItem(it, 1) - obj := PyObject2Go(value) - out[keyStr] = obj - } - return out -} - -func PyList2Go(obj *C.PyObject) []any { - out := []any{} - for i := 0; i < int(C.PyList_Size(obj)); i++ { - item := C._go_PyList_GetItem(obj, C.int(i)) - out = append(out, PyObject2Go(item)) - } - return out -} - -func PyObject2Go(obj *C.PyObject) any { - if C._go_PyDict_Check(obj) != 0 { - return PyDict2Go(obj) - } else if C._go_PyList_Check(obj) != 0 { - return PyList2Go(obj) - } else if C._go_PyUnicode_Check(obj) != 0 { - s := C._go_PyUnicode_AsUTF8(obj) - return C.GoString(s) - } else if C._go_PyFloat_Check(obj) != 0 { - return C.PyFloat_AsDouble(obj) - } else if C._go_PyLong_Check(obj) != 0 { - return C.PyLong_AsLong(obj) - } //TODO: other types - return nil -} - -func Go2PyObject(data any) *C.PyObject { - - switch value := data.(type) { - case map[string]any: - out := C.PyDict_New() - for k, v := range value { - vObj := Go2PyObject(v) - C.PyDict_SetItemString(out, C.CString(k), vObj) - C.Py_DECREF(vObj) - } - return out - case []any: - out := C.PyList_New(0) - for _, v := range value { - vObj := Go2PyObject(v) - C.PyList_Append(out, vObj) - C.Py_DECREF(vObj) - } - return out - case primitive.A: - out := C.PyList_New(0) - for _, v := range value { - vObj := Go2PyObject(v) - C.PyList_Append(out, vObj) - C.Py_DECREF(vObj) - } - return out - case int64: - return C.PyLong_FromLong(C.long(int64(value))) - case int32: - return C.PyLong_FromLong(C.long(int64(value))) - case float32: - return C.PyFloat_FromDouble(C.double(float64(value))) - case float64: - return C.PyFloat_FromDouble(C.double(float64(value))) - case string: - return C.PyUnicode_FromString(C.CString(value)) - default: - fmt.Printf("Unknown type: %#v\n", value) - } - return C.Py_None -} - -func main() {} diff --git a/test/benchmark/compact_test.go b/test/benchmark/compact_test.go index 4571407..304f260 100644 --- a/test/benchmark/compact_test.go +++ b/test/benchmark/compact_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/benchtop/util" ) @@ -17,20 +17,20 @@ const ( NumDeleteKeys = 200 ) -func BenchmarkCompactBson(b *testing.B) { - var compactbsoname = "test.bson" + util.RandomString(5) - defer os.RemoveAll(compactbsoname) +func BenchmarkCompactJson(b *testing.B) { + var compactjsoname = "test.json" + util.RandomString(5) + defer os.RemoveAll(compactjsoname) - b.Log("BenchmarkScaleWriteBson start") + b.Log("BenchmarkScaleWriteJson start") - compactbsonDriver, err := bsontable.NewBSONDriver(compactbsoname) + compactjsonDriver, err := jsontable.NewJSONDriver(compactjsoname) if err != nil { b.Fatal(err) } - columns := []benchtop.ColumnDef{{Key: "data", Type: benchtop.Bytes}} + columns := []benchtop.ColumnDef{{Key: "data"}} - compactbsonTable, err := compactbsonDriver.New(compactbsoname, columns) + compactjsonTable, err := compactjsonDriver.New(compactjsoname, columns) if err != nil { b.Fatal(err) } @@ -49,12 +49,12 @@ func BenchmarkCompactBson(b *testing.B) { }() b.Log("start load") - if err := compactbsonTable.Load(inputChan); err != nil { + if err := compactjsonTable.Load(inputChan); err != nil { b.Fatal(err) } b.Log("Load completed successfully") - keys, err := compactbsonTable.Keys() + keys, err := compactjsonTable.Keys() if err != nil { b.Fatal(err) } @@ -68,7 +68,7 @@ func BenchmarkCompactBson(b *testing.B) { deleted := 0 for key := range keys { if _, exists := randomIndexSet[count]; exists { - if err := compactbsonTable.DeleteRow(key.Key); err != nil { + if err := compactjsonTable.DeleteRow(key.Key); err != nil { b.Fatal(err) } deleted++ @@ -80,11 +80,11 @@ func BenchmarkCompactBson(b *testing.B) { b.Log("start compact") b.ResetTimer() - if err := compactbsonTable.Compact(); err != nil { + if err := compactjsonTable.Compact(); err != nil { b.Fatal(err) } - keysAfterCompact, err := compactbsonTable.Keys() + keysAfterCompact, err := compactjsonTable.Keys() if err != nil { b.Fatal(err) } diff --git a/test/benchmark/fetch_test.go b/test/benchmark/fetch_test.go index 331722e..3cd11c0 100644 --- a/test/benchmark/fetch_test.go +++ b/test/benchmark/fetch_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/benchtop/util" ) @@ -17,19 +17,19 @@ const ( ) func BenchmarkFetch(b *testing.B) { - var fetchname = "test.bson" + util.RandomString(5) - defer os.RemoveAll(fetchname) // Clean up + var fetchname = "test.json" + util.RandomString(5) + defer os.RemoveAll(fetchname) - b.Log("BenchmarkScaleWriteBson start") + b.Log("BenchmarkScaleWriteJson start") - compactbsonDriver, err := bsontable.NewBSONDriver(fetchname) + compactjsonDriver, err := jsontable.NewJSONDriver(fetchname) if err != nil { b.Fatal(err) } - columns := []benchtop.ColumnDef{{Key: "data", Type: benchtop.Bytes}} + columns := []benchtop.ColumnDef{{Key: "data"}} - compactbsonTable, err := compactbsonDriver.New(fetchname, columns) + compactjsonTable, err := compactjsonDriver.New(fetchname, columns) if err != nil { b.Fatal(err) } @@ -48,17 +48,17 @@ func BenchmarkFetch(b *testing.B) { }() b.Log("start load") - if err := compactbsonTable.Load(inputChan); err != nil { + if err := compactjsonTable.Load(inputChan); err != nil { b.Fatal(err) } b.Log("Load completed successfully") - keys, err := compactbsonTable.Keys() + keys, err := compactjsonTable.Keys() if err != nil { b.Fatal(err) } - outStruct := compactbsonTable.Fetch(keys, 5) + outStruct := compactjsonTable.Fetch(keys, 5) keyCount := 0 for _ = range outStruct { //b.Log("KEY: ", keys) diff --git a/test/benchmark/remove_test.go b/test/benchmark/remove_test.go index 44114c9..7e59442 100644 --- a/test/benchmark/remove_test.go +++ b/test/benchmark/remove_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/benchtop/util" "github.com/bmeg/grip/log" @@ -19,18 +19,18 @@ const ( ) func BenchmarkRemove(b *testing.B) { - var removename = "test.bson" + util.RandomString(5) + var removename = "test.json" + util.RandomString(5) defer os.RemoveAll(removename) // Clean up - b.Log("BenchmarkScaleWriteBson start") + b.Log("BenchmarkScaleWriteJson start") - compactbsonDriver, err := bsontable.NewBSONDriver(removename) + compactjsonDriver, err := jsontable.NewJSONDriver(removename) if err != nil { b.Fatal(err) } - columns := []benchtop.ColumnDef{{Key: "data", Type: benchtop.Bytes}} + columns := []benchtop.ColumnDef{{Key: "data"}} - compactbsonTable, err := compactbsonDriver.New(removename, columns) + compactjsonTable, err := compactjsonDriver.New(removename, columns) if err != nil { b.Fatal(err) } @@ -49,12 +49,12 @@ func BenchmarkRemove(b *testing.B) { }() b.Log("start load") - if err := compactbsonTable.Load(inputChan); err != nil { + if err := compactjsonTable.Load(inputChan); err != nil { b.Fatal(err) } b.Log("Load completed successfully") - bT, _ := compactbsonTable.(*bsontable.BSONTable) + bT, _ := compactjsonTable.(*jsontable.JSONTable) pKey := benchtop.NewPosKey(bT.TableId, []byte("key_5")) val, closer, err := bT.Pb.Db.Get(pKey) if err != nil { @@ -66,30 +66,30 @@ func BenchmarkRemove(b *testing.B) { closer.Close() offset, size := benchtop.ParsePosValue(val) - data, err := compactbsonTable.GetRow(benchtop.RowLoc{Offset: offset, Size: size, Label: 0}) + data, err := compactjsonTable.GetRow(benchtop.RowLoc{Offset: offset, Size: size, Label: 0}) b.Log("DATA BEFORE: ", data) if len(data) == 0 { b.Fatal("Expected data to be in key_5 but none was found") } - keys, err := compactbsonTable.Keys() + keys, err := compactjsonTable.Keys() if err != nil { b.Fatal(err) } - outStruct := compactbsonTable.Remove(keys, 5) + outStruct := compactjsonTable.Remove(keys, 5) keyCount := 0 for _ = range outStruct { keyCount++ } - keys, err = compactbsonTable.Keys() + keys, err = compactjsonTable.Keys() if err != nil { b.Fatal(err) } - data, err = compactbsonTable.GetRow(benchtop.RowLoc{Offset: offset, Size: size, Label: 0}) + data, err = compactjsonTable.GetRow(benchtop.RowLoc{Offset: offset, Size: size, Label: 0}) b.Log("DATA AFTER: ", data) if len(data) != 0 { b.Fatalf("Expected data to be empty for key_5 but %#v was found\n", data) @@ -99,7 +99,7 @@ func BenchmarkRemove(b *testing.B) { b.Error("Unexpected Key: ", key) } - scaChan := compactbsonTable.Scan(true, nil) + scaChan := compactjsonTable.Scan(true, nil) for elem := range scaChan { fmt.Println("ELEM: ", elem) } diff --git a/test/benchmark/scale_test.go b/test/benchmark/scale_test.go index bcbe0bb..df456d6 100644 --- a/test/benchmark/scale_test.go +++ b/test/benchmark/scale_test.go @@ -6,50 +6,50 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/benchtop/util" "github.com/bmeg/grip/log" "github.com/cockroachdb/pebble" ) -var Bsonname = "test.bson" + util.RandomString(5) -var bsonTable *bsontable.BSONTable -var bsonDriver *bsontable.BSONDriver +var Jsonname = "test.json" + util.RandomString(5) +var jsonTable *jsontable.JSONTable +var jsonDriver *jsontable.JSONDriver const ( scalenumKeys = 100000 scalevalueSize = 5024 ) -func BenchmarkScaleWriteBson(b *testing.B) { - b.Log("BenchmarkScaleWriteBson start") +func BenchmarkScaleWriteJson(b *testing.B) { + b.Log("BenchmarkScaleWriteJson start") var err error - if bsonDriver == nil { - driver, err := bsontable.NewBSONDriver(Bsonname) + if jsonDriver == nil { + driver, err := jsontable.NewJSONDriver(Jsonname) if err != nil { b.Fatal(err) } var ok bool - bsonDriver, ok = driver.(*bsontable.BSONDriver) + jsonDriver, ok = driver.(*jsontable.JSONDriver) if !ok { - b.Fatal("Failed to assert type *benchtop.BSONDriver") + b.Fatal("Failed to assert type *benchtop.JSONDriver") } } - columns := []benchtop.ColumnDef{{Key: "data", Type: benchtop.Bytes}} + columns := []benchtop.ColumnDef{{Key: "data"}} - if bsonTable == nil { - table, err := bsonDriver.New(Bsonname, columns) + if jsonTable == nil { + table, err := jsonDriver.New(Jsonname, columns) if err != nil { b.Fatal(err) } var ok bool - bsonTable, ok = table.(*bsontable.BSONTable) + jsonTable, ok = table.(*jsontable.JSONTable) if !ok { - b.Fatal("Failed to assert type *benchtop.BSONDriver") + b.Fatal("Failed to assert type *benchtop.JSONDriver") } } @@ -66,28 +66,28 @@ func BenchmarkScaleWriteBson(b *testing.B) { close(inputChan) }() - err = bsonTable.Load(inputChan) + err = jsonTable.Load(inputChan) if err != nil { b.Fatal(err) } } } -func BenchmarkRandomReadBson(b *testing.B) { +func BenchmarkRandomReadJson(b *testing.B) { var err error - if bsonDriver == nil { - driver, err := bsontable.NewBSONDriver(Bsonname) + if jsonDriver == nil { + driver, err := jsontable.NewJSONDriver(Jsonname) if err != nil { b.Fatal(err) } var ok bool - bsonDriver, ok = driver.(*bsontable.BSONDriver) + jsonDriver, ok = driver.(*jsontable.JSONDriver) if !ok { - b.Fatal("Failed to assert type *benchtop.BSONDriver") + b.Fatal("Failed to assert type *benchtop.JSONDriver") } } - ot, err := bsonDriver.Get(Bsonname) + ot, err := jsonDriver.Get(Jsonname) if err != nil { b.Log(err) } @@ -99,7 +99,7 @@ func BenchmarkRandomReadBson(b *testing.B) { b.ResetTimer() OTKEYS, _ := ot.Keys() - bT, _ := ot.(*bsontable.BSONTable) + bT, _ := ot.(*jsontable.JSONTable) for key := range OTKEYS { if _, exists := randomIndexSet[count]; exists { @@ -126,20 +126,20 @@ func BenchmarkRandomReadBson(b *testing.B) { } -func BenchmarkRandomKeysBson(b *testing.B) { +func BenchmarkRandomKeysJson(b *testing.B) { var err error - if bsonDriver == nil { - driver, err := bsontable.NewBSONDriver(Bsonname) + if jsonDriver == nil { + driver, err := jsontable.NewJSONDriver(Jsonname) if err != nil { b.Fatal(err) } var ok bool - bsonDriver, ok = driver.(*bsontable.BSONDriver) + jsonDriver, ok = driver.(*jsontable.JSONDriver) if !ok { - b.Fatal("Failed to assert type *benchtop.BSONDriver") + b.Fatal("Failed to assert type *benchtop.JSONDriver") } } - ot, err := bsonDriver.Get(Bsonname) + ot, err := jsonDriver.Get(Jsonname) if err != nil { b.Log(err) } @@ -161,6 +161,6 @@ func BenchmarkRandomKeysBson(b *testing.B) { count++ } b.Log("READS: ", len(selectedValues), "COUNT: ", count) - os.RemoveAll(Bsonname) + os.RemoveAll(Jsonname) } diff --git a/test/integration/basic_test.go b/test/integration/basic_test.go index 541396c..324c211 100644 --- a/test/integration/basic_test.go +++ b/test/integration/basic_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/util" "github.com/bmeg/grip/log" "github.com/cockroachdb/pebble" @@ -31,14 +31,14 @@ func TestOpenClose(t *testing.T) { name := "test.data" + util.RandomString(5) defer os.RemoveAll(name) - dr, err := bsontable.NewBSONDriver(name) + dr, err := jsontable.NewJSONDriver(name) if err != nil { t.Error(err) } _, err = dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "other", Type: benchtop.String}, + {Key: "field1"}, + {Key: "other"}, }) if err != nil { @@ -46,7 +46,7 @@ func TestOpenClose(t *testing.T) { } dr.Close() - or, err := bsontable.NewBSONDriver(name) + or, err := jsontable.NewJSONDriver(name) if err != nil { t.Error(err) } @@ -65,19 +65,19 @@ func TestInsert(t *testing.T) { dbname := "test.data" + util.RandomString(5) defer os.RemoveAll(dbname) - dr, err := bsontable.NewBSONDriver(dbname) + dr, err := jsontable.NewJSONDriver(dbname) if err != nil { t.Error(err) } ts, err := dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "other", Type: benchtop.String}, + {Key: "field1"}, + {Key: "other"}, }) if err != nil { t.Error(err) } - bT, _ := ts.(*bsontable.BSONTable) + bT, _ := ts.(*jsontable.JSONTable) for k, r := range data { loc, err := bT.AddRow(benchtop.Row{Id: []byte(k), TableName: "table_1", Data: r}) if err != nil { @@ -136,14 +136,14 @@ func TestDeleteTable(t *testing.T) { name := "test.data" + util.RandomString(5) defer os.RemoveAll(name) - dr, err := bsontable.NewBSONDriver(name) + dr, err := jsontable.NewJSONDriver(name) if err != nil { t.Error(err) } _, err = dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "other", Type: benchtop.String}, + {Key: "field1"}, + {Key: "other"}, }) if err != nil { t.Error(err) @@ -156,7 +156,7 @@ func TestDeleteTable(t *testing.T) { dr.Close() - or, err := bsontable.NewBSONDriver(name) + or, err := jsontable.NewJSONDriver(name) if err != nil { t.Error(err) } diff --git a/test/integration/cols_test.go b/test/integration/cols_test.go index f427872..c17c527 100644 --- a/test/integration/cols_test.go +++ b/test/integration/cols_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/util" ) @@ -13,38 +13,38 @@ func TestGetAllColls(t *testing.T) { name := "test.data" + util.RandomString(5) defer os.RemoveAll(name) - dr, err := bsontable.NewBSONDriver(name) + dr, err := jsontable.NewJSONDriver(name) if err != nil { t.Error(err) } _, err = dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "name1", Type: benchtop.String}, + {Key: "field1"}, + {Key: "name1"}, }) if err != nil { t.Error(err) } _, err = dr.New("table_2", []benchtop.ColumnDef{ - {Key: "field2", Type: benchtop.Double}, - {Key: "name2", Type: benchtop.String}, + {Key: "field2"}, + {Key: "name2"}, }) if err != nil { t.Error(err) } _, err = dr.New("table_3", []benchtop.ColumnDef{ - {Key: "field3", Type: benchtop.Double}, - {Key: "name3", Type: benchtop.String}, + {Key: "field3"}, + {Key: "name3"}, }) if err != nil { t.Error(err) } _, err = dr.New("table_4", []benchtop.ColumnDef{ - {Key: "field3", Type: benchtop.Double}, - {Key: "name3", Type: benchtop.String}, + {Key: "field3"}, + {Key: "name3"}, }) if err != nil { t.Error(err) diff --git a/test/integration/compact_test.go b/test/integration/compact_test.go index 03fccdc..d5e8cac 100644 --- a/test/integration/compact_test.go +++ b/test/integration/compact_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/benchtop/util" ) @@ -14,20 +14,20 @@ func TestCompact(t *testing.T) { dbname := "test_compact.data" + util.RandomString(5) defer os.RemoveAll(dbname) - dr, err := bsontable.NewBSONDriver(dbname) + dr, err := jsontable.NewJSONDriver(dbname) if err != nil { t.Fatal(err) } ts, err := dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "name", Type: benchtop.String}, + {Key: "field1"}, + {Key: "name"}, }) if err != nil { t.Fatal(err) } - bT, _ := ts.(*bsontable.BSONTable) + bT, _ := ts.(*jsontable.JSONTable) for k, r := range fixtures.ScanData { loc, err := bT.AddRow(benchtop.Row{Id: []byte(k), TableName: "table_1", Data: r}) if err != nil { @@ -51,7 +51,7 @@ func TestCompact(t *testing.T) { t.Fatal(err) } - beforeStat, err := os.Stat(dbname + "/TABLES/" + table.(*bsontable.BSONTable).FileName) + beforeStat, err := os.Stat(dbname + "/TABLES/" + table.(*jsontable.BSONTable).FileName) if err != nil { t.Fatal(err) } diff --git a/test/integration/delete_test.go b/test/integration/delete_test.go index d3fc8d6..ecf7bd2 100644 --- a/test/integration/delete_test.go +++ b/test/integration/delete_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/util" ) @@ -14,14 +14,14 @@ func TestDelete(t *testing.T) { dbname := "test.data" + util.RandomString(5) defer os.RemoveAll(dbname) - dr, err := bsontable.NewBSONDriver(dbname) + dr, err := jsontable.NewJSONDriver(dbname) if err != nil { t.Error(err) } ts, err := dr.New("table_1", []benchtop.ColumnDef{ - {Key: "data", Type: benchtop.Int64}, - {Key: "id", Type: benchtop.String}, + {Key: "data"}, + {Key: "id"}, }) if err != nil { @@ -29,7 +29,7 @@ func TestDelete(t *testing.T) { } totalCount := 100 - bT, _ := ts.(*bsontable.BSONTable) + bT, _ := ts.(*jsontable.JSONTable) for i := 0; i < totalCount; i++ { key := fmt.Sprintf("key_%d", i) loc, err := bT.AddRow(benchtop.Row{Id: []byte(key), Data: map[string]any{ diff --git a/test/integration/marshal_test.go b/test/integration/marshal_test.go index 46da82a..ca77c5f 100644 --- a/test/integration/marshal_test.go +++ b/test/integration/marshal_test.go @@ -11,7 +11,7 @@ func TestMarshal(t *testing.T) { tinfo := benchtop.TableInfo{ Columns: []benchtop.ColumnDef{ - {Key: "columnA", Type: benchtop.String}, + {Key: "columnA"}, }, TableId: 42, } @@ -36,8 +36,5 @@ func TestMarshal(t *testing.T) { if tinfo.Columns[i].Key != out.Columns[i].Key { t.Errorf("invalid unmarshal") } - if tinfo.Columns[i].Type != out.Columns[i].Type { - t.Errorf("invalid unmarshal") - } } } diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 5df359f..134bb0c 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" "github.com/bmeg/benchtop/filters" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/test/fixtures" "github.com/bmeg/grip/gripql" @@ -86,20 +86,20 @@ func TestScan(t *testing.T) { dbname := "test.data" + util.RandomString(5) defer os.RemoveAll(dbname) - dr, err := bsontable.NewBSONDriver(dbname) + dr, err := jsontable.NewJSONDriver(dbname) if err != nil { t.Error(err) } ts, err := dr.New("table_1", []benchtop.ColumnDef{ - {Key: "field1", Type: benchtop.Double}, - {Key: "name", Type: benchtop.String}, + {Key: "field1"}, + {Key: "name"}, }) if err != nil { t.Error(err) } - bT, _ := ts.(*bsontable.BSONTable) + bT, _ := ts.(*jsontable.JSONTable) for k, r := range fixtures.ScanData { loc, err := bT.AddRow(benchtop.Row{Id: []byte(k), Data: r}) if err != nil { diff --git a/test/speed_test/marshal_test.go b/test/speed_test/marshal_test.go new file mode 100644 index 0000000..d024e6e --- /dev/null +++ b/test/speed_test/marshal_test.go @@ -0,0 +1,7 @@ +package test + +import "testing" + +func TestMarshal(t *testing.T) { + +} diff --git a/test/vector/vector_search_test.go b/test/vector/vector_search_test.go index c3ca506..d8b2896 100644 --- a/test/vector/vector_search_test.go +++ b/test/vector/vector_search_test.go @@ -8,8 +8,8 @@ import ( "time" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" "github.com/bmeg/benchtop/distqueue" + "github.com/bmeg/benchtop/jsontable" ) // RandomString generates a random string of length n. @@ -32,13 +32,13 @@ func TestInsert(t *testing.T) { dbname := "test_index." + RandomString(5) - driver, err := bsontable.NewBSONDriver(dbname) + driver, err := jsontable.NewJSONDriver(dbname) if err != nil { t.Error(err) } - table, err := driver.New("VECTORS", []benchtop.ColumnDef{{Key: "embedding", Type: benchtop.VectorArray}}) + table, err := driver.New("VECTORS", []benchtop.ColumnDef{{Key: "embedding"}}) if err != nil { t.Error(err) } diff --git a/types.go b/types.go deleted file mode 100644 index 6e88a79..0000000 --- a/types.go +++ /dev/null @@ -1,16 +0,0 @@ -package benchtop - -import "fmt" - -func CheckType(val any, t FieldType) (any, error) { - switch t { - case Int64: - if x, ok := val.(int32); !ok { - return int64(x), nil - } - if _, ok := val.(int64); !ok { - return val, fmt.Errorf("not int64") - } - } - return val, nil -}