Rename module BSONTable to JSONTable to reflect new underlying driver

README.md

@@ -50,15 +50,15 @@ Written using [Pebble](https://github.com/cockroachdb/)
 |type|t|<[]byte> |
 |Desc|prefix|user ID|
 
-The user ID is provided by the user, but should be checked to ensure it is unique. 
+The user ID is provided by the user, but should be checked to ensure it is unique.
 
 **Value**
 |bytes|0:4|4:...|
 |-|-|-------|
 |type|[]byte|
-|Desc|BSON formatted Column definitions|
+|Desc|Json formatted Column definitions|
 
-First is the Table system ID, which is used as a prefix during key lookup. Then rest 
+First is the Table system ID, which is used as a prefix during key lookup. Then rest
 of the bytes describe a list of columns and their data types.
 
 #### Table ID
@@ -68,7 +68,7 @@ of the bytes describe a list of columns and their data types.
 |type|T|uint32|
 |Desc|prefix|system table ID|
 
-The generated ID for a table. 
+The generated ID for a table.
 
 **Value**
 |bytes|0:4|4:...|
@@ -94,4 +94,4 @@ These map the user specified ID to a data block specified with offset and size.
 
 
 ### Data file format
-Sequentially written [BSON](https://bsonspec.org/) entries.
+Sequentially written [JSON](https://www.json.org/json-en.html/) entries.

cmdline/benchtop/cmds/get/main.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/bmeg/benchtop"
-	"github.com/bmeg/benchtop/bsontable"
+	"github.com/bmeg/benchtop/jsontable"
 	"github.com/bmeg/grip/log"
 	"github.com/cockroachdb/pebble"
 	"github.com/spf13/cobra"
@@ -22,7 +22,7 @@ var Cmd = &cobra.Command{
 		tableName := args[1]
 		keys := args[2:]
 
-		driver, err := bsontable.NewBSONDriver(dbPath)
+		driver, err := jsontable.NewJSONDriver(dbPath)
 		if err != nil {
 			return err
 		}
@@ -32,7 +32,7 @@ var Cmd = &cobra.Command{
 			return err
 		}
 
-		TS, _ := driver.(*bsontable.BSONDriver)
+		TS, _ := driver.(*jsontable.JSONDriver)
 		for _, key := range keys {
 			val, closer, err := TS.Pb.Db.Get([]byte(key))
 			if err != nil {

cmdline/benchtop/cmds/keys/main.go

@@ -3,7 +3,7 @@ package keys
 import (
 	"fmt"
 
-	"github.com/bmeg/benchtop/bsontable"
+	"github.com/bmeg/benchtop/jsontable"
 	"github.com/spf13/cobra"
 )
 
@@ -17,7 +17,7 @@ var Cmd = &cobra.Command{
 		dbPath := args[0]
 		tableName := args[1]
 
-		driver, err := bsontable.NewBSONDriver(dbPath)
+		driver, err := jsontable.NewJSONDriver(dbPath)
 		if err != nil {
 			return err
 		}

cmdline/benchtop/cmds/load/main.go

@@ -5,7 +5,7 @@ import (
 	"log"
 
 	"github.com/bmeg/benchtop"
-	"github.com/bmeg/benchtop/bsontable"
+	"github.com/bmeg/benchtop/jsontable"
 	"github.com/bmeg/benchtop/util"
 	"github.com/schollz/progressbar/v3"
 	"github.com/spf13/cobra"
@@ -24,7 +24,7 @@ var Cmd = &cobra.Command{
 		tableName := args[1]
 		filePath := args[2]
 
-		driver, err := bsontable.NewBSONDriver(dbPath)
+		driver, err := jsontable.NewJSONDriver(dbPath)
 		if err != nil {
 			return err
 		}

cmdline/benchtop/cmds/tables/main.go

@@ -3,7 +3,7 @@ package tables
 import (
 	"fmt"
 
-	"github.com/bmeg/benchtop/bsontable"
+	"github.com/bmeg/benchtop/jsontable"
 	"github.com/spf13/cobra"
 )
 
@@ -16,7 +16,7 @@ var Cmd = &cobra.Command{
 
 		dbPath := args[0]
 
-		driver, err := bsontable.NewBSONDriver(dbPath)
+		driver, err := jsontable.NewJSONDriver(dbPath)
 		if err != nil {
 			return err
 		}

examples/vecload.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/bmeg/benchtop"
-	"github.com/bmeg/benchtop/bsontable"
+	"github.com/bmeg/benchtop/jsontable"
 	"github.com/bmeg/benchtop/util"
 
 	"github.com/schollz/progressbar/v3"
@@ -19,7 +19,7 @@ func main() {
 	file := flag.Arg(0)
 	dbPath := flag.Arg(1)
 
-	db, err := bsontable.NewBSONDriver(dbPath)
+	db, err := jsontable.NewJSONDriver(dbPath)
 	if err != nil {
 		fmt.Printf("Error: %s", err)
 		return

interface.go

@@ -1,29 +1,5 @@
 package benchtop
 
-import (
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/bsontype"
-)
-
-type OperatorType string
-
-const (
-	OP_EQ         OperatorType = "=="
-	OP_NEQ        OperatorType = "!="
-	OP_GT         OperatorType = ">"
-	OP_LT         OperatorType = "<"
-	OP_GTE        OperatorType = ">="
-	OP_LTE        OperatorType = "<="
-	OP_INSIDE     OperatorType = "INSIDE"
-	OP_OUTSIDE    OperatorType = "OUTSIDE"
-	OP_BETWEEN    OperatorType = "BETWEEN"
-	OP_WITHIN     OperatorType = "WITHIN"
-	OP_WITHOUT    OperatorType = "WITHOUT"
-	OP_CONTAINS   OperatorType = "CONTAINS"
-	OP_STARTSWITH OperatorType = "STARTSWITH"
-	OP_ENDSWITH   OperatorType = "ENDSWITH"
-)
-
 type TableInfo struct {
 	FileName string      `json:"fileName"`
 	Columns  []ColumnDef `json:"columns"`
@@ -33,10 +9,23 @@ type TableInfo struct {
 }
 
 type ColumnDef struct {
-	Key  string    `json:"key"`
-	Type FieldType `json:"type"`
+	Key string `json:"key"`
+	// Type FieldType `json:"type"` Remove this for now since not using bson anymore
 }
 
+/*
+	 Keep this code as a reminder for what the table field type architecture when bson was used
+		 type FieldType bsontype.Type
+
+		 const (
+			Double      FieldType = FieldType(bson.TypeDouble)
+			Int64       FieldType = FieldType(bson.TypeInt64)
+			String      FieldType = FieldType(bson.TypeString)
+			Bytes       FieldType = FieldType(bson.TypeBinary)
+			VectorArray FieldType = FieldType(bson.TypeArray)
+		 )
+*/
+
 type TableDriver interface {
 	New(name string, columns []ColumnDef) (TableStore, error)
 	Get(name string) (TableStore, error)
@@ -93,13 +82,3 @@ type TableStore interface {
 	Compact() error
 	Close()
 }
-
-type FieldType bsontype.Type
-
-const (
-	Double      FieldType = FieldType(bson.TypeDouble)
-	Int64       FieldType = FieldType(bson.TypeInt64)
-	String      FieldType = FieldType(bson.TypeString)
-	Bytes       FieldType = FieldType(bson.TypeBinary)
-	VectorArray FieldType = FieldType(bson.TypeArray)
-)

jsontable/cache.go

@@ -0,0 +1,57 @@
+package jsontable
+
+import (
+	"bytes"
+	"context"
+	"time"
+
+	"github.com/bmeg/benchtop"
+	"github.com/bmeg/benchtop/pebblebulk"
+	"github.com/bmeg/grip/log"
+	"github.com/maypok86/otter/v2"
+)
+
+func (dr *JSONDriver) PreloadCache() error {
+	var keys []string
+	prefix := []byte{benchtop.PosPrefix}
+	L_Start := time.Now()
+
+	err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error {
+		for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() {
+			_, id := benchtop.ParsePosKey(it.Key())
+			keys = append(keys, string(id))
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	bulkLoader := otter.BulkLoaderFunc[string, benchtop.RowLoc](func(ctx context.Context, keys []string) (map[string]benchtop.RowLoc, error) {
+		result := make(map[string]benchtop.RowLoc, len(keys))
+		err := dr.Pb.View(func(it *pebblebulk.PebbleIterator) error {
+			for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() {
+				tableId, id := benchtop.ParsePosKey(it.Key())
+				val, err := it.Value()
+				if err != nil {
+					log.Errorf("Err on it.Value() in bulkLoader: %v", err)
+					continue
+				}
+				offset, size := benchtop.ParsePosValue(val)
+				result[string(id)] = benchtop.RowLoc{Offset: offset, Size: size, Label: tableId}
+
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		return result, nil
+	})
+
+	_, err = dr.PageCache.BulkGet(context.Background(), keys, bulkLoader)
+	if err == nil {
+		log.Debugf("Successfully loaded %d keys in RowLoc cache in %s", len(keys), (time.Now().Sub(L_Start).String()))
+	}
+	return err
+}