Skip to content

store hex format for binary data in sqlite #771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 28, 2024
Merged

store hex format for binary data in sqlite #771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions service/apinode/api/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ func (s *httpServer) queryTask(c *gin.Context) {
resp.States = append(resp.States, &StateLog{
State: "assigned",
Time: assignedTask.CreatedAt,
ProverID: "did:io:" + strings.TrimPrefix(assignedTask.Prover.String(), "0x"),
ProverID: "did:io:" + strings.TrimPrefix(assignedTask.Prover, "0x"),
})

// Get settled state
Expand All @@ -300,7 +300,7 @@ func (s *httpServer) queryTask(c *gin.Context) {
State: "settled",
Time: settledTask.CreatedAt,
Comment: "The task has been completed. Please check the generated proof in the corresponding DApp contract.",
Tx: settledTask.Tx.String(),
Tx: settledTask.Tx,
})
c.JSON(http.StatusOK, resp)
return
Expand Down
28 changes: 14 additions & 14 deletions service/apinode/db/sqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,26 @@ type scannedBlockNumber struct {

type AssignedTask struct {
gorm.Model
TaskID common.Hash `gorm:"uniqueIndex:assigned_task_uniq,not null"`
Prover common.Address
TaskID string `gorm:"uniqueIndex:assigned_task_uniq,not null"`
Prover string
}

type SettledTask struct {
gorm.Model
TaskID common.Hash `gorm:"uniqueIndex:settled_task_uniq,not null"`
Tx common.Hash `gorm:"not null"`
TaskID string `gorm:"uniqueIndex:settled_task_uniq,not null"`
Tx string `gorm:"not null"`
}

type ProjectDevice struct {
gorm.Model
ProjectID string `gorm:"uniqueIndex:project_device_uniq,not null"`
DeviceAddress common.Address `gorm:"uniqueIndex:project_device_uniq,not null"`
ProjectID string `gorm:"uniqueIndex:project_device_uniq,not null"`
DeviceAddress string `gorm:"uniqueIndex:project_device_uniq,not null"`
}

func (p *DB) UpsertAssignedTask(taskID common.Hash, prover common.Address) error {
t := AssignedTask{
TaskID: taskID,
Prover: prover,
TaskID: taskID.Hex(),
Prover: prover.Hex(),
}
err := p.sqlite.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "task_id"}},
Expand All @@ -48,8 +48,8 @@ func (p *DB) UpsertAssignedTask(taskID common.Hash, prover common.Address) error

func (p *DB) UpsertSettledTask(taskID, tx common.Hash) error {
t := SettledTask{
TaskID: taskID,
Tx: tx,
TaskID: taskID.Hex(),
Tx: tx.Hex(),
}
err := p.sqlite.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "task_id"}},
Expand All @@ -60,7 +60,7 @@ func (p *DB) UpsertSettledTask(taskID, tx common.Hash) error {

func (p *DB) FetchAssignedTask(taskID common.Hash) (*AssignedTask, error) {
t := AssignedTask{}
if err := p.sqlite.Where("task_id = ?", taskID).First(&t).Error; err != nil {
if err := p.sqlite.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, nil
}
Expand All @@ -71,7 +71,7 @@ func (p *DB) FetchAssignedTask(taskID common.Hash) (*AssignedTask, error) {

func (p *DB) FetchSettledTask(taskID common.Hash) (*SettledTask, error) {
t := SettledTask{}
if err := p.sqlite.Where("task_id = ?", taskID).First(&t).Error; err != nil {
if err := p.sqlite.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, nil
}
Expand All @@ -83,7 +83,7 @@ func (p *DB) FetchSettledTask(taskID common.Hash) (*SettledTask, error) {
func (p *DB) UpsertProjectDevice(projectID *big.Int, address common.Address) error {
t := ProjectDevice{
ProjectID: projectID.String(),
DeviceAddress: address,
DeviceAddress: address.Hex(),
}
err := p.sqlite.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "project_id"}, {Name: "device_address"}},
Expand All @@ -94,7 +94,7 @@ func (p *DB) UpsertProjectDevice(projectID *big.Int, address common.Address) err

func (p *DB) IsDeviceApproved(projectID *big.Int, address common.Address) (bool, error) {
t := ProjectDevice{}
if err := p.sqlite.Where("project_id = ?", projectID.String()).Where("device_address = ?", address).First(&t).Error; err != nil {
if err := p.sqlite.Where("project_id = ?", projectID.String()).Where("device_address = ?", address.Hex()).First(&t).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return false, nil
}
Expand Down
43 changes: 24 additions & 19 deletions service/prover/db/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"time"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/pkg/errors"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
Expand All @@ -20,23 +21,23 @@ type scannedBlockNumber struct {

type project struct {
gorm.Model
ProjectID string `gorm:"uniqueIndex:project_id_project,not null"`
URI string `gorm:"not null"`
Hash common.Hash `gorm:"not null"`
ProjectID string `gorm:"uniqueIndex:project_id_project,not null"`
URI string `gorm:"not null"`
Hash string `gorm:"not null"`
}

type projectFile struct {
gorm.Model
ProjectID string `gorm:"uniqueIndex:project_id_project_file,not null"`
File []byte `gorm:"not null"`
Hash common.Hash `gorm:"not null"`
ProjectID string `gorm:"uniqueIndex:project_id_project_file,not null"`
File string `gorm:"not null"`
Hash string `gorm:"not null"`
}

type task struct {
gorm.Model
TaskID common.Hash `gorm:"uniqueIndex:task_uniq,not null"`
Processed bool `gorm:"index:unprocessed_task,not null,default:false"`
Error string `gorm:"not null,default:''"`
TaskID string `gorm:"uniqueIndex:task_uniq,not null"`
Processed bool `gorm:"index:unprocessed_task,not null,default:false"`
Error string `gorm:"not null,default:''"`
}

type DB struct {
Expand Down Expand Up @@ -74,14 +75,14 @@ func (p *DB) Project(projectID *big.Int) (string, common.Hash, error) {
if err := p.db.Where("project_id = ?", projectID.String()).First(&t).Error; err != nil {
return "", common.Hash{}, errors.Wrap(err, "failed to query project")
}
return t.URI, t.Hash, nil
return t.URI, common.HexToHash(t.Hash), nil
}

func (p *DB) UpsertProject(projectID *big.Int, uri string, hash common.Hash) error {
t := project{
ProjectID: projectID.String(),
URI: uri,
Hash: hash,
Hash: hash.Hex(),
}
err := p.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "project_id"}},
Expand All @@ -98,14 +99,18 @@ func (p *DB) ProjectFile(projectID *big.Int) ([]byte, common.Hash, error) {
}
return nil, common.Hash{}, errors.Wrap(err, "failed to query project file")
}
return t.File, t.Hash, nil
f, err := hexutil.Decode(t.File)
if err != nil {
return nil, common.Hash{}, errors.Wrap(err, "failed to decode file from hex format")
}
return f, common.HexToHash(t.Hash), nil
}

func (p *DB) UpsertProjectFile(projectID *big.Int, file []byte, hash common.Hash) error {
t := projectFile{
ProjectID: projectID.String(),
File: file,
Hash: hash,
File: hexutil.Encode(file),
Hash: hash.Hex(),
}
err := p.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "project_id"}},
Expand All @@ -119,7 +124,7 @@ func (p *DB) CreateTask(taskID common.Hash, prover common.Address) error {
return nil
}
t := &task{
TaskID: taskID,
TaskID: taskID.Hex(),
Processed: false,
}
err := p.db.Clauses(clause.OnConflict{
Expand All @@ -136,18 +141,18 @@ func (p *DB) ProcessTask(taskID common.Hash, err error) error {
if err != nil {
t.Error = err.Error()
}
err = p.db.Model(t).Where("task_id = ?", taskID).Updates(t).Error
err = p.db.Model(t).Where("task_id = ?", taskID.Hex()).Updates(t).Error
return errors.Wrap(err, "failed to update task")
}

func (p *DB) DeleteTask(taskID, tx common.Hash) error {
err := p.db.Where("task_id = ?", taskID).Delete(&task{}).Error
err := p.db.Where("task_id = ?", taskID.Hex()).Delete(&task{}).Error
return errors.Wrap(err, "failed to delete task")
}

func (p *DB) ProcessedTask(taskID common.Hash) (bool, string, time.Time, error) {
t := task{}
if err := p.db.Where("task_id = ?", taskID).First(&t).Error; err != nil {
if err := p.db.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return false, "", time.Now(), nil
}
Expand All @@ -164,7 +169,7 @@ func (p *DB) UnprocessedTask() (common.Hash, error) {
}
return common.Hash{}, errors.Wrap(err, "failed to query unprocessed task")
}
return t.TaskID, nil
return common.HexToHash(t.TaskID), nil
}

func New(localDBDir string, prover common.Address) (*DB, error) {
Expand Down
22 changes: 13 additions & 9 deletions service/sequencer/db/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ type scannedBlockNumber struct {

type prover struct {
gorm.Model
Prover common.Address `gorm:"uniqueIndex:prover,not null"`
Prover string `gorm:"uniqueIndex:prover,not null"`
}

type task struct {
gorm.Model
TaskID common.Hash `gorm:"uniqueIndex:task_uniq,not null"`
Assigned bool `gorm:"index:unassigned_task,not null,default:false"`
TaskID string `gorm:"uniqueIndex:task_uniq,not null"`
Assigned bool `gorm:"index:unassigned_task,not null,default:false"`
}

type DB struct {
Expand Down Expand Up @@ -63,15 +63,15 @@ func (p *DB) Provers() ([]common.Address, error) {
}
res := make([]common.Address, 0, len(ts))
for _, t := range ts {
res = append(res, t.Prover)
res = append(res, common.HexToAddress(t.Prover))
}
metrics.ProverMtc.Set(float64(len(ts)))
return res, nil
}

func (p *DB) UpsertProver(addr common.Address) error {
t := prover{
Prover: addr,
Prover: addr.Hex(),
}
err := p.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "prover"}},
Expand All @@ -82,7 +82,7 @@ func (p *DB) UpsertProver(addr common.Address) error {

func (p *DB) CreateTask(taskID common.Hash) error {
t := &task{
TaskID: taskID,
TaskID: taskID.Hex(),
Assigned: false,
}
if err := p.db.Create(t).Error; err != nil {
Expand All @@ -95,15 +95,19 @@ func (p *DB) AssignTasks(ids []common.Hash) error {
if len(ids) == 0 {
return nil
}
sids := make([]string, 0, len(ids))
for _, id := range ids {
sids = append(sids, id.Hex())
}
t := &task{
Assigned: true,
}
err := p.db.Model(t).Where("task_id IN ?", ids).Updates(t).Error
err := p.db.Model(t).Where("task_id IN ?", sids).Updates(t).Error
return errors.Wrap(err, "failed to assign tasks")
}

func (p *DB) DeleteTask(taskID, tx common.Hash) error {
err := p.db.Where("task_id = ?", taskID).Delete(&task{}).Error
err := p.db.Where("task_id = ?", taskID.Hex()).Delete(&task{}).Error
return errors.Wrap(err, "failed to delete task")
}

Expand All @@ -114,7 +118,7 @@ func (p *DB) UnassignedTasks(limit int) ([]common.Hash, error) {
}
ids := []common.Hash{}
for _, t := range ts {
ids = append(ids, t.TaskID)
ids = append(ids, common.HexToHash(t.TaskID))
}
return ids, nil
}
Expand Down