qb/database/database.go

package database

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"gordenko.dev/dima/qb/atree"
	"gordenko.dev/dima/qb/freelist"
	"gordenko.dev/dima/qb/storage"
)

func JoinSnapshotFileName(dir string, snapshotNumber int) string {
	return filepath.Join(dir, fmt.Sprintf("%d.snapshot", snapshotNumber))
}

func JoinWALFileName(dir string, snapshotNumber int) string {
	return filepath.Join(dir, fmt.Sprintf("%d.wal", snapshotNumber))
}

// type metricLockEntry struct {
// 	XLock     bool
// 	RLocks    int
// 	WaitQueue []any
// }

type Database struct {
	mutex           sync.Mutex
	workerSignalCh  chan struct{}
	workerQueue     []any
	rLocksToRelease []uint32
	metrics         map[uint32]*_metric
	//metricLockEntries map[uint32]*metricLockEntry
	dataFreeList   *freelist.FreeList
	indexFreeList  *freelist.FreeList
	dir            string
	databaseName   string
	snapshotNumber int
	storage        *storage.Writer
	atree          *atree.Atree
	tcpPort        int
	logfile        *os.File
	logger         *log.Logger
	exitCh         chan struct{}
	waitGroup      *sync.WaitGroup
}

type Options struct {
	TCPPort      int
	Dir          string
	DatabaseName string
	RedoDir      string
	Logfile      *os.File
	ExitCh       chan struct{}
	WaitGroup    *sync.WaitGroup
}

func New(opt Options) (_ *Database, err error) {
	if opt.TCPPort <= 0 {
		return nil, errors.New("TCPPort option is required")
	}
	if opt.Dir == "" {
		return nil, errors.New("Dir option is required")
	}
	if opt.DatabaseName == "" {
		return nil, errors.New("DatabaseName option is required")
	}
	if opt.RedoDir == "" {
		return nil, errors.New("RedoDir option is required")
	}
	if opt.Logfile == nil {
		return nil, errors.New("Logfile option is required")
	}
	if opt.ExitCh == nil {
		return nil, errors.New("ExitCh option is required")
	}
	if opt.WaitGroup == nil {
		return nil, errors.New("WaitGroup option is required")
	}

	// deltaFreeListFile, err := os.OpenFile(
	// 	filepath.Join(opt.Dir, opt.DatabaseName+".deltafree"),
	// 	os.O_RDWR|os.O_CREATE,
	// 	0666,
	// )
	// if err != nil {
	// 	return nil, err
	// }

	dataFreeList, err := freelist.New(freelist.Options{
		PageSize:      2048,
		BaseFilePath:  filepath.Join(opt.Dir, opt.DatabaseName+".free_data_base"),
		DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_data_delta"),
	})

	indexFreeList, err := freelist.New(freelist.Options{
		PageSize:      2048,
		BaseFilePath:  filepath.Join(opt.Dir, opt.DatabaseName+".free_index_base"),
		DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_index_delta"),
	})

	s := &Database{
		workerSignalCh: make(chan struct{}, 1),
		dir:            opt.Dir,
		databaseName:   opt.DatabaseName,
		metrics:        make(map[uint32]*_metric),
		//metricLockEntries: make(map[uint32]*metricLockEntry),
		dataFreeList:  dataFreeList,
		indexFreeList: indexFreeList,
		tcpPort:       opt.TCPPort,
		logfile:       opt.Logfile,
		logger:        log.New(opt.Logfile, "", log.LstdFlags),
		exitCh:        opt.ExitCh,
		waitGroup:     opt.WaitGroup,
	}
	return s, nil
}

func (s *Database) ListenAndServe() (err error) {
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", s.tcpPort))
	if err != nil {
		return fmt.Errorf("net.Listen: %s; port=%d", err, s.tcpPort)
	}

	s.atree, err = atree.New(atree.Options{
		Dir:          s.dir,
		DatabaseName: s.databaseName,
	})
	if err != nil {
		return fmt.Errorf("atree.New: %s", err)
	}
	s.atree.Run()

	go s.worker()

	s.recovery()

	s.logger.Println("database started")
	for {
		// Listen for an incoming connection.
		conn, err := listener.Accept()
		if err != nil {
			s.logger.Printf("listener.Accept: %s\n", err)
			time.Sleep(time.Second)
		} else {
			go s.handleTCPConn(conn)
		}
	}
}

func (s *Database) recovery() {
	// FIX
	// advisor, err := recovery.NewRecoveryAdvisor(recovery.RecoveryAdvisorOptions{
	// 	Dir:            s.dir,
	// 	VerifySnapshot: s.verifySnapshot,
	// })
	// if err != nil {
	// 	panic(err)
	// }

	// recipe, err := advisor.GetRecipe()
	// if err != nil {
	// 	qb.Abort(qb.GetRecoveryRecipeFailed, err)
	// }

	// var logNumber int

	// if recipe != nil {
	// 	if recipe.Snapshot != "" {
	// 		err = s.loadSnapshot(recipe.Snapshot)
	// 		if err != nil {
	// 			qb.Abort(qb.LoadSnapshotFailed, err)
	// 		}
	// 	}
	// 	for _, changesFileName := range recipe.Changes {
	// 		err = s.replayChanges(changesFileName)
	// 		if err != nil {
	// 			qb.Abort(qb.ReplayChangesFailed, err)
	// 		}
	// 	}
	// 	logNumber = recipe.LogNumber
	// }

	// s.storage, err = storage.NewWriter(storage.WriterOptions{
	// 	Dir:                 s.dir,
	// 	LogNumber:           logNumber,
	// 	AppendToWorkerQueue: s.appendJobToWorkerQueue,
	// 	FreeList:            s.freeList,
	// 	Atree:               s.atree,
	// 	ExitCh:              s.exitCh,
	// 	WaitGroup:           s.waitGroup,
	// })
	// if err != nil {
	// 	qb.Abort(qb.CreateChangesWriterFailed, err)

	// }
	// go s.storage.Run()

	// // fileNames, err := s.searchREDOFiles()
	// // if err != nil {
	// // 	qb.Abort(qb.SearchREDOFilesFailed, err)
	// // }

	// // if len(fileNames) > 0 {
	// // 	for _, fileName := range fileNames {
	// // 		err = s.replayREDOFile(fileName)
	// // 		if err != nil {
	// // 			qb.Abort(qb.ReplayREDOFileFailed, err)
	// // 		}
	// // 	}

	// // 	for _, fileName := range fileNames {
	// // 		err = os.Remove(fileName)
	// // 		if err != nil {
	// // 			qb.Abort(qb.RemoveREDOFileFailed, err)
	// // 		}
	// // 	}
	// // }

	// if recipe != nil {
	// 	if recipe.CompleteSnapshot {
	// 		err = s.dumpSnapshot(logNumber)
	// 		if err != nil {
	// 			qb.Abort(qb.DumpSnapshotFailed, err)
	// 		}
	// 	}

	// 	for _, fileName := range recipe.ToDelete {
	// 		err = os.Remove(fileName)
	// 		if err != nil {
	// 			qb.Abort(qb.RemoveRecipeFileFailed, err)
	// 		}
	// 	}
	// }
}

//

// зробити object?

func (s *Database) replayChanges(snapshotNumber int) (err error) {
	snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber))
	if err != nil {
		return
	}

	walReader, err := storage.NewWALReader(storage.WALReaderOptions{
		FileName:   JoinWALFileName(s.dir, snapshotNumber),
		BufferSize: 8 * 1024 * 1024,
	})
	if err != nil {
		return err
	}

	walReplayer, err := NewWALReplayer(WALReplayerOptions{
		WALReader:      walReader,
		Metrics:        snapshot.Metrics,
		FreeIndexPages: snapshot.IndexPageNumbers,
		FreeDataPages:  snapshot.DataPageNumbers,
	})
	if err != nil {
		return
	}

	err = walReplayer.Replay()
	if err != nil {
		return
	}

	err = s.storage.WritePagesToAtree(walReplayer.IndexPages(), walReplayer.DataPages())
	if err != nil {
		return err
	}

	// FIX free pages sync

	return nil
}

func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
	for metricID, x := range replayMetrics {
		s.metrics[metricID] = x.ToMetric()
	}
}

//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
// 	file, err := os.Open(fileName)
// 	if err != nil {
// 		return
// 	}
// 	defer file.Close()

// 	stat, err := file.Stat()
// 	if err != nil {
// 		return
// 	}

// 	if stat.Size() <= 4 {
// 		return false, nil
// 	}

// 	var (
// 		payloadSize = stat.Size() - 4
// 		hash        = crc32.NewIEEE()
// 	)

// 	_, err = io.CopyN(hash, file, payloadSize)
// 	if err != nil {
// 		return
// 	}
// 	calculatedCRC := hash.Sum32()

// 	storedCRC, err := bin.ReadUint32(file)
// 	if err != nil {
// 		return
// 	}
// 	if storedCRC != calculatedCRC {
// 		return false, fmt.Errorf("strored CRC %d not equal calculated CRC %d",
// 			storedCRC, calculatedCRC)
// 	}
// 	return true, nil
// }