dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-13 22:43:17 +00:00
parent 1d1fdb1f0b
commit 685c48f94f
28 changed files with 2492 additions and 2657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
storage/wal_writer.go
View File

@@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"log"
"math"
"os"
"path/filepath"
@@ -12,6 +11,7 @@ import (
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/inbox"
)
const (
@@ -42,10 +42,6 @@ type FreeList interface {
GetPageNumber() (uint32, error)
}
func JoinChangesFileName(dir string, logNumber int) string {
return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber))
}
type MetricsState struct {
//Metrics []Metric
WaitCh chan struct{}
@@ -63,32 +59,37 @@ type Changes struct {
type Writer struct {
mutex sync.Mutex
inbox *inbox.Inbox
workerInbox *inbox.Inbox
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
//atree *atree.Atree
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
appendToWorkerQueue func(any)
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
dir string
databaseName string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
//signalCh chan struct{}
}
type WriterOptions struct {
Dir string
Inbox *inbox.Inbox
WorkerInbox *inbox.Inbox
Dir string
DatabaseName string
//SnapshotNumber int // номер журнала
AppendToWorkerQueue func(any)
DataFreeList *freelist.FreeList
IndexFreeList *freelist.FreeList
WAL string
DataFreeList *freelist.FreeList
IndexFreeList *freelist.FreeList
//Atree *atree.Atree
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
@@ -101,12 +102,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
// if (opt.DataPageSize % 2) != 0 {
// return nil, errors.New("DataPageSize must be multiple of 2")
// }
if opt.Inbox == nil {
return nil, errors.New("Inbox option is required")
}
if opt.WorkerInbox == nil {
return nil, errors.New("WorkerInbox option is required")
}
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
if opt.AppendToWorkerQueue == nil {
return nil, errors.New("AppendToWorkerQueue option is required")
if opt.DatabaseName == "" {
return nil, errors.New("DatabaseName option is required")
}
// if opt.AppendToWorkerQueue == nil {
// return nil, errors.New("AppendToWorkerQueue option is required")
// }
if opt.DataFreeList == nil {
return nil, errors.New("DataFreeList option is required")
}
@@ -123,9 +133,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
return nil, errors.New("WaitGroup option is required")
}
indexPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
FilePath: qb.GetIndexFilePath(opt.Dir, opt.DatabaseName),
PageSize: IndexPageSize,
FreeList: opt.IndexFreeList,
})
dataPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
FilePath: qb.GetDataFilePath(opt.Dir, opt.DatabaseName),
PageSize: DataPageSize,
FreeList: opt.DataFreeList,
})
pageIngester, err := NewPageIngester(PageIngesterOptions{
GetIndexPageNumber: s.getIndexPageNumber,
GetDataPageNumber: s.getDataPageNumber,
GetIndexPageNumber: indexPageNoProvider.GetPageNumber,
GetDataPageNumber: dataPageNoProvider.GetPageNumber,
})
if err != nil {
return nil, err
@@ -137,27 +159,31 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
})
s := &Writer{
dir: opt.Dir,
appendToWorkerQueue: opt.AppendToWorkerQueue,
dataFreeList: opt.DataFreeList,
indexFreeList: opt.IndexFreeList,
writePreparer: writePreparer,
//atree: opt.Atree,
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
signalCh: make(chan struct{}, 1),
inbox: opt.Inbox,
workerInbox: opt.WorkerInbox,
dir: opt.Dir,
databaseName: opt.DatabaseName,
dataFreeList: opt.DataFreeList,
indexFreeList: opt.IndexFreeList,
writePreparer: writePreparer,
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
}
if opt.SnapshotNumber <= 0 {
log.Fatalln("FIX FUCK NUMBER")
}
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, opt.SnapshotNumber),
os.O_APPEND|os.O_WRONLY,
filePerm,
)
if err != nil {
return nil, err
if opt.WAL != "" {
s.wal, err = os.OpenFile(filepath.Join(opt.Dir, opt.WAL), os.O_APPEND|os.O_WRONLY, filePerm)
if err != nil {
return nil, err
}
} else {
s.wal, err = os.OpenFile(
qb.GetWALFilePath(opt.Dir, opt.DatabaseName, 0), // .wal_0
os.O_CREATE|os.O_APPEND|os.O_WRONLY,
filePerm,
)
if err != nil {
return nil, err
}
}
return s, nil
}
@@ -165,7 +191,7 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
func (s *Writer) Run() {
for {
select {
case <-s.signalCh:
case <-s.inbox.Ready():
if err := s.packAndWrite(); err != nil {
qb.Abort(qb.FailedWriteToTxLog, err)
}
@@ -208,16 +234,16 @@ func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
}
func (s *Writer) packAndWrite() (err error) {
s.mutex.Lock()
//s.mutex.Lock()
//isExited := s.isExited
input := s.input
s.input = nil
// var exitWaitGroup *sync.WaitGroup
// if s.isExited {
// exitWaitGroup = s.waitGroup
// }
s.mutex.Unlock()
//s.mutex.Unlock()
input := s.inbox.Drain()
prepared := s.writePreparer.Prepare(input)
@@ -234,7 +260,11 @@ func (s *Writer) packAndWrite() (err error) {
}
// 4. Пишу в atree сторінки
err = s.WritePagesToAtree(prepared.WriteToIndex, prepared.WriteToData)
err = WriteDataPages(s.dataFile, prepared.WriteToData)
if err != nil {
return
}
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
if err != nil {
return
}
@@ -262,7 +292,7 @@ func (s *Writer) packAndWrite() (err error) {
snapshotNumberCh := make(chan int, 1)
s.appendToWorkerQueue(Changes{
s.workerInbox.Push(Changes{
Commits: prepared.Commits,
SnapshotNumberCh: snapshotNumberCh,
FrozenIndexPagesCount: s.indexFreeList.Pages(),
@@ -285,7 +315,7 @@ func (s *Writer) packAndWrite() (err error) {
var err error
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, snapshotNumber),
qb.GetWALFilePath(s.dir, s.databaseName, snapshotNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
)
@@ -293,24 +323,13 @@ func (s *Writer) packAndWrite() (err error) {
return fmt.Errorf("create new changes file: %s", err)
}
} else {
s.appendToWorkerQueue(Changes{
s.workerInbox.Push(Changes{
Commits: prepared.Commits,
})
}
return nil
}
func (s *Writer) Append(req any) {
s.mutex.Lock()
s.input = append(s.input, req)
s.mutex.Unlock()
select {
case s.signalCh <- struct{}{}:
default:
}
}
// func (s *Writer) reset() {
// s.buf.Reset()
// s.buf.Write([]byte{
@@ -350,9 +369,8 @@ type PageToWrite struct {
Content []byte
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) {
for _, p := range dataPages {
func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages {
if len(p.Content) != DataPageSize {
return fmt.Errorf("wrong data page size: %d", len(p.Content))
}
@@ -360,7 +378,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
off = int(p.PageNo-1) * DataPageSize
n int
)
n, err = s.dataFile.WriteAt(p.Content, int64(off))
n, err = file.WriteAt(p.Content, int64(off))
if err != nil {
return
}
@@ -368,7 +386,11 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
return fmt.Errorf("write %d instead of %d", n, DataPageSize)
}
}
for _, p := range indexPages {
return
}
func WriteIndexPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages {
if len(p.Content) != IndexPageSize {
return fmt.Errorf("wrong index page size: %d", len(p.Content))
}
@@ -376,7 +398,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
off = int(p.PageNo-1) * IndexPageSize
n int
)
n, err = s.indexFile.WriteAt(p.Content, int64(off))
n, err = file.WriteAt(p.Content, int64(off))
if err != nil {
return
}
@@ -384,7 +406,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
return fmt.Errorf("write %d instead of %d", n, IndexPageSize)
}
}
return nil
return
}
// API
@@ -401,13 +423,6 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
// ValuesBuf []byte
// }
func (s *Writer) sendSignal() {
select {
case s.signalCh <- struct{}{}:
default:
}
}
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {