dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-02-23 19:37:05 +00:00
parent 6011535509
commit 271046231c
15 changed files with 948 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
atree/io.go
View File

@@ -3,16 +3,15 @@ package atree
import (
"errors"
"fmt"
"hash/crc32"
"io/fs"
"math"
"os"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree/redo"
"gordenko.dev/dima/qb/bin"
)
// fix - додати ID, щоб потім перемістити із тимчасового буфера в pages, або звільнити
type AllocatedPage struct {
PageNo uint32
Data []byte
@@ -26,114 +25,38 @@ type readResult struct {
// INDEX PAGES
func (s *Atree) DeleteIndexPages(pageNumbers []uint32) {
func (s *Atree) DeletePages(pageNumbers []uint32) {
s.mutex.Lock()
for _, pageNo := range pageNumbers {
delete(s.indexPages, pageNo)
delete(s.pages, pageNo)
}
s.mutex.Unlock()
}
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock()
p, ok := s.indexPages[pageNo]
if ok {
p.ReferenceCount++
s.mutex.Unlock()
return p.Buf, nil
}
resultCh := make(chan readResult, 1)
s.indexWaits[pageNo] = append(s.indexWaits[pageNo], resultCh)
if len(s.indexWaits[pageNo]) == 1 {
s.indexPagesToRead = append(s.indexPagesToRead, pageNo)
s.mutex.Unlock()
select {
case s.readSignalCh <- struct{}{}:
default:
}
} else {
s.mutex.Unlock()
}
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, IndexPageSize)
}
return result.Data, result.Err
}
func (s *Atree) releaseIndexPage(pageNo uint32) {
s.mutex.Lock()
defer s.mutex.Unlock()
p, ok := s.indexPages[pageNo]
if ok {
if p.ReferenceCount > 0 {
p.ReferenceCount--
return
} else {
qb.Abort(
qb.ReferenceCountBug,
fmt.Errorf("call releaseIndexPage on page %d with reference count = %d",
pageNo, p.ReferenceCount),
)
}
}
}
func (s *Atree) allocIndexPage() AllocatedPage {
var (
allocated = AllocatedPage{
Data: make([]byte, IndexPageSize),
}
)
allocated.PageNo = s.indexFreelist.ReservePage()
s.mutex.Lock()
if allocated.PageNo > 0 {
allocated.IsReused = true
} else {
if s.allocatedIndexPagesQty == math.MaxUint32 {
qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index"))
}
s.allocatedIndexPagesQty++
allocated.PageNo = s.allocatedIndexPagesQty
}
s.indexPages[allocated.PageNo] = &_page{
PageNo: allocated.PageNo,
Buf: allocated.Data,
ReferenceCount: 1,
}
s.mutex.Unlock()
return allocated
}
// DATA PAGES
func (s *Atree) DeleteDataPages(pageNumbers []uint32) {
s.mutex.Lock()
for _, pageNo := range pageNumbers {
delete(s.dataPages, pageNo)
}
s.mutex.Unlock()
return s.fetchPage(pageNo, PageTypeIndex)
}
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
return s.fetchPage(pageNo, PageTypeData)
}
func (s *Atree) fetchPage(pageNo uint32, pageTypeIdx byte) ([]byte, error) {
s.mutex.Lock()
p, ok := s.dataPages[pageNo]
p, ok := s.pages[pageNo]
if ok {
if p.Buf[pageTypeIdx] != pageTypeIdx {
return nil, fmt.Errorf("wrong pageType %d instead of %d", p.Buf[pageTypeIdx], pageTypeIdx)
}
p.ReferenceCount++
s.mutex.Unlock()
return p.Buf, nil
}
resultCh := make(chan readResult, 1)
s.dataWaits[pageNo] = append(s.dataWaits[pageNo], resultCh)
if len(s.dataWaits[pageNo]) == 1 {
s.dataPagesToRead = append(s.dataPagesToRead, pageNo)
s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh)
if len(s.pageWaits[pageNo]) == 1 {
s.pagesToRead = append(s.pagesToRead, pageNo)
s.mutex.Unlock()
select {
@@ -143,18 +66,19 @@ func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
} else {
s.mutex.Unlock()
}
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, DataPageSize)
result.Err = s.verifyCRC(result.Data, PageSize)
}
return result.Data, result.Err
}
func (s *Atree) releaseDataPage(pageNo uint32) {
func (s *Atree) releasePage(pageNo uint32) {
s.mutex.Lock()
defer s.mutex.Unlock()
p, ok := s.dataPages[pageNo]
p, ok := s.pages[pageNo]
if ok {
if p.ReferenceCount > 0 {
p.ReferenceCount--
@@ -162,43 +86,102 @@ func (s *Atree) releaseDataPage(pageNo uint32) {
} else {
qb.Abort(
qb.ReferenceCountBug,
fmt.Errorf("call releaseDataPage on page %d with reference count = %d",
fmt.Errorf("call releasePage on page %d with reference count = %d",
pageNo, p.ReferenceCount),
)
}
}
}
func (s *Atree) allocDataPage() AllocatedPage {
func (s *Atree) allocPage() AllocatedPage {
var (
allocated = AllocatedPage{
Data: make([]byte, DataPageSize),
Data: make([]byte, PageSize),
}
)
allocated.PageNo = s.dataFreelist.ReservePage()
allocated.PageNo = s.freelist.ReservePage()
s.mutex.Lock()
if allocated.PageNo > 0 {
allocated.IsReused = true
s.mutex.Lock()
} else {
s.mutex.Lock()
if s.allocatedDataPagesQty == math.MaxUint32 {
qb.Abort(qb.MaxAtreeSizeExceeded,
errors.New("no space in Atree index"))
if s.allocatedPagesQty == math.MaxUint32 {
qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index"))
}
s.allocatedDataPagesQty++
allocated.PageNo = s.allocatedDataPagesQty
s.allocatedPagesQty++
allocated.PageNo = s.allocatedPagesQty
}
s.dataPages[allocated.PageNo] = &_page{
PageNo: allocated.PageNo,
Buf: allocated.Data,
ReferenceCount: 1,
}
// s.pages[allocated.PageNo] = &_page{
// // fix pageType
// PageNo: allocated.PageNo,
// Buf: allocated.Data,
// ReferenceCount: 1,
// }
s.mutex.Unlock()
return allocated
}
// // fix - без freelist
// func (s *Atree) allocIndexPage() AllocatedPage {
// var (
// allocated = AllocatedPage{
// Data: make([]byte, PageSize),
// }
// )
// allocated.PageNo = s.freelist.ReservePage()
// s.mutex.Lock()
// if allocated.PageNo > 0 {
// allocated.IsReused = true
// } else {
// if s.allocatedPagesQty == math.MaxUint32 {
// qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index"))
// }
// s.allocatedPagesQty++
// allocated.PageNo = s.allocatedPagesQty
// }
// s.pages[allocated.PageNo] = &_page{
// // fix pageType
// PageNo: allocated.PageNo,
// Buf: allocated.Data,
// ReferenceCount: 1,
// }
// s.mutex.Unlock()
// return allocated
// }
// func (s *Atree) allocDataPage() AllocatedPage {
// var (
// allocated = AllocatedPage{
// Data: make([]byte, PageSize),
// }
// )
// allocated.PageNo = s.freelist.ReservePage()
// s.mutex.Lock()
// if allocated.PageNo > 0 {
// allocated.IsReused = true
// } else {
// if s.allocatedPagesQty == math.MaxUint32 {
// qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index"))
// }
// s.allocatedPagesQty++
// allocated.PageNo = s.allocatedPagesQty
// }
// s.pages[allocated.PageNo] = &_page{
// // fix pageType
// PageNo: allocated.PageNo,
// Buf: allocated.Data,
// ReferenceCount: 1,
// }
// s.mutex.Unlock()
// return allocated
// }
// DATA PAGES
// READ
func (s *Atree) pageReader() {
@@ -212,27 +195,25 @@ func (s *Atree) pageReader() {
func (s *Atree) readPages() {
s.mutex.Lock()
if len(s.indexPagesToRead) == 0 && len(s.dataPagesToRead) == 0 {
if len(s.pagesToRead) == 0 {
s.mutex.Unlock()
return
}
indexPagesToRead := s.indexPagesToRead
s.indexPagesToRead = nil
dataPagesToRead := s.dataPagesToRead
s.dataPagesToRead = nil
pagesToRead := s.pagesToRead
s.pagesToRead = nil
s.mutex.Unlock()
for _, pageNo := range dataPagesToRead {
buf := make([]byte, DataPageSize)
off := (pageNo - 1) * DataPageSize
n, err := s.dataFile.ReadAt(buf, int64(off))
if n != DataPageSize {
err = fmt.Errorf("read %d instead of %d", n, DataPageSize)
for _, pageNo := range pagesToRead {
buf := make([]byte, PageSize)
off := (pageNo - 1) * PageSize
n, err := s.file.ReadAt(buf, int64(off))
if n != PageSize {
err = fmt.Errorf("read %d instead of %d", n, PageSize)
}
s.mutex.Lock()
resultChannels := s.dataWaits[pageNo]
delete(s.dataWaits, pageNo)
resultChannels := s.pageWaits[pageNo]
delete(s.pageWaits, pageNo)
if err != nil {
s.mutex.Unlock()
@@ -242,7 +223,8 @@ func (s *Atree) readPages() {
}
}
} else {
s.dataPages[pageNo] = &_page{
s.pages[pageNo] = &_page{
// fix - page type
PageNo: pageNo,
Buf: buf,
ReferenceCount: len(resultChannels),
@@ -255,41 +237,6 @@ func (s *Atree) readPages() {
}
}
}
for _, pageNo := range indexPagesToRead {
buf := make([]byte, IndexPageSize)
off := (pageNo - 1) * IndexPageSize
n, err := s.indexFile.ReadAt(buf, int64(off))
if n != IndexPageSize {
err = fmt.Errorf("read %d instead of %d", n, IndexPageSize)
}
s.mutex.Lock()
resultChannels := s.indexWaits[pageNo]
delete(s.indexWaits, pageNo)
if err != nil {
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Err: err,
}
}
} else {
s.indexPages[pageNo] = &_page{
PageNo: pageNo,
Buf: buf,
ReferenceCount: len(resultChannels),
}
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Data: buf,
}
}
}
}
}
// WRITE
@@ -307,9 +254,8 @@ func (s *Atree) pageWriter() {
}
type WriteTask struct {
WaitCh chan struct{}
DataPage redo.PageToWrite
IndexPages []redo.PageToWrite
WaitCh chan struct{}
Pages []PageToWrite
}
func (s *Atree) appendWriteTaskToQueue(task WriteTask) {
@@ -330,31 +276,15 @@ func (s *Atree) writeTasks() error {
s.mutex.Unlock()
for _, task := range tasks {
// data page
p := task.DataPage
if len(p.Data) != DataPageSize {
return fmt.Errorf("wrong data page %d size: %d",
p.PageNo, len(p.Data))
}
off := (p.PageNo - 1) * DataPageSize
n, err := s.dataFile.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
if n != len(p.Data) {
return fmt.Errorf("write %d instead of %d", n, len(p.Data))
}
// index pages
for _, p := range task.IndexPages {
if len(p.Data) != IndexPageSize {
return fmt.Errorf("wrong index page %d size: %d",
for _, p := range task.Pages {
if len(p.Data) != PageSize {
return fmt.Errorf("wrong page %d size: %d",
p.PageNo, len(p.Data))
}
bin.PutUint32(p.Data[indexCRC32Idx:], crc32.ChecksumIEEE(p.Data[:indexCRC32Idx]))
bin.PutUint32(p.Data[crc32Idx:], calcChecksum(p.Data[:crc32Idx]))
off := (p.PageNo - 1) * IndexPageSize
n, err := s.indexFile.WriteAt(p.Data, int64(off))
off := (p.PageNo - 1) * PageSize
n, err := s.file.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
@@ -416,7 +346,7 @@ func (s *Atree) ApplyREDO(task WriteTask) {
func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var (
pos = pageSize - 4
calculatedCRC = crc32.ChecksumIEEE(data[:pos])
calculatedCRC = calcChecksum(data[:pos])
storedCRC = bin.GetUint32(data[pos:])
)
if calculatedCRC != storedCRC {