diff --git a/atree/atree.go b/atree/atree.go index 1b2403c..c20cacf 100644 --- a/atree/atree.go +++ b/atree/atree.go @@ -39,6 +39,7 @@ const ( ) type FreeList interface { + // використовується в allocPage ReservePage() uint32 } diff --git a/atree/io.go b/atree/io.go index 7ea86e8..6f486af 100644 --- a/atree/io.go +++ b/atree/io.go @@ -8,7 +8,7 @@ import ( "math" "os" - diploma "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/atree/redo" "gordenko.dev/dima/qb/bin" ) @@ -74,8 +74,8 @@ func (s *Atree) releaseIndexPage(pageNo uint32) { p.ReferenceCount-- return } else { - diploma.Abort( - diploma.ReferenceCountBug, + qb.Abort( + qb.ReferenceCountBug, fmt.Errorf("call releaseIndexPage on page %d with reference count = %d", pageNo, p.ReferenceCount), ) @@ -91,15 +91,12 @@ func (s *Atree) allocIndexPage() AllocatedPage { ) allocated.PageNo = s.indexFreelist.ReservePage() + s.mutex.Lock() if allocated.PageNo > 0 { allocated.IsReused = true - - s.mutex.Lock() } else { - s.mutex.Lock() if s.allocatedIndexPagesQty == math.MaxUint32 { - diploma.Abort(diploma.MaxAtreeSizeExceeded, - errors.New("no space in Atree index")) + qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index")) } s.allocatedIndexPagesQty++ allocated.PageNo = s.allocatedIndexPagesQty @@ -163,8 +160,8 @@ func (s *Atree) releaseDataPage(pageNo uint32) { p.ReferenceCount-- return } else { - diploma.Abort( - diploma.ReferenceCountBug, + qb.Abort( + qb.ReferenceCountBug, fmt.Errorf("call releaseDataPage on page %d with reference count = %d", pageNo, p.ReferenceCount), ) @@ -186,7 +183,7 @@ func (s *Atree) allocDataPage() AllocatedPage { } else { s.mutex.Lock() if s.allocatedDataPagesQty == math.MaxUint32 { - diploma.Abort(diploma.MaxAtreeSizeExceeded, + qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index")) } s.allocatedDataPagesQty++ @@ -303,7 +300,7 @@ func (s *Atree) pageWriter() { case <-s.writeSignalCh: err := s.writeTasks() if err != nil { - diploma.Abort(diploma.WriteToAtreeFailed, err) + qb.Abort(qb.WriteToAtreeFailed, err) } } } diff --git a/database/database.go b/database/database.go index 8ae5025..5f7489c 100644 --- a/database/database.go +++ b/database/database.go @@ -431,12 +431,12 @@ func (s *Database) replayChangesRecord(untyped any) error { metric.LastPageNo = rec.DataPageNo // delete free pages if rec.IsDataPageReused { - s.dataFreeList.DeletePages([]uint32{ + s.dataFreeList.DeleteReservedPages([]uint32{ rec.DataPageNo, }) } if len(rec.ReusedIndexPages) > 0 { - s.indexFreeList.DeletePages(rec.ReusedIndexPages) + s.indexFreeList.DeleteReservedPages(rec.ReusedIndexPages) } } diff --git a/database/proc.go b/database/proc.go index e9432f2..62cf9a6 100644 --- a/database/proc.go +++ b/database/proc.go @@ -535,13 +535,13 @@ func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWith metric.LastPageNo = rec.DataPageNo if rec.IsDataPageReused { - s.dataFreeList.DeletePages([]uint32{ + s.dataFreeList.DeleteReservedPages([]uint32{ rec.DataPageNo, }) } if len(rec.ReusedIndexPages) > 0 { - s.indexFreeList.DeletePages(rec.ReusedIndexPages) + s.indexFreeList.DeleteReservedPages(rec.ReusedIndexPages) } if !extended.HoldLock { diff --git a/freelist/freelist.go b/freelist/freelist.go index e2c8d15..4b68fcf 100644 --- a/freelist/freelist.go +++ b/freelist/freelist.go @@ -1,29 +1,46 @@ package freelist import ( + "bytes" "fmt" + "os" + "slices" "sync" - "github.com/RoaringBitmap/roaring/v2" + "gordenko.dev/dima/qb/bin" +) + +const ( + pageSize = 1024 + crcSize = 4 + ptrSize = 4 + pointersOnPage = (pageSize - crcSize) / ptrSize + flushTreshold = 1000 //= int(float(pageSize/ptrSize) * 1.3) // кількість вільних сторінок коли вже треба писати на диск ) type FreeList struct { - mutex sync.Mutex - free *roaring.Bitmap - reserved *roaring.Bitmap + mutex sync.Mutex + file *os.File + pages int + //free *roaring.Bitmap + //reserved *roaring.Bitmap + free []uint32 + reserved []uint32 } func New() *FreeList { return &FreeList{ - free: roaring.New(), - reserved: roaring.New(), + //free: roaring.New(), + //reserved: roaring.New(), } } func (s *FreeList) Restore(serialized []byte) error { - err := s.free.UnmarshalBinary(serialized) - if err != nil { - return fmt.Errorf("UnmarshalBinary: %s", err) + if (len(serialized) % ptrSize) != 0 { + return fmt.Errorf("wrong size") + } + for i := 0; i < len(serialized); i += ptrSize { + s.free = append(s.free, bin.GetUint32(serialized[i:])) } return nil } @@ -33,10 +50,60 @@ func (s *FreeList) AddPages(pageNumbers []uint32) { return } s.mutex.Lock() - s.free.AddMany(pageNumbers) + s.free = append(s.free, pageNumbers...) + if len(s.free) > flushTreshold { + err := s.save() + if err != nil { + // ABORT + } + } s.mutex.Unlock() } +func (s *FreeList) save() error { + buf := make([]byte, pageSize) + //w := bytes.NewBuffer(nil) + //s.mutex.Lock() + i := crcSize + for _, pageNo := range s.free[:pointersOnPage] { + bin.PutUint32(buf[i:], pageNo) + i += ptrSize + } + // fix checksum + fileOffset := int64(s.pages-1) * pageSize + n, err := s.file.WriteAt(buf, fileOffset) + if err != nil { + return err + } + if n != pageSize { + return fmt.Errorf("written size %d bytes not equal page size %d", n, pageSize) + } + s.free = s.free[pointersOnPage:] + s.pages++ + return nil +} + +func (s *FreeList) loadLastPage() error { + var ( + buf = make([]byte, pageSize) + offset = int64(s.pages-1) * pageSize + ) + n, err := s.file.ReadAt(buf, offset) + if err != nil { + return fmt.Errorf("file.Seek: %s", err) + } + if n != s.pages { + return fmt.Errorf("read size %d bytes not equal page size %d", n, pageSize) + } + // check crc + //var pageNumbers []uint32 + for i := crcSize; i < len(buf); i += ptrSize { + s.free = append(s.free, bin.GetUint32(buf[i:])) + } + s.pages-- + return nil +} + // ReserveDataPage - аллокатор резервирует страницу, но не удаляет до визова // DeleteFromFree, ибо транзакция может не завершится, а между віделением страници // и падением транзакции - будет создан init файл. @@ -44,29 +111,46 @@ func (s *FreeList) ReservePage() (pageNo uint32) { s.mutex.Lock() defer s.mutex.Unlock() - if s.free.IsEmpty() { - return + if len(s.free) == 0 { + if s.pages == 0 { + return + } + if err := s.loadLastPage(); err != nil { + // ABORT + return + } } - pageNo = s.free.Minimum() - s.free.Remove(pageNo) - s.reserved.Add(pageNo) + lastIdx := len(s.free) + pageNo = s.free[lastIdx] + s.free = s.free[:lastIdx] + s.reserved = append(s.reserved, pageNo) return } // Удаляет ранее зарезервированные страницы -func (s *FreeList) DeletePages(pageNumbers []uint32) { +func (s *FreeList) DeleteReservedPages(pageNumbers []uint32) { + var cleared []uint32 s.mutex.Lock() - for _, pageNo := range pageNumbers { - s.reserved.Remove(pageNo) - s.free.Remove(pageNo) // прокрута TransactionLog + for _, reservedPageNo := range s.reserved { + if slices.Contains(pageNumbers, reservedPageNo) { + // delete + } else { + cleared = append(cleared, reservedPageNo) + } } + s.reserved = cleared s.mutex.Unlock() } func (s *FreeList) Serialize() ([]byte, error) { + w := bytes.NewBuffer(nil) s.mutex.Lock() - defer s.mutex.Unlock() - tmp := roaring.Or(s.free, s.reserved) - tmp.RunOptimize() - return tmp.ToBytes() + for _, pageNo := range s.free { + bin.WriteUint32(w, pageNo) + } + for _, pageNo := range s.reserved { + bin.WriteUint32(w, pageNo) + } + s.mutex.Unlock() + return w.Bytes(), nil } diff --git a/diploma.go b/qb.go similarity index 99% rename from diploma.go rename to qb.go index 53889c3..1be7f8d 100644 --- a/diploma.go +++ b/qb.go @@ -1,4 +1,4 @@ -package diploma +package qb import ( "fmt"