dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-19 07:25:46 +03:00
parent 06823ee547
commit 2d73ce1727
52 changed files with 5173 additions and 547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
storage/cursor.go
View File

@@ -6,6 +6,8 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/util"
)
type BackwardCursor struct {
@@ -29,6 +31,7 @@ type BackwardCursorOptions struct {
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
fmt.Printf("footer: % x\n", opt.PageData[DataPageSize-DataPageFooterSize:])
switch opt.MetricType {
case qb.Instant, qb.Cumulative:
// ok
@@ -90,6 +93,7 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
}
prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
fmt.Printf("CURSOR: prev pageNo %d\n", prevPageNo)
if prevPageNo == 0 {
return 0, 0, true, nil
}
@@ -129,8 +133,51 @@ func (s *BackwardCursor) Close() {
s.releasePage(s.pageNo)
}
// HELPER
func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
size, _ := bin.GetUint16(page[timestampsSizeIdx:])
if int(size) > DataPagePayloadSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
pos := DataPagePayloadSize - int(size)
d := enc.NewTimeDeltaDecompressor()
d.RestoreFromEnd(page[pos:DataPagePayloadSize])
//func (s *BackwardCursor) makeDecompressors() error {
payload := page[pos:DataPagePayloadSize]
//}
fmt.Printf("PAGE timestamps (CURSOR) %d:\n% x\n", len(payload), payload)
return d, nil
}
func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
size, _ := bin.GetUint16(page[valuesSizeIdx:])
if int(size) > DataPagePayloadSize {
return nil, fmt.Errorf("bug: invalid values size %d", size)
}
d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(page[:size])
return d, nil
}
func VerifyDataPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}
func VerifyIndexPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}

55
storage/misc.go
View File

@@ -1,55 +0,0 @@
package storage
import (
"fmt"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/util"
)
func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
size, _ := bin.GetUint16(page[timestampsSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
pos := DataPagePayloadSize - int(size)
d := enc.NewTimeDeltaDecompressor()
d.RestoreFromEnd(page[pos:DataPagePayloadSize])
return d, nil
}
func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
size, _ := bin.GetUint16(page[valuesSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(page[:size])
return d, nil
}
func VerifyDataPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}
func VerifyIndexPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}

148
storage/navigation.go
View File

@@ -1,6 +1,8 @@
package storage
import bin "gordenko.dev/dima/bin/little"
import (
bin "gordenko.dev/dima/bin/little"
)
const (
PageNoSize = 4
@@ -61,16 +63,6 @@ func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bo
}
}
// func getPrevPageNo(buf []byte) uint32 {
// pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
// return pageNo
// }
// func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
// pageNo, _ = bin.GetUint32(buf)
// return
// }
func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: level.Buffer,
@@ -82,17 +74,6 @@ func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo ui
return
}
// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: records,
// timestamp: timestamp,
// }
// count := len(records) / IndexRecordSize
// elemIdx, _ := BinarySearch(count, comparator)
// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
// return
// }
func FindPageOnIndexPage(page []byte, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: page,
@@ -123,6 +104,108 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN
return
}
type DeleteSinceOnIndexPageResult struct {
Buffer []byte
RecordsCount int
PageNumbers []uint32
//Idx int
}
func DeleteSinceOnIndexPage(buf []byte, since uint32) DeleteSinceOnIndexPageResult {
comparator := ValueAtComparator{
buf: buf,
timestamp: since,
}
count, _ := bin.GetUint16(buf[indexRecordsCountIdx:])
elemIdx, _ := BinarySearch(int(count), comparator)
newbuf := make([]byte, IndexPageSize)
copy(newbuf, buf[:elemIdx*IndexRecordSize]) // [0..idx)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
deleteCount := int(count) - elemIdx
var pageNumbers []uint32
for range deleteCount {
pageNo, _ := bin.GetUint32(buf[pos:])
pageNumbers = append(pageNumbers, pageNo)
pos += IndexRecordSize
}
return DeleteSinceOnIndexPageResult{
Buffer: newbuf,
RecordsCount: elemIdx,
PageNumbers: pageNumbers,
//Idx: elemIdx,
}
}
type DeleteOnIndexTailResult struct {
RecordsCount int
PageNumbers []uint32
}
func DeleteSinceOnIndexTail(level IndexLevelTail, since uint32) DeleteOnIndexTailResult {
comparator := ValueAtComparator{
buf: level.Buffer,
timestamp: since,
}
elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
deleteCount := level.RecordsCount - elemIdx
var pageNumbers []uint32
for range deleteCount {
pageNo, _ := bin.GetUint32(level.Buffer[pos:])
pageNumbers = append(pageNumbers, pageNo)
pos += IndexRecordSize
}
return DeleteOnIndexTailResult{
RecordsCount: elemIdx,
PageNumbers: pageNumbers,
//Idx: elemIdx,
}
}
func DeleteUntilOnIndexTail(level IndexLevelTail, until uint32) DeleteOnIndexTailResult {
comparator := ValueAtComparator{
buf: level.Buffer,
timestamp: until,
}
elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
pos := 4 // timestamp size
deleteCount := elemIdx + 1
var pageNumbers []uint32
for range deleteCount {
pageNo, _ := bin.GetUint32(level.Buffer[pos:])
pageNumbers = append(pageNumbers, pageNo)
pos += IndexRecordSize
}
copy(level.Buffer, level.Buffer[:])
return DeleteOnIndexTailResult{
RecordsCount: elemIdx,
PageNumbers: pageNumbers,
//Idx: elemIdx,
}
}
// func getPrevPageNo(buf []byte) uint32 {
// pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
// return pageNo
// }
// func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
// pageNo, _ = bin.GetUint32(buf)
// return
// }
// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: records,
// timestamp: timestamp,
// }
// count := len(records) / IndexRecordSize
// elemIdx, _ := BinarySearch(count, comparator)
// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
// return
// }
// func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
// comparator := ValueAtComparator{
// buf: buf,
@@ -167,16 +250,17 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN
// return
// }
// func listPageNumbers(buf []byte) (pageNumbers []uint32) {
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// pos := indexFooterIdx - PageNoSize
// for range qty {
// pageNo, _ := bin.GetUint32(buf[pos:])
// pageNumbers = append(pageNumbers, pageNo)
// pos -= PageNoSize
// }
// return
// }
// from index page
func ListPageNumbers(buf []byte) (pageNumbers []uint32) {
count, _ := bin.GetUint16(buf[indexRecordsCountIdx:])
pos := timestampSize
for range count {
pageNo, _ := bin.GetUint32(buf[pos:])
pageNumbers = append(pageNumbers, pageNo)
pos += IndexRecordSize
}
return
}
// // include since timestamp
// func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {

32
storage/replay_metric.go
View File

@@ -51,6 +51,7 @@ type ReplayMeasuresAppendWithGrowResult struct {
}
func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ ReplayMeasuresAppendWithGrowResult) {
//fmt.Printf("%#v\n", rec)
var (
reusedIndexPages int
reusedDataPages int
@@ -164,17 +165,34 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
// HELPERS
func (s *ReplayMetric) composeHeadIndexPage(levelIdx int, head *IndexPageTail) PageToWrite {
level := s.IndexLevelTails[levelIdx]
// розраховую pos, з якого буду дописувати хвіст
pos := level.RecordsCount * IndexRecordSize
// створюю копію сторінки
page := make([]byte, IndexPageSize)
copy(page, level.Buffer[:pos]) // поточні дані
var (
page = make([]byte, IndexPageSize)
pos int
prevCount int
)
//fmt.Printf("composeHeadIndexPage: levels=%d\n", len(s.IndexLevelTails))
//fmt.Printf("head: % x\n", head.Records)
if levelIdx < len(s.IndexLevelTails) {
level := s.IndexLevelTails[levelIdx]
//fmt.Printf("level (%d): % x\n", level.RecordsCount, level.Buffer)
// розраховую pos, з якого буду дописувати хвіст
pos = level.RecordsCount * IndexRecordSize
// створюю копію сторінки
copy(page, level.Buffer[:pos]) // поточні дані
prevCount = level.RecordsCount
} else {
//
s.IndexLevelTails = append(s.IndexLevelTails, IndexLevelTail{
Buffer: page,
RecordsCount: len(head.Records) / IndexRecordSize,
})
}
copy(page[pos:], head.Records)
// запечатати сторінку
calculatedCRC := SealIndexPage(SealIndexPageIn{
Content: page,
RecordsCount: level.RecordsCount + len(head.Records)/IndexRecordSize,
RecordsCount: prevCount + len(head.Records)/IndexRecordSize,
ZeroLevel: levelIdx == 0,
})
// перевірка CRC

4
storage/storage.go
View File

@@ -17,7 +17,7 @@ var (
prevPageIdx = DataPageSize - 12
// index page
IndexPageSize = 1024
IndexPageSize = 24 // 1024
IndexPagePayloadSize = IndexPageSize - IndexPageFooterSize
//indexPageIncSize = IndexPageIncSize
@@ -27,7 +27,7 @@ var (
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / IndexRecordSize
// timestampSize = 4
timestampSize = 4
// pairSize = timestampSize + PageNoSize
// dataFooterIdx = timestampsSizeIdx
)

7
storage/write_preparer.go
View File

@@ -2,7 +2,6 @@ package storage
import (
"bytes"
"fmt"
"io"
bin "gordenko.dev/dima/bin/little"
@@ -270,10 +269,10 @@ func finalizePacket(w *bytes.Buffer, checksum uint32) []byte {
start := 9 - bin.CountVarSize(bodySize)
bin.PutVarSize(packet[start:], bodySize)
fmt.Printf("checksum: %d\n", checksum)
fmt.Printf("bodySize: %d\n", bodySize)
//fmt.Printf("checksum: %d\n", checksum)
//fmt.Printf("bodySize: %d\n", bodySize)
//fmt.Printf("start: %d\n", start)
fmt.Printf("body: % x\n", packet[9:len(packet)-4])
//fmt.Printf("body: % x\n", packet[9:len(packet)-4])
return packet[start:]
}

42
storage/writer.go
View File

@@ -23,7 +23,7 @@ const (
filePerm = 0770
dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB
dumpSnapshotAfterNBytes = 20 * 1024 // 1024 * 1024 * 1024 // 1 GB
writeBufferSize = 4 * 1024 * 1024
)
@@ -64,19 +64,18 @@ type Writer struct {
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
//atree *atree.Atree
dir string
databaseName string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
dir string
databaseName string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
}
type WriterOptions struct {
@@ -225,7 +224,7 @@ func (s *Writer) packAndWrite() (err error) {
prepared := s.writePreparer.Prepare(input)
//fmt.Printf("prepared: %#v\n", prepared)
fmt.Println("prepared")
//fmt.Println("prepared")
// 3. Пишу на диск WAL (append)
n, err := s.wal.Write(prepared.Packet)
@@ -239,7 +238,8 @@ func (s *Writer) packAndWrite() (err error) {
if err = s.wal.Sync(); err != nil {
return
}
fmt.Println("synced to wal")
s.written += int64(len(prepared.Packet))
//fmt.Println("synced to wal")
// 4. Пишу в atree сторінки
if len(prepared.WriteToData) > 0 {
@@ -250,7 +250,7 @@ func (s *Writer) packAndWrite() (err error) {
if err = s.dataFile.Sync(); err != nil {
return
}
fmt.Println("written to data")
//fmt.Println("written to data")
}
if len(prepared.WriteToIndex) > 0 {
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
@@ -260,7 +260,7 @@ func (s *Writer) packAndWrite() (err error) {
if err = s.indexFile.Sync(); err != nil {
return
}
fmt.Println("written to index")
//fmt.Println("written to index")
}
// 6. відправляю input - worker-у
@@ -296,6 +296,8 @@ func (s *Writer) packAndWrite() (err error) {
snapshotNumber := <-snapshotNumberCh
//fmt.Println("got snapshot number:", snapshotNumber)
// копіюю сторінки із delta файла в base файл і потім роблю Truncate
err = s.indexFreeList.Merge()
if err != nil {
@@ -305,6 +307,7 @@ func (s *Writer) packAndWrite() (err error) {
if err != nil {
return
}
//fmt.Println("free lists merged")
var err error
s.wal, err = os.OpenFile(
@@ -315,11 +318,12 @@ func (s *Writer) packAndWrite() (err error) {
if err != nil {
return fmt.Errorf("create new changes file: %s", err)
}
//fmt.Println("new wal created")
} else {
s.workerInbox.Push(Changes{
Commits: prepared.Commits,
})
fmt.Println("pushed commits")
//fmt.Println("pushed commits")
}
return nil
}