dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 23:12:03 +03:00
parent 181031a753
commit b32279deaf
35 changed files with 1395 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
worker/snapshot.go
View File

@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"os"
"path/filepath"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
@@ -30,11 +29,10 @@ dataFreeList - Nb
CRC32 - 4b
*/
const readBufferSize = 4 * 1024 * 1024 // 8mb
const readBufferSize = 4 * 1024 * 1024
type WriteSnapshotIn struct {
SnapshotNumber int
Dir string
WriteBufferSize int
Metrics map[uint32]*Metric
FrozenIndexPagesCount int
@@ -43,24 +41,21 @@ type WriteSnapshotIn struct {
DataPageNumbers []uint32
}
func WriteSnapshot(in WriteSnapshotIn) (err error) {
var (
fileName = filepath.Join(in.Dir, fmt.Sprintf("%d.snapshot", in.SnapshotNumber))
hasher = util.NewHasher()
)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0666) //0770)
func writeSnapshot(filePath string, in WriteSnapshotIn) (err error) {
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
return
qb.Abort(qb.WriteSnapshotFailed, err)
}
dst := io.MultiWriter(bufio.NewWriterSize(file, in.WriteBufferSize), hasher)
var (
bufferedWriter = bufio.NewWriterSize(file, in.WriteBufferSize)
hasher = util.NewHasher()
)
dst := io.MultiWriter(bufferedWriter, hasher)
//
_, err = bin.WriteVarSize(dst, len(in.Metrics))
if err != nil {
return
}
for metricID, metric := range in.Metrics {
err = bin.WriteUint32(dst, metricID)
if err != nil {
@@ -81,13 +76,16 @@ func WriteSnapshot(in WriteSnapshotIn) (err error) {
if err != nil {
return
}
// CRC32
bin.WriteUint32(file, hasher.Sum32())
err = file.Close()
err = bufferedWriter.Flush()
if err != nil {
return
}
// CRC32
err = bin.WriteUint32(file, hasher.Sum32())
if err != nil {
return
}
return file.Close()
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
@@ -116,7 +114,6 @@ func WriteSnapshot(in WriteSnapshotIn) (err error) {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
type ReadSnapshotOut struct {
@@ -127,7 +124,7 @@ type ReadSnapshotOut struct {
DataPageNumbers []uint32
}
func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
var (
metrics = make(map[uint32]*storage.ReplayMetric)
frozenIndexPagesCount int
@@ -137,7 +134,7 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
hasher = util.NewHasher()
)
file, err := os.Open(fileName)
file, err := os.Open(filePath)
if err != nil {
return
}
@@ -148,25 +145,16 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
return
}
fileSize := stat.Size()
if fileSize == 0 {
return ReadSnapshotOut{
Metrics: metrics,
}, nil
}
if fileSize < 4 {
err = fmt.Errorf("%s is corrupted", fileName)
err = fmt.Errorf("snapshot %s is corrupted", filePath)
return
}
payloadSize := fileSize - 4
// 2. Створюємо великий буфер для NVMe (4 MB)
bufferedReader := bufio.NewReaderSize(file, readBufferSize)
// 3. Обмежуємо читання лише розміром Payload
limitReader := io.LimitReader(bufferedReader, payloadSize)
// 4. Створюємо хеш та обгортку TeeReader, яка рахує CRC32 на льоту
src := io.TeeReader(limitReader, hasher)
@@ -189,6 +177,7 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("metricID", metricID)
metricType, err = bin.ReadByte(src)
if err != nil {
return
@@ -198,10 +187,13 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("m.MetricType", m.MetricType)
fmt.Println("m.FracDigits", m.FracDigits)
m.LastPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
fmt.Println("m.LastPageNo", m.LastPageNo)
m.TimestampsSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
@@ -210,7 +202,8 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
err = bin.ReadNInto(src, buf[len(buf)-m.TimestampsSize:])
fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize)
err = bin.ReadNInto(src, buf[storage.DataPagePayloadSize-m.TimestampsSize:storage.DataPagePayloadSize])
if err != nil {
return
}
@@ -218,11 +211,13 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Printf("% x\n", buf)
//
levelsCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
fmt.Println("levels count:", levelsCount)
for range levelsCount {
var (
buf = make([]byte, storage.IndexPageSize)
@@ -232,10 +227,12 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("records count:", count)
err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize])
if err != nil {
return
}
fmt.Printf("% x\n", buf)
m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: count,
@@ -258,12 +255,10 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
calculatedCRC := hasher.Sum32()
if expectedCRC != calculatedCRC {
err = fmt.Errorf("%s is corrupted. Calculated CRC %d not equal expected CRC %d",
fileName, calculatedCRC, expectedCRC)
filePath, calculatedCRC, expectedCRC)
return
}
@@ -283,6 +278,7 @@ func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (
if err != nil {
return
}
fmt.Println("write frozenPagesCount:", frozenPagesCount)
_, err = bin.WriteVarSize(dst, len(pageNumbers))
if err != nil {
return
@@ -305,6 +301,7 @@ func readFreePages(src io.Reader) (_ int, _ []uint32, err error) {
if err != nil {
return
}
fmt.Println("read frozenPagesCount:", frozenPagesCount)
pageNumbersCount, err := bin.ReadVarSize(src)
if err != nil {
return