wp
This commit is contained in:
92
storage/misc.go
Normal file
92
storage/misc.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb"
|
||||
"gordenko.dev/dima/qb/enc"
|
||||
)
|
||||
|
||||
// func (s *BackwardCursor) makeDecompressors() error {
|
||||
// timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
|
||||
// valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
|
||||
|
||||
// payloadSize := timestampsSize + valuesSize
|
||||
|
||||
// if payloadSize > dataFooterIdx {
|
||||
// return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
|
||||
// s.pageNo, payloadSize)
|
||||
// }
|
||||
|
||||
// s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
|
||||
// s.pageData[:timestampsSize],
|
||||
// )
|
||||
|
||||
// vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
|
||||
|
||||
// switch s.metricType {
|
||||
// case qb.Instant:
|
||||
// s.valueDecompressor = enc.NewInstantDeltaDecompressor(
|
||||
// vbuf, s.fracDigits)
|
||||
|
||||
// case qb.Cumulative:
|
||||
// s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
|
||||
// vbuf, s.fracDigits)
|
||||
|
||||
// default:
|
||||
// return fmt.Errorf("bug: wrong metricType %d", s.metricType)
|
||||
// }
|
||||
// return nil
|
||||
// }
|
||||
|
||||
func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
|
||||
qb.TimestampDecompressor, qb.ValueDecompressor, error,
|
||||
) {
|
||||
|
||||
valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
|
||||
|
||||
payloadSize := timestampsSize + valuesSize
|
||||
|
||||
if payloadSize > dataFooterIdx {
|
||||
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
|
||||
payloadSize)
|
||||
}
|
||||
|
||||
timestampDecompressor := enc.NewTimeDeltaDecompressor(
|
||||
pageData[:timestampsSize],
|
||||
)
|
||||
|
||||
vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
|
||||
|
||||
var valueDecompressor qb.ValueDecompressor
|
||||
switch metricType {
|
||||
case qb.Instant:
|
||||
valueDecompressor = enc.NewInstantDeltaDecompressor(
|
||||
vbuf, fracDigits)
|
||||
|
||||
case qb.Cumulative:
|
||||
valueDecompressor = enc.NewCumulativeDeltaDecompressor(
|
||||
vbuf, fracDigits)
|
||||
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
|
||||
}
|
||||
return timestampDecompressor, valueDecompressor, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func CreateTimeDeltaDecompressor(page []byte) qb.TimestampDecompressor {
|
||||
size, _ := bin.GetUint16(page[timestampsSizeIdx:])
|
||||
pos := DataPagePayloadSize - int(size)
|
||||
d := enc.NewTimeDeltaDecompressor()
|
||||
d.RestoreFromEnd(page[pos:DataPagePayloadSize])
|
||||
return d
|
||||
}
|
||||
|
||||
func CreateValueDeltaDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) qb.ValueDecompressor {
|
||||
size, _ := bin.GetUint16(page[valuesSizeIdx:])
|
||||
d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
|
||||
d.RestoreFromEnd(page[:size])
|
||||
return d
|
||||
}
|
||||
@@ -77,23 +77,20 @@ type Writer struct {
|
||||
isExited bool
|
||||
exitCh chan struct{}
|
||||
waitGroup *sync.WaitGroup
|
||||
//signalCh chan struct{}
|
||||
}
|
||||
|
||||
type WriterOptions struct {
|
||||
Inbox *inbox.Inbox
|
||||
WorkerInbox *inbox.Inbox
|
||||
Dir string
|
||||
DatabaseName string
|
||||
//SnapshotNumber int // номер журнала
|
||||
Inbox *inbox.Inbox
|
||||
WorkerInbox *inbox.Inbox
|
||||
Dir string
|
||||
DatabaseName string
|
||||
WAL string
|
||||
DataFreeList *freelist.FreeList
|
||||
IndexFreeList *freelist.FreeList
|
||||
DataFile *os.File
|
||||
IndexFile *os.File
|
||||
//Atree *atree.Atree
|
||||
ExitCh chan struct{}
|
||||
WaitGroup *sync.WaitGroup
|
||||
ExitCh chan struct{}
|
||||
WaitGroup *sync.WaitGroup
|
||||
}
|
||||
|
||||
func NewWriter(opt WriterOptions) (*Writer, error) {
|
||||
@@ -245,17 +242,26 @@ func (s *Writer) packAndWrite() (err error) {
|
||||
fmt.Println("synced to wal")
|
||||
|
||||
// 4. Пишу в atree сторінки
|
||||
err = WriteDataPages(s.dataFile, prepared.WriteToData)
|
||||
if err != nil {
|
||||
return
|
||||
if len(prepared.WriteToData) > 0 {
|
||||
err = WriteDataPages(s.dataFile, prepared.WriteToData)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = s.dataFile.Sync(); err != nil {
|
||||
return
|
||||
}
|
||||
fmt.Println("written to data")
|
||||
}
|
||||
fmt.Println("written to data")
|
||||
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
|
||||
if err != nil {
|
||||
return
|
||||
if len(prepared.WriteToIndex) > 0 {
|
||||
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = s.indexFile.Sync(); err != nil {
|
||||
return
|
||||
}
|
||||
fmt.Println("written to index")
|
||||
}
|
||||
fmt.Println("written to index")
|
||||
|
||||
// 6. відправляю input - worker-у
|
||||
|
||||
var (
|
||||
@@ -413,62 +419,3 @@ func WriteIndexPages(file *os.File, pages []PageToWrite) (err error) {
|
||||
// TimestampsBuf []byte
|
||||
// ValuesBuf []byte
|
||||
// }
|
||||
|
||||
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
|
||||
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
|
||||
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
|
||||
// // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні
|
||||
// // без звернення до диску.
|
||||
// report := s.atree.AppendDataPages(atree.AppendDataPagesReq{
|
||||
// LastPageNo: req.LastPageNo,
|
||||
// Legs: req.Legs,
|
||||
// DataPages: req.Pages,
|
||||
// })
|
||||
|
||||
// m := AppendedPages{
|
||||
// MetricID: req.MetricID,
|
||||
// Timestamp: req.Timestamp,
|
||||
// Value: req.Value,
|
||||
// NewRootPageNo: report.NewRootPageNo,
|
||||
// LastPageNo: report.LastPageNo,
|
||||
// Pages: report.Pages,
|
||||
// TimestampsChunks: req.TimestampsChunks,
|
||||
// TimestampsSize: int(req.TimestampsSize),
|
||||
// ValuesChunks: req.ValuesChunks,
|
||||
// ValuesSize: int(req.ValuesSize),
|
||||
// }
|
||||
// m.Pack(w)
|
||||
// }
|
||||
|
||||
// helpers
|
||||
|
||||
// type Metric struct {
|
||||
// MetricID uint32
|
||||
// MetricType qb.MetricType
|
||||
// FracDigits byte
|
||||
// LastPageNo uint32
|
||||
// Since uint32
|
||||
// SinceValue float64
|
||||
// Until uint32
|
||||
// UntilValue float64
|
||||
// Timestamps []byte // payload FIX
|
||||
// Values []byte // payload FIX add lastDelta + h ?
|
||||
// }
|
||||
|
||||
// func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
|
||||
// remaining := size
|
||||
// for _, buf := range chunks {
|
||||
// if remaining < len(buf) {
|
||||
// buf = buf[:remaining]
|
||||
// }
|
||||
// _, err = dst.Write(buf)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// remaining -= len(buf)
|
||||
// if remaining == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
Reference in New Issue
Block a user