dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-09 08:16:16 +03:00
parent 964fe4b4a8
commit 7007b1b6fd
18 changed files with 1334 additions and 1026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
enc/cumdelta.go
View File

@@ -1,6 +1,7 @@
package enc
import (
"io"
"log"
"math"
@@ -47,32 +48,32 @@ type CumulativeDeltaCompressor struct {
}
// Після відновлення із снапшота
func NewCumulativeDeltaCompressor(buf []byte, payloadSize int, fracDigits byte) *CumulativeDeltaCompressor {
func NewCumulativeDeltaCompressor(buf []byte, fracDigits byte) *CumulativeDeltaCompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
}
s := &CumulativeDeltaCompressor{
return &CumulativeDeltaCompressor{
buf: buf,
pos: payloadSize, // перший вільний байт
coef: coef,
}
}
func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) {
if payloadSize > 0 {
u64, _, err := bin.GetVarUint64(buf)
s.pos = payloadSize - 1 // завжди показує на h
// base value на початку
u64, _, err := bin.GetVarUint64(s.buf)
if err != nil {
log.Fatalf("bug: get base value: %s", err)
}
s.baseValue = float64(u64) / s.coef
s.pos--
s.h = s.buf[s.pos]
var n int
s.lastDelta, n, err = bin.ReverseGetVarUint64(s.buf[:s.pos])
s.lastDelta, _, err = bin.ReverseGetVarUint64(s.buf[:s.pos-1])
if err != nil {
log.Fatalf("bug: get last delta: %s", err)
}
s.pos -= n
}
return s
}
// func (s *CumulativeDeltaCompressor) Size() int {
@@ -80,83 +81,73 @@ func NewCumulativeDeltaCompressor(buf []byte, payloadSize int, fracDigits byte)
// }
func (s *CumulativeDeltaCompressor) Evaluate(value float64) (compressionWay int, requiredSpace int) {
var (
delta = uint64((value-s.baseValue)*s.coef + eps)
)
delta := uint64((value-s.baseValue)*s.coef + eps) // fix - if delta 0, no eps
requiredSpace = s.pos
// fix - requiredSpace - all space
if s.pos > 0 {
if s.h < 128 {
// run
if delta == s.lastDelta && s.h < 127 {
compressionWay = incrementRun
requiredSpace += bin.CountVarUint64(uint64(s.lastDelta)) + // current delta
hSize
} else {
compressionWay = endSeries
requiredSpace += bin.CountVarUint64(uint64(s.lastDelta)) + // previous delta
hSize + // h - end of run
bin.CountVarUint64(uint64(delta)) + // new literal
hSize // new h
requiredSpace += bin.CountVarUint64(uint64(delta)) + hSize
}
} else {
// literal
if delta != s.lastDelta {
if s.h < 255 {
compressionWay = incrementLiteral
requiredSpace += bin.CountVarUint64(uint64(s.lastDelta)) + // previous delta
bin.CountVarUint64(uint64(delta)) + // new delta
hSize
requiredSpace += bin.CountVarUint64(uint64(delta))
} else {
compressionWay = endSeries
requiredSpace += bin.CountVarUint64(uint64(s.lastDelta)) + // previous delta
hSize + // h - end of literal
bin.CountVarUint64(uint64(delta)) + // new literal
hSize // new h
requiredSpace += bin.CountVarUint64(uint64(delta)) + hSize
}
} else {
compressionWay = startRun
if s.h > 128 {
requiredSpace += hSize // h - end of literal
requiredSpace += hSize // h - end of run
}
requiredSpace += bin.CountVarUint64(uint64(delta)) + // new delta
hSize // new h
}
}
} else {
// encode base value
compressionWay = addBaseValue
requiredSpace = bin.CountVarUint64(uint64(value*s.coef)) + // base value
bin.CountVarUint64(uint64(delta)) + // new delta
+hSize // new h
bin.CountVarUint64(0) + // new delta
hSize // new h
}
return
}
// pos завжди вказує на h
func (s *CumulativeDeltaCompressor) Compress(compressionWay int, value float64) {
delta := uint64((value-s.baseValue)*s.coef + eps)
switch compressionWay {
case incrementRun:
s.h++
case incrementLiteral:
// write previous delta and increment counter
n, _ := bin.ReversePutVarUint64(s.buf[s.pos:], uint64(s.lastDelta))
s.pos += n
s.lastDelta = delta
s.h++
case endSeries:
n, _ := bin.ReversePutVarUint64(s.buf[s.pos:], uint64(s.lastDelta))
// write previous delta and increment counter
n, _ := bin.ReversePutVarUint64(s.buf[s.pos:], delta)
s.pos += n
// write h - end of run
s.buf[s.pos] = s.h
s.pos++
// start new literal (length=1)
case endSeries:
s.lastDelta = delta
s.h = 128
n, _ := bin.ReversePutVarUint64(s.buf[s.pos:], delta)
s.pos += n
s.h = 128 // start new literal (length=1)
case startRun:
if s.h > 128 {
// write h - end of literal (because length > 1)
s.pos -= 1 + bin.CountVarUint64(s.lastDelta)
s.h--
s.buf[s.pos] = s.h
s.pos++
// run delta
n, _ := bin.ReversePutVarUint64(s.buf[s.pos:], delta)
s.pos += n
} else {
}
// start new run (length=2)
s.h = 0
@@ -165,10 +156,14 @@ func (s *CumulativeDeltaCompressor) Compress(compressionWay int, value float64)
n, _ := bin.PutVarUint64(s.buf[s.pos:], uint64(value*s.coef))
s.pos += n
s.baseValue = value
// start new literal (length=1)
s.lastDelta = 0
n, _ = bin.ReversePutVarUint64(s.buf[s.pos:], s.lastDelta)
s.pos += n
// start new literal (length=1)
s.h = 128
}
s.buf[s.pos] = s.h
}
func (s *CumulativeDeltaCompressor) DeleteLast() {
@@ -186,10 +181,11 @@ func (s *CumulativeDeltaCompressor) CaptureState() {
qb.Abort(qb.RepeatableLock, nil)
}
// позиція посувається вліво, отже може перескочити на попередній chunk
pos := s.pos - hSize - bin.CountVarUint64(s.lastDelta)
s.state = &CumulativeDeltaCapturedState{
H: s.h,
LastDelta: s.lastDelta,
Payload: s.buf[:s.pos],
Payload: s.buf[:pos],
}
}
@@ -205,31 +201,65 @@ func (s *CumulativeDeltaCompressor) Offset() int {
return 0
}
// Snapshot - для створення снапшота.
func (s *CumulativeDeltaCompressor) Snapshot() (left []byte, right []byte) {
func (s *CumulativeDeltaCompressor) Size() int {
if s.state == nil {
left = s.buf[:s.pos]
right = s.encodeTail(s.lastDelta, s.h)
return s.pos
} else {
left = s.state.Payload
right = s.encodeTail(s.state.LastDelta, s.state.H)
return len(s.state.Payload)
}
return
}
func (s *CumulativeDeltaCompressor) encodeTail(lastDelta uint64, h byte) []byte {
tail := make([]byte, 10) // max var uint64 + h
n, _ := bin.PutVarUint64(tail, lastDelta)
tail[n] = h
return tail[:n+1]
// Snapshot - для створення снапшота.
func (s *CumulativeDeltaCompressor) Payload() []byte {
if s.state == nil {
return s.buf[:s.pos]
} else {
return s.state.Payload
}
}
func (s *CumulativeDeltaCompressor) Rotate(newbuf []byte) (payloadSize int) {
n, _ := bin.PutVarUint64(s.buf[s.pos:], s.lastDelta)
s.pos += n
s.buf[s.pos] = s.h
s.pos++
payloadSize = s.pos
func (s *CumulativeDeltaCompressor) WritePayloadTo(w io.Writer) (err error) {
if s.state == nil {
err = bin.WriteUint16(w, uint16(s.pos))
if err != nil {
return
}
_, err = w.Write(s.buf[:s.pos])
return
} else {
size := bin.CountVarUint64(s.state.LastDelta) + hSize + len(s.state.Payload)
err = bin.WriteUint16(w, uint16(size))
if err != nil {
return
}
_, err = w.Write(s.state.Payload)
if err != nil {
return
}
_, err = bin.WriteVarUint64(w, s.state.LastDelta)
if err != nil {
return
}
w.Write([]byte{
s.state.H,
})
return
}
}
// func (s *CumulativeDeltaCompressor) encodeTail(lastDelta uint64, h byte) []byte {
// tail := make([]byte, 10) // max var uint64 + h
// n, _ := bin.PutVarUint64(tail, lastDelta)
// tail[n] = h
// return tail[:n+1]
// }
func (s *CumulativeDeltaCompressor) Rotate(newbuf []byte) {
// n, _ := bin.PutVarUint64(s.buf[s.pos:], s.lastDelta)
// s.pos += n
// s.buf[s.pos] = s.h
// s.pos++
// payloadSize = s.pos
// УВАГА!
// state не чіпаємо
s.buf = newbuf
@@ -237,7 +267,14 @@ func (s *CumulativeDeltaCompressor) Rotate(newbuf []byte) (payloadSize int) {
s.baseValue = 0
s.lastDelta = 0
s.h = 0
return
}
func (s *CumulativeDeltaCompressor) LastValue() float64 {
if s.state == nil {
return s.baseValue + float64(s.lastDelta)*s.coef
} else {
return s.baseValue + float64(s.state.LastDelta)*s.coef
}
}
func (s *CumulativeDeltaCompressor) CreateDecompressor() qb.ValueDecompressor {
@@ -259,10 +296,6 @@ func (s *CumulativeDeltaCompressor) CreateDecompressor() qb.ValueDecompressor {
})
}
func (s *CumulativeDeltaCompressor) Chunks() []byte {
return s.buf
}
// DECOMPRESSOR
type CumulativeDeltaDecompressor struct {