dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tmp

Browse Source
This commit is contained in:
dima
2026-06-05 19:43:01 +00:00
parent db5ecd3dfd
commit 05eea3b279
16 changed files with 1187 additions and 724 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
chunkenc/cumdelta.go
View File

@@ -5,6 +5,7 @@ import (
"log"
"math"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/conbuf"
)
@@ -38,7 +39,7 @@ v1 v2 h-byte(literal, 2) v3 h-byte(run, 2)
*/
type ReverseCumulativeDeltaCompressor struct {
buf *conbuf.ContinuousBuffer
buf []byte
coef float64
pos int
baseValue float64
@@ -48,7 +49,7 @@ type ReverseCumulativeDeltaCompressor struct {
state *CumulativeDeltaBound
}
func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaCompressor {
func NewReverseCumulativeDeltaCompressor(buf []byte, size int, fracDigits byte) *ReverseCumulativeDeltaCompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
@@ -59,13 +60,13 @@ func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int,
coef: coef,
}
if size > 0 {
u64, _, err := s.buf.GetVarUint64(0)
u64, _, err := bin.GetVarUint64(s.buf)
if err != nil {
log.Fatalf("bug: get base value: %s", err)
}
s.baseValue = float64(u64) / s.coef
s.h = s.buf.GetByte(s.pos - 1)
s.lastDelta, s.lastDeltaSize, err = s.buf.ReverseGetVarUint64(s.pos - 2)
s.h = s.buf[s.pos-1]
s.lastDelta, s.lastDeltaSize, err = bin.ReverseGetVarUint64(s.buf[:s.pos-2])
if err != nil {
log.Fatalf("bug: get last delta: %s", err)
}
@@ -84,7 +85,8 @@ func (s *ReverseCumulativeDeltaCompressor) CalcRequiredSpace(value float64) int
func (s *ReverseCumulativeDeltaCompressor) Append(value float64) {
if s.pos == 0 {
// base value
s.pos += s.buf.PutVarUint64(s.pos, uint64(value*s.coef))
n, _ := bin.PutVarUint64(s.buf[s.pos:], uint64(value*s.coef))
s.pos += n
s.baseValue = value
s.appendNewLiteral(0)
} else {
@@ -95,7 +97,7 @@ func (s *ReverseCumulativeDeltaCompressor) Append(value float64) {
if s.h < 127 {
// increase counter
s.h++
s.buf.SetByte(s.pos-1, s.h)
s.buf[s.pos-1] = s.h
} else {
// не можу збільшити - буде переповнення. Додаю новий literal блок
// counter overflow
@@ -134,37 +136,37 @@ func (s *ReverseCumulativeDeltaCompressor) convertLastFromLiteralToRun() {
// Зменшую кількість елементів в literal блоці
s.h--
s.pos -= 1 + s.lastDeltaSize
s.buf.SetByte(s.pos, s.h) // закриваю literal блок
s.buf[s.pos] = s.h // закриваю literal блок
s.pos++
s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, s.lastDelta)
s.lastDeltaSize, _ = bin.ReversePutVarUint64(s.buf[s.pos:], s.lastDelta)
s.pos += s.lastDeltaSize
s.h = 0 // run блок, довжини 2
s.buf.SetByte(s.pos, s.h)
s.buf[s.pos] = s.h
s.pos++
}
func (s *ReverseCumulativeDeltaCompressor) convertLiteralToRun() {
// Знімаю flagLiteral, а лічильник 0 дорівнює 2 елементам в серії.
s.h = 0
s.buf.SetByte(s.pos-1, s.h)
s.buf[s.pos-1] = s.h
}
func (s *ReverseCumulativeDeltaCompressor) appendDeltaToLiteral(delta uint64) {
s.h++ // збільшую к-сть дельт
s.lastDelta = delta
s.pos--
s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, delta)
s.lastDeltaSize, _ = bin.ReversePutVarUint64(s.buf[s.pos:], delta)
s.pos += s.lastDeltaSize
s.buf.SetByte(s.pos, s.h)
s.buf[s.pos] = s.h
s.pos++
}
func (s *ReverseCumulativeDeltaCompressor) appendNewLiteral(delta uint64) {
s.h = flagLiteral
s.lastDelta = delta
s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, delta)
s.lastDeltaSize, _ = bin.ReversePutVarUint64(s.buf[s.pos:], delta)
s.pos += s.lastDeltaSize
s.buf.SetByte(s.pos, flagLiteral) // literal, length = 1
s.buf[s.pos] = flagLiteral // literal, length = 1
s.pos++
}
@@ -176,7 +178,7 @@ type CumulativeDeltaBound struct {
Pos int
H byte
LastDelta uint64
Chunks [][]byte
Chunks []byte
}
// delta h
@@ -186,15 +188,11 @@ func (s *ReverseCumulativeDeltaCompressor) Lock() {
}
// позиція посувається вліво, отже може перескочити на попередній chunk
pos := s.pos - 1 - s.lastDeltaSize
chunksQty := pos / conbuf.ChunkSize
if (pos % conbuf.ChunkSize) > 0 {
chunksQty++
}
s.state = &CumulativeDeltaBound{
Pos: pos,
H: s.h,
LastDelta: s.lastDelta,
Chunks: s.buf.Chunks()[:chunksQty],
Chunks: s.buf[:s.pos], // fix check ?
}
}
@@ -210,9 +208,9 @@ func (s *ReverseCumulativeDeltaCompressor) Offset() int {
return 0
}
func (s *ReverseCumulativeDeltaCompressor) Snapshot() ([][]byte, int) {
func (s *ReverseCumulativeDeltaCompressor) Snapshot() ([]byte, int) {
if s.state == nil {
return s.buf.Chunks(), s.Size()
return s.buf, s.Size()
}
// ВАЖЛИВО!
// Треба відтворити стан останнього чанка
@@ -246,7 +244,7 @@ func (s *ReverseCumulativeDeltaCompressor) CreateDecompressor(fracDigits byte) q
func (s *ReverseCumulativeDeltaCompressor) Renew() {
// УВАГА!
// state не чіпаємо
s.buf = conbuf.New(nil)
s.buf = make([]byte, minBufferSize)
s.pos = 0
//
s.baseValue = 0
@@ -255,14 +253,14 @@ func (s *ReverseCumulativeDeltaCompressor) Renew() {
s.h = 0
}
func (s *ReverseCumulativeDeltaCompressor) Chunks() [][]byte {
return s.buf.Chunks()
func (s *ReverseCumulativeDeltaCompressor) Chunks() []byte {
return s.buf
}
// DECOMPRESSOR
type ReverseCumulativeDeltaDecompressor struct {
buf *conbuf.ContinuousBuffer
buf []byte
coef float64
pos int
bound int
@@ -273,12 +271,12 @@ type ReverseCumulativeDeltaDecompressor struct {
done bool
}
func NewReverseCumulativeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaDecompressor {
func NewReverseCumulativeDeltaDecompressor(buf []byte, size int, fracDigits byte) *ReverseCumulativeDeltaDecompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
}
u64, n, err := buf.GetVarUint64(0)
u64, n, err := bin.GetVarUint64(buf)
if err != nil {
log.Fatalf("bug: get base value: %s", err)
}
@@ -335,7 +333,7 @@ func (s *ReverseCumulativeDeltaDecompressor) NextValue() (value float64, done bo
}
func (s *ReverseCumulativeDeltaDecompressor) readHeader() {
h := s.buf.GetByte(s.pos)
h := s.buf[s.pos]
s.pos--
s.decodeHeaderByte(h)
// fmt.Println("h:", h)
@@ -343,7 +341,7 @@ func (s *ReverseCumulativeDeltaDecompressor) readHeader() {
// fmt.Println("pending:", s.pending)
}
func (s *ReverseCumulativeDeltaDecompressor) readValue() {
u64, n, err := s.buf.ReverseGetVarUint64(s.pos)
u64, n, err := bin.ReverseGetVarUint64(s.buf[:s.pos])
if err != nil {
log.Fatalln(err)
}