dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 23:12:03 +03:00
parent 181031a753
commit b32279deaf
35 changed files with 1395 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
enc/enc_test.go
View File

@@ -31,9 +31,25 @@ var (
}{
{
Nums: []float64{
0,
},
Name: "add 1st value zero",
Offset: 0,
ChangeSize: 3,
Buf: []byte{
0x8f, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 0,
LastDelta: 0,
},
{
Nums: []float64{
1.5,
},
Name: "add 1st value",
Name: "add 1st value non zero",
Offset: 0,
ChangeSize: 3,
Buf: []byte{

68
enc/time_delta.go
View File

@@ -1,6 +1,7 @@
package enc
import (
"fmt"
"io"
"log"
@@ -159,7 +160,7 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
}
return qb.TimeEvaluationReport{
RewindOffset: rewindOffset,
Offset: s.pos + rewindOffset,
Offset: len(s.buf) - s.pos - rewindOffset,
ChangeSize: i,
TotalSpace: s.pos - rewindOffset + i,
}
@@ -185,13 +186,21 @@ func (s *TimeDeltaCompressor) Append(rewindOffset int, change []byte, timestamp
// }
func (s *TimeDeltaCompressor) getState() qb.TimeDeltaCapturedState {
bound := s.pos + 1 + bin.CountVarUint64(uint64(s.lastDelta))
return qb.TimeDeltaCapturedState{
H: s.buf[s.pos],
state := qb.TimeDeltaCapturedState{
LastUnixtime: s.lastUnixtime,
LastDelta: s.lastDelta,
Payload: s.buf[bound:],
}
if s.pos < len(s.buf) {
var bound int
if s.lastDelta > 0 {
state.H = s.buf[s.pos]
bound = s.pos + 1 + bin.CountVarUint64(uint64(s.lastDelta))
} else {
bound = s.pos // only since encoded
}
state.Payload = s.buf[bound:]
}
return state
}
func (s *TimeDeltaCompressor) CaptureState() {
@@ -204,6 +213,7 @@ func (s *TimeDeltaCompressor) ForgetCapturedState() {
}
func (s *TimeDeltaCompressor) Tail(offset int) []byte {
fmt.Println("time tail:", s.pos, len(s.buf), offset)
return s.buf[s.pos : len(s.buf)-offset]
}
@@ -220,11 +230,19 @@ func (s *TimeDeltaCompressor) Size() int {
return len(s.buf) - s.pos
}
func (s *TimeDeltaCompressor) StoredSize() int {
func (s *TimeDeltaCompressor) CommitedSize() int {
if s.state == nil {
return len(s.buf) - s.pos
} else {
return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
if len(s.state.Payload) > 0 {
if s.state.LastDelta > 0 {
return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
} else {
return 4 // since
}
} else {
return 0
}
}
}
@@ -240,9 +258,9 @@ func (s *TimeDeltaCompressor) StoredSize() int {
// }
// }
func (s *TimeDeltaCompressor) WriteStoredTo(w io.Writer) (err error) {
func (s *TimeDeltaCompressor) WriteCommitedTo(w io.Writer) (err error) {
if s.state == nil {
_, err = w.Write(s.buf[:s.pos])
_, err = w.Write(s.buf[s.pos:])
return
} else {
_, err = w.Write(s.state.Payload)
@@ -308,14 +326,18 @@ func NewTimeDeltaDecompressor() *TimeDeltaDecompressor {
// викликається для повних сторінок
func (s *TimeDeltaDecompressor) RestoreFromEnd(buf []byte) {
s.buf = buf
var err error
s.lastUnixtime, err = bin.GetUint32(buf[len(buf)-4:])
if err != nil {
log.Fatalf("bug: get last unixtime: %s", err)
}
if len(buf) > 4 {
s.readHeader()
s.readDelta()
if len(s.buf) > 0 {
var err error
s.lastUnixtime, err = bin.GetUint32(buf[len(buf)-4:])
if err != nil {
log.Fatalf("bug: get last unixtime: %s", err)
}
if len(buf) > 4 {
s.readHeader()
s.readDelta()
}
} else {
s.done = true
}
// fmt.Printf("h: % x\n", s.buf[s.pos])
// fmt.Printf("lastUnixtime: %d\n", s.lastUnixtime)
@@ -326,10 +348,14 @@ func (s *TimeDeltaDecompressor) RestoreFromEnd(buf []byte) {
// викликається для data level tail
func (s *TimeDeltaDecompressor) RestoreFromState(state qb.TimeDeltaCapturedState) {
s.buf = state.Payload
s.lastUnixtime = state.LastUnixtime
if state.LastDelta > 0 {
s.lastDelta = state.LastDelta
s.decodeHeaderByte(state.H)
if len(s.buf) > 0 {
s.lastUnixtime = state.LastUnixtime
if state.LastDelta > 0 {
s.lastDelta = state.LastDelta
s.decodeHeaderByte(state.H)
}
} else {
s.done = true
}
}

64
enc/value_delta.go
View File

@@ -1,6 +1,7 @@
package enc
import (
"fmt"
"io"
"log"
"math"
@@ -48,9 +49,6 @@ func NewValueDeltaCompressor(metricType qb.MetricType, fracDigits byte, buf []by
s.toFloat64 = toInstantFloat64
}
if payloadSize > 0 {
// fmt.Printf("% x\n", buf)
// fmt.Printf("% x\n", buf[:s.pos])
// fmt.Printf("pos: %d\n", s.pos)
u64, _, err := bin.GetVarUint64(s.buf)
if err != nil {
log.Fatalf("bug: get base value: %s", err)
@@ -145,13 +143,21 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
// pos завжди вказує на h
func (s *ValueDeltaCompressor) Append(rewindOffset int, change []byte, value float64, delta uint64) {
// fmt.Printf("append value:\n")
// fmt.Printf("rewind offset: %d\n", rewindOffset)
// fmt.Printf("change: % x\n", change)
// fmt.Printf("value: %v\n", value)
// fmt.Printf("delta: %d\n", delta)
if s.pos > 0 {
s.lastDelta = delta
} else {
s.baseValue = value
}
//fmt.Printf("buf before: % x\n", s.buf[:s.pos])
copy(s.buf[s.pos-rewindOffset:], change)
s.pos += len(change) - rewindOffset
//fmt.Printf("buf after: % x\n", s.buf[:s.pos])
//fmt.Printf("buf after: % x\n", s.buf[:s.pos])
}
func (s *ValueDeltaCompressor) DeleteLast() {
@@ -159,11 +165,20 @@ func (s *ValueDeltaCompressor) DeleteLast() {
}
func (s *ValueDeltaCompressor) getState() qb.ValueDeltaCapturedState {
bound := s.pos - 1 - bin.CountVarUint64(s.lastDelta)
return qb.ValueDeltaCapturedState{
H: s.buf[s.pos-1],
LastDelta: s.lastDelta,
Payload: s.buf[:bound],
if s.pos > 0 {
// fmt.Println("getState")
// fmt.Printf("pos: %d\n", s.pos)
// fmt.Printf("buf: % x\n", s.buf[:s.pos])
// fmt.Printf("h: %d\n", s.buf[s.pos-1])
// fmt.Printf("lastDelta: %d\n", s.lastDelta)
bound := s.pos - 1 - bin.CountVarUint64(s.lastDelta)
return qb.ValueDeltaCapturedState{
H: s.buf[s.pos-1],
LastDelta: s.lastDelta,
Payload: s.buf[:bound],
}
} else {
return qb.ValueDeltaCapturedState{}
}
}
@@ -185,11 +200,15 @@ func (s *ValueDeltaCompressor) Size() int {
return s.pos
}
func (s *ValueDeltaCompressor) StoredSize() int {
func (s *ValueDeltaCompressor) CommitedSize() int {
if s.state == nil {
return len(s.buf) - s.pos
return s.pos
} else {
return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
if len(s.state.Payload) > 0 {
return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
} else {
return 0
}
}
}
@@ -202,7 +221,7 @@ func (s *ValueDeltaCompressor) StoredSize() int {
// }
// }
func (s *ValueDeltaCompressor) WriteStoredTo(w io.Writer) (err error) {
func (s *ValueDeltaCompressor) WriteCommitedTo(w io.Writer) (err error) {
if s.state == nil {
_, err = w.Write(s.buf[:s.pos])
return
@@ -289,17 +308,26 @@ func NewValueDeltaDecompressor(metricType qb.MetricType, fracDigits byte) *Value
func (s *ValueDeltaDecompressor) RestoreFromEnd(buf []byte) {
s.buf = buf
s.pos = len(buf) // first free
s.readBaseValue()
s.readHeader()
s.readValue()
if len(buf) > 0 {
s.readBaseValue()
s.readHeader()
s.readValue()
} else {
s.done = true
}
}
func (s *ValueDeltaDecompressor) RestoreFromState(state qb.ValueDeltaCapturedState) {
fmt.Printf("%#v\n", state)
s.buf = state.Payload
s.pos = len(s.buf) // first free
s.readBaseValue()
s.lastValue = s.baseValue + s.toFloat64(state.LastDelta, s.coef)
s.decodeHeaderByte(state.H)
if len(s.buf) > 0 {
s.readBaseValue()
s.lastValue = s.baseValue + s.toFloat64(state.LastDelta, s.coef)
s.decodeHeaderByte(state.H)
} else {
s.done = true
}
}
func (s *ValueDeltaDecompressor) readBaseValue() {