package enc import ( "bytes" "fmt" "math" "slices" "testing" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" ) func equalFloatSlices(a, b []float64, epsilon float64) bool { return slices.EqualFunc(a, b, func(x, y float64) bool { return math.Abs(x-y) <= epsilon }) } // CUMULATIVE func TestCumulativeDeltaCompressor(t *testing.T) { var ( testCases = []struct { Nums []float64 Name string Offset int ChangeSize int Buf []byte BaseValue float64 LastDelta uint64 }{ { Nums: []float64{ 1.5, }, Name: "add 1st value", Offset: 0, ChangeSize: 3, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.5, }, Name: "literal switch to run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x00, // run (len = 2) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.6, 1.6, }, Name: "literal decrease by 1 and switch to run", Offset: 2, ChangeSize: 3, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x81, // delta 1 0x00, // run (len = 2) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 1, }, { Nums: []float64{ 1.5, 1.5, 1.5, }, Name: "increment run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x01, // run (len = 3) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, Name: "run switch to literal", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x01, // run (len = 3) 0x81, // delta 1 0x80, // literal (len = 1) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 1, }, { Nums: repeatFloat64(1.5, 129), Name: "increment run to full fill h-byte", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x7f, // run (len = 129) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: repeatFloat64(1.5, 130), Name: "run switch to literal after h-byte overflow", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x7f, // run (len = 129) 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.6, 1.7, }, Name: "increment literal", Offset: 1, ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 0x81, // delta 1 0x82, // delta 2 0x82, // literal (len = 3) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 2, }, } ) for _, testCase := range testCases { var ( tmp = make([]byte, tmpValueSize) metricType = qb.Cumulative fracDigits byte = 1 buf = make([]byte, 8) payloadSize = 0 report qb.ValueEvaluationReport ) c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) } if report.Offset != testCase.Offset { t.Fatalf("%s: got offset %d are not equal expected %d", testCase.Name, report.Offset, testCase.Offset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", testCase.Name, report.ChangeSize, testCase.ChangeSize) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%s: got buf % x are not equal expected % x", testCase.Name, buf, testCase.Buf) } if c.baseValue != testCase.BaseValue { t.Fatalf("%s: got baseValue %v are not equal expected %v", testCase.Name, c.baseValue, testCase.BaseValue) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%s: got lastDelta %d are not equal expected %d", testCase.Name, c.lastDelta, testCase.LastDelta) } } } func TestCumulativeDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []float64 //RepeatLastDeltaNTimes int }{ { // addBaseValue Nums: []float64{ 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.6, 1.6, }, }, { // incrementRun Nums: []float64{ 1.5, 1.5, 1.5, }, }, { // endSeries run Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, }, { // incrementLiteral Nums: []float64{ 1.5, 1.6, }, }, { // ... Nums: []float64{ 1.5, 1.5, 1.5, // 1.5, // 1.6, // 1.7, // 1.8, // 1.9, // 2.0, // 2.0, // 2.0, // 2.2, // }, }, } ) for caseIdx, testCase := range testCases { var ( metricType = qb.Cumulative fracDigits byte = 1 tmp = make([]byte, tmpValueSize) buf = make([]byte, 16) payloadSize = 0 decodedNums []float64 ) c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) } // c.CaptureState() fmt.Println("---------------") d := c.CreateDecompressor() //fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) fmt.Println(testCase.Nums) fmt.Println(decodedNums) fmt.Println() if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } // INSTANT func TestInstantDeltaCompressor(t *testing.T) { var ( testCases = []struct { Nums []float64 Name string Offset int ChangeSize int Buf []byte BaseValue float64 LastDelta uint64 }{ { Nums: []float64{ 1.5, }, Name: "add 1st value", Offset: 0, ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ -1.5, }, Name: "add 1st value (negative)", Offset: 0, ChangeSize: 3, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: -1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.5, }, Name: "literal switch to run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x00, // run (len = 2) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.6, 1.6, }, Name: "literal decrease by 1 and switch to run", Offset: 2, ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x82, // delta 1 0x00, // run (len = 2) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(1), }, { Nums: []float64{ 1.5, 1.4, 1.4, }, Name: "literal decrease by 1 and switch to run (negative delta)", Offset: 2, ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x81, // delta -1 0x00, // run (len = 2) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(-1), }, { Nums: []float64{ 1.5, 1.5, 1.5, }, Name: "increment run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x01, // run (len = 3) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, Name: "run switch to literal", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x01, // run (len = 3) 0x82, // delta 1 0x80, // literal (len = 1) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(1), }, { Nums: []float64{ 1.5, 1.5, 1.5, 1.4, }, Name: "run switch to literal (negative delta)", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x01, // run (len = 3) 0x81, // delta -1 0x80, // literal (len = 1) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(-1), }, { Nums: repeatFloat64(1.5, 129), Name: "increment run to full fill h-byte", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x7f, // run (len = 129) 0x00, 0x00, 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: repeatFloat64(1.5, 130), Name: "run switch to literal after h-byte overflow", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x7f, // run (len = 129) 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: 0, }, { Nums: []float64{ 1.5, 1.6, 1.7, }, Name: "increment literal", Offset: 1, ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x82, // delta 1 0x84, // delta 2 0x82, // literal (len = 3) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(2), }, { Nums: []float64{ 1.5, 1.4, 1.3, }, Name: "increment literal (negative delta)", Offset: 1, ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 0x81, // delta -1 0x83, // delta -2 0x82, // literal (len = 3) 0x00, 0x00, 0x00, }, BaseValue: 1.5, LastDelta: bin.EncodeZigZag(-2), }, } ) for _, testCase := range testCases { // fmt.Println("-----") // fmt.Println("nums", testCase.Nums) var ( tmp = make([]byte, tmpValueSize) metricType = qb.Instant fracDigits byte = 1 buf = make([]byte, 8) payloadSize = 0 report qb.ValueEvaluationReport ) c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) } if report.Offset != testCase.Offset { t.Fatalf("%s: got offset %d are not equal expected %d", testCase.Name, report.Offset, testCase.Offset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", testCase.Name, report.ChangeSize, testCase.ChangeSize) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%s: got buf % x are not equal expected % x", testCase.Name, buf, testCase.Buf) } if c.baseValue != testCase.BaseValue { t.Fatalf("%s: got baseValue %v are not equal expected %v", testCase.Name, c.baseValue, testCase.BaseValue) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%s: got lastDelta %d are not equal expected %d", testCase.Name, c.lastDelta, testCase.LastDelta) } } } func TestInstantDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []float64 //RepeatLastDeltaNTimes int }{ { // addBaseValue Nums: []float64{ 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.6, 1.6, }, }, { // incrementRun Nums: []float64{ 1.5, 1.5, 1.5, }, }, { // endSeries run Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, }, { // incrementLiteral Nums: []float64{ 1.5, 1.6, }, }, { // ... Nums: []float64{ 1.5, 1.5, 1.5, // 1.5, // 1.6, // 1.7, // 1.8, // 1.9, // 2.0, // 2.0, // 2.0, // 2.2, // }, }, } ) for caseIdx, testCase := range testCases { var ( metricType = qb.Instant fracDigits byte = 1 tmp = make([]byte, tmpValueSize) buf = make([]byte, 16) payloadSize = 0 decodedNums []float64 ) c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) } // c.CaptureState() fmt.Println("---------------") d := c.CreateDecompressor() //fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) fmt.Println(testCase.Nums) fmt.Println(decodedNums) fmt.Println() if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } // TIME DELTA func TestTimeDeltaCompressor(t *testing.T) { var ( testCases = []struct { Name string Nums []uint32 Buf []byte LastUnixtime uint32 LastDelta uint32 Offset int ChangeSize int }{ { Nums: []uint32{ 1780777000, }, Name: "add 1st value", Offset: 0, ChangeSize: 4, Buf: []byte{ 0x00, 0x00, 0x00, 0x00, 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777000, LastDelta: 0, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 }, Name: "add 1st delta", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x00, 0x00, 0x80, // h-byte (literal, len=1) 0xbc, // delta (60) 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777060, LastDelta: 60, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 }, Name: "literal changed to run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x00, // h-byte (run, len=2) 0xbc, // delta 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777120, LastDelta: 60, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777130, // +70 1780777200, // +70 }, Name: "literal decrease by 1 and switch to run", Offset: 2, ChangeSize: 3, Buf: []byte{ 0x00, // h-byte (run, len=2) 0xc6, // delta 70 0x80, // h-byte (literal, len=1) 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777200, LastDelta: 70, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 }, Name: "increment run", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x01, // h-byte (run, len=2) 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777180, LastDelta: 60, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 1780777200, // +20 }, Name: "switch run to literal", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x80, // h-byte (literal, len=1) 0x94, // delta 20 0x01, // h-byte (run, len=3) 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777200, LastDelta: 20, }, { Nums: generateProgression(1780777000, 60, 130), Name: "increment run up to full filled h-byte", Offset: 1, ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x7f, // h-byte (run, len=129) 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777000 + 129*60, LastDelta: 60, }, { Nums: generateProgression(1780777000, 60, 131), Name: "run switch to literal after h-byte overflow", Offset: 0, ChangeSize: 2, Buf: []byte{ 0x80, // h-byte (literal, len=1) 0xbc, // delta 60 0x7f, // h-byte (run, len=129) 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777000 + 130*60, LastDelta: 60, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777130, // +70 }, Name: "increment literal", Offset: 1, ChangeSize: 2, Buf: []byte{ 0x00, 0x81, // h-byte (literal, len=2) 0xc6, // delta 70 0xbc, // delta 60 0x28, 0x80, 0x24, 0x6a, // since }, LastUnixtime: 1780777130, LastDelta: 70, }, } ) for _, testCase := range testCases { //fmt.Println("----------") var ( tmp = make([]byte, tmpTimeSize) buf = make([]byte, 8) payloadSize = 0 report qb.TimeEvaluationReport ) c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) // fmt.Println("----------\npos", c.pos) // fmt.Printf("buf: % x\n", c.buf[c.pos:]) // pretty.Println(report) // fmt.Printf("change: % x\n", tmp[:report.ChangeSize]) c.Append(report.Offset, tmp[:report.ChangeSize], num) } if report.Offset != testCase.Offset { t.Fatalf("%s: got offset %d are not equal expected %d", testCase.Name, report.Offset, testCase.Offset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", testCase.Name, report.ChangeSize, testCase.ChangeSize) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%s: got buf % x are not equal expected % x", testCase.Name, buf, testCase.Buf) } if c.lastUnixtime != testCase.LastUnixtime { t.Fatalf("%s: got lastUnixtime %d are not equal expected %d", testCase.Name, c.lastUnixtime, testCase.LastUnixtime) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%s: got lastDelta %d are not equal expected %d", testCase.Name, c.lastDelta, testCase.LastDelta) } } } func TestTimeDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []uint32 //RepeatLastDeltaNTimes int }{ { Nums: []uint32{ 1780777000, }, }, { // add1stDelta Nums: []uint32{ 1780777000, 1780777060, // +60 }, }, { // startRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 }, }, { // incrementRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 }, }, { // endRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 1780777200, // +20 }, }, { // incrementLiteral Nums: []uint32{ 1780777000, 1780777060, // +60 1780777122, // +62 }, }, { // ... Nums: []uint32{ 1780777000, 1780777010, // +10 1780777120, // +10 1780777130, // +10 1780777135, // +5 1780777141, // +6 1780777148, // +7 1780777156, // +8 1780777165, // +9 1780777174, // +9 1780777183, // +9 1780777190, // +7 }, }, } ) for caseIdx, testCase := range testCases { var ( tmp = make([]byte, tmpTimeSize) buf = make([]byte, 16) payloadSize = 0 decodedNums []uint32 ) c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) c.Append(report.Offset, tmp[:report.ChangeSize], num) } // c.CaptureState() d := c.CreateDecompressor() //fmt.Println(testCase.Nums) // fmt.Println("---------------") // fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) if !slices.Equal(testCase.Nums, decodedNums) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } func TestVarInt64(t *testing.T) { arr := make([]byte, 9) n, err := bin.PutVarInt64(arr, -15) if err != nil { t.Fatal(err) } fmt.Println(arr[:n]) } func repeatFloat64(num float64, n int) []float64 { nums := make([]float64, n) for i := range nums { nums[i] = num } return nums } func generateProgression(start uint32, delta uint32, n int) []uint32 { nums := make([]uint32, n) nums[0] = start for i := 1; i < n; i++ { nums[i] = nums[i-1] + delta } return nums }