package txlog import ( "bytes" bin "gordenko.dev/dima/bin/little" ) const recordSize = 8 const chunkSize = 24 /* Format appended measures: 1b - tx type 4b - metricID Nb - qty of filled data pages (varsize) [ 4b - pageNo 1b - reused 4b - prevPageNo 4b - page crc32 2b - timestamps size Nb - timestamps payload 2b - values size Nb - values payload ] 2b - timestamps offset (for the 1st data page only) 2b - values offset (for the 1st data page only) Nb - timestamps size unfilled (varsize) Nb - timestamps payload unfilled Nb - values size unfilled (varsize) Nb - values payload unfilled Nb - qty of index levels (varsize) [ Nb - qty of level filled pages (varsize) [ 4b - pageNo 1b - reused 4b - page crc32 Nb - records qty (varsize) Nb - records payload ] Nb - records qty unfilled (varsize) Nb - records payload unfilled ] */ func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) { bin.WriteUint16(w, uint16(size)) var ( chunkIdx = offset / chunkSize byteIdx = offset % chunkSize ) for _, chunk := range chunks[chunkIdx:] { available := len(chunk) - byteIdx if available <= size { w.Write(chunk[byteIdx:]) size -= available if size == 0 { return } byteIdx = 0 } else { w.Write(chunk[byteIdx : byteIdx+size]) return } } } ///////////////////////////////////////// type DataPage struct { PageNo uint32 Reused bool PrevPageNo uint32 Checksum uint32 Timestamps []byte Values []byte } type IndexPage struct { PageNo uint32 Reused bool Checksum uint32 Records []byte } type IndexLevel struct { Pages []IndexPage Records []byte // unfilled } // Для декодінга type TxAppendedMeasures struct { MetricID uint32 DataPages []DataPage // offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька // останніх байтів unfilled даних. Потрібно для коректного recovery. // Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled) TimestampsOffset int ValuesOffset int Timestamps []byte // unfilled Values []byte // unfilled IndexLevels []IndexLevel } func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) { s.MetricID, err = bin.ReadUint32(r) if err != nil { return } dataPagesQty, err := bin.ReadVarSize(r) if err != nil { return } for range dataPagesQty { var ( p DataPage timestampsSize, valuesSize int //b byte ) p.PageNo, err = bin.ReadUint32(r) if err != nil { return } p.Reused, err = bin.ReadBool(r) if err != nil { return } p.PrevPageNo, err = bin.ReadUint32(r) if err != nil { return } p.Checksum, err = bin.ReadUint32(r) if err != nil { return } timestampsSize, err = bin.ReadUint16AsInt(r) if err != nil { return } p.Timestamps, err = bin.ReadN(r, timestampsSize) if err != nil { return } valuesSize, err = bin.ReadUint16AsInt(r) if err != nil { return } p.Values, err = bin.ReadN(r, valuesSize) if err != nil { return } s.DataPages = append(s.DataPages, p) } s.TimestampsOffset, err = bin.ReadUint16AsInt(r) if err != nil { return } s.ValuesOffset, err = bin.ReadUint16AsInt(r) if err != nil { return } var ( timestampsSize, valuesSize int ) timestampsSize, err = bin.ReadVarSize(r) if err != nil { return } s.Timestamps, err = bin.ReadN(r, timestampsSize) if err != nil { return } valuesSize, err = bin.ReadVarSize(r) if err != nil { return } s.Values, err = bin.ReadN(r, valuesSize) if err != nil { return } if len(s.DataPages) == 0 { return } indexLevelsQty, err := bin.ReadVarSize(r) if err != nil { return } for range indexLevelsQty { var ( indexPagesQty int level IndexLevel recordsQty int ) indexPagesQty, err = bin.ReadVarSize(r) if err != nil { return } for range indexPagesQty { var ( p IndexPage recordsQty int ) p.PageNo, err = bin.ReadUint32(r) if err != nil { return } p.Reused, err = bin.ReadBool(r) if err != nil { return } p.Checksum, err = bin.ReadUint32(r) if err != nil { return } recordsQty, err = bin.ReadVarSize(r) if err != nil { return } p.Records, err = bin.ReadN(r, recordsQty*recordSize) if err != nil { return } level.Pages = append(level.Pages, p) } recordsQty, err = bin.ReadVarSize(r) if err != nil { return } level.Records, err = bin.ReadN(r, recordsQty*recordSize) if err != nil { return } s.IndexLevels = append(s.IndexLevels, level) } return }