package storage import ( "bytes" "io" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" ) type HeadDataPage struct { PageNo uint32 Reused bool PrevPageNo uint32 Checksum uint32 TimestampsOffset int // offset on 1st page Timestamps []byte ValuesOffset int // offset on 1st page Values []byte } type HeadIndexPage struct { PageNo uint32 Reused bool Checksum uint32 Records []byte // offset не потрібен, оскільки Records додаються в кінець } type ChangedIndexLevel struct { HeadIndexPage *HeadIndexPage IndexPages []SealedIndexPage TailRecords []byte // payload only } type MeasuresAppendWithGrowRecord struct { MetricID uint32 HeadDataPage HeadDataPage DataPages []SealedDataPage ChangedIndexLevels []ChangedIndexLevel TailTimestamps []byte // data level tail TailValues []byte // data level tail } /* Format appended measures: 1b - tx type 4b - metricID 4b - pageNo 1b - reused 4b - prevPageNo 4b - page crc32 Nb - (varsize) timestamps offset Nb - (varsize) timestamps tail size on 1st filled data page Nb - timestamps tail on 1st filled data page Nb - (varsize) values offset Nb - (varsize) values tail size on 1st filled data page Nb - values tail on 1st filled data page Nb - (varsize) filled data pages qty [ 4b - pageNo 1b - reused Nb - (data page size) page content ] Nb - (varsize) timestamps payload size on tail Nb - timestamps payload Nb - (varsize) values payload size on tail Nb - values payload Nb - (varsize) qty of index levels [ // NOTE! // - idx = 0 - zeroLevel // - skipped = maxRecordsOnIndexPage - records count 4b - pageNo of 1st index page 1b - reused 4b - page crc32 Nb - (varsize) records tail count on 1st filled index page Nb - records tail on 1st filled index page Nb - (varsize) qty of level filled pages [ 4b - pageNo 1b - reused Nb - (index page size) page content ] Nb - (varsize) size of records level tail Nb - level tail records ] */ func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) { w.WriteByte(CodeMeasuresAppendWithGrow) bin.WriteUint32(w, s.MetricID) // completed data page completed := s.HeadDataPage bin.WriteUint32(w, completed.PageNo) bin.WriteBool(w, completed.Reused) bin.WriteUint32(w, completed.PrevPageNo) bin.WriteUint32(w, completed.Checksum) bin.WriteVarSize(w, completed.TimestampsOffset) bin.WriteVarSize(w, len(completed.Timestamps)) w.Write(completed.Timestamps) bin.WriteVarSize(w, completed.ValuesOffset) bin.WriteVarSize(w, len(completed.Values)) w.Write(completed.Values) // data pages bin.WriteVarSize(w, len(s.DataPages)) for _, p := range s.DataPages { bin.WriteUint32(w, p.PageNo) bin.WriteBool(w, p.Reused) bin.WriteVarSize(w, len(p.Content)) w.Write(p.Content) } // data tail bin.WriteVarSize(w, len(s.TailTimestamps)) w.Write(s.TailTimestamps) bin.WriteVarSize(w, len(s.TailValues)) w.Write(s.TailValues) // changed levels bin.WriteVarSize(w, len(s.ChangedIndexLevels)) for _, level := range s.ChangedIndexLevels { // completed index page completed := level.HeadIndexPage bin.WriteUint32(w, completed.PageNo) bin.WriteBool(w, completed.Reused) bin.WriteUint32(w, completed.Checksum) bin.WriteVarSize(w, len(completed.Records)/indexRecordSize) w.Write(completed.Records) // full index pages for _, p := range level.IndexPages { bin.WriteUint32(w, p.PageNo) bin.WriteBool(w, p.Reused) bin.WriteVarSize(w, len(p.Content)) w.Write(p.Content) } // level tail bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize) w.Write(level.TailRecords) } } func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) { var ( size int recordsCount int ) s.MetricID, err = bin.ReadUint32(r) if err != nil { return } s.HeadDataPage.PageNo, err = bin.ReadUint32(r) if err != nil { return } s.HeadDataPage.Reused, err = bin.ReadBool(r) if err != nil { return } s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r) if err != nil { return } s.HeadDataPage.Checksum, err = bin.ReadUint32(r) if err != nil { return } s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } s.HeadDataPage.Timestamps, err = bin.ReadN(r, size) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } s.HeadDataPage.Values, err = bin.ReadN(r, size) if err != nil { return } // data pages dataPagesQty, err := bin.ReadVarSize(r) if err != nil { return } for range dataPagesQty { var ( p SealedDataPage ) p.PageNo, err = bin.ReadUint32(r) if err != nil { return } p.Reused, err = bin.ReadBool(r) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } p.Content, err = bin.ReadN(r, size) if err != nil { return } s.DataPages = append(s.DataPages, p) } // data tail size, err = bin.ReadVarSize(r) if err != nil { return } s.TailTimestamps, err = bin.ReadN(r, size) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } s.TailValues, err = bin.ReadN(r, size) if err != nil { return } // changed levels changedIndexLevelsCount, err := bin.ReadVarSize(r) if err != nil { return } for range changedIndexLevelsCount { var ( indexPagesCount int level ChangedIndexLevel ) // completed index page level.HeadIndexPage.PageNo, err = bin.ReadUint32(r) if err != nil { return } level.HeadIndexPage.Reused, err = bin.ReadBool(r) if err != nil { return } level.HeadIndexPage.Checksum, err = bin.ReadUint32(r) if err != nil { return } recordsCount, err = bin.ReadVarSize(r) if err != nil { return } level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize) if err != nil { return } // indexPagesCount, err = bin.ReadVarSize(r) if err != nil { return } for range indexPagesCount { var p SealedIndexPage p.PageNo, err = bin.ReadUint32(r) if err != nil { return } p.Reused, err = bin.ReadBool(r) if err != nil { return } p.Checksum, err = bin.ReadUint32(r) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } p.Content, err = bin.ReadN(r, size) if err != nil { return } level.IndexPages = append(level.IndexPages, p) } recordsCount, err = bin.ReadVarSize(r) if err != nil { return } level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize) if err != nil { return } s.ChangedIndexLevels = append(s.ChangedIndexLevels, level) } return } type MeasuresAppendRecord struct { MetricID uint32 TimestampsOffset int Timestamps []byte ValuesOffset int Values []byte } func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) { w.WriteByte(CodeMeasuresAppendWithGrow) bin.WriteUint32(w, s.MetricID) bin.WriteVarSize(w, s.TimestampsOffset) bin.WriteVarSize(w, len(s.Timestamps)) w.Write(s.Timestamps) bin.WriteVarSize(w, s.ValuesOffset) bin.WriteVarSize(w, len(s.Values)) w.Write(s.Values) } func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) { var size int s.MetricID, err = bin.ReadUint32(r) if err != nil { return } s.TimestampsOffset, err = bin.ReadVarSize(r) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } s.Timestamps, err = bin.ReadN(r, size) if err != nil { return } s.ValuesOffset, err = bin.ReadVarSize(r) if err != nil { return } size, err = bin.ReadVarSize(r) if err != nil { return } s.Values, err = bin.ReadN(r, size) if err != nil { return } return } type MetricAddRecord struct { MetricID uint32 MetricType qb.MetricType FracDigits int } func (s MetricAddRecord) Pack(w io.Writer) { arr := []byte{ CodeMetricAdd, 0, 0, 0, 0, // byte(s.MetricType), byte(s.FracDigits), } bin.PutUint32(arr[1:], s.MetricID) w.Write(arr) } func (s *MetricAddRecord) Parse(src io.Reader) (err error) { arr, err := bin.ReadN(src, 6) if err != nil { return } s.MetricID, _ = bin.GetUint32(arr) s.MetricType = qb.MetricType(arr[4]) s.FracDigits = int(arr[5]) return nil } type MetricDeleteRecord struct { MetricID uint32 FreeIndexPages []uint32 FreeDataPages []uint32 } func (s MetricDeleteRecord) Pack(w io.Writer) { arr := []byte{ CodeMetricDelete, 0, 0, 0, 0, // metricID } bin.PutUint32(arr[1:], s.MetricID) w.Write(arr) // fix // bin.WriteVarSize(w, len(s.FreePageNumbers)) // for _, pageNo := range s.FreePageNumbers { // bin.WriteUint32(w, pageNo) // } } func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) { s.MetricID, err = bin.ReadUint32(src) if err != nil { return } // free pages // fix // qty, err := bin.ReadVarSize(src) // if err != nil { // return // } // for range qty { // var pageNo uint32 // pageNo, err = bin.ReadUint32(src) // if err != nil { // return // } // //s.FreePageNumbers = append(s.FreePageNumbers, pageNo) // } return nil } // fix - add changed pages type DeletedMeasures struct { MetricID uint32 FreePageNumbers []uint32 } func (s DeletedMeasures) Pack(w io.Writer) { arr := []byte{ CodeMeasuresDelete, 0, 0, 0, 0, } bin.PutUint32(arr[1:], s.MetricID) w.Write(arr) bin.WriteVarSize(w, len(s.FreePageNumbers)) for _, pageNo := range s.FreePageNumbers { bin.WriteUint32(w, pageNo) } } func (s *DeletedMeasures) Parse(src io.Reader) (err error) { s.MetricID, err = bin.ReadUint32(src) if err != nil { return } // free pages qty, err := bin.ReadVarSize(src) if err != nil { return } for range qty { var pageNo uint32 pageNo, err = bin.ReadUint32(src) if err != nil { return } s.FreePageNumbers = append(s.FreePageNumbers, pageNo) } return nil } // type AppendedMeasure struct { // MetricID uint32 // Timestamp uint32 // Value float64 // } // func (s AppendedMeasure) Pack(w io.Writer) { // arr := []byte{ // CodeAppendedMeasure, // 0, 0, 0, 0, // metricID // 0, 0, 0, 0, // timestamp // 0, 0, 0, 0, 0, 0, 0, 0, // value // } // bin.PutUint32(arr[1:], s.MetricID) // bin.PutUint32(arr[5:], s.Timestamp) // bin.PutFloat64(arr[9:], s.Value) // w.Write(arr) // } // func (s *AppendedMeasure) Parse(src io.Reader) (err error) { // arr, err := bin.ReadN(src, 16) // if err != nil { // return // } // s.MetricID, _ = bin.GetUint32(arr[0:]) // s.Timestamp, _ = bin.GetUint32(arr[4:]) // s.Value, _ = bin.GetFloat64(arr[8:]) // return nil // } // type AppendedMeasures struct { // MetricID uint32 // Measures []proto.Measure // } // func (s AppendedMeasures) Pack(w io.Writer) { // arr := []byte{ // CodeAppendedMeasures, // 0, 0, 0, 0, // metricID // 0, 0, // qty // } // bin.PutUint32(arr[1:], s.MetricID) // bin.PutUint16(arr[5:], uint16(len(s.Measures))) // w.Write(arr) // for _, measure := range s.Measures { // bin.WriteUint32(w, measure.Timestamp) // bin.WriteFloat64(w, measure.Value) // } // } // func (s *AppendedMeasures) Parse(src io.Reader) (err error) { // s.MetricID, err = bin.ReadUint32(src) // if err != nil { // return // } // qty, err := bin.ReadUint16(src) // if err != nil { // return // } // for range qty { // var measure proto.Measure // measure.Timestamp, err = bin.ReadUint32(src) // if err != nil { // return // } // measure.Value, err = bin.ReadFloat64(src) // if err != nil { // return // } // s.Measures = append(s.Measures, measure) // } // return nil // } // type AppendedPagesReq struct { // Legs []atree.PathLeg // MetricID uint32 // Timestamp uint32 // last measure // Value float64 // last measure // //RootPageNo uint32 // LastPageNo uint32 // Pages []atree.NotLinkedDataPage // TimestampsChunks [][]byte // TimestampsSize uint16 // ValuesChunks [][]byte // ValuesSize uint16 // } /* 4b metricID 4b timestamp 8b value 4b LastPageNo (0 if no new root page) 4b newRootPageNo (0 if no new root page) Nb - varsize (pages qty) [ 4b pageNo 1b isReused Xb page (page size) ] 2b - timestamps size (not filled page) Nb - timestamps payload 2b - values size (not filled page) Nb - values payload */ // type AppendedPages struct { // MetricID uint32 // Timestamp uint32 // last measure // Value float64 // last measure // NewRootPageNo uint32 // LastPageNo uint32 // Pages []atree.PageToWrite // TimestampsChunks [][]byte // TimestampsSize int // ValuesChunks [][]byte // ValuesSize int // } // func (s *AppendedPages) Pack(w io.Writer) { // w.Write([]byte{ // CodeAppendedPages, // }) // bin.WriteUint32(w, s.MetricID) // bin.WriteUint32(w, s.Timestamp) // bin.WriteFloat64(w, s.Value) // bin.WriteUint32(w, s.LastPageNo) // bin.WriteUint32(w, s.NewRootPageNo) // // // bin.WriteVarSize(w, len(s.Pages)) // for _, p := range s.Pages { // bin.WriteUint32(w, p.PageNo) // if p.IsReused { // w.Write([]byte{1}) // } else { // w.Write([]byte{0}) // } // w.Write(p.Data) // } // // timestamps // writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w) // // values // writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w) // } // func (s *AppendedPages) Parse(src io.Reader) (err error) { // s.MetricID, err = bin.ReadUint32(src) // if err != nil { // return // } // s.Timestamp, err = bin.ReadUint32(src) // if err != nil { // return // } // s.Value, err = bin.ReadFloat64(src) // if err != nil { // return // } // s.LastPageNo, err = bin.ReadUint32(src) // if err != nil { // return // } // s.NewRootPageNo, err = bin.ReadUint32(src) // if err != nil { // return // } // pagesQty, err := bin.ReadVarSize(src) // if err != nil { // return // } // for range pagesQty { // var page atree.PageToWrite // page.PageNo, err = bin.ReadUint32(src) // if err != nil { // return // } // page.IsReused, err = bin.ReadBool(src) // if err != nil { // return // } // page.Data, err = bin.ReadN(src, atree.PageSize) // if err != nil { // return // } // s.Pages = append(s.Pages, page) // } // s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src) // if err != nil { // return // } // s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src) // if err != nil { // return // } // return nil // }