dima
/
diploma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tmp

Browse Source
main
Dima Gordenko 2 months ago
parent 5e49c66e15
commit 37893fc805
11 changed files with 59 additions and 110 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 72
      atree/aggregate.go
  2. 4
      atree/select.go
  3. 2
      client/client.go
  4. 83
      database/api.go
  5. BIN
      database_linux
  6. BIN
      loadtest_linux
  7. 8
      proto/proto.go
  8. BIN
      requests_linux
  9. BIN
      testdir/1.changes
  10. BIN
      testdir/test.data
  11. BIN
      testdir/test.index

72
atree/aggregate.go
Unescape View File

@ -12,7 +12,6 @@ import (
type InstantAggregator struct {
firstHourOfDay int
lastDayOfMonth int
time2period func(uint32) uint32
currentPeriod uint32
since uint32
@ -26,13 +25,11 @@ type InstantAggregator struct {
type InstantAggregatorOptions struct {
GroupBy diploma.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func NewInstantAggregator(opt InstantAggregatorOptions) (*InstantAggregator, error) {
s := &InstantAggregator{
firstHourOfDay: opt.FirstHourOfDay,
lastDayOfMonth: opt.LastDayOfMonth,
}
switch opt.GroupBy {
@ -48,17 +45,9 @@ func NewInstantAggregator(opt InstantAggregatorOptions) (*InstantAggregator, err
case diploma.GroupByMonth:
if s.firstHourOfDay > 0 {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingFHDAndLDM
} else {
s.time2period = s.groupByMonthUsingFHD
}
s.time2period = s.groupByMonthUsingFHD
} else {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingLDM
} else {
s.time2period = groupByMonth
}
s.time2period = groupByMonth
}
default:
@ -145,32 +134,10 @@ func (s *InstantAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 {
return uint32(tm.Unix())
}
func (s *InstantAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func (s *InstantAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 {
// ВАЖНО!
// Сперва проверяю время.
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
// CUMULATIVE
type CumulativeAggregator struct {
firstHourOfDay int
lastDayOfMonth int
time2period func(uint32) uint32
currentPeriod uint32
since uint32
@ -183,13 +150,11 @@ type CumulativeAggregator struct {
type CumulativeAggregatorOptions struct {
GroupBy diploma.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func NewCumulativeAggregator(opt CumulativeAggregatorOptions) (*CumulativeAggregator, error) {
s := &CumulativeAggregator{
firstHourOfDay: opt.FirstHourOfDay,
lastDayOfMonth: opt.LastDayOfMonth,
}
switch opt.GroupBy {
@ -205,17 +170,9 @@ func NewCumulativeAggregator(opt CumulativeAggregatorOptions) (*CumulativeAggreg
case diploma.GroupByMonth:
if s.firstHourOfDay > 0 {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingFHDAndLDM
} else {
s.time2period = s.groupByMonthUsingFHD
}
s.time2period = s.groupByMonthUsingFHD
} else {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingLDM
} else {
s.time2period = groupByMonth
}
s.time2period = groupByMonth
}
default:
@ -291,27 +248,6 @@ func (s *CumulativeAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 {
return uint32(tm.Unix())
}
func (s *CumulativeAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func (s *CumulativeAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 {
// ВАЖНО!
// Сперва проверяю время.
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func groupByHour(timestamp uint32) uint32 {
return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "h").Unix())
}

4
atree/select.go
Unescape View File

@ -430,7 +430,6 @@ type FindInstantPeriodsReq struct {
Until uint32
GroupBy octopus.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func (s *Atree) FindInstantPeriods(req FindInstantPeriodsReq) error {
@ -442,7 +441,6 @@ func (s *Atree) FindInstantPeriods(req FindInstantPeriodsReq) error {
aggregator, err := NewInstantAggregator(InstantAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
return err
@ -499,7 +497,6 @@ type FindCumulativePeriodsReq struct {
Until uint32
GroupBy octopus.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func (s *Atree) FindCumulativePeriods(req FindCumulativePeriodsReq) error {
@ -511,7 +508,6 @@ func (s *Atree) FindCumulativePeriods(req FindCumulativePeriodsReq) error {
aggregator, err := NewCumulativeAggregator(CumulativeAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
return err

2
client/client.go
Unescape View File

@ -447,7 +447,6 @@ func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]Inst
byte(req.GroupBy),
req.AggregateFuncs,
byte(req.FirstHourOfDay),
byte(req.LastDayOfMonth),
}
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since)
@ -546,7 +545,6 @@ func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (
0, 0, 0, 0, // until
byte(req.GroupBy),
byte(req.FirstHourOfDay),
byte(req.LastDayOfMonth),
}
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since)

83
database/api.go
Unescape View File

@ -724,12 +724,12 @@ func (s *Database) ListInstantMeasures(conn net.Conn, req proto.ListInstantMeasu
return nil
}
var since, until uint32
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(req.Since, req.Until, req.FirstHourOfDay)
} else {
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(req.Since, req.Until, req.FirstHourOfDay)
}
resultCh := make(chan instantMeasuresResult, 1)
@ -800,13 +800,26 @@ type cumulativeMeasuresResult struct {
}
func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativeMeasuresReq) error {
if req.Since > req.Until {
reply(conn, proto.ErrInvalidRange)
return nil
}
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
resultCh := make(chan cumulativeMeasuresResult, 1)
responseWriter := atree.NewCumulativeMeasureWriter(conn)
s.appendJobToWorkerQueue(tryListCumulativeMeasuresReq{
MetricID: req.MetricID,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
ResponseWriter: responseWriter,
ResultCh: resultCh,
})
@ -820,8 +833,8 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
case UntilFound:
err := s.atree.ContinueIterateCumulativeByTreeCursor(atree.ContinueIterateCumulativeByTreeCursorReq{
FracDigits: result.FracDigits,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
LastPageNo: result.PageNo,
EndTimestamp: result.EndTimestamp,
EndValue: result.EndValue,
@ -837,8 +850,8 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
case UntilNotFound:
err := s.atree.FindAndIterateCumulativeByTreeCursor(atree.FindAndIterateCumulativeByTreeCursorReq{
FracDigits: result.FracDigits,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
RootPageNo: result.PageNo,
ResponseWriter: responseWriter,
})
@ -868,6 +881,11 @@ type instantPeriodsResult struct {
}
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
if req.Since > req.Until {
reply(conn, proto.ErrInvalidRange)
return nil
}
var (
since = req.Since
until = req.Until
@ -876,16 +894,11 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
if req.LastDayOfMonth > 0 {
// fix
}
resultCh := make(chan instantPeriodsResult, 1)
aggregator, err := atree.NewInstantAggregator(atree.InstantAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
reply(conn, proto.ErrUnexpected)
@ -896,8 +909,8 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
s.appendJobToWorkerQueue(tryListInstantPeriodsReq{
MetricID: req.MetricID,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
Aggregator: aggregator,
ResponseWriter: responseWriter,
ResultCh: resultCh,
@ -915,8 +928,8 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
Aggregator: aggregator,
ResponseWriter: responseWriter,
LastPageNo: result.PageNo,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
})
s.metricRUnlock(req.MetricID)
@ -931,11 +944,10 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
FracDigits: result.FracDigits,
ResponseWriter: responseWriter,
RootPageNo: result.PageNo,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
s.metricRUnlock(req.MetricID)
@ -964,12 +976,24 @@ type cumulativePeriodsResult struct {
}
func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
if req.Since > req.Until {
reply(conn, proto.ErrInvalidRange)
return nil
}
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
resultCh := make(chan cumulativePeriodsResult, 1)
aggregator, err := atree.NewCumulativeAggregator(atree.CumulativeAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
reply(conn, proto.ErrUnexpected)
@ -980,8 +1004,8 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
s.appendJobToWorkerQueue(tryListCumulativePeriodsReq{
MetricID: req.MetricID,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
Aggregator: aggregator,
ResponseWriter: responseWriter,
ResultCh: resultCh,
@ -999,8 +1023,8 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
Aggregator: aggregator,
ResponseWriter: responseWriter,
LastPageNo: result.PageNo,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
})
s.metricRUnlock(req.MetricID)
@ -1015,11 +1039,10 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
FracDigits: result.FracDigits,
ResponseWriter: responseWriter,
RootPageNo: result.PageNo,
Since: req.Since,
Until: req.Until,
Since: since,
Until: until,
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
s.metricRUnlock(req.MetricID)

BIN
database_linux
View File

Binary file not shown.

BIN
loadtest_linux
View File

Binary file not shown.

8
proto/proto.go
Unescape View File

@ -130,7 +130,6 @@ type ListInstantPeriodsReq struct {
GroupBy octopus.GroupBy
AggregateFuncs byte
FirstHourOfDay int
LastDayOfMonth int
}
type ListCumulativePeriodsReq struct {
@ -139,7 +138,6 @@ type ListCumulativePeriodsReq struct {
Until uint32
GroupBy octopus.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
type Metric struct {
@ -255,7 +253,6 @@ func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) {
m.GroupBy = octopus.GroupBy(arr[12])
m.AggregateFuncs = arr[13]
m.FirstHourOfDay = int(arr[14])
m.LastDayOfMonth = int(arr[15])
return
}
@ -265,7 +262,6 @@ func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) {
m.Until = bin.GetUint32(arr[8:])
m.GroupBy = octopus.GroupBy(arr[12])
m.FirstHourOfDay = int(arr[13])
m.LastDayOfMonth = int(arr[14])
return
}
@ -369,7 +365,7 @@ func ReadListCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListCumulativ
}
func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriodsReq, err error) {
arr, err := r.ReadN(16)
arr, err := r.ReadN(15)
if err != nil {
err = fmt.Errorf("read req: %s", err)
return
@ -378,7 +374,7 @@ func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriod
}
func ReadListCumulativePeriodsReq(r *bufreader.BufferedReader) (m ListCumulativePeriodsReq, err error) {
arr, err := r.ReadN(15)
arr, err := r.ReadN(14)
if err != nil {
err = fmt.Errorf("read req: %s", err)
return

BIN
requests_linux
View File

Binary file not shown.

BIN
testdir/1.changes
View File

Binary file not shown.

BIN
testdir/test.data
View File

Binary file not shown.

BIN
testdir/test.index
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save