this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

SetupSlog GOLOG_ROTATE_KEEP to keep some old logs, rm older

authored by

Brian Olson and committed by
Brian Olson
0a3197db a5a00fc0

+319 -4
+6 -3
cmd/bigsky/main.go
··· 229 229 env = "dev" 230 230 } 231 231 if cctx.Bool("jaeger") { 232 - url := "http://localhost:14268/api/traces" 233 - exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) 232 + jaegerUrl := "http://localhost:14268/api/traces" 233 + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl))) 234 234 if err != nil { 235 235 return err 236 236 } ··· 294 294 signals := make(chan os.Signal, 1) 295 295 signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 296 296 297 - // TODO: set slog default from param/env 297 + _, err := cliutil.SetupSlog(cliutil.LogOptions{}) 298 + if err != nil { 299 + return err 300 + } 298 301 299 302 // start observability/tracing (OTEL and jaeger) 300 303 if err := setupOTEL(cctx); err != nil {
+6 -1
cmd/gosky/main.go
··· 81 81 }, 82 82 } 83 83 84 - // TODO: slog.SetDefault from param/env 84 + _, err := cliutil.SetupSlog(cliutil.LogOptions{}) 85 + if err != nil { 86 + fmt.Fprintf(os.Stderr, "logging setup error: %s\n", err.Error()) 87 + os.Exit(1) 88 + return 89 + } 85 90 86 91 app.Commands = []*cli.Command{ 87 92 accountCmd,
+307
util/cliutil/util.go
··· 2 2 3 3 import ( 4 4 "encoding/json" 5 + "errors" 5 6 "fmt" 7 + "io" 8 + "io/fs" 9 + "log/slog" 6 10 "net/http" 7 11 "os" 8 12 "path/filepath" 13 + "regexp" 14 + "sort" 15 + "strconv" 9 16 "strings" 10 17 "time" 11 18 ··· 230 237 231 238 return db, nil 232 239 } 240 + 241 + type LogOptions struct { 242 + // e.g. 1_000_000_000 243 + LogRotateBytes int64 244 + 245 + // path to write to, if rotating, %T gets UnixMilli at file open time 246 + // NOTE: substitution is simple replace("%T", "") 247 + LogPath string 248 + 249 + // text|json 250 + LogFormat string 251 + 252 + // info|debug|warn|error 253 + LogLevel string 254 + 255 + // Keep N old logs (not including current); <0 disables removal, 0==remove all old log files immediately 256 + KeepOld int 257 + } 258 + 259 + // SetupSlog integrates passed in options and env vars. 260 + // 261 + // passing default cliutil.LogOptions{} is ok. 262 + // 263 + // GOLOG_LOG_LEVEL=info|debug|warn|error 264 + // 265 + // GOLOG_LOG_FMT=text|json 266 + // 267 + // GOLOG_FILE=path (or "-" or "" for stdout), %T gets UnixMilli; if a path with '/', {prefix}/current becomes a link to active log file 268 + // 269 + // GOLOG_ROTATE_BYTES=int maximum size of log chunk before rotating 270 + // 271 + // GOLOG_ROTATE_KEEP=int keep N olg logs (not including current) 272 + // 273 + // (env vars derived from ipfs logging library) 274 + func SetupSlog(options LogOptions) (*slog.Logger, error) { 275 + var hopts slog.HandlerOptions 276 + hopts.Level = slog.LevelInfo 277 + hopts.AddSource = true 278 + if options.LogLevel == "" { 279 + options.LogLevel = os.Getenv("GOLOG_LOG_LEVEL") 280 + } 281 + if options.LogLevel == "" { 282 + hopts.Level = slog.LevelInfo 283 + options.LogLevel = "info" 284 + } else { 285 + level := strings.ToLower(options.LogLevel) 286 + switch level { 287 + case "debug": 288 + hopts.Level = slog.LevelDebug 289 + case "info": 290 + hopts.Level = slog.LevelInfo 291 + case "warn": 292 + hopts.Level = slog.LevelWarn 293 + case "error": 294 + hopts.Level = slog.LevelError 295 + default: 296 + return nil, fmt.Errorf("unknown log level: %#v", options.LogLevel) 297 + } 298 + } 299 + if options.LogFormat == "" { 300 + options.LogFormat = os.Getenv("GOLOG_LOG_FMT") 301 + } 302 + if options.LogFormat == "" { 303 + options.LogFormat = "text" 304 + } else { 305 + format := strings.ToLower(options.LogFormat) 306 + if format == "json" || format == "text" { 307 + // ok 308 + } else { 309 + return nil, fmt.Errorf("invalid log format: %#v", options.LogFormat) 310 + } 311 + options.LogFormat = format 312 + } 313 + 314 + if options.LogPath == "" { 315 + options.LogPath = os.Getenv("GOLOG_FILE") 316 + } 317 + if options.LogRotateBytes == 0 { 318 + rotateBytesStr := os.Getenv("GOLOG_ROTATE_BYTES") 319 + if rotateBytesStr != "" { 320 + rotateBytes, err := strconv.ParseInt(rotateBytesStr, 10, 64) 321 + if err != nil { 322 + return nil, fmt.Errorf("invalid GOLOG_ROTATE_BYTES value: %w", err) 323 + } 324 + options.LogRotateBytes = rotateBytes 325 + } 326 + } 327 + if options.KeepOld == 0 { 328 + keepOldUnset := true 329 + keepOldStr := os.Getenv("GOLOG_ROTATE_KEEP") 330 + if keepOldStr != "" { 331 + keepOld, err := strconv.ParseInt(keepOldStr, 10, 64) 332 + if err != nil { 333 + return nil, fmt.Errorf("invalid GOLOG_ROTATE_KEEP value: %w", err) 334 + } 335 + keepOldUnset = false 336 + options.KeepOld = int(keepOld) 337 + } 338 + if keepOldUnset { 339 + options.KeepOld = 2 340 + } 341 + } 342 + var out io.Writer 343 + if (options.LogPath == "") || (options.LogPath == "-") { 344 + out = os.Stdout 345 + } else if options.LogRotateBytes != 0 { 346 + out = &logRotateWriter{ 347 + rotateBytes: options.LogRotateBytes, 348 + outPathTemplate: options.LogPath, 349 + keep: options.KeepOld, 350 + } 351 + } else { 352 + var err error 353 + out, err = os.Create(options.LogPath) 354 + if err != nil { 355 + return nil, fmt.Errorf("%s: %w", options.LogPath, err) 356 + } 357 + } 358 + var handler slog.Handler 359 + switch options.LogFormat { 360 + case "text": 361 + handler = slog.NewTextHandler(out, &hopts) 362 + case "json": 363 + handler = slog.NewJSONHandler(out, &hopts) 364 + default: 365 + return nil, fmt.Errorf("unknown log format: %#v", options.LogFormat) 366 + } 367 + logger := slog.New(handler) 368 + slog.SetDefault(logger) 369 + return logger, nil 370 + } 371 + 372 + type logRotateWriter struct { 373 + currentWriter io.WriteCloser 374 + 375 + // how much has been written to current log file 376 + currentBytes int64 377 + 378 + // e.g. path/to/logs/foo%T 379 + currentPath string 380 + 381 + // e.g. path/to/logs/current 382 + currentPathCurrent string 383 + 384 + rotateBytes int64 385 + 386 + outPathTemplate string 387 + 388 + // keep the most recent N log files (not including current) 389 + keep int 390 + } 391 + 392 + var currentMatcher = regexp.MustCompile("current_\\d+") 393 + 394 + func (w *logRotateWriter) cleanOldLogs() { 395 + if w.keep < 0 { 396 + // old log removal is disabled 397 + return 398 + } 399 + // w.currentPath was recently set as the new log 400 + dirpart, _ := filepath.Split(w.currentPath) 401 + // find old logs 402 + templateDirPart, templateNamePart := filepath.Split(w.outPathTemplate) 403 + if dirpart != templateDirPart { 404 + fmt.Fprintf(os.Stderr, "current dir part %#v != template dir part %#v\n", w.currentPath, w.outPathTemplate) 405 + return 406 + } 407 + // build a regexp that is string literal parts with \d+ replacing the UnixMilli part 408 + templateNameParts := strings.Split(templateNamePart, "%T") 409 + var sb strings.Builder 410 + first := true 411 + for _, part := range templateNameParts { 412 + if first { 413 + first = false 414 + } else { 415 + sb.WriteString("\\d+") 416 + } 417 + sb.WriteString(regexp.QuoteMeta(part)) 418 + } 419 + tmre, err := regexp.Compile(sb.String()) 420 + if err != nil { 421 + fmt.Fprintf(os.Stderr, "failed to compile old log template regexp: %#v\n", err) 422 + return 423 + } 424 + dir, err := os.ReadDir(dirpart) 425 + if err != nil { 426 + fmt.Fprintf(os.Stderr, "failed to read old log template dir: %#v\n", err) 427 + return 428 + } 429 + var found []fs.FileInfo 430 + for _, ent := range dir { 431 + name := ent.Name() 432 + if tmre.MatchString(name) || currentMatcher.MatchString(name) { 433 + fi, err := ent.Info() 434 + if err != nil { 435 + continue 436 + } 437 + found = append(found, fi) 438 + } 439 + } 440 + if len(found) <= w.keep { 441 + // not too many, nothing to do 442 + return 443 + } 444 + foundMtimeLess := func(i, j int) bool { 445 + return found[i].ModTime().Before(found[j].ModTime()) 446 + } 447 + sort.Slice(found, foundMtimeLess) 448 + drops := found[:len(found)-w.keep] 449 + for _, fi := range drops { 450 + fullpath := filepath.Join(dirpart, fi.Name()) 451 + err = os.Remove(fullpath) 452 + if err != nil { 453 + fmt.Fprintf(os.Stderr, "failed to rm old log: %#v\n", err) 454 + // but keep going 455 + } 456 + // maybe it would be safe to debug-log old log removal from within the logging infrastructure? 457 + } 458 + } 459 + 460 + func (w *logRotateWriter) closeOldLog() []error { 461 + if w.currentWriter == nil { 462 + return nil 463 + } 464 + var earlyWeakErrors []error 465 + err := w.currentWriter.Close() 466 + if err != nil { 467 + earlyWeakErrors = append(earlyWeakErrors, err) 468 + } 469 + w.currentWriter = nil 470 + w.currentBytes = 0 471 + w.currentPath = "" 472 + if w.currentPathCurrent != "" { 473 + err = os.Remove(w.currentPathCurrent) // not really an error until something else goes wrong 474 + if err != nil { 475 + earlyWeakErrors = append(earlyWeakErrors, err) 476 + } 477 + w.currentPathCurrent = "" 478 + } 479 + return earlyWeakErrors 480 + } 481 + 482 + func (w *logRotateWriter) openNewLog(earlyWeakErrors []error) (badErr error, weakErrors []error) { 483 + nowMillis := time.Now().UnixMilli() 484 + nows := strconv.FormatInt(nowMillis, 10) 485 + w.currentPath = strings.Replace(w.outPathTemplate, "%T", nows, -1) 486 + var err error 487 + w.currentWriter, err = os.Create(w.currentPath) 488 + if err != nil { 489 + earlyWeakErrors = append(earlyWeakErrors, err) 490 + return errors.Join(earlyWeakErrors...), nil 491 + } 492 + w.cleanOldLogs() 493 + dirpart, _ := filepath.Split(w.currentPath) 494 + if dirpart != "" { 495 + w.currentPathCurrent = filepath.Join(dirpart, "current") 496 + fi, err := os.Stat(w.currentPathCurrent) 497 + if err == nil && fi.Mode().IsRegular() { 498 + // move aside unknown "current" from a previous run 499 + // see also currentMatcher regexp current_\d+ 500 + err = os.Rename(w.currentPathCurrent, w.currentPathCurrent+"_"+nows) 501 + if err != nil { 502 + // not crucial if we can't move aside "current" 503 + // TODO: log warning ... but not from inside log writer? 504 + earlyWeakErrors = append(earlyWeakErrors, err) 505 + } 506 + } 507 + err = os.Link(w.currentPath, w.currentPathCurrent) 508 + if err != nil { 509 + // not crucial if we can't make "current" link 510 + // TODO: log warning ... but not from inside log writer? 511 + earlyWeakErrors = append(earlyWeakErrors, err) 512 + } 513 + } 514 + return nil, earlyWeakErrors 515 + } 516 + 517 + func (w *logRotateWriter) Write(p []byte) (n int, err error) { 518 + var earlyWeakErrors []error 519 + if int64(len(p))+w.currentBytes > w.rotateBytes { 520 + // next write would be over the limit 521 + earlyWeakErrors = w.closeOldLog() 522 + } 523 + if w.currentWriter == nil { 524 + // start new log file 525 + var err error 526 + err, earlyWeakErrors = w.openNewLog(earlyWeakErrors) 527 + if err != nil { 528 + return 0, err 529 + } 530 + } 531 + var wrote int 532 + wrote, err = w.currentWriter.Write(p) 533 + w.currentBytes += int64(wrote) 534 + if err != nil { 535 + earlyWeakErrors = append(earlyWeakErrors, err) 536 + return wrote, errors.Join(earlyWeakErrors...) 537 + } 538 + return wrote, nil 539 + }