At work, we recently took down our site for a 6 hour maintenance window, during which, we wanted to show our users a pretty maintenance page. But as a SaaS company, we also have an API, and legacy stuff means it's a bit intermixed with the human facing routes. So after staring at Akamai documentation for a bit too long, I decided a hello world Go http server would be the easiest path to serving the customized maintenance pages.
So I deployed the thing into our cluster with 3 replicas,
ran a quick and dirty load test against it with
github.com/rakyll/hey,
and just left it running.
Come the day of our maintenance window, we switched traffic over. The traffic it served out with approx 1000 req/sec, and gradually crept up to around 1700 req/sec at the end of our window, api traffic remained steady, it was the ui traffic that almost doubled.
What was more surprising was just how little resources it needed. Unconstrained by resource limits, the 3 replicas each ran with just 100 millicores cpu and 800 mebibyte of memory. From our metrics, we could also see p99 latency for serving the html page was 200ms, compared to mere microseconds for the api responses. I think it may have to do with the page being 10kb.
I may have to reevaluate how resource hungry http servers need to be.
Not very interesting, but the code is included below:
1package main
2
3import (
4 "context"
5 "embed"
6 "io"
7 "log/slog"
8 "net/http"
9 "strings"
10 "time"
11
12 "github.com/work/otelsdk"
13 "go.opentelemetry.io/otel"
14 "go.opentelemetry.io/otel/attribute"
15 "go.opentelemetry.io/otel/metric"
16)
17
18var (
19 //go:embed assets
20 assets embed.FS
21
22 //go:embed maintenance.html
23 maintenanceHTML string
24
25 maintenanceAPI = `{
26 "msg": "... some message ..."
27}`
28
29 maintenanceCLI = `{
30 "alerts": [{
31 "msg": "... some message ...",
32 "type": "info"
33 }]
34}`
35)
36
37func main() {
38 exp, err := otelsdk.Init(context.Background(), otelsdk.Config{})
39 if err != nil {
40 panic(err)
41 }
42 defer exp.Shutdown(context.Background())
43
44 meter := otel.GetMeterProvider().Meter("maintenance-page")
45 pageHist, _ := meter.Float64Histogram("maintenance.page.latency", metric.WithUnit("s"))
46 mahtml := []metric.RecordOption{metric.WithAttributeSet(attribute.NewSet(attribute.String("page", "html")))}
47 maapi := []metric.RecordOption{metric.WithAttributeSet(attribute.NewSet(attribute.String("page", "api")))}
48 macli := []metric.RecordOption{metric.WithAttributeSet(attribute.NewSet(attribute.String("page", "cli")))}
49
50 http.Handle("/assets/", http.FileServer(http.FS(assets)))
51 http.HandleFunc("/-/healthcheck", func(rw http.ResponseWriter, r *http.Request) {
52 io.WriteString(rw, "ok")
53 })
54 http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
55 if strings.HasPrefix(r.UserAgent(), "GoogleHC") {
56 io.WriteString(rw, "ok")
57 return
58 }
59
60 t0 := time.Now()
61 response := maintenanceHTML
62 mattrs := mahtml
63
64 // ensure akamai doesn't cache our maintenance page
65 rw.Header().Set("cache-control", "no-cache")
66 rw.Header().Set("retry-after", "3600") // seconds, maintenance takes a long time
67 // fun
68 rw.Header().Set("server", "work")
69 rw.Header().Set("x-recruiting", "come join us @ careers page")
70
71 if r.Header.Get("x-work-cli-version") != "" {
72 rw.Header().Set("content-type", "application/json")
73 response = maintenanceCLI
74 mattrs = macli
75 } else if strings.HasPrefix(r.URL.Path, "/api") || strings.HasPrefix(r.URL.Path, "/rest") || r.Header.Get("accept") == "application/json" {
76 rw.Header().Set("content-type", "application/json")
77 response = maintenanceAPI
78 mattrs = maapi
79 }
80
81 rw.WriteHeader(http.StatusServiceUnavailable)
82 io.WriteString(rw, response)
83
84 pageHist.Record(r.Context(), float64(time.Since(t0).Seconds()), mattrs...)
85 })
86
87 slog.Info("starting on :8080")
88 http.ListenAndServe(":8080", nil)
89}