From 141c18cf29da73ebae33769d5a3f26fee7e45267 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=D0=B0ramtamt=C4=81m?=
<7326800+tarampampam@users.noreply.github.com>
Date: Fri, 5 Jul 2024 07:59:07 -0700
Subject: [PATCH] feat: Add HTML/CSS/JS minification on the fly (#293)
---
Dockerfile | 5 +-
README.md | 38 ++++----
go.mod | 2 +
go.sum | 7 ++
internal/cli/app.go | 3 +-
.../cli/{update_readme.go => app_generate.go} | 7 +-
internal/cli/build/command.go | 25 +++--
internal/cli/serve/command.go | 17 ++--
internal/cli/shared/flags.go | 8 ++
internal/cli/shared/flags_test.go | 9 ++
internal/config/config.go | 3 +
internal/config/config_test.go | 1 +
internal/http/handlers/error_page/handler.go | 8 ++
.../http/handlers/error_page/handler_test.go | 2 +-
internal/template/minify.go | 23 +++++
internal/template/minify_test.go | 94 +++++++++++++++++++
16 files changed, 213 insertions(+), 39 deletions(-)
rename internal/cli/{update_readme.go => app_generate.go} (72%)
create mode 100644 internal/template/minify.go
create mode 100644 internal/template/minify_test.go
diff --git a/Dockerfile b/Dockerfile
index 4be9c34..fbabb34 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,7 +32,10 @@ FROM develop AS compile
# can be passed with any prefix (like `v1.2.3@GITHASH`), e.g.: `docker build --build-arg "APP_VERSION=v1.2.3" .`
ARG APP_VERSION="undefined@docker"
-RUN --mount=type=bind,source=.,target=/src set -x \
+# copy the source code
+COPY . /src
+
+RUN set -x \
&& go generate ./... \
&& CGO_ENABLED=0 LDFLAGS="-s -w -X gh.tarampamp.am/error-pages/internal/appmeta.version=${APP_VERSION}" \
go build -trimpath -ldflags "${LDFLAGS}" -o /tmp/error-pages ./cmd/error-pages/ \
diff --git a/README.md b/README.md
index e532090..a8b2a78 100644
--- a/README.md
+++ b/README.md
@@ -610,11 +610,11 @@ Test completed successfully. Here is the output:
Running 15s test @ http://127.0.0.1:8080/
12 threads and 400 connections
Thread Stats Avg Stdev Max +/- Stdev
- Latency 3.54ms 4.90ms 74.57ms 86.55%
- Req/Sec 16.47k 2.89k 38.11k 69.46%
- 2967567 requests in 15.09s, 44.70GB read
-Requests/sec: 196596.49
-Transfer/sec: 2.96GB
+ Latency 4.52ms 6.43ms 94.34ms 85.44%
+ Req/Sec 15.76k 2.83k 29.64k 69.20%
+ 2839632 requests in 15.09s, 32.90GB read
+Requests/sec: 188185.61
+Transfer/sec: 2.18GB
Starting the test to bomb DIFFERENT PAGES (codes). Please, be patient...
Test completed successfully. Here is the output:
@@ -622,11 +622,11 @@ Test completed successfully. Here is the output:
Running 15s test @ http://127.0.0.1:8080/
12 threads and 400 connections
Thread Stats Avg Stdev Max +/- Stdev
- Latency 4.25ms 6.03ms 74.23ms 86.97%
- Req/Sec 14.29k 2.75k 32.16k 69.63%
- 2563245 requests in 15.07s, 38.47GB read
-Requests/sec: 170062.69
-Transfer/sec: 2.55GB
+ Latency 6.75ms 13.71ms 252.66ms 91.94%
+ Req/Sec 14.06k 3.25k 26.39k 71.98%
+ 2534473 requests in 15.10s, 29.22GB read
+Requests/sec: 167899.78
+Transfer/sec: 1.94GB
```
@@ -678,6 +678,7 @@ The following flags are supported:
| `--proxy-headers="…"` | HTTP headers listed here will be proxied from the original request to the error page response (comma-separated list) | `X-Request-Id,X-Trace-Id,X-Amzn-Trace-Id` | `PROXY_HTTP_HEADERS` |
| `--rotation-mode="…"` | Templates automatic rotation mode (disabled/random-on-startup/random-on-each-request/random-hourly/random-daily) | `disabled` | `TEMPLATES_ROTATION_MODE` |
| `--read-buffer-size="…"` | Per-connection buffer size in bytes for reading requests, this also limits the maximum header size (increase this buffer if your clients send multi-KB Request URIs and/or multi-KB headers (e.g., large cookies), note that increasing this value will increase memory consumption) | `5120` | `READ_BUFFER_SIZE` |
+| `--disable-minification` | Disable the minification of HTML pages, including CSS, SVG, and JS (may be useful for debugging) | `false` | `DISABLE_MINIFICATION` |
### `build` command (aliases: `b`)
@@ -691,14 +692,15 @@ $ error-pages [GLOBAL FLAGS] build [COMMAND FLAGS] [ARGUMENTS...]
The following flags are supported:
-| Name | Description | Default value | Environment variables |
-|---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------:|:---------------------:|
-| `--add-template="…"` | To add a new template, provide the path to the file using this flag (the filename without the extension will be used as the template name) | `[]` | *none* |
-| `--disable-template="…"` | Disable the specified template by its name (useful to disable the built-in templates and use only custom ones) | `[]` | *none* |
-| `--add-code="…"` | To add a new HTTP status code, provide the code and its message/description using this flag (the format should be '%code%=%message%/%description%'; the code may contain a wildcard '*' to cover multiple codes at once, for example, '4**' will cover all 4xx codes unless a more specific code is described previously) | `map[]` | *none* |
-| `--disable-l10n` | Disable localization of error pages (if the template supports localization) | `false` | `DISABLE_L10N` |
-| `--index` (`-i`) | Generate index.html file with links to all error pages | `false` | *none* |
-| `--target-dir="…"` (`--out`, `--dir`, `-o`) | Directory to put the built error pages into | `.` | *none* |
+| Name | Description | Default value | Environment variables |
+|---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------:|:----------------------:|
+| `--add-template="…"` | To add a new template, provide the path to the file using this flag (the filename without the extension will be used as the template name) | `[]` | *none* |
+| `--disable-template="…"` | Disable the specified template by its name (useful to disable the built-in templates and use only custom ones) | `[]` | *none* |
+| `--add-code="…"` | To add a new HTTP status code, provide the code and its message/description using this flag (the format should be '%code%=%message%/%description%'; the code may contain a wildcard '*' to cover multiple codes at once, for example, '4**' will cover all 4xx codes unless a more specific code is described previously) | `map[]` | *none* |
+| `--disable-l10n` | Disable localization of error pages (if the template supports localization) | `false` | `DISABLE_L10N` |
+| `--index` (`-i`) | Generate index.html file with links to all error pages | `false` | *none* |
+| `--target-dir="…"` (`--out`, `--dir`, `-o`) | Directory to put the built error pages into | `.` | *none* |
+| `--disable-minification` | Disable the minification of HTML pages, including CSS, SVG, and JS (may be useful for debugging) | `false` | `DISABLE_MINIFICATION` |
### `healthcheck` command (aliases: `chk`, `health`, `check`)
diff --git a/go.mod b/go.mod
index 467baec..7929038 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.22
require (
github.com/stretchr/testify v1.9.0
+ github.com/tdewolff/minify/v2 v2.20.35
github.com/urfave/cli-docs/v3 v3.0.0-alpha5
github.com/urfave/cli/v3 v3.0.0-alpha9
github.com/valyala/fasthttp v1.55.0
@@ -19,6 +20,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/tdewolff/parse/v2 v2.7.15 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
diff --git a/go.sum b/go.sum
index 26ec34a..92db187 100644
--- a/go.sum
+++ b/go.sum
@@ -26,6 +26,13 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tdewolff/minify/v2 v2.20.35 h1:/Vq/oivpkFyi2PViD25XHZZbJz+eO4OmPSgePex1kBU=
+github.com/tdewolff/minify/v2 v2.20.35/go.mod h1:L1VYef/jwKw6Wwyk5A+T0mBjjn3mMPgmjjA688RNsxU=
+github.com/tdewolff/parse/v2 v2.7.15 h1:hysDXtdGZIRF5UZXwpfn3ZWRbm+ru4l53/ajBRGpCTw=
+github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
+github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/urfave/cli-docs/v3 v3.0.0-alpha5 h1:H1oWnR2/GN0dNm2PVylws+GxSOD6YOwW/jI5l78YfPk=
github.com/urfave/cli-docs/v3 v3.0.0-alpha5/go.mod h1:AIqom6Q60U4tiqHp41i7+/AB2XHgi1WvQ7jOFlccmZ4=
github.com/urfave/cli/v3 v3.0.0-alpha9 h1:P0RMy5fQm1AslQS+XCmy9UknDXctOmG/q/FZkUFnJSo=
diff --git a/internal/cli/app.go b/internal/cli/app.go
index ecf807d..ce89ff2 100644
--- a/internal/cli/app.go
+++ b/internal/cli/app.go
@@ -6,7 +6,6 @@ import (
"runtime"
"strings"
- _ "github.com/urfave/cli-docs/v3" // required for `go generate` to work
"github.com/urfave/cli/v3"
"gh.tarampamp.am/error-pages/internal/appmeta"
@@ -17,7 +16,7 @@ import (
"gh.tarampamp.am/error-pages/internal/logger"
)
-//go:generate go run update_readme.go
+//go:generate go run app_generate.go
// NewApp creates a new console application.
func NewApp(appName string) *cli.Command { //nolint:funlen
diff --git a/internal/cli/update_readme.go b/internal/cli/app_generate.go
similarity index 72%
rename from internal/cli/update_readme.go
rename to internal/cli/app_generate.go
index c108ca3..6592c7e 100644
--- a/internal/cli/update_readme.go
+++ b/internal/cli/app_generate.go
@@ -1,5 +1,4 @@
-//go:build ignore
-// +build ignore
+//go:build generate
package main
@@ -17,8 +16,10 @@ func main() {
if stat, err := os.Stat(readmePath); err == nil && stat.Mode().IsRegular() {
if err = cliDocs.ToTabularToFileBetweenTags(cli.NewApp(""), "error-pages", readmePath); err != nil {
panic(err)
+ } else {
+ println("✔ cli docs updated successfully")
}
} else if err != nil {
- println("readme file not found, cli docs not updated:", err.Error())
+ println("⚠ readme file not found, cli docs not updated:", err.Error())
}
}
diff --git a/internal/cli/build/command.go b/internal/cli/build/command.go
index 657f312..28ab5dd 100644
--- a/internal/cli/build/command.go
+++ b/internal/cli/build/command.go
@@ -39,11 +39,12 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit
cmd command
cfg = config.New()
- addTplFlag = shared.AddTemplatesFlag
- disableTplFlag = shared.DisableTemplateNamesFlag
- addCodeFlag = shared.AddHTTPCodesFlag
- disableL10nFlag = shared.DisableL10nFlag
- createIndexFlag = cli.BoolFlag{
+ addTplFlag = shared.AddTemplatesFlag
+ disableTplFlag = shared.DisableTemplateNamesFlag
+ addCodeFlag = shared.AddHTTPCodesFlag
+ disableL10nFlag = shared.DisableL10nFlag
+ disableMinificationFlag = shared.DisableMinificationFlag
+ createIndexFlag = cli.BoolFlag{
Name: "index",
Aliases: []string{"i"},
Usage: "Generate index.html file with links to all error pages",
@@ -81,6 +82,7 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit
Usage: "Build the static error pages and put them into a specified directory",
Action: func(ctx context.Context, c *cli.Command) error {
cfg.L10n.Disable = c.Bool(disableL10nFlag.Name)
+ cfg.DisableMinification = c.Bool(disableMinificationFlag.Name)
cmd.opt.createIndex = c.Bool(createIndexFlag.Name)
cmd.opt.targetDirAbsPath, _ = filepath.Abs(c.String(targetDirFlag.Name)) // an error checked by [os.Stat] validator
@@ -140,13 +142,14 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit
&disableL10nFlag,
&createIndexFlag,
&targetDirFlag,
+ &disableMinificationFlag,
},
}
return cmd.c
}
-func (cmd *command) Run( //nolint:funlen
+func (cmd *command) Run( //nolint:funlen,gocognit
ctx context.Context,
log *logger.Logger,
cfg *config.Config,
@@ -172,13 +175,21 @@ func (cmd *command) Run( //nolint:funlen
var outFilePath = path.Join(cmd.opt.targetDirAbsPath, templateName, code+".html")
- if content, renderErr := appTemplate.Render(templateContent, appTemplate.Props{
+ if content, renderErr := appTemplate.Render(templateContent, appTemplate.Props{ //nolint:nestif
Code: uint16(codeAsUint),
Message: codeDescription.Message,
Description: codeDescription.Description,
L10nDisabled: cfg.L10n.Disable,
ShowRequestDetails: false,
}); renderErr == nil {
+ if !cfg.DisableMinification {
+ if mini, minErr := appTemplate.MiniHTML(content); minErr != nil {
+ log.Warn("Cannot minify the content", logger.Error(minErr))
+ } else {
+ content = mini
+ }
+ }
+
if err := os.WriteFile(outFilePath, []byte(content), os.FileMode(0664)); err != nil { //nolint:mnd
return err
}
diff --git a/internal/cli/serve/command.go b/internal/cli/serve/command.go
index 4a4cf61..497b26a 100644
--- a/internal/cli/serve/command.go
+++ b/internal/cli/serve/command.go
@@ -38,13 +38,14 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit,gocy
)
var (
- addrFlag = shared.ListenAddrFlag
- portFlag = shared.ListenPortFlag
- addTplFlag = shared.AddTemplatesFlag
- disableTplFlag = shared.DisableTemplateNamesFlag
- addCodeFlag = shared.AddHTTPCodesFlag
- disableL10nFlag = shared.DisableL10nFlag
- jsonFormatFlag = cli.StringFlag{
+ addrFlag = shared.ListenAddrFlag
+ portFlag = shared.ListenPortFlag
+ addTplFlag = shared.AddTemplatesFlag
+ disableTplFlag = shared.DisableTemplateNamesFlag
+ addCodeFlag = shared.AddHTTPCodesFlag
+ disableL10nFlag = shared.DisableL10nFlag
+ disableMinificationFlag = shared.DisableMinificationFlag
+ jsonFormatFlag = cli.StringFlag{
Name: "json-format",
Usage: "Override the default error page response in JSON format (Go templates are supported; the error " +
"page will use this template if the client requests JSON content type)",
@@ -182,6 +183,7 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit,gocy
cfg.RespondWithSameHTTPCode = c.Bool(sendSameHTTPCodeFlag.Name)
cfg.RotationMode, _ = config.ParseRotationMode(c.String(rotationModeFlag.Name))
cfg.ShowDetails = c.Bool(showDetailsFlag.Name)
+ cfg.DisableMinification = c.Bool(disableMinificationFlag.Name)
{ // override default JSON, XML, and PlainText formats
if c.IsSet(jsonFormatFlag.Name) {
@@ -303,6 +305,7 @@ func NewCommand(log *logger.Logger) *cli.Command { //nolint:funlen,gocognit,gocy
&proxyHeadersListFlag,
&rotationModeFlag,
&readBufferSizeFlag,
+ &disableMinificationFlag,
},
}
diff --git a/internal/cli/shared/flags.go b/internal/cli/shared/flags.go
index a2d2a8a..1e44823 100644
--- a/internal/cli/shared/flags.go
+++ b/internal/cli/shared/flags.go
@@ -146,3 +146,11 @@ var DisableL10nFlag = cli.BoolFlag{
Category: CategoryOther,
OnlyOnce: true,
}
+
+var DisableMinificationFlag = cli.BoolFlag{
+ Name: "disable-minification",
+ Usage: "Disable the minification of HTML pages, including CSS, SVG, and JS (may be useful for debugging)",
+ Sources: cli.EnvVars("DISABLE_MINIFICATION"),
+ Category: CategoryOther,
+ OnlyOnce: true,
+}
diff --git a/internal/cli/shared/flags_test.go b/internal/cli/shared/flags_test.go
index 3528b39..e381344 100644
--- a/internal/cli/shared/flags_test.go
+++ b/internal/cli/shared/flags_test.go
@@ -216,3 +216,12 @@ func TestDisableL10nFlag(t *testing.T) {
assert.Equal(t, "disable-l10n", flag.Name)
assert.Contains(t, flag.Sources.String(), "DISABLE_L10N")
}
+
+func TestDisableMinificationFlag(t *testing.T) {
+ t.Parallel()
+
+ var flag = shared.DisableMinificationFlag
+
+ assert.Equal(t, "disable-minification", flag.Name)
+ assert.Contains(t, flag.Sources.String(), "DISABLE_MINIFICATION")
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 032cc4f..323d0b7 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -56,6 +56,9 @@ type Config struct {
// ShowDetails determines whether to show additional details in the error response, extracted from the
// incoming request (if supported by the template).
ShowDetails bool
+
+ // DisableMinification determines whether to disable minification of the rendered content (e.g., HTML, CSS) or not.
+ DisableMinification bool
}
const defaultJSONFormat string = `{
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index f141647..11ea8ba 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -24,6 +24,7 @@ func TestNew(t *testing.T) {
assert.NotEmpty(t, cfg.TemplateName)
assert.True(t, cfg.Templates.Has(cfg.TemplateName))
assert.Equal(t, uint16(http.StatusNotFound), cfg.DefaultCodeToRender)
+ assert.False(t, cfg.DisableMinification)
})
t.Run("changing cfg1 should not affect cfg2", func(t *testing.T) {
diff --git a/internal/http/handlers/error_page/handler.go b/internal/http/handlers/error_page/handler.go
index d698bfb..fc52c80 100644
--- a/internal/http/handlers/error_page/handler.go
+++ b/internal/http/handlers/error_page/handler.go
@@ -175,6 +175,14 @@ func New(cfg *config.Config, log *logger.Logger) (_ fasthttp.RequestHandler, clo
err.Error(),
))
} else {
+ if !cfg.DisableMinification {
+ if mini, minErr := template.MiniHTML(content); minErr != nil {
+ log.Warn("HTML minification failed", logger.Error(minErr))
+ } else {
+ content = mini
+ }
+ }
+
cache.Put(tpl, tplProps, []byte(content))
write(ctx, log, content)
diff --git a/internal/http/handlers/error_page/handler_test.go b/internal/http/handlers/error_page/handler_test.go
index c274723..7175601 100644
--- a/internal/http/handlers/error_page/handler_test.go
+++ b/internal/http/handlers/error_page/handler_test.go
@@ -48,7 +48,7 @@ func TestHandler(t *testing.T) {
wantStatusCode: http.StatusOK,
wantHeaders: map[string]string{"Content-Type": "text/html; charset=utf-8"},
wantBodyIncludes: []string{
- "",
+ "",
"407: Proxy Authentication Required",
"Proxy Authentication Required",
},
diff --git a/internal/template/minify.go b/internal/template/minify.go
new file mode 100644
index 0000000..e1affaf
--- /dev/null
+++ b/internal/template/minify.go
@@ -0,0 +1,23 @@
+package template
+
+import (
+ "github.com/tdewolff/minify/v2"
+ "github.com/tdewolff/minify/v2/css"
+ "github.com/tdewolff/minify/v2/html"
+ "github.com/tdewolff/minify/v2/js"
+ "github.com/tdewolff/minify/v2/svg"
+)
+
+var htmlMinify = func() *minify.M { //nolint:gochecknoglobals
+ var m = minify.New()
+
+ m.AddFunc("text/css", css.Minify)
+ m.Add("text/html", &html.Minifier{KeepDocumentTags: true, KeepEndTags: true, KeepQuotes: true})
+ m.AddFunc("image/svg+xml", svg.Minify)
+ m.AddFunc("application/javascript", js.Minify)
+
+ return m
+}()
+
+// MiniHTML minifies HTML data, including inline CSS, SVG and JS.
+func MiniHTML(data string) (string, error) { return htmlMinify.String("text/html", data) }
diff --git a/internal/template/minify_test.go b/internal/template/minify_test.go
new file mode 100644
index 0000000..993f886
--- /dev/null
+++ b/internal/template/minify_test.go
@@ -0,0 +1,94 @@
+package template_test
+
+import (
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "gh.tarampamp.am/error-pages/internal/template"
+)
+
+func TestMiniHTML(t *testing.T) {
+ t.Parallel()
+
+ var wg sync.WaitGroup
+
+ for range 100 { // race condition provocation
+ wg.Add(1)
+
+ go func() {
+ defer wg.Done()
+
+ for give, want := range map[string]string{
+ "": "",
+ `<!-- Simple HTML page -->
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Test
+
+
+ Test
+
+