ai cooking
Add runtime env loader for .env and age-encrypted secrets with tests (#459)
* ugh
* Stream decrypted env directly into godotenv parser
* Walk to git root for .env and secrets/envtest lookup
* Clarify manual env merge for decrypted secrets
* let me load encyptedsecrets
* seems ot work
* simplify
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
15152e89
6a43ff4e
+232
-13
+16
-12
getsecret.sh
+16
-12
getsecret.sh
···
8
8
./getsecret.sh put [secrets-dir] [namespace]
9
9
10
10
Commands:
11
-
get Fetch each Kubernetes Secret named by a *.env file in secrets-dir
12
-
and write it back to that file.
11
+
get Fetch every Kubernetes Secret in the namespace and write each
12
+
one to secrets-dir/<secret-name>.env.
13
13
put Apply each *.env file in secrets-dir to the Kubernetes Secret with
14
14
the same basename.
15
15
···
34
34
sync_get() {
35
35
local secrets_dir="$1"
36
36
local namespace="$2"
37
-
local files=()
37
+
local secret_names=()
38
38
39
39
mkdir -p "${secrets_dir}"
40
-
shopt -s nullglob
41
-
files=("${secrets_dir}"/*.env)
42
-
shopt -u nullglob
43
40
44
-
if [[ ${#files[@]} -eq 0 ]]; then
45
-
echo "error: no .env files found in ${secrets_dir}" >&2
41
+
mapfile -t secret_names < <(
42
+
kubectl get secrets \
43
+
-n "${namespace}" \
44
+
-o json \
45
+
| jq -r '.items | sort_by(.metadata.name)[] | .metadata.name'
46
+
)
47
+
48
+
if [[ ${#secret_names[@]} -eq 0 ]]; then
49
+
echo "error: no secrets found in namespace '${namespace}'" >&2
46
50
exit 1
47
51
fi
48
52
49
-
local file
50
-
for file in "${files[@]}"; do
51
-
local secret_name
53
+
local secret_name
54
+
for secret_name in "${secret_names[@]}"; do
55
+
local file
52
56
local tmp_file
53
-
secret_name="$(basename "${file}" .env)"
57
+
file="${secrets_dir}/${secret_name}.env"
54
58
tmp_file="$(mktemp)"
55
59
56
60
kubectl get secret "${secret_name}" \
+4
-1
go.mod
+4
-1
go.mod
···
7
7
require github.com/samber/lo v1.53.0
8
8
9
9
require (
10
+
filippo.io/age v1.2.1
10
11
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
11
12
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
12
13
github.com/clerk/clerk-sdk-go/v2 v2.5.1
13
14
github.com/gobwas/ws v1.4.0
15
+
github.com/hashicorp/go-retryablehttp v0.7.8
14
16
github.com/invopop/jsonschema v0.13.0
17
+
github.com/joho/godotenv v1.5.1
15
18
github.com/microsoft/ApplicationInsights-Go v0.4.4
16
19
github.com/openai/openai-go/v3 v3.29.0
17
20
github.com/openclosed-dev/slogan v0.2.0
···
24
27
25
28
require (
26
29
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect
30
+
filippo.io/edwards25519 v1.1.0 // indirect
27
31
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
28
32
github.com/bahlo/generic-list-go v0.2.0 // indirect
29
33
github.com/buger/jsonparser v1.1.1 // indirect
···
32
36
github.com/gobwas/pool v0.2.1 // indirect
33
37
github.com/gofrs/uuid v3.3.0+incompatible // indirect
34
38
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
35
-
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
36
39
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
37
40
github.com/woodsbury/decimal128 v1.3.0 // indirect
38
41
golang.org/x/sys v0.42.0 // indirect
+6
go.sum
+6
go.sum
···
1
1
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ=
2
2
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8=
3
+
filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o=
4
+
filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
5
+
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
6
+
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
3
7
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
4
8
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
5
9
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
···
82
86
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
83
87
github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E=
84
88
github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
89
+
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
90
+
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
85
91
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
86
92
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
87
93
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+4
internal/config/config.go
+4
internal/config/config.go
+99
internal/config/envload.go
+99
internal/config/envload.go
···
1
+
package config
2
+
3
+
import (
4
+
"errors"
5
+
"fmt"
6
+
"os"
7
+
"path/filepath"
8
+
"sync"
9
+
10
+
"filippo.io/age"
11
+
"filippo.io/age/agessh"
12
+
"github.com/joho/godotenv"
13
+
)
14
+
15
+
var envLoadOnce sync.Once
16
+
17
+
// asumes you are running from root of repo.
18
+
func loadRuntimeEnv() error {
19
+
var loadErr error
20
+
envLoadOnce.Do(func() {
21
+
// does not error on not found (or any file.open error)
22
+
if err := godotenv.Load(); err != nil {
23
+
if !errors.Is(err, os.ErrNotExist) {
24
+
loadErr = fmt.Errorf("load .env: %w", err)
25
+
return
26
+
}
27
+
}
28
+
29
+
if err := loadEncryptedEnv("secrets/envtest"); err != nil {
30
+
loadErr = err
31
+
}
32
+
})
33
+
return loadErr
34
+
}
35
+
36
+
func loadEncryptedEnv(path string) error {
37
+
identities, err := loadSSHIdentities()
38
+
if err != nil {
39
+
if errors.Is(err, os.ErrNotExist) {
40
+
return nil
41
+
}
42
+
return fmt.Errorf("load ssh identity for %q: %w", path, err)
43
+
}
44
+
45
+
return decryptDotEnv(path, identities)
46
+
}
47
+
48
+
func decryptDotEnv(path string, identities []age.Identity) error {
49
+
ciphertext, err := os.Open(path)
50
+
if err != nil {
51
+
if errors.Is(err, os.ErrNotExist) {
52
+
return nil
53
+
}
54
+
return fmt.Errorf("read encrypted env %q: %w", path, err)
55
+
}
56
+
defer func() {
57
+
_ = ciphertext.Close()
58
+
}()
59
+
60
+
reader, err := age.Decrypt(ciphertext, identities...)
61
+
if err != nil {
62
+
return fmt.Errorf("decrypt env %q: %w", path, err)
63
+
}
64
+
65
+
// no load for read so manually merge in entries from parse
66
+
entries, err := godotenv.Parse(reader)
67
+
if err != nil {
68
+
return fmt.Errorf("parse decrypted env %q: %w", path, err)
69
+
}
70
+
for key, value := range entries {
71
+
if _, exists := os.LookupEnv(key); !exists {
72
+
_ = os.Setenv(key, value)
73
+
}
74
+
}
75
+
return nil
76
+
}
77
+
78
+
func loadSSHIdentities() ([]age.Identity, error) {
79
+
home, err := os.UserHomeDir()
80
+
if err != nil || home == "" {
81
+
return []age.Identity{}, nil
82
+
}
83
+
path := filepath.Join(home, ".ssh", "id_ed25519")
84
+
85
+
key, err := os.ReadFile(path)
86
+
if err != nil {
87
+
if errors.Is(err, os.ErrNotExist) {
88
+
return []age.Identity{}, nil
89
+
}
90
+
return nil, err
91
+
}
92
+
93
+
identity, err := agessh.ParseIdentity(key)
94
+
if err != nil {
95
+
return nil, fmt.Errorf("parse ssh identity %q: %w", path, err)
96
+
}
97
+
98
+
return []age.Identity{identity}, nil
99
+
}
+99
internal/config/envload_test.go
+99
internal/config/envload_test.go
···
1
+
package config
2
+
3
+
import (
4
+
"bytes"
5
+
"io"
6
+
"os"
7
+
"path/filepath"
8
+
"sync"
9
+
"testing"
10
+
11
+
"filippo.io/age"
12
+
agessh "filippo.io/age/agessh"
13
+
)
14
+
15
+
const testSSHPrivateKey = `-----BEGIN OPENSSH PRIVATE KEY-----
16
+
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
17
+
QyNTUxOQAAACB7qx7CGF0+RlAe2W0yhkiKlf71UMVcDaxCDfkSqtRO1QAAAJhNAJ9JTQCf
18
+
SQAAAAtzc2gtZWQyNTUxOQAAACB7qx7CGF0+RlAe2W0yhkiKlf71UMVcDaxCDfkSqtRO1Q
19
+
AAAEAIOeRpdKSm4SAwH+TzGtR01RQoGiR/PSEns26+wH1GXXurHsIYXT5GUB7ZbTKGSIqV
20
+
/vVQxVwNrEIN+RKq1E7VAAAAEXJvb3RAZTIzYzBmOTM1ZGFmAQIDBA==
21
+
-----END OPENSSH PRIVATE KEY-----`
22
+
23
+
const testSSHPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurHsIYXT5GUB7ZbTKGSIqV/vVQxVwNrEIN+RKq1E7V test@careme"
24
+
25
+
func TestLoadRuntimeEnvLoadsDotAndEncryptedWithoutOverride(t *testing.T) {
26
+
envLoadOnce = sync.Once{}
27
+
t.Setenv("KEEP", "already")
28
+
29
+
tmp := t.TempDir()
30
+
t.Setenv("HOME", tmp)
31
+
oldWD, err := os.Getwd()
32
+
if err != nil {
33
+
t.Fatalf("Getwd() error = %v", err)
34
+
}
35
+
if err := os.Chdir(tmp); err != nil {
36
+
t.Fatalf("Chdir(%q) error = %v", tmp, err)
37
+
}
38
+
t.Cleanup(func() {
39
+
_ = os.Chdir(oldWD)
40
+
})
41
+
42
+
if err := os.WriteFile(".env", []byte("DOTENV_KEY=plain\nKEEP=from-dotenv\n"), 0o600); err != nil {
43
+
t.Fatalf("WriteFile(.env) error = %v", err)
44
+
}
45
+
46
+
sshDir := filepath.Join(tmp, ".ssh")
47
+
if err := os.MkdirAll(sshDir, 0o700); err != nil {
48
+
t.Fatalf("MkdirAll(.ssh) error = %v", err)
49
+
}
50
+
if err := os.WriteFile(filepath.Join(sshDir, "id_ed25519"), []byte(testSSHPrivateKey), 0o600); err != nil {
51
+
t.Fatalf("WriteFile(id_ed25519) error = %v", err)
52
+
}
53
+
54
+
if err := os.MkdirAll(filepath.Join("secrets"), 0o700); err != nil {
55
+
t.Fatalf("MkdirAll(secrets) error = %v", err)
56
+
}
57
+
58
+
ciphertext, err := encryptWithRecipient("SECRET_KEY=encrypted\n", testSSHPublicKey)
59
+
if err != nil {
60
+
t.Fatalf("encryptWithRecipient() error = %v", err)
61
+
}
62
+
if err := os.WriteFile(filepath.Join("secrets", "envtest"), ciphertext, 0o600); err != nil {
63
+
t.Fatalf("WriteFile(secrets/envtest) error = %v", err)
64
+
}
65
+
66
+
if err := loadRuntimeEnv(); err != nil {
67
+
t.Fatalf("loadRuntimeEnv() error = %v", err)
68
+
}
69
+
70
+
if got := os.Getenv("DOTENV_KEY"); got != "plain" {
71
+
t.Fatalf("DOTENV_KEY = %q, want %q", got, "plain")
72
+
}
73
+
if got := os.Getenv("SECRET_KEY"); got != "encrypted" {
74
+
t.Fatalf("SECRET_KEY = %q, want %q", got, "encrypted")
75
+
}
76
+
if got := os.Getenv("KEEP"); got != "already" {
77
+
t.Fatalf("KEEP = %q, want %q", got, "already")
78
+
}
79
+
}
80
+
81
+
func encryptWithRecipient(plaintext, publicKey string) ([]byte, error) {
82
+
recipient, err := agessh.ParseRecipient(publicKey)
83
+
if err != nil {
84
+
return nil, err
85
+
}
86
+
87
+
var ciphertext bytes.Buffer
88
+
writer, err := age.Encrypt(&ciphertext, recipient)
89
+
if err != nil {
90
+
return nil, err
91
+
}
92
+
if _, err := io.WriteString(writer, plaintext); err != nil {
93
+
return nil, err
94
+
}
95
+
if err := writer.Close(); err != nil {
96
+
return nil, err
97
+
}
98
+
return ciphertext.Bytes(), nil
99
+
}
secrets/envprod
secrets/envprod
This is a binary file and will not be displayed.
secrets/envtest
secrets/envtest
This is a binary file and will not be displayed.
+4
secrets/recipients.txt
+4
secrets/recipients.txt
···
1
+
# curl https://github.com/paulgmiller.keys
2
+
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICzyp4BO0dtNGui6npejdWfHxrwx2c2euwxceYkeZG2h
3
+
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILXPw5GIXpVl0JY+4Fes0qmsg8zIbEcz1b0ICpoXczqd
4
+
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAAgtC8MPTEhwVzcLOXl+3CYwSxwAEiGLcSEZxglfsj