this repo has no description
feat: retrieve CLI environment variables from `process.env` and `.env*` files (#937)
Co-authored-by: James Anderson <james@eli.cx>
authored by
Victor Berchet
James Anderson
and committed by
GitHub
32ba91a6
54c47e5b
+231
-127
+15
.changeset/quiet-snails-hope.md
+15
.changeset/quiet-snails-hope.md
···
1
+
---
2
+
"@opennextjs/cloudflare": minor
3
+
---
4
+
5
+
feat: retrieve CLI environment variables from `process.env` and `.env*` files
6
+
7
+
Recommended usage on CI:
8
+
9
+
- Add your secrets to `process.env` (i.e. `CF_ACCOUNT_ID`)
10
+
- Add public values to the wrangler config `wrangler.jsonc` (i.e. `R2_CACHE_PREFIX_ENV_NAME`)
11
+
12
+
Recommended usage for local dev:
13
+
14
+
- Add your secrets to either a `.dev.vars*` or `.env*` file (i.e. `CF_ACCOUNT_ID`)
15
+
- Add public values to the wrangler config `wrangler.jsonc` (i.e. `R2_CACHE_PREFIX_ENV_NAME`)
+4
packages/cloudflare/src/api/cloudflare-context.ts
+4
packages/cloudflare/src/api/cloudflare-context.ts
···
340
340
341
341
const { env, cf, ctx } = await getPlatformProxy({
342
342
...options,
343
+
// The `env` passed to the fetch handler does not contain variables from `.env*` files.
344
+
// because we invoke wrangler with `CLOUDFLARE_LOAD_DEV_VARS_FROM_DOT_ENV`=`"false"`.
345
+
// Initializing `envFiles` with an empty list is the equivalent for this API call.
346
+
envFiles: [],
343
347
environment,
344
348
});
345
349
return {
+1
-1
packages/cloudflare/src/cli/build/utils/extract-project-env-vars.ts
+1
-1
packages/cloudflare/src/cli/build/utils/extract-project-env-vars.ts
+17
-14
packages/cloudflare/src/cli/commands/deploy.ts
+17
-14
packages/cloudflare/src/cli/commands/deploy.ts
···
23
23
printHeaders("deploy");
24
24
25
25
const { config } = await retrieveCompiledConfig();
26
-
const options = getNormalizedOptions(config);
26
+
const buildOpts = getNormalizedOptions(config);
27
27
28
28
const wranglerConfig = readWranglerConfig(args);
29
29
30
-
const envVars = await getEnvFromPlatformProxy({
31
-
configPath: args.wranglerConfigPath,
32
-
environment: args.env,
33
-
});
30
+
const envVars = await getEnvFromPlatformProxy(config, buildOpts);
34
31
35
-
const deploymentMapping = await getDeploymentMapping(options, config, envVars);
32
+
await populateCache(
33
+
buildOpts,
34
+
config,
35
+
wranglerConfig,
36
+
{
37
+
target: "remote",
38
+
environment: args.env,
39
+
wranglerConfigPath: args.wranglerConfigPath,
40
+
cacheChunkSize: args.cacheChunkSize,
41
+
shouldUsePreviewId: false,
42
+
},
43
+
envVars
44
+
);
36
45
37
-
await populateCache(options, config, wranglerConfig, {
38
-
target: "remote",
39
-
environment: args.env,
40
-
wranglerConfigPath: args.wranglerConfigPath,
41
-
cacheChunkSize: args.cacheChunkSize,
42
-
shouldUsePreviewId: false,
43
-
});
46
+
const deploymentMapping = await getDeploymentMapping(buildOpts, config, envVars);
44
47
45
48
runWrangler(
46
-
options,
49
+
buildOpts,
47
50
[
48
51
"deploy",
49
52
...args.wranglerArgs,
+57
-5
packages/cloudflare/src/cli/commands/helpers.ts
+57
-5
packages/cloudflare/src/cli/commands/helpers.ts
···
1
+
import { type BuildOptions } from "@opennextjs/aws/build/helper.js";
1
2
import { getPlatformProxy, type GetPlatformProxyOptions } from "wrangler";
2
3
4
+
import { extractProjectEnvVars } from "../build/utils/extract-project-env-vars.js";
5
+
3
6
export type WorkerEnvVar = Record<keyof CloudflareEnv, string | undefined>;
4
7
5
8
/**
6
-
* Return the string env vars from the worker.
9
+
* Returns the env vars to use by the CLI.
10
+
*
11
+
* The environments variables are returned from a combination of `process.env`, wrangler config, and `.env*` files.
12
+
*
13
+
* Recommended usage on CI:
14
+
*
15
+
* - Add you secrets to `process.env` (i.e. `CF_ACCOUNT_ID`)
16
+
* - Add public values to the wrangler config `wrangler.jsonc` (i.e. `R2_CACHE_PREFIX_ENV_NAME`)
17
+
*
18
+
* Note: `.dev.vars*` and `.env*` should not be checked in.
19
+
*
20
+
* Recommended usage for local dev:
21
+
*
22
+
* - Add you secrets to either a `.dev.vars*` or `.env*` file (i.e. `CF_ACCOUNT_ID`)
23
+
* - Add public values to the wrangler config `wrangler.jsonc` (i.e. `R2_CACHE_PREFIX_ENV_NAME`)
24
+
*
25
+
* Note: `.env*` files are also used by `next dev` while `.dev.var*` files are only loaded by `wrangler`.
26
+
*
27
+
* Loading details:
28
+
*
29
+
* 1. The variables are first initialized from `process.env`
30
+
* 2. They are then augmented/replaced with variables from the wrangler config (`wrangler.jsonc` and `.dev.vars*`)
31
+
* 3. They are then augmented with variables from `.env*` files (existing values are not replaced).
7
32
*
8
33
* @param options Options to pass to `getPlatformProxy`, i.e. to set the environment
34
+
* @param buildOpts Open Next build options
9
35
* @returns the env vars
10
36
*/
11
-
export async function getEnvFromPlatformProxy(options: GetPlatformProxyOptions) {
12
-
const envVars = {} as WorkerEnvVar;
13
-
const proxy = await getPlatformProxy<CloudflareEnv>(options);
37
+
export async function getEnvFromPlatformProxy(options: GetPlatformProxyOptions, buildOpts: BuildOptions) {
38
+
// 1. Start from `process.env`
39
+
const envVars = process.env;
40
+
41
+
// 2. Apply vars from workers `env`
42
+
const proxy = await getPlatformProxy<CloudflareEnv>({
43
+
...options,
44
+
// Next.js uses a different mechanism to load `.env*` files from wrangler.
45
+
// We prevent wrangler for loading the files and handle that in `getEnvFromPlatformProxy`.
46
+
envFiles: [],
47
+
});
48
+
14
49
Object.entries(proxy.env).forEach(([key, value]) => {
15
50
if (typeof value === "string") {
51
+
// filter out bindings by only considering string values
16
52
envVars[key as keyof CloudflareEnv] = value;
17
53
}
18
54
});
55
+
19
56
await proxy.dispose();
20
-
return envVars;
57
+
58
+
// 3. Apply new vars from `.env*` files
59
+
let mode: "production" | "development" | "test" = "production";
60
+
if (envVars.NEXTJS_ENV === "development") {
61
+
mode = "development";
62
+
} else if (envVars.NEXTJS_ENV === "test") {
63
+
mode = "test";
64
+
}
65
+
66
+
const dotEnvVars = extractProjectEnvVars(mode, buildOpts);
67
+
68
+
for (const varName in dotEnvVars) {
69
+
envVars[varName] ??= dotEnvVars[varName];
70
+
}
71
+
72
+
return envVars as unknown as WorkerEnvVar;
21
73
}
22
74
23
75
/**
+89
-81
packages/cloudflare/src/cli/commands/populate-cache.ts
+89
-81
packages/cloudflare/src/cli/commands/populate-cache.ts
···
37
37
import { normalizePath } from "../build/utils/normalize-path.js";
38
38
import type { WranglerTarget } from "../utils/run-wrangler.js";
39
39
import { runWrangler } from "../utils/run-wrangler.js";
40
-
import { getEnvFromPlatformProxy, quoteShellMeta } from "./helpers.js";
40
+
import { getEnvFromPlatformProxy, quoteShellMeta, type WorkerEnvVar } from "./helpers.js";
41
41
import type { WithWranglerArgs } from "./utils.js";
42
42
import {
43
43
getNormalizedOptions,
···
48
48
withWranglerPassthroughArgs,
49
49
} from "./utils.js";
50
50
51
+
/**
52
+
* Implementation of the `opennextjs-cloudflare populateCache` command.
53
+
*
54
+
* @param args
55
+
*/
56
+
async function populateCacheCommand(
57
+
target: "local" | "remote",
58
+
args: WithWranglerArgs<{ cacheChunkSize?: number }>
59
+
) {
60
+
printHeaders(`populate cache - ${target}`);
61
+
62
+
const { config } = await retrieveCompiledConfig();
63
+
const buildOpts = getNormalizedOptions(config);
64
+
65
+
const wranglerConfig = readWranglerConfig(args);
66
+
const envVars = await getEnvFromPlatformProxy(config, buildOpts);
67
+
68
+
await populateCache(
69
+
buildOpts,
70
+
config,
71
+
wranglerConfig,
72
+
{
73
+
target,
74
+
environment: args.env,
75
+
wranglerConfigPath: args.wranglerConfigPath,
76
+
cacheChunkSize: args.cacheChunkSize,
77
+
shouldUsePreviewId: false,
78
+
},
79
+
envVars
80
+
);
81
+
}
82
+
83
+
export async function populateCache(
84
+
buildOpts: BuildOptions,
85
+
config: OpenNextConfig,
86
+
wranglerConfig: WranglerConfig,
87
+
populateCacheOptions: PopulateCacheOptions,
88
+
envVars: WorkerEnvVar
89
+
) {
90
+
const { incrementalCache, tagCache } = config.default.override ?? {};
91
+
92
+
if (!existsSync(buildOpts.outputDir)) {
93
+
logger.error("Unable to populate cache: Open Next build not found");
94
+
process.exit(1);
95
+
}
96
+
97
+
if (!config.dangerous?.disableIncrementalCache && incrementalCache) {
98
+
const name = await resolveCacheName(incrementalCache);
99
+
switch (name) {
100
+
case R2_CACHE_NAME:
101
+
await populateR2IncrementalCache(buildOpts, wranglerConfig, populateCacheOptions, envVars);
102
+
break;
103
+
case KV_CACHE_NAME:
104
+
await populateKVIncrementalCache(buildOpts, wranglerConfig, populateCacheOptions, envVars);
105
+
break;
106
+
case STATIC_ASSETS_CACHE_NAME:
107
+
populateStaticAssetsIncrementalCache(buildOpts);
108
+
break;
109
+
default:
110
+
logger.info("Incremental cache does not need populating");
111
+
}
112
+
}
113
+
114
+
if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) {
115
+
const name = await resolveCacheName(tagCache);
116
+
switch (name) {
117
+
case D1_TAG_NAME:
118
+
populateD1TagCache(buildOpts, wranglerConfig, populateCacheOptions);
119
+
break;
120
+
default:
121
+
logger.info("Tag cache does not need populating");
122
+
}
123
+
}
124
+
}
125
+
51
126
async function resolveCacheName(
52
127
value:
53
128
| IncludedIncrementalCache
···
130
205
};
131
206
132
207
async function populateR2IncrementalCache(
133
-
options: BuildOptions,
208
+
buildOpts: BuildOptions,
134
209
config: WranglerConfig,
135
-
populateCacheOptions: PopulateCacheOptions
210
+
populateCacheOptions: PopulateCacheOptions,
211
+
envVars: WorkerEnvVar
136
212
) {
137
213
logger.info("\nPopulating R2 incremental cache...");
138
214
···
146
222
throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`);
147
223
}
148
224
149
-
const envVars = await getEnvFromPlatformProxy(populateCacheOptions);
150
225
const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME];
151
226
152
-
const assets = getCacheAssets(options);
227
+
const assets = getCacheAssets(buildOpts);
153
228
154
229
for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
155
230
const cacheKey = computeCacheKey(key, {
···
158
233
cacheType: isFetch ? "fetch" : "cache",
159
234
});
160
235
runWrangler(
161
-
options,
236
+
buildOpts,
162
237
[
163
238
"r2 object put",
164
239
quoteShellMeta(normalizePath(path.join(bucket, cacheKey))),
···
178
253
}
179
254
180
255
async function populateKVIncrementalCache(
181
-
options: BuildOptions,
256
+
buildOpts: BuildOptions,
182
257
config: WranglerConfig,
183
-
populateCacheOptions: PopulateCacheOptions
258
+
populateCacheOptions: PopulateCacheOptions,
259
+
envVars: WorkerEnvVar
184
260
) {
185
261
logger.info("\nPopulating KV incremental cache...");
186
262
···
189
265
throw new Error(`No KV binding ${JSON.stringify(KV_CACHE_BINDING_NAME)} found!`);
190
266
}
191
267
192
-
const envVars = await getEnvFromPlatformProxy(populateCacheOptions);
193
268
const prefix = envVars[KV_CACHE_PREFIX_ENV_NAME];
194
269
195
-
const assets = getCacheAssets(options);
270
+
const assets = getCacheAssets(buildOpts);
196
271
197
272
const chunkSize = Math.max(1, populateCacheOptions.cacheChunkSize ?? 25);
198
273
const totalChunks = Math.ceil(assets.length / chunkSize);
···
200
275
logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`);
201
276
202
277
for (const i of tqdm(Array.from({ length: totalChunks }, (_, i) => i))) {
203
-
const chunkPath = path.join(options.outputDir, "cloudflare", `cache-chunk-${i}.json`);
278
+
const chunkPath = path.join(buildOpts.outputDir, "cloudflare", `cache-chunk-${i}.json`);
204
279
205
280
const kvMapping = assets
206
281
.slice(i * chunkSize, (i + 1) * chunkSize)
···
216
291
writeFileSync(chunkPath, JSON.stringify(kvMapping));
217
292
218
293
runWrangler(
219
-
options,
294
+
buildOpts,
220
295
[
221
296
"kv bulk put",
222
297
quoteShellMeta(chunkPath),
···
238
313
}
239
314
240
315
function populateD1TagCache(
241
-
options: BuildOptions,
316
+
buildOpts: BuildOptions,
242
317
config: WranglerConfig,
243
318
populateCacheOptions: PopulateCacheOptions
244
319
) {
···
250
325
}
251
326
252
327
runWrangler(
253
-
options,
328
+
buildOpts,
254
329
[
255
330
"d1 execute",
256
331
D1_TAG_BINDING_NAME,
···
278
353
);
279
354
280
355
logger.info(`Successfully populated static assets cache`);
281
-
}
282
-
283
-
export async function populateCache(
284
-
options: BuildOptions,
285
-
config: OpenNextConfig,
286
-
wranglerConfig: WranglerConfig,
287
-
populateCacheOptions: PopulateCacheOptions
288
-
) {
289
-
const { incrementalCache, tagCache } = config.default.override ?? {};
290
-
291
-
if (!existsSync(options.outputDir)) {
292
-
logger.error("Unable to populate cache: Open Next build not found");
293
-
process.exit(1);
294
-
}
295
-
296
-
if (!config.dangerous?.disableIncrementalCache && incrementalCache) {
297
-
const name = await resolveCacheName(incrementalCache);
298
-
switch (name) {
299
-
case R2_CACHE_NAME:
300
-
await populateR2IncrementalCache(options, wranglerConfig, populateCacheOptions);
301
-
break;
302
-
case KV_CACHE_NAME:
303
-
await populateKVIncrementalCache(options, wranglerConfig, populateCacheOptions);
304
-
break;
305
-
case STATIC_ASSETS_CACHE_NAME:
306
-
populateStaticAssetsIncrementalCache(options);
307
-
break;
308
-
default:
309
-
logger.info("Incremental cache does not need populating");
310
-
}
311
-
}
312
-
313
-
if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) {
314
-
const name = await resolveCacheName(tagCache);
315
-
switch (name) {
316
-
case D1_TAG_NAME:
317
-
populateD1TagCache(options, wranglerConfig, populateCacheOptions);
318
-
break;
319
-
default:
320
-
logger.info("Tag cache does not need populating");
321
-
}
322
-
}
323
-
}
324
-
325
-
/**
326
-
* Implementation of the `opennextjs-cloudflare populateCache` command.
327
-
*
328
-
* @param args
329
-
*/
330
-
async function populateCacheCommand(
331
-
target: "local" | "remote",
332
-
args: WithWranglerArgs<{ cacheChunkSize?: number }>
333
-
) {
334
-
printHeaders(`populate cache - ${target}`);
335
-
336
-
const { config } = await retrieveCompiledConfig();
337
-
const options = getNormalizedOptions(config);
338
-
339
-
const wranglerConfig = readWranglerConfig(args);
340
-
341
-
await populateCache(options, config, wranglerConfig, {
342
-
target,
343
-
environment: args.env,
344
-
wranglerConfigPath: args.wranglerConfigPath,
345
-
cacheChunkSize: args.cacheChunkSize,
346
-
shouldUsePreviewId: false,
347
-
});
348
356
}
349
357
350
358
/**
+17
-9
packages/cloudflare/src/cli/commands/preview.ts
+17
-9
packages/cloudflare/src/cli/commands/preview.ts
···
1
1
import type yargs from "yargs";
2
2
3
3
import { runWrangler } from "../utils/run-wrangler.js";
4
+
import { getEnvFromPlatformProxy } from "./helpers.js";
4
5
import { populateCache, withPopulateCacheOptions } from "./populate-cache.js";
5
6
import type { WithWranglerArgs } from "./utils.js";
6
7
import {
···
22
23
printHeaders("preview");
23
24
24
25
const { config } = await retrieveCompiledConfig();
25
-
const options = getNormalizedOptions(config);
26
+
const buildOpts = getNormalizedOptions(config);
26
27
27
28
const wranglerConfig = readWranglerConfig(args);
29
+
const envVars = await getEnvFromPlatformProxy(config, buildOpts);
28
30
29
-
await populateCache(options, config, wranglerConfig, {
30
-
target: args.remote ? "remote" : "local",
31
-
environment: args.env,
32
-
wranglerConfigPath: args.wranglerConfigPath,
33
-
cacheChunkSize: args.cacheChunkSize,
34
-
shouldUsePreviewId: args.remote,
35
-
});
31
+
await populateCache(
32
+
buildOpts,
33
+
config,
34
+
wranglerConfig,
35
+
{
36
+
target: args.remote ? "remote" : "local",
37
+
environment: args.env,
38
+
wranglerConfigPath: args.wranglerConfigPath,
39
+
cacheChunkSize: args.cacheChunkSize,
40
+
shouldUsePreviewId: args.remote,
41
+
},
42
+
envVars
43
+
);
36
44
37
-
runWrangler(options, ["dev", ...args.wranglerArgs], { logging: "all" });
45
+
runWrangler(buildOpts, ["dev", ...args.wranglerArgs], { logging: "all" });
38
46
}
39
47
40
48
/**
+3
-3
packages/cloudflare/src/cli/commands/skew-protection.ts
+3
-3
packages/cloudflare/src/cli/commands/skew-protection.ts
···
43
43
/**
44
44
* Compute the deployment mapping for a deployment.
45
45
*
46
-
* @param options Build options
46
+
* @param buildOpts Build options
47
47
* @param config OpenNext config
48
48
* @param workerEnvVars Worker Environment variables (taken from the wrangler config files)
49
49
* @returns Deployment mapping or undefined
50
50
*/
51
51
export async function getDeploymentMapping(
52
-
options: BuildOptions,
52
+
buildOpts: BuildOptions,
53
53
config: OpenNextConfig,
54
54
workerEnvVars: WorkerEnvVar
55
55
): Promise<Record<string, string> | undefined> {
···
62
62
// in the wrangler config files
63
63
const envVars = { ...workerEnvVars, ...process.env };
64
64
65
-
const nextConfig = loadConfig(path.join(options.appBuildOutputPath, ".next"));
65
+
const nextConfig = loadConfig(path.join(buildOpts.appBuildOutputPath, ".next"));
66
66
const deploymentId = nextConfig.deploymentId;
67
67
68
68
if (!deploymentId) {
+23
-14
packages/cloudflare/src/cli/commands/upload.ts
+23
-14
packages/cloudflare/src/cli/commands/upload.ts
···
23
23
printHeaders("upload");
24
24
25
25
const { config } = await retrieveCompiledConfig();
26
-
const options = getNormalizedOptions(config);
26
+
const buildOpts = getNormalizedOptions(config);
27
27
28
28
const wranglerConfig = readWranglerConfig(args);
29
29
30
-
const envVars = await getEnvFromPlatformProxy({
31
-
configPath: args.wranglerConfigPath,
32
-
environment: args.env,
33
-
});
30
+
const envVars = await getEnvFromPlatformProxy(
31
+
{
32
+
configPath: args.wranglerConfigPath,
33
+
environment: args.env,
34
+
},
35
+
buildOpts
36
+
);
34
37
35
-
const deploymentMapping = await getDeploymentMapping(options, config, envVars);
38
+
const deploymentMapping = await getDeploymentMapping(buildOpts, config, envVars);
36
39
37
-
await populateCache(options, config, wranglerConfig, {
38
-
target: "remote",
39
-
environment: args.env,
40
-
wranglerConfigPath: args.wranglerConfigPath,
41
-
cacheChunkSize: args.cacheChunkSize,
42
-
shouldUsePreviewId: false,
43
-
});
40
+
await populateCache(
41
+
buildOpts,
42
+
config,
43
+
wranglerConfig,
44
+
{
45
+
target: "remote",
46
+
environment: args.env,
47
+
wranglerConfigPath: args.wranglerConfigPath,
48
+
cacheChunkSize: args.cacheChunkSize,
49
+
shouldUsePreviewId: false,
50
+
},
51
+
envVars
52
+
);
44
53
45
54
runWrangler(
46
-
options,
55
+
buildOpts,
47
56
[
48
57
"versions upload",
49
58
...args.wranglerArgs,
+5
pnpm-lock.yaml
+5
pnpm-lock.yaml
···
3307
3307
'@next/env@15.4.5':
3308
3308
resolution: {integrity: sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ==}
3309
3309
3310
+
'@next/env@15.5.4':
3311
+
resolution: {integrity: sha512-27SQhYp5QryzIT5uO8hq99C69eLQ7qkzkDPsk3N+GuS2XgOgoYEeOav7Pf8Tn4drECOVDsDg8oj+/DVy8qQL2A==}
3312
+
3310
3313
'@next/eslint-plugin-next@14.2.14':
3311
3314
resolution: {integrity: sha512-kV+OsZ56xhj0rnTn6HegyTGkoa16Mxjrpk7pjWumyB2P8JVQb8S9qtkjy/ye0GnTr4JWtWG4x/2qN40lKZ3iVQ==}
3312
3315
···
12609
12612
'@next/env@15.4.2-canary.29': {}
12610
12613
12611
12614
'@next/env@15.4.5': {}
12615
+
12616
+
'@next/env@15.5.4': {}
12612
12617
12613
12618
'@next/eslint-plugin-next@14.2.14':
12614
12619
dependencies: