feat: add override options for s3 uploader.

This commit is contained in:
Yufan Sheng 2024-06-28 02:33:46 +08:00
parent 815d12e137
commit 61e25c17a1
Signed by: syhily
GPG Key ID: 9D18A22A7DCD5A9B
7 changed files with 102 additions and 81 deletions

View File

@ -176,19 +176,19 @@ Or you can host on your own machine. Use [Dockerfile](./Dockerfile) to build an
The comment system is leverage the [Artalk](https://artalk.js.org), a self-hosted comment system.
You should host it on your own machine.
## TODO Checklist
## Short-Term TODO Checklist
- [ ] Move the S3 uploader as a separate npm package. Use OpenDAL.
- [ ] Move comments into the new Astro content layer.
- [ ] Check article grammar errors by using ChatGPT. Remain **54** posts.
- [ ] Add music to the articles. Remain **54** posts.
- [ ] Slide share components integration.
### Comments TODO Checklist
## Long-Term TODO Checklist
- [ ] Use self-developed comment solution.
- [ ] Support modification after commenting in 60 minutes even if you have refreshed the page.
- [ ] Use self-developed duoshuo as the comment's solution.
### Long-Term Goals
- [ ] Support login into the blog for managing the comments.
- [ ] Slide share components integration.
- [ ] Add han.js support for better typography.
- [ ] Drop bootstrap, in favor of tailwind css.
@ -228,6 +228,7 @@ The source codes used from third party projects are:
from [yuaanlin/yual.in](https://github.com/yuaanlin/yual.in/blob/main/pages/og_image/%5Bslug%5D.tsx)
with [permission](licenses/LICENSE.yuaanlin.jpg)
- [images.ts](src/helpers/images.ts)
and [config.ts](src/content/config.ts)
from [zce/velite](https://github.com/zce/velite/blob/main/src/assets.ts)
with [license](licenses/LICENSE.zce.txt)
- [images.ts](src/helpers/images.ts)

View File

@ -1,5 +1,5 @@
{
"$schema": "https://biomejs.dev/schemas/1.8.2/schema.json",
"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
"formatter": {
"enabled": false
},

98
package-lock.json generated
View File

@ -25,9 +25,9 @@
"devDependencies": {
"@astrojs/check": "^0.7.0",
"@aws-sdk/client-s3": "^3.600.0",
"@biomejs/biome": "^1.8.2",
"@biomejs/biome": "^1.8.3",
"@napi-rs/canvas": "^0.1.53",
"@types/lodash": "^4.17.5",
"@types/lodash": "^4.17.6",
"@types/luxon": "^3.4.2",
"@types/node": "^20.14.9",
"@types/pg": "^8.11.6",
@ -1586,9 +1586,9 @@
}
},
"node_modules/@biomejs/biome": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.8.2.tgz",
"integrity": "sha512-XafCzLgs0xbH0bCjYKxQ63ig2V86fZQMq1jiy5pyLToWk9aHxA8GAUxyBtklPHtPYZPGEPOYglQHj4jyfUp+Iw==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.8.3.tgz",
"integrity": "sha512-/uUV3MV+vyAczO+vKrPdOW0Iaet7UnJMU4bNMinggGJTAnBPjCoLEYcyYtYHNnUNYlv4xZMH6hVIQCAozq8d5w==",
"dev": true,
"hasInstallScript": true,
"license": "MIT OR Apache-2.0",
@ -1603,20 +1603,20 @@
"url": "https://opencollective.com/biome"
},
"optionalDependencies": {
"@biomejs/cli-darwin-arm64": "1.8.2",
"@biomejs/cli-darwin-x64": "1.8.2",
"@biomejs/cli-linux-arm64": "1.8.2",
"@biomejs/cli-linux-arm64-musl": "1.8.2",
"@biomejs/cli-linux-x64": "1.8.2",
"@biomejs/cli-linux-x64-musl": "1.8.2",
"@biomejs/cli-win32-arm64": "1.8.2",
"@biomejs/cli-win32-x64": "1.8.2"
"@biomejs/cli-darwin-arm64": "1.8.3",
"@biomejs/cli-darwin-x64": "1.8.3",
"@biomejs/cli-linux-arm64": "1.8.3",
"@biomejs/cli-linux-arm64-musl": "1.8.3",
"@biomejs/cli-linux-x64": "1.8.3",
"@biomejs/cli-linux-x64-musl": "1.8.3",
"@biomejs/cli-win32-arm64": "1.8.3",
"@biomejs/cli-win32-x64": "1.8.3"
}
},
"node_modules/@biomejs/cli-darwin-arm64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.8.2.tgz",
"integrity": "sha512-l9msLsTcSIAPqMsPIhodQmb50sEfaXPLQ0YW4cdj6INmd8iaOh/V9NceQb2366vACTJgcWDQ2RzlvURek1T68g==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.8.3.tgz",
"integrity": "sha512-9DYOjclFpKrH/m1Oz75SSExR8VKvNSSsLnVIqdnKexj6NwmiMlKk94Wa1kZEdv6MCOHGHgyyoV57Cw8WzL5n3A==",
"cpu": [
"arm64"
],
@ -1631,9 +1631,9 @@
}
},
"node_modules/@biomejs/cli-darwin-x64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.8.2.tgz",
"integrity": "sha512-Fc4y/FuIxRSiB3TJ+y27vFDE/HJt4QgBuymktsIKEcBZvnKfsRjxvzVDunccRn4xbKgepnp+fn6BoS+ZIg/I3Q==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.8.3.tgz",
"integrity": "sha512-UeW44L/AtbmOF7KXLCoM+9PSgPo0IDcyEUfIoOXYeANaNXXf9mLUwV1GeF2OWjyic5zj6CnAJ9uzk2LT3v/wAw==",
"cpu": [
"x64"
],
@ -1648,9 +1648,9 @@
}
},
"node_modules/@biomejs/cli-linux-arm64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.8.2.tgz",
"integrity": "sha512-Q99qwP0qibkZxm2kfnt37OxeIlliDYf5ogi3zX9ij2DULzc+KtPA9Uj0wCljcJofOBsBYaHc7597Q+Bf/251ww==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.8.3.tgz",
"integrity": "sha512-fed2ji8s+I/m8upWpTJGanqiJ0rnlHOK3DdxsyVLZQ8ClY6qLuPc9uehCREBifRJLl/iJyQpHIRufLDeotsPtw==",
"cpu": [
"arm64"
],
@ -1665,9 +1665,9 @@
}
},
"node_modules/@biomejs/cli-linux-arm64-musl": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.8.2.tgz",
"integrity": "sha512-WpT41QJJvkZa1eZq0WmD513zkC6AYaMI39HJKmKeiUeX2NZirG+bxv1YRDhqkns1NbBqo3+qrJqBkPmOW+xAVA==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.8.3.tgz",
"integrity": "sha512-9yjUfOFN7wrYsXt/T/gEWfvVxKlnh3yBpnScw98IF+oOeCYb5/b/+K7YNqKROV2i1DlMjg9g/EcN9wvj+NkMuQ==",
"cpu": [
"arm64"
],
@ -1682,9 +1682,9 @@
}
},
"node_modules/@biomejs/cli-linux-x64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.8.2.tgz",
"integrity": "sha512-bjhhUVFchFid2gOjrvBe4fg8BShcpyFQTHuB/QQnfGxs1ddrGP30yq3fHfc6S6MoCcz9Tjd3Zzq1EfWfyy5iHA==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.8.3.tgz",
"integrity": "sha512-I8G2QmuE1teISyT8ie1HXsjFRz9L1m5n83U1O6m30Kw+kPMPSKjag6QGUn+sXT8V+XWIZxFFBoTDEDZW2KPDDw==",
"cpu": [
"x64"
],
@ -1699,9 +1699,9 @@
}
},
"node_modules/@biomejs/cli-linux-x64-musl": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.8.2.tgz",
"integrity": "sha512-rk1Wj4d3LIlAlIAS1m2jlyfOjkNbuY1lfwKvWIAeZC51yDMzwhRD7cReE5PE+jqLDtq60PX38hDPeKd7nA1S6A==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.8.3.tgz",
"integrity": "sha512-UHrGJX7PrKMKzPGoEsooKC9jXJMa28TUSMjcIlbDnIO4EAavCoVmNQaIuUSH0Ls2mpGMwUIf+aZJv657zfWWjA==",
"cpu": [
"x64"
],
@ -1716,9 +1716,9 @@
}
},
"node_modules/@biomejs/cli-win32-arm64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.8.2.tgz",
"integrity": "sha512-EUbqmCmNWT5xhnxHrCAEBzJB1AnLqxTYoRjlxiCMzGvsy5jQzhCanJ8CT9kNsApW3pfPWBWkoTa7qrwWmwnEGA==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.8.3.tgz",
"integrity": "sha512-J+Hu9WvrBevfy06eU1Na0lpc7uR9tibm9maHynLIoAjLZpQU3IW+OKHUtyL8p6/3pT2Ju5t5emReeIS2SAxhkQ==",
"cpu": [
"arm64"
],
@ -1733,9 +1733,9 @@
}
},
"node_modules/@biomejs/cli-win32-x64": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.8.2.tgz",
"integrity": "sha512-n9H5oRUCk1uNezMgyJh9+hZdtfD8PXLLeq8DUzTycIhl0I1BulIoZ/uxWgRVDFDwAR1JHu1AykISCRFNGnc4iA==",
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.8.3.tgz",
"integrity": "sha512-/PJ59vA1pnQeKahemaQf4Nyj7IKUvGQSc3Ze1uIGi+Wvr1xF7rGobSrAAG01T/gUDG21vkDsZYM03NAmPiVkqg==",
"cpu": [
"x64"
],
@ -4058,9 +4058,9 @@
}
},
"node_modules/@types/lodash": {
"version": "4.17.5",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz",
"integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==",
"version": "4.17.6",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.6.tgz",
"integrity": "sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==",
"dev": true,
"license": "MIT"
},
@ -4706,9 +4706,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001637",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001637.tgz",
"integrity": "sha512-1x0qRI1mD1o9e+7mBI7XtzFAP4XszbHaVWsMiGbSPLYekKTJF7K+FNk6AsXH4sUpc+qrsI3pVgf1Jdl/uGkuSQ==",
"version": "1.0.30001638",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001638.tgz",
"integrity": "sha512-5SuJUJ7cZnhPpeLHaH0c/HPAnAHZvS6ElWyHK9GSIbVOQABLzowiI2pjmpvZ1WEbkyz46iFd4UXlOHR5SqgfMQ==",
"funding": [
{
"type": "opencollective",
@ -9701,9 +9701,9 @@
}
},
"node_modules/tsconfck": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.0.tgz",
"integrity": "sha512-CMjc5zMnyAjcS9sPLytrbFmj89st2g+JYtY/c02ug4Q+CZaAtCgbyviI0n1YvjZE/pzoc6FbNsINS13DOL1B9w==",
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.1.tgz",
"integrity": "sha512-00eoI6WY57SvZEVjm13stEVE90VkEdJAFGgpFLTsZbJyW/LwFQ7uQxJHWpZ2hzSWgCPKc9AnBnNP+0X7o3hAmQ==",
"license": "MIT",
"bin": {
"tsconfck": "bin/tsconfck.js"
@ -10044,9 +10044,9 @@
}
},
"node_modules/vite": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.3.1.tgz",
"integrity": "sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==",
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.3.2.tgz",
"integrity": "sha512-6lA7OBHBlXUxiJxbO5aAY2fsHHzDr1q7DvXYnyZycRs2Dz+dXBWuhpWHvmljTRTpQC2uvGmUFFkSHF2vGo90MA==",
"license": "MIT",
"dependencies": {
"esbuild": "^0.21.3",

View File

@ -57,9 +57,9 @@
"devDependencies": {
"@astrojs/check": "^0.7.0",
"@aws-sdk/client-s3": "^3.600.0",
"@biomejs/biome": "^1.8.2",
"@biomejs/biome": "^1.8.3",
"@napi-rs/canvas": "^0.1.53",
"@types/lodash": "^4.17.5",
"@types/lodash": "^4.17.6",
"@types/luxon": "^3.4.2",
"@types/node": "^20.14.9",
"@types/pg": "^8.11.6",

View File

@ -1,4 +1,5 @@
import {
DeleteObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
NoSuchBucket,
@ -19,6 +20,9 @@ const S3Options = z
paths: z.array(z.string()).min(1),
// Whether to keep the original files after uploading.
keep: z.boolean().default(false),
// Whether to override the existing files on S3.
// It will be override only when the content-length don't match the file size by default.
override: z.boolean().default(false),
// The S3 region, set it if you use AWS S3 service.
region: z.string().min(1).default('auto'),
// The endpoint, set it if you use 3rd-party S3 service.
@ -54,21 +58,33 @@ const parseOptions = (opts: z.input<typeof S3Options>, logger: AstroIntegrationL
}
};
class Context {
class Uploader {
private client: S3Client;
private bucket: string;
private root: string;
private options: z.infer<typeof S3Options>;
constructor(client: S3Client, bucket: string, root: string) {
constructor(client: S3Client, options: z.infer<typeof S3Options>) {
this.client = client;
this.bucket = bucket;
this.root = root;
this.options = options;
}
async isExist(key: string): Promise<boolean> {
const headCmd = new HeadObjectCommand({ Bucket: this.bucket, Key: path.posix.join(this.root, key) });
private key(key: string): string {
return path.posix.join(this.options.root, key);
}
private async delete(key: string): Promise<void> {
const deleteCmd = new DeleteObjectCommand({ Bucket: this.options.bucket, Key: this.key(key) });
await this.client.send(deleteCmd);
}
async isExist(key: string, size: number): Promise<boolean> {
const headCmd = new HeadObjectCommand({ Bucket: this.options.bucket, Key: this.key(key) });
try {
await this.client.send(headCmd);
const { ContentLength } = await this.client.send(headCmd);
// The file checksum should be uploaded with file. So we only check content length here.
if (this.options.override || (ContentLength !== undefined && ContentLength !== size)) {
await this.delete(key);
return false;
}
return true;
} catch (error) {
if (error instanceof NotFound) {
@ -81,8 +97,8 @@ class Context {
async write(key: string, body: Buffer) {
const contentType = mime.getType(key);
const putCmd = new PutObjectCommand({
Bucket: this.bucket,
Key: path.posix.join(this.root, key),
Bucket: this.options.bucket,
Key: this.key(key),
Body: body,
ContentType: contentType === null ? undefined : contentType,
});
@ -95,7 +111,8 @@ export const uploader = (opts: z.input<typeof S3Options>): AstroIntegration => (
name: 'S3 Uploader',
hooks: {
'astro:build:done': async ({ dir, logger }: { dir: URL; logger: AstroIntegrationLogger }) => {
const { paths, keep, region, endpoint, bucket, root, accessKey, secretAccessKey } = parseOptions(opts, logger);
const options = parseOptions(opts, logger);
const { paths, keep, region, endpoint, bucket, accessKey, secretAccessKey } = options;
const client = new S3Client({
region: region,
endpoint: endpoint,
@ -119,9 +136,9 @@ export const uploader = (opts: z.input<typeof S3Options>): AstroIntegration => (
logger.info(`Start to upload static files in dir ${paths} to S3 compatible backend.`);
const context = new Context(client, bucket, root);
const uploader = new Uploader(client, options);
for (const current of paths) {
await uploadFile(context, logger, current, dir.pathname);
await uploadFile(uploader, logger, current, dir.pathname);
if (!keep) {
rimrafSync(path.join(dir.pathname, current));
}
@ -137,18 +154,19 @@ const normalizePath = (current: string): string => {
return current.includes(path.win32.sep) ? current.split(path.win32.sep).join(path.posix.sep) : current;
};
const uploadFile = async (context: Context, logger: AstroIntegrationLogger, current: string, root: string) => {
const uploadFile = async (uploader: Uploader, logger: AstroIntegrationLogger, current: string, root: string) => {
const filePath = path.join(root, current);
const isFile = !fs.statSync(filePath).isDirectory();
const fileStats = fs.statSync(filePath);
const isFile = !fileStats.isDirectory();
const uploadAction = async (key: string) => {
logger.info(`Start to upload file: ${key}`);
const body = fs.readFileSync(filePath);
await context.write(key, body);
await uploader.write(key, body);
};
if (isFile) {
const key = normalizePath(current);
if (await context.isExist(key)) {
if (await uploader.isExist(key, fileStats.size)) {
logger.info(`${key} exists on backend, skip.`);
} else {
await uploadAction(key);
@ -159,7 +177,7 @@ const uploadFile = async (context: Context, logger: AstroIntegrationLogger, curr
if (next.startsWith('.')) {
continue;
}
await uploadFile(context, logger, path.join(current, next), root);
await uploadFile(uploader, logger, path.join(current, next), root);
}
}
};

View File

@ -12,5 +12,5 @@ require (
require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/image v0.17.0 // indirect
golang.org/x/image v0.18.0 // indirect
)

View File

@ -21,6 +21,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=