From b6f3bc34215609f0cb32035b155c9d2237b7e816 Mon Sep 17 00:00:00 2001 From: Yufan Sheng Date: Fri, 28 Jun 2024 02:33:46 +0800 Subject: [PATCH] feat: add override options for s3 uploader. --- README.md | 19 ++++----- biome.json | 2 +- package-lock.json | 98 ++++++++++++++++++++++----------------------- package.json | 4 +- plugins/uploader.ts | 56 +++++++++++++++++--------- tools/go.mod | 2 +- tools/go.sum | 2 + 7 files changed, 102 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index f15740e..f6a7c76 100644 --- a/README.md +++ b/README.md @@ -176,19 +176,19 @@ Or you can host on your own machine. Use [Dockerfile](./Dockerfile) to build an The comment system is leverage the [Artalk](https://artalk.js.org), a self-hosted comment system. You should host it on your own machine. -## TODO Checklist +## Short-Term TODO Checklist +- [ ] Move the S3 uploader as a separate npm package. Use OpenDAL. +- [ ] Move comments into the new Astro content layer. - [ ] Check article grammar errors by using ChatGPT. Remain **54** posts. - [ ] Add music to the articles. Remain **54** posts. + +## Long-Term TODO Checklist + +- [ ] Use self-developed comment solution. + - [ ] Support modification after commenting in 60 minutes even if you have refreshed the page. + - [ ] Support login into the blog for managing the comments. - [ ] Slide share components integration. - -### Comments TODO Checklist - -- [ ] Support modification after commenting in 60 minutes even if you have refreshed the page. -- [ ] Use self-developed duoshuo as the comment's solution. - -### Long-Term Goals - - [ ] Add han.js support for better typography. - [ ] Drop bootstrap, in favor of tailwind css. @@ -228,6 +228,7 @@ The source codes used from third party projects are: from [yuaanlin/yual.in](https://github.com/yuaanlin/yual.in/blob/main/pages/og_image/%5Bslug%5D.tsx) with [permission](licenses/LICENSE.yuaanlin.jpg) - [images.ts](src/helpers/images.ts) + and [config.ts](src/content/config.ts) from [zce/velite](https://github.com/zce/velite/blob/main/src/assets.ts) with [license](licenses/LICENSE.zce.txt) - [images.ts](src/helpers/images.ts) diff --git a/biome.json b/biome.json index e594ef1..a4ed198 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/1.8.2/schema.json", + "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", "formatter": { "enabled": false }, diff --git a/package-lock.json b/package-lock.json index 69a17b9..00cf6e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,9 +25,9 @@ "devDependencies": { "@astrojs/check": "^0.7.0", "@aws-sdk/client-s3": "^3.600.0", - "@biomejs/biome": "^1.8.2", + "@biomejs/biome": "^1.8.3", "@napi-rs/canvas": "^0.1.53", - "@types/lodash": "^4.17.5", + "@types/lodash": "^4.17.6", "@types/luxon": "^3.4.2", "@types/node": "^20.14.9", "@types/pg": "^8.11.6", @@ -1586,9 +1586,9 @@ } }, "node_modules/@biomejs/biome": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.8.2.tgz", - "integrity": "sha512-XafCzLgs0xbH0bCjYKxQ63ig2V86fZQMq1jiy5pyLToWk9aHxA8GAUxyBtklPHtPYZPGEPOYglQHj4jyfUp+Iw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.8.3.tgz", + "integrity": "sha512-/uUV3MV+vyAczO+vKrPdOW0Iaet7UnJMU4bNMinggGJTAnBPjCoLEYcyYtYHNnUNYlv4xZMH6hVIQCAozq8d5w==", "dev": true, "hasInstallScript": true, "license": "MIT OR Apache-2.0", @@ -1603,20 +1603,20 @@ "url": "https://opencollective.com/biome" }, "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "1.8.2", - "@biomejs/cli-darwin-x64": "1.8.2", - "@biomejs/cli-linux-arm64": "1.8.2", - "@biomejs/cli-linux-arm64-musl": "1.8.2", - "@biomejs/cli-linux-x64": "1.8.2", - "@biomejs/cli-linux-x64-musl": "1.8.2", - "@biomejs/cli-win32-arm64": "1.8.2", - "@biomejs/cli-win32-x64": "1.8.2" + "@biomejs/cli-darwin-arm64": "1.8.3", + "@biomejs/cli-darwin-x64": "1.8.3", + "@biomejs/cli-linux-arm64": "1.8.3", + "@biomejs/cli-linux-arm64-musl": "1.8.3", + "@biomejs/cli-linux-x64": "1.8.3", + "@biomejs/cli-linux-x64-musl": "1.8.3", + "@biomejs/cli-win32-arm64": "1.8.3", + "@biomejs/cli-win32-x64": "1.8.3" } }, "node_modules/@biomejs/cli-darwin-arm64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.8.2.tgz", - "integrity": "sha512-l9msLsTcSIAPqMsPIhodQmb50sEfaXPLQ0YW4cdj6INmd8iaOh/V9NceQb2366vACTJgcWDQ2RzlvURek1T68g==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.8.3.tgz", + "integrity": "sha512-9DYOjclFpKrH/m1Oz75SSExR8VKvNSSsLnVIqdnKexj6NwmiMlKk94Wa1kZEdv6MCOHGHgyyoV57Cw8WzL5n3A==", "cpu": [ "arm64" ], @@ -1631,9 +1631,9 @@ } }, "node_modules/@biomejs/cli-darwin-x64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.8.2.tgz", - "integrity": "sha512-Fc4y/FuIxRSiB3TJ+y27vFDE/HJt4QgBuymktsIKEcBZvnKfsRjxvzVDunccRn4xbKgepnp+fn6BoS+ZIg/I3Q==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.8.3.tgz", + "integrity": "sha512-UeW44L/AtbmOF7KXLCoM+9PSgPo0IDcyEUfIoOXYeANaNXXf9mLUwV1GeF2OWjyic5zj6CnAJ9uzk2LT3v/wAw==", "cpu": [ "x64" ], @@ -1648,9 +1648,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.8.2.tgz", - "integrity": "sha512-Q99qwP0qibkZxm2kfnt37OxeIlliDYf5ogi3zX9ij2DULzc+KtPA9Uj0wCljcJofOBsBYaHc7597Q+Bf/251ww==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.8.3.tgz", + "integrity": "sha512-fed2ji8s+I/m8upWpTJGanqiJ0rnlHOK3DdxsyVLZQ8ClY6qLuPc9uehCREBifRJLl/iJyQpHIRufLDeotsPtw==", "cpu": [ "arm64" ], @@ -1665,9 +1665,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.8.2.tgz", - "integrity": "sha512-WpT41QJJvkZa1eZq0WmD513zkC6AYaMI39HJKmKeiUeX2NZirG+bxv1YRDhqkns1NbBqo3+qrJqBkPmOW+xAVA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.8.3.tgz", + "integrity": "sha512-9yjUfOFN7wrYsXt/T/gEWfvVxKlnh3yBpnScw98IF+oOeCYb5/b/+K7YNqKROV2i1DlMjg9g/EcN9wvj+NkMuQ==", "cpu": [ "arm64" ], @@ -1682,9 +1682,9 @@ } }, "node_modules/@biomejs/cli-linux-x64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.8.2.tgz", - "integrity": "sha512-bjhhUVFchFid2gOjrvBe4fg8BShcpyFQTHuB/QQnfGxs1ddrGP30yq3fHfc6S6MoCcz9Tjd3Zzq1EfWfyy5iHA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.8.3.tgz", + "integrity": "sha512-I8G2QmuE1teISyT8ie1HXsjFRz9L1m5n83U1O6m30Kw+kPMPSKjag6QGUn+sXT8V+XWIZxFFBoTDEDZW2KPDDw==", "cpu": [ "x64" ], @@ -1699,9 +1699,9 @@ } }, "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.8.2.tgz", - "integrity": "sha512-rk1Wj4d3LIlAlIAS1m2jlyfOjkNbuY1lfwKvWIAeZC51yDMzwhRD7cReE5PE+jqLDtq60PX38hDPeKd7nA1S6A==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.8.3.tgz", + "integrity": "sha512-UHrGJX7PrKMKzPGoEsooKC9jXJMa28TUSMjcIlbDnIO4EAavCoVmNQaIuUSH0Ls2mpGMwUIf+aZJv657zfWWjA==", "cpu": [ "x64" ], @@ -1716,9 +1716,9 @@ } }, "node_modules/@biomejs/cli-win32-arm64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.8.2.tgz", - "integrity": "sha512-EUbqmCmNWT5xhnxHrCAEBzJB1AnLqxTYoRjlxiCMzGvsy5jQzhCanJ8CT9kNsApW3pfPWBWkoTa7qrwWmwnEGA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.8.3.tgz", + "integrity": "sha512-J+Hu9WvrBevfy06eU1Na0lpc7uR9tibm9maHynLIoAjLZpQU3IW+OKHUtyL8p6/3pT2Ju5t5emReeIS2SAxhkQ==", "cpu": [ "arm64" ], @@ -1733,9 +1733,9 @@ } }, "node_modules/@biomejs/cli-win32-x64": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.8.2.tgz", - "integrity": "sha512-n9H5oRUCk1uNezMgyJh9+hZdtfD8PXLLeq8DUzTycIhl0I1BulIoZ/uxWgRVDFDwAR1JHu1AykISCRFNGnc4iA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.8.3.tgz", + "integrity": "sha512-/PJ59vA1pnQeKahemaQf4Nyj7IKUvGQSc3Ze1uIGi+Wvr1xF7rGobSrAAG01T/gUDG21vkDsZYM03NAmPiVkqg==", "cpu": [ "x64" ], @@ -4058,9 +4058,9 @@ } }, "node_modules/@types/lodash": { - "version": "4.17.5", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz", - "integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==", + "version": "4.17.6", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.6.tgz", + "integrity": "sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==", "dev": true, "license": "MIT" }, @@ -4706,9 +4706,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001637", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001637.tgz", - "integrity": "sha512-1x0qRI1mD1o9e+7mBI7XtzFAP4XszbHaVWsMiGbSPLYekKTJF7K+FNk6AsXH4sUpc+qrsI3pVgf1Jdl/uGkuSQ==", + "version": "1.0.30001638", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001638.tgz", + "integrity": "sha512-5SuJUJ7cZnhPpeLHaH0c/HPAnAHZvS6ElWyHK9GSIbVOQABLzowiI2pjmpvZ1WEbkyz46iFd4UXlOHR5SqgfMQ==", "funding": [ { "type": "opencollective", @@ -9701,9 +9701,9 @@ } }, "node_modules/tsconfck": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.0.tgz", - "integrity": "sha512-CMjc5zMnyAjcS9sPLytrbFmj89st2g+JYtY/c02ug4Q+CZaAtCgbyviI0n1YvjZE/pzoc6FbNsINS13DOL1B9w==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.1.tgz", + "integrity": "sha512-00eoI6WY57SvZEVjm13stEVE90VkEdJAFGgpFLTsZbJyW/LwFQ7uQxJHWpZ2hzSWgCPKc9AnBnNP+0X7o3hAmQ==", "license": "MIT", "bin": { "tsconfck": "bin/tsconfck.js" @@ -10044,9 +10044,9 @@ } }, "node_modules/vite": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.1.tgz", - "integrity": "sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.2.tgz", + "integrity": "sha512-6lA7OBHBlXUxiJxbO5aAY2fsHHzDr1q7DvXYnyZycRs2Dz+dXBWuhpWHvmljTRTpQC2uvGmUFFkSHF2vGo90MA==", "license": "MIT", "dependencies": { "esbuild": "^0.21.3", diff --git a/package.json b/package.json index 65d4a7d..9823854 100644 --- a/package.json +++ b/package.json @@ -57,9 +57,9 @@ "devDependencies": { "@astrojs/check": "^0.7.0", "@aws-sdk/client-s3": "^3.600.0", - "@biomejs/biome": "^1.8.2", + "@biomejs/biome": "^1.8.3", "@napi-rs/canvas": "^0.1.53", - "@types/lodash": "^4.17.5", + "@types/lodash": "^4.17.6", "@types/luxon": "^3.4.2", "@types/node": "^20.14.9", "@types/pg": "^8.11.6", diff --git a/plugins/uploader.ts b/plugins/uploader.ts index f312486..1794aaf 100644 --- a/plugins/uploader.ts +++ b/plugins/uploader.ts @@ -1,4 +1,5 @@ import { + DeleteObjectCommand, HeadBucketCommand, HeadObjectCommand, NoSuchBucket, @@ -19,6 +20,9 @@ const S3Options = z paths: z.array(z.string()).min(1), // Whether to keep the original files after uploading. keep: z.boolean().default(false), + // Whether to override the existing files on S3. + // It will be override only when the content-length don't match the file size by default. + override: z.boolean().default(false), // The S3 region, set it if you use AWS S3 service. region: z.string().min(1).default('auto'), // The endpoint, set it if you use 3rd-party S3 service. @@ -54,21 +58,33 @@ const parseOptions = (opts: z.input, logger: AstroIntegrationL } }; -class Context { +class Uploader { private client: S3Client; - private bucket: string; - private root: string; + private options: z.infer; - constructor(client: S3Client, bucket: string, root: string) { + constructor(client: S3Client, options: z.infer) { this.client = client; - this.bucket = bucket; - this.root = root; + this.options = options; } - async isExist(key: string): Promise { - const headCmd = new HeadObjectCommand({ Bucket: this.bucket, Key: path.posix.join(this.root, key) }); + private key(key: string): string { + return path.posix.join(this.options.root, key); + } + + private async delete(key: string): Promise { + const deleteCmd = new DeleteObjectCommand({ Bucket: this.options.bucket, Key: this.key(key) }); + await this.client.send(deleteCmd); + } + + async isExist(key: string, size: number): Promise { + const headCmd = new HeadObjectCommand({ Bucket: this.options.bucket, Key: this.key(key) }); try { - await this.client.send(headCmd); + const { ContentLength } = await this.client.send(headCmd); + // The file checksum should be uploaded with file. So we only check content length here. + if (this.options.override || (ContentLength !== undefined && ContentLength !== size)) { + await this.delete(key); + return false; + } return true; } catch (error) { if (error instanceof NotFound) { @@ -81,8 +97,8 @@ class Context { async write(key: string, body: Buffer) { const contentType = mime.getType(key); const putCmd = new PutObjectCommand({ - Bucket: this.bucket, - Key: path.posix.join(this.root, key), + Bucket: this.options.bucket, + Key: this.key(key), Body: body, ContentType: contentType === null ? undefined : contentType, }); @@ -95,7 +111,8 @@ export const uploader = (opts: z.input): AstroIntegration => ( name: 'S3 Uploader', hooks: { 'astro:build:done': async ({ dir, logger }: { dir: URL; logger: AstroIntegrationLogger }) => { - const { paths, keep, region, endpoint, bucket, root, accessKey, secretAccessKey } = parseOptions(opts, logger); + const options = parseOptions(opts, logger); + const { paths, keep, region, endpoint, bucket, accessKey, secretAccessKey } = options; const client = new S3Client({ region: region, endpoint: endpoint, @@ -119,9 +136,9 @@ export const uploader = (opts: z.input): AstroIntegration => ( logger.info(`Start to upload static files in dir ${paths} to S3 compatible backend.`); - const context = new Context(client, bucket, root); + const uploader = new Uploader(client, options); for (const current of paths) { - await uploadFile(context, logger, current, dir.pathname); + await uploadFile(uploader, logger, current, dir.pathname); if (!keep) { rimrafSync(path.join(dir.pathname, current)); } @@ -137,18 +154,19 @@ const normalizePath = (current: string): string => { return current.includes(path.win32.sep) ? current.split(path.win32.sep).join(path.posix.sep) : current; }; -const uploadFile = async (context: Context, logger: AstroIntegrationLogger, current: string, root: string) => { +const uploadFile = async (uploader: Uploader, logger: AstroIntegrationLogger, current: string, root: string) => { const filePath = path.join(root, current); - const isFile = !fs.statSync(filePath).isDirectory(); + const fileStats = fs.statSync(filePath); + const isFile = !fileStats.isDirectory(); const uploadAction = async (key: string) => { logger.info(`Start to upload file: ${key}`); const body = fs.readFileSync(filePath); - await context.write(key, body); + await uploader.write(key, body); }; if (isFile) { const key = normalizePath(current); - if (await context.isExist(key)) { + if (await uploader.isExist(key, fileStats.size)) { logger.info(`${key} exists on backend, skip.`); } else { await uploadAction(key); @@ -159,7 +177,7 @@ const uploadFile = async (context: Context, logger: AstroIntegrationLogger, curr if (next.startsWith('.')) { continue; } - await uploadFile(context, logger, path.join(current, next), root); + await uploadFile(uploader, logger, path.join(current, next), root); } } }; diff --git a/tools/go.mod b/tools/go.mod index 84e9dab..8586bfd 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -12,5 +12,5 @@ require ( require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/image v0.17.0 // indirect + golang.org/x/image v0.18.0 // indirect ) diff --git a/tools/go.sum b/tools/go.sum index 3255eb9..fe73c9f 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -21,6 +21,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco= golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=