minio and photowall

.gitignore

@@ -111,4 +111,7 @@ temp/
 # End of https://www.gitignore.io/api/node
 
 .env
+.idea/
 build/
+
+src/scripts/photowall/assets/*.jpg

package.json

@@ -9,7 +9,7 @@
   "private": true,
   "scripts": {
     "build": "tsc",
-    "start": "ts-node-dev --respawn src/index.ts",
+    "start": "ts-node-dev --respawn --files src/index.ts",
     "start:prod": "ts-node --files src/index.ts",
     "lint:fix": "eslint src/** types.ts --fix",
     "lint": "eslint src/** types.ts"
@@ -18,13 +18,16 @@
     "@hapi/boom": "^9.1.0",
     "@hapi/joi": "^17.1.1",
     "@koa/cors": "^3.0.0",
-    "@koa/router": "^8.0.8",
+    "@koa/multer": "^3.0.0",
+    "@koa/router": "^10.0.0",
     "axios": "^0.19.2",
     "discord.js": "^12.2.0",
     "dotenv": "^8.2.0",
     "koa": "^2.11.0",
     "koa-bodyparser": "^4.3.0",
     "koa-logger-winston": "^0.0.2",
+    "minio": "^7.0.16",
+    "multer": "^1.4.2",
     "pg": "^8.2.1",
     "shortid": "^2.2.15",
     "typeorm": "^0.2.25",
@@ -37,7 +40,10 @@
     "@types/koa-bodyparser": "^4.3.0",
     "@types/koa-logger-winston": "^0.0.2",
     "@types/koa__cors": "^3.0.1",
+    "@types/koa__multer": "^2.0.2",
     "@types/koa__router": "^8.0.2",
+    "@types/minio": "^7.0.6",
+    "@types/multer": "^1.4.4",
     "@types/node": "^13.13.4",
     "@types/shortid": "0.0.29",
     "@types/ws": "^7.2.4",

src/entities/PhotowallEntry.ts

@@ -0,0 +1,23 @@
+import {
+  Entity,
+  Column,
+  getConnection,
+  PrimaryGeneratedColumn,
+  Repository,
+} from "typeorm";
+
+@Entity()
+export class PhotowallEntry {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column()
+  fileId: string;
+
+  @Column()
+  published: boolean;
+}
+
+export const PhotowallEntryRepository = async (): Promise<
+  Repository<PhotowallEntry>
+> => getConnection().getRepository(PhotowallEntry);

src/index.ts

@@ -11,6 +11,8 @@ dotenv.config();
 import "./db"; // Imported for side effects
 import { forceValidToken, logger, errorHandler } from "./middleware";
 import routes from "./routes";
+import { PhotowallEntryRepository } from "./entities/PhotowallEntry";
+import { seedPhotowall } from "./scripts/photowall/seedPhotowall";
 
 const app = new Koa();
 const HTTP_PORT = process.env["HTTP_PORT"] ?? 3000;
@@ -50,3 +52,13 @@ app.use(routes.routes());
 console.log(routes.stack.map((s) => s.path));
 
 app.listen(HTTP_PORT, () => console.log("Running on port " + HTTP_PORT));
+
+const seed = async (): Promise<void> => {
+  const photoWallRepo = await PhotowallEntryRepository();
+  const hasResult = await photoWallRepo.findOne();
+  if (!hasResult) {
+    await seedPhotowall();
+  }
+};
+
+setTimeout(() => seed().then(() => console.log("Seeded")), 1000);

src/lib/minio/client.ts

@@ -0,0 +1,13 @@
+import { Client } from "minio";
+
+const { ACCESS_KEY, SECRET_KEY, ENDPOINT, MINIO_PORT } = process.env;
+
+export const client = new Client({
+  accessKey: ACCESS_KEY,
+  secretKey: SECRET_KEY,
+  endPoint: ENDPOINT,
+  port: Number(MINIO_PORT),
+  useSSL: false,
+});
+
+export const BUCKET_NAME = "yestheoryfam";

src/lib/minio/index.ts

@@ -0,0 +1,2 @@
+export * from "./client";
+export * from "./upload";

src/lib/minio/upload.ts

@@ -0,0 +1,13 @@
+import { client, BUCKET_NAME } from "./client";
+import shortId from "shortid";
+import mime from "mime";
+import { File } from "@koa/multer";
+
+export const upload = async (file: File): Promise<string> => {
+  const id = shortId.generate();
+  const extension = mime.getExtension(file.mimetype);
+  const uploadName = `${id}.${extension}`;
+  await client.putObject(BUCKET_NAME, uploadName, file.buffer, file.size);
+
+  return uploadName;
+};

src/routes/files/download.ts

@@ -0,0 +1,13 @@
+import Router from "@koa/router";
+import { BUCKET_NAME, client } from "../../lib/minio";
+
+const router = new Router<KoaState>();
+
+router.get("/:fileId", async (ctx) => {
+  const { fileId } = ctx.params;
+  ctx.body = await client.getObject(BUCKET_NAME, fileId);
+  ctx.status = 200;
+});
+// Download
+
+export default router;

src/routes/files/index.ts

@@ -0,0 +1,10 @@
+import Router from "@koa/router";
+import upload from "./upload";
+import download from "./download";
+
+const router = new Router();
+
+router.use("/upload", upload.routes(), upload.allowedMethods());
+router.use("/download", download.routes(), download.allowedMethods());
+
+export default router;

src/routes/files/upload.ts

@@ -0,0 +1,45 @@
+import Router from "@koa/router";
+import multer from "@koa/multer";
+import { upload } from "../../lib/minio";
+import {
+  PhotowallEntry,
+  PhotowallEntryRepository,
+} from "../../entities/PhotowallEntry";
+
+const multerM = multer();
+
+const router = new Router<KoaState>();
+
+const postPhotowallUpload = async (fileId: string): Promise<void> => {
+  const repo = await PhotowallEntryRepository();
+  const entry = new PhotowallEntry();
+  entry.fileId = fileId;
+  // TODO false + review process
+  entry.published = true;
+  await repo.save(entry);
+};
+
+const uploads = [
+  {
+    directory: "photowall",
+    postUpload: postPhotowallUpload,
+  },
+  {
+    directory: "blog",
+  },
+];
+
+for (const { directory, postUpload } of uploads) {
+  // Make directory more flexible but with validation
+  router.post("/" + directory, multerM.single("file"), async (ctx) => {
+    const { file } = ctx.request;
+    const fileId = await upload(file);
+    if (postUpload) {
+      await postUpload(fileId);
+    }
+
+    ctx.status = 200;
+  });
+}
+
+export default router;

src/routes/index.ts

@@ -2,12 +2,16 @@ import Router from "@koa/router";
 import botActions from "./botActions";
 import blogs from "./blog";
 import isMod from "./isMod";
+import files from "./files";
+import photowall from "./photowall";
 
 const router = new Router();
 
 router.use("/bot-actions", botActions.routes(), botActions.allowedMethods());
 router.use("/blogs", blogs.routes(), blogs.allowedMethods());
 router.use("/is-mod", isMod.routes(), isMod.allowedMethods());
+router.use("/files", files.routes(), files.allowedMethods());
+router.use("/photowall", photowall.routes(), photowall.allowedMethods());
 router.get("/", (ctx) => (ctx.body = "OK"));
 
 export default router;

src/routes/photowall/images.ts

@@ -0,0 +1,43 @@
+import Router from "@koa/router";
+import { PhotowallEntryRepository } from "../../entities/PhotowallEntry";
+
+const pageSize = 20;
+const router = new Router<KoaState>();
+
+interface PhotowallPageResult {
+  page: number;
+  hasNext: boolean;
+  imageIds: string[];
+}
+
+const getImages = async (page = 0): Promise<PhotowallPageResult> => {
+  const skip = page * pageSize;
+  const repo = await PhotowallEntryRepository();
+  const photowallEntries = await repo.find({
+    where: {
+      published: true,
+    },
+    skip,
+    take: pageSize + 1,
+    select: ["fileId"],
+  });
+
+  return {
+    page,
+    hasNext: photowallEntries.length > pageSize,
+    imageIds: photowallEntries.map((entry) => entry.fileId),
+  };
+};
+
+router.get("/", async (ctx) => {
+  ctx.body = await getImages();
+  ctx.status = 200;
+});
+
+router.get("/:page", async (ctx) => {
+  const { page } = ctx.params;
+  ctx.body = await getImages(page);
+  ctx.status = 200;
+});
+
+export default router;

src/routes/photowall/index.ts

@@ -0,0 +1,8 @@
+import Router from "@koa/router";
+import images from "./images";
+
+const router = new Router();
+
+router.use("/images", images.routes(), images.allowedMethods());
+
+export default router;

src/scripts/photowall/seedPhotowall.ts

@@ -0,0 +1,32 @@
+import { BUCKET_NAME, client } from "../../lib/minio";
+import {
+  PhotowallEntry,
+  PhotowallEntryRepository,
+} from "../../entities/PhotowallEntry";
+
+import { promises as fsPromises } from "fs";
+
+export const seedPhotowall = async (): Promise<void> => {
+  const directory = "src/scripts/photowall/assets/";
+  const files = await fsPromises.readdir(directory);
+  const repo = await PhotowallEntryRepository();
+  const entries = [];
+  for (const file of files) {
+    if (!(file.endsWith(".jpg") || file.endsWith(".jpeg"))) {
+      continue;
+    }
+
+    const path = directory + file;
+    const stat = await fsPromises.stat(path);
+    if (stat.isDirectory()) {
+      continue;
+    }
+
+    await client.fPutObject(BUCKET_NAME, file, path, {});
+    const entry = new PhotowallEntry();
+    entry.fileId = file;
+    entry.published = true;
+    entries.push(entry);
+  }
+  await repo.save(entries);
+};

tsconfig.json

@@ -25,7 +25,7 @@
   },
   "include": [
     "./types.ts",
-    "./src",
+    "./src"
   ],
   "exclude": [
     "node_modules",