enhance: AVIF support (misskey-dev#9281)

* chore: Make image/avif browsersafe

* server side

* change FileInfoService

* ✌️

* avifはMastodonでは絶望的 see misskey-dev#9283

Co-authored-by: syuilo <[email protected]>
Co-authored-by: tamaina <[email protected]>

Author: 3 people

packages/backend/src/const.ts

@@ -12,6 +12,7 @@ export const FILE_TYPE_BROWSERSAFE = [
 	'image/gif',
 	'image/jpeg',
 	'image/webp',
+	'image/avif',
 	'image/apng',
 	'image/bmp',
 	'image/tiff',

packages/backend/src/misc/get-file-info.ts

@@ -63,7 +63,18 @@ export async function getFileInfo(path: string, opts: {
 	let height: number | undefined;
 	let orientation: number | undefined;
 
-	if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) {
+	if ([
+		'image/png',
+		'image/gif',
+		'image/jpeg',
+		'image/webp',
+		'image/avif',
+		'image/apng',
+		'image/bmp',
+		'image/tiff',
+		'image/svg+xml',
+		'image/vnd.adobe.photoshop',
+	].includes(type.mime)) {
 		const imageSize = await detectImageSize(path).catch(e => {
 			warnings.push(`detectImageSize failed: ${e}`);
 			return undefined;
@@ -90,7 +101,15 @@ export async function getFileInfo(path: string, opts: {
 
 	let blurhash: string | undefined;
 
-	if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) {
+	if ([
+		'image/jpeg',
+		'image/gif',
+		'image/png',
+		'image/apng',
+		'image/webp',
+		'image/avif',
+		'image/svg+xml',
+	].includes(type.mime)) {
 		blurhash = await getBlurhash(path).catch(e => {
 			warnings.push(`getBlurhash failed: ${e}`);
 			return undefined;
@@ -145,7 +164,11 @@ async function detectSensitivity(source: string, mime: string, sensitiveThreshol
 		return [sensitive, porn];
 	}
 
-	if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
+	if ([
+		'image/jpeg',
+		'image/png',
+		'image/webp',
+	].includes(mime)) {
 		const result = await detectSensitive(source);
 		if (result) {
 			[sensitive, porn] = judgePrediction(result);

packages/backend/src/misc/is-mime-image.ts

@@ -2,7 +2,7 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 
 const dictionary = {
 	'safe-file': FILE_TYPE_BROWSERSAFE,
-	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'],
+	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
 };
 
 export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);

packages/backend/src/models/repositories/drive-file.ts

@@ -61,7 +61,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
 			}
 		}
 
-		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/svg+xml'].includes(file.type);
+		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type);
 
 		return thumbnail ? (file.thumbnailUrl || (isImage ? (file.webpublicUrl || file.url) : null)) : (file.webpublicUrl || file.url);
 	},

packages/backend/src/server/file/send-drive-file.ts

@@ -59,7 +59,7 @@ export default async function(ctx: Koa.Context) {
 
 				const convertFile = async () => {
 					if (isThumbnail) {
-						if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(mime)) {
+						if (['image/jpeg', 'image/webp', 'image/png', 'image/avif', 'image/svg+xml'].includes(mime)) {
 							return await convertToWebp(path, 498, 280);
 						} else if (mime.startsWith('video/')) {
 							return await GenerateVideoThumbnail(path);

packages/backend/src/services/drive/add-file.ts

@@ -48,6 +48,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h
 			if (type === 'image/jpeg') ext = '.jpg';
 			if (type === 'image/png') ext = '.png';
 			if (type === 'image/webp') ext = '.webp';
+			if (type === 'image/avif') ext = '.avif';
 			if (type === 'image/apng') ext = '.apng';
 			if (type === 'image/vnd.mozilla.apng') ext = '.apng';
 		}
@@ -171,7 +172,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
 		}
 	}
 
-	if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
+	if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) {
 		logger.debug('web image and thumbnail not created (not an required file)');
 		return {
 			webpublic: null,
@@ -196,7 +197,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
 		}
 
 		satisfyWebpublic = !!(
-			type !== 'image/svg+xml' && type !== 'image/webp' &&
+			type !== 'image/svg+xml' && type !== 'image/webp' && type !== 'image/avif' &&
 			!(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) &&
 			metadata.width && metadata.width <= 2048 &&
 			metadata.height && metadata.height <= 2048
@@ -216,7 +217,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
 		logger.info('creating web image');
 
 		try {
-			if (['image/jpeg', 'image/webp'].includes(type)) {
+			if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
 				webpublic = await convertSharpToJpeg(img, 2048, 2048);
 			} else if (['image/png'].includes(type)) {
 				webpublic = await convertSharpToPng(img, 2048, 2048);
@@ -238,7 +239,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
 	let thumbnail: IImage | null = null;
 
 	try {
-		if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
+		if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) {
 			thumbnail = await convertSharpToWebp(img, 498, 280);
 		} else {
 			logger.debug('thumbnail not created (not an required file)');

packages/client/src/const.ts

@@ -7,6 +7,7 @@ export const FILE_TYPE_BROWSERSAFE = [
 	'image/gif',
 	'image/jpeg',
 	'image/webp',
+	'image/avif',
 	'image/apng',
 	'image/bmp',
 	'image/tiff',

packages/client/src/pages/user/index.photos.vue

@@ -47,6 +47,8 @@ function thumbnail(image: misskey.entities.DriveFile): string {
 onMounted(() => {
 	const image = [
 		'image/jpeg',
+		'image/webp',
+		'image/avif',
 		'image/png',
 		'image/gif',
 		'image/apng',