Skip to content

Commit

Permalink
feat: ability to configure and use image thumbnails
Browse files Browse the repository at this point in the history
  • Loading branch information
guqing committed Sep 3, 2024
1 parent 97f653b commit 36d9267
Show file tree
Hide file tree
Showing 10 changed files with 326 additions and 51 deletions.
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ repositories {
}

dependencies {
implementation platform('run.halo.tools.platform:plugin:2.17.0-SNAPSHOT')
implementation platform('run.halo.tools.platform:plugin:2.19.0-SNAPSHOT')
compileOnly 'run.halo.app:api'

implementation platform('software.amazon.awssdk:bom:2.19.8')
Expand Down
74 changes: 74 additions & 0 deletions src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package run.halo.s3os;

import static run.halo.app.infra.FileCategoryMatcher.IMAGE;

import java.time.Duration;
import java.time.Instant;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import run.halo.app.core.extension.attachment.Attachment;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.controller.Controller;
import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.Reconciler;

/**
* <p>This {@link Reconciler} used to check thumbnail status are generated
* if not, update the attachment to trigger thumbnail generation by halo</p>
*/
@Component
@RequiredArgsConstructor
public class AttachmentThumbnailReconciler implements Reconciler<Reconciler.Request> {
static final String REQUEST_GEN_THUMBNAIL = "s3os.halo.run/request-gen-thumbnail";
private final ExtensionClient client;

@Override
public Result reconcile(Request request) {
client.fetch(Attachment.class, request.name())
.ifPresent(attachment -> {
var annotations = MetadataUtil.nullSafeAnnotations(attachment);
var requestTime = annotations.get(REQUEST_GEN_THUMBNAIL);
if (isMadeWithIn1Day(requestTime)) {
// skip if request is made within 1 day
return;
}

if (!isImage(attachment)) {
// skip non-image attachments
return;
}

var status = attachment.getStatus();
if (status == null || status.getThumbnails() == null) {
// update to trigger attachment thumbnail generation
annotations.put(REQUEST_GEN_THUMBNAIL, Instant.now().toString());
client.update(attachment);
}
});
return Result.doNotRetry();
}

static boolean isMadeWithIn1Day(String requestTimeStr) {
if (StringUtils.isBlank(requestTimeStr)) {
return false;
}
var requestTime = Instant.parse(requestTimeStr);
return Duration.between(requestTime, Instant.now()).minusDays(1).isNegative();
}

public static boolean isImage(Attachment attachment) {
Assert.notNull(attachment, "Attachment must not be null");
var mediaType = attachment.getSpec().getMediaType();
return mediaType != null && IMAGE.match(mediaType);
}

@Override
public Controller setupWith(ControllerBuilder builder) {
return builder
.extension(new Attachment())
.build();
}
}
4 changes: 2 additions & 2 deletions src/main/java/run/halo/s3os/S3LinkServiceImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public Mono<S3ListResult> listObjects(String policyName, String continuationToke
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap((configMap) -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation());
return Mono.using(() -> handler.buildS3Client(properties),
// 执行 listObjects
Expand Down Expand Up @@ -231,7 +231,7 @@ public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap(configMap -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
return Mono.using(() -> handler.buildS3Client(properties),
(s3Client) -> Mono.fromCallable(
() -> s3Client.headObject(
Expand Down
49 changes: 8 additions & 41 deletions src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.time.Duration;
import java.util.HashMap;
Expand All @@ -23,7 +22,6 @@
import org.springframework.lang.Nullable;
import org.springframework.web.server.ServerErrorException;
import org.springframework.web.server.ServerWebInputException;
import org.springframework.web.util.UriUtils;
import reactor.core.Exceptions;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
Expand All @@ -38,7 +36,6 @@
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.infra.utils.JsonUtils;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.awscore.presigner.SdkPresigner;
import software.amazon.awssdk.core.SdkResponse;
Expand All @@ -47,16 +44,7 @@
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.utils.SdkAutoCloseable;
Expand All @@ -79,7 +67,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
public Mono<Attachment> upload(UploadContext uploadContext) {
return Mono.just(uploadContext).filter(context -> this.shouldHandle(context.policy()))
.flatMap(context -> {
final var properties = getProperties(context.configMap());
final var properties = S3OsProperties.convertFrom(context.configMap());
return upload(context, properties)
.subscribeOn(Schedulers.boundedElastic())
.map(objectDetail -> this.buildAttachment(properties, objectDetail))
Expand All @@ -102,7 +90,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {
log.info("Skip deleting object {} from S3.", objectKey);
return Mono.just(context);
}
var properties = getProperties(deleteContext.configMap());
var properties = S3OsProperties.convertFrom(deleteContext.configMap());
return Mono.using(() -> buildS3Client(properties),
client -> Mono.fromCallable(
() -> client.deleteObject(DeleteObjectRequest.builder()
Expand All @@ -123,7 +111,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {

@Override
public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap configMap,
Duration ttl) {
Duration ttl) {
if (!this.shouldHandle(policy)) {
return Mono.empty();
}
Expand All @@ -132,7 +120,7 @@ public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap co
return Mono.error(new IllegalArgumentException(
"Cannot obtain object key from attachment " + attachment.getMetadata().getName()));
}
var properties = getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);

return Mono.using(() -> buildS3Presigner(properties),
s3Presigner -> {
Expand Down Expand Up @@ -168,8 +156,8 @@ public Mono<URI> getPermalink(Attachment attachment, Policy policy, ConfigMap co
// fallback to default handler for backward compatibility
return Mono.empty();
}
var properties = getProperties(configMap);
var objectURL = getObjectURL(properties, objectKey);
var properties = S3OsProperties.convertFrom(configMap);
var objectURL = properties.toObjectURL(objectKey);
var urlSuffix = getUrlSuffixAnnotation(attachment);
if (StringUtils.isNotBlank(urlSuffix)) {
objectURL += urlSuffix;
Expand All @@ -195,13 +183,8 @@ private String getUrlSuffixAnnotation(Attachment attachment) {
return annotations.get(URL_SUFFIX_ANNO_KEY);
}

S3OsProperties getProperties(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}

Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) {
String externalLink = getObjectURL(properties, objectDetail.uploadState.objectKey);
String externalLink = properties.toObjectURL(objectDetail.uploadState.objectKey);
var urlSuffix = UrlUtils.findUrlSuffix(properties.getUrlSuffixes(),
objectDetail.uploadState.fileName);

Expand Down Expand Up @@ -229,22 +212,6 @@ Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail)
return attachment;
}

String getObjectURL(S3OsProperties properties, String objectKey) {
String objectURL;
if (StringUtils.isBlank(properties.getDomain())) {
String host;
if (properties.getEnablePathStyleAccess()) {
host = properties.getEndpoint() + "/" + properties.getBucket();
} else {
host = properties.getBucket() + "." + properties.getEndpoint();
}
objectURL = properties.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

S3Client buildS3Client(S3OsProperties properties) {
return S3Client.builder()
.region(Region.of(properties.getRegion()))
Expand Down
1 change: 1 addition & 0 deletions src/main/java/run/halo/s3os/S3OsPlugin.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/
@Component
public class S3OsPlugin extends BasePlugin {
public static final String POLICY_SETTING_NAME = "s3os-policy-template-setting";

public S3OsPlugin(PluginContext pluginContext) {
super(pluginContext);
Expand Down
42 changes: 36 additions & 6 deletions src/main/java/run/halo/s3os/S3OsProperties.java
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package run.halo.s3os;

import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import org.springframework.util.StringUtils;

import java.time.LocalDate;
import org.springframework.web.util.UriUtils;
import run.halo.app.extension.ConfigMap;
import run.halo.app.infra.utils.JsonUtils;

@Data
class S3OsProperties {
public class S3OsProperties {

private String bucket;

Expand Down Expand Up @@ -49,6 +50,14 @@ class S3OsProperties {

private List<urlSuffixItem> urlSuffixes;

private ThumbnailParam thumbnailParam;

public record ThumbnailParam(String type, String pattern) {
public boolean hasPattern() {
return StringUtils.hasText(type) && StringUtils.hasText(pattern);
}
}

@Data
@AllArgsConstructor
@NoArgsConstructor
Expand Down Expand Up @@ -103,19 +112,40 @@ public void setRandomStringLength(String randomStringLength) { // if you use In
if (length >= 4 && length <= 16) {
this.randomStringLength = length;
}
} catch (NumberFormatException ignored) {
}
catch (NumberFormatException ignored) { }
}

public void setRegion(String region) {
if (!StringUtils.hasText(region)) {
this.region = "Auto";
}else {
} else {
this.region = region;
}
}

public void setEndpoint(String endpoint) {
this.endpoint = UrlUtils.removeHttpPrefix(endpoint);
}

public String toObjectURL(String objectKey) {
String objectURL;
if (!StringUtils.hasText(this.getDomain())) {
String host;
if (this.getEnablePathStyleAccess()) {
host = this.getEndpoint() + "/" + this.getBucket();
} else {
host = this.getBucket() + "." + this.getEndpoint();
}
objectURL = this.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = this.getProtocol() + "://" + this.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

public static S3OsProperties convertFrom(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}
}
Loading

0 comments on commit 36d9267

Please sign in to comment.