Skip to content

Commit

Permalink
Merge pull request #95 from Netflix/feature/base85
Browse files Browse the repository at this point in the history 
Added support for Ascii85 encoding
  • Loading branch information
@sumanth-pasupuleti
sumanth-pasupuleti authored Oct 13, 2020
2 parents 0e8b973 + 1c0f424 commit 26109f3
Show file tree
Hide file tree
Showing 8 changed files with 109 additions and 68 deletions.
1 change: 1 addition & 0 deletions evcache-core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ dependencies {
compile group:"org.apache.httpcomponents", name:"httpclient", version:"latest.release"
compile group:"joda-time", name:"joda-time", version:"latest.release"
compile group:"javax.annotation", name:"javax.annotation-api", version:"latest.release"
compile group:"com.github.fzakaria", name:"ascii85", version:"latest.release"

testCompile group:"org.testng", name:"testng", version:"latest.release"
testCompile group:"com.beust", name:"jcommander", version:"1.72"
Expand Down
47 changes: 25 additions & 22 deletions evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ private EVCacheLatch addOrSet(boolean replaceItem, String key, CachedData value,
// identify that evcache client whose primary node is the destination ip for the key being processed
evCacheClients = evCacheClients.stream().filter(client ->
destinationIps.contains(((InetSocketAddress) client.getNodeLocator()
.getPrimary(getEVCacheKey(key).getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength()))
.getPrimary(getEVCacheKey(key).getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder()))
.getSocketAddress()).getAddress().getHostAddress())
).collect(Collectors.toList());
}
Expand Down
44 changes: 33 additions & 11 deletions evcache-core/src/main/java/com/netflix/evcache/EVCacheKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@ public class EVCacheKey {
// like max.hash.length. So changing max.hash.length alone would not necessarily trigger hash recalculation, but
// one would have to change the hashing algorithm in order to having hashing properties taken into account.
// This is to make such a hashing property change very obvious and not subtle.
private final Map<HashingAlgorithm, String> hashedKeysByAlgorithm;
private final Map<HashingAlgorithm, String> hashedKeysByAlgorithmForDuet;
private final Map<String, String> hashedKeysByAlgorithm;
private final Map<String, String> hashedKeysByAlgorithmForDuet;
private final String encoder;

public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgorithm hashingAlgorithmAtAppLevel, Property<Boolean> shouldEncodeHashKeyAtAppLevel, Property<Integer> maxDigestBytesAtAppLevel, Property<Integer> maxHashLengthAtAppLevel) {
this(appName, key, canonicalKey, hashingAlgorithmAtAppLevel, shouldEncodeHashKeyAtAppLevel, maxDigestBytesAtAppLevel, maxHashLengthAtAppLevel, null);
}

public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgorithm hashingAlgorithmAtAppLevel, Property<Boolean> shouldEncodeHashKeyAtAppLevel, Property<Integer> maxDigestBytesAtAppLevel, Property<Integer> maxHashLengthAtAppLevel, String encoder) {
super();
this.appName = appName;
this.key = key;
Expand All @@ -31,6 +36,7 @@ public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgori
this.shouldEncodeHashKeyAtAppLevel = shouldEncodeHashKeyAtAppLevel;
this.maxDigestBytesAtAppLevel = maxDigestBytesAtAppLevel;
this.maxHashLengthAtAppLevel = maxHashLengthAtAppLevel;
this.encoder = encoder;
hashedKeysByAlgorithm = new HashMap<>();
hashedKeysByAlgorithmForDuet = new HashMap<>();
}
Expand Down Expand Up @@ -59,11 +65,11 @@ private String getCanonicalKeyForDuet() {

@Deprecated
public String getHashKey() {
return getHashKey(hashingAlgorithmAtAppLevel, null == shouldEncodeHashKeyAtAppLevel ? null : shouldEncodeHashKeyAtAppLevel.get(), null == maxDigestBytesAtAppLevel ? null : maxDigestBytesAtAppLevel.get(), null == maxHashLengthAtAppLevel ? null : maxHashLengthAtAppLevel.get());
return getHashKey(hashingAlgorithmAtAppLevel, null == shouldEncodeHashKeyAtAppLevel ? null : shouldEncodeHashKeyAtAppLevel.get(), null == maxDigestBytesAtAppLevel ? null : maxDigestBytesAtAppLevel.get(), null == maxHashLengthAtAppLevel ? null : maxHashLengthAtAppLevel.get(), encoder);
}

// overlays app level hashing and client level hashing
public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String baseEnoder) {
if (hashingAlgorithm == HashingAlgorithm.NO_HASHING) {
return null;
}
Expand All @@ -83,36 +89,52 @@ public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Bool
if (null == maxHashLength) {
maxHashLength = this.maxHashLengthAtAppLevel.get();
}

if(null == baseEnoder) {
baseEnoder = encoder;
}

return isDuet ? getHashKeyForDuet(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength) : getHashKey(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength);
return isDuet ? getHashKeyForDuet(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder) : getHashKey(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder);
}

// overlays app level hashing algorithm and client level hashing algorithm
public String getDerivedKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
public String getDerivedKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String baseEnoder) {
// this overlay of hashingAlgorithm helps determine if there at all needs to be hashing performed, otherwise, will return canonical key
if (null == hashingAlgorithm) {
hashingAlgorithm = hashingAlgorithmAtAppLevel;
}

return null == hashingAlgorithm || hashingAlgorithm == HashingAlgorithm.NO_HASHING ? getCanonicalKey(isDuet) : getHashKey(isDuet, hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength);
return null == hashingAlgorithm || hashingAlgorithm == HashingAlgorithm.NO_HASHING ? getCanonicalKey(isDuet) : getHashKey(isDuet, hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder);
}

private String getHashKey(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
private String getHashKey(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String encoder) {
if (null == hashingAlgorithm) {
return null;
}

final String key = hashingAlgorithm.toString()+ maxDigestBytes != null ? maxDigestBytes.toString() : "-" + maxHashLength != null ? maxHashLength.toString() : "-" + encoder != null ? encoder : "-";
String val = hashedKeysByAlgorithm.get(key);
if(val == null) {
val = KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), hashingAlgorithm, maxDigestBytes, maxHashLength, encoder);
hashedKeysByAlgorithm.put(key , val);
}
// TODO: Once the issue around passing hashedKey in bytes[] is figured, we will start using (nullable) shouldEncodeHashKey, and call KeyHasher.getHashedKeyInBytes() accordingly
return hashedKeysByAlgorithm.computeIfAbsent(hashingAlgorithm, ha -> KeyHasher.getHashedKeyEncoded(canonicalKey, ha, maxDigestBytes, maxHashLength));
return val;
}

private String getHashKeyForDuet(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
private String getHashKeyForDuet(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String encoder) {
if (null == hashingAlgorithm) {
return null;
}

final String key = hashingAlgorithm.toString()+ maxDigestBytes != null ? maxDigestBytes.toString() : "-" + maxHashLength != null ? maxHashLength.toString() : "-" + encoder != null ? encoder : "-";
String val = hashedKeysByAlgorithmForDuet.get(key);
if(val == null) {
val = KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), hashingAlgorithm, maxDigestBytes, maxHashLength, encoder);
hashedKeysByAlgorithmForDuet.put(key , val);
}
// TODO: Once the issue around passing hashedKey in bytes[] is figured, we will start using (nullable) shouldEncodeHashKey, and call KeyHasher.getHashedKeyInBytes() accordingly
return hashedKeysByAlgorithmForDuet.computeIfAbsent(hashingAlgorithm, ha -> KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), ha, maxDigestBytes, maxHashLength));
return val;
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion evcache-core/src/main/java/com/netflix/evcache/event/EVCacheEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public Collection<String> getCanonicalKeys() {
public Collection<MemcachedNode> getMemcachedNode(EVCacheKey evckey) {
final Collection<MemcachedNode> nodeList = new ArrayList<MemcachedNode>(clients.size());
for(EVCacheClient client : clients) {
String key = evckey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
String key = evckey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder());
nodeList.add(client.getNodeLocator().getPrimary(key));
}
return nodeList;
Expand Down
7 changes: 7 additions & 0 deletions evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ public class EVCacheClient {
private final Property<Integer> maxDigestBytes;
private final Property<Integer> maxHashLength;
private final Property<Integer> chunkSize, writeBlock;
private final Property<String> encoderBase;

private final ChunkTranscoder chunkingTranscoder;
private final SerializingTranscoder decodingTranscoder;
private static final int SPECIAL_BYTEARRAY = (8 << 8);
Expand Down Expand Up @@ -153,6 +155,7 @@ public class EVCacheClient {
this.shouldEncodeHashKey = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".hash.encode", Boolean.class).orElse(null);
this.maxDigestBytes = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".max.digest.bytes", Integer.class).orElse(null);
this.maxHashLength = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".max.hash.length", Integer.class).orElse(null);
this.encoderBase = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".hash.encoder", String.class).orElse("base64");
ping();
}

Expand All @@ -176,6 +179,10 @@ public Boolean shouldEncodeHashKey() {
return this.shouldEncodeHashKey.get();
}

public String getBaseEncoder() {
return this.encoderBase.get();
}

public Integer getMaxDigestBytes() {
return this.maxDigestBytes.get();
}
Expand Down
10 changes: 5 additions & 5 deletions evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheClientUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
Boolean firstStatus = null;
for (EVCacheClient client : clients) {
CachedData cd1;
if (evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength()) != null) {
if (evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder()) != null) {
if(cdHashed == null) {
final EVCacheValue val = new EVCacheValue(evcKey.getCanonicalKey(client.isDuetClient()), cd.getData(), cd.getFlags(), timeToLive, System.currentTimeMillis());
cdHashed = evcacheValueTranscoder.encode(val);
Expand All @@ -47,7 +47,7 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
} else {
cd1 = cd;
}
String key = evcKey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
String key = evcKey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder());
final Future<Boolean> f = client.add(key, timeToLive, cd1, latch);
if (log.isDebugEnabled()) log.debug("ADD : Op Submitted : APP " + _appName + ", key " + key + "; future : " + f + "; client : " + client);
if(fixMissing) {
Expand All @@ -71,12 +71,12 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
private EVCacheLatch fixup(EVCacheClient sourceClient, EVCacheClient[] destClients, EVCacheKey evcKey, int timeToLive, Policy policy) {
final EVCacheLatchImpl latch = new EVCacheLatchImpl(policy, destClients.length, _appName);
try {
final CachedData readData = sourceClient.get(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength()), ct, false, false);
final CachedData readData = sourceClient.get(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength(), sourceClient.getBaseEncoder()), ct, false, false);

if(readData != null) {
sourceClient.touch(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength()), timeToLive);
sourceClient.touch(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength(), sourceClient.getBaseEncoder()), timeToLive);
for(EVCacheClient destClient : destClients) {
destClient.set(evcKey.getDerivedKey(destClient.isDuetClient(), destClient.getHashingAlgorithm(), destClient.shouldEncodeHashKey(), destClient.getMaxDigestBytes(), destClient.getMaxHashLength()), readData, timeToLive, latch);
destClient.set(evcKey.getDerivedKey(destClient.isDuetClient(), destClient.getHashingAlgorithm(), destClient.shouldEncodeHashKey(), destClient.getMaxDigestBytes(), destClient.getMaxHashLength(), destClient.getBaseEncoder()), readData, timeToLive, latch);
}
}
latch.await(_operationTimeout, TimeUnit.MILLISECONDS);
Expand Down
Loading

0 comments on commit 26109f3

Please sign in to comment.