resolve merge

IABTechLab · Jul 16, 2024 · eaf02ed · eaf02ed
2 parents da2a957 + 0152384
commit eaf02ed
Show file tree

Hide file tree

Showing 15 changed files with 638 additions and 10 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-shared</artifactId>
-    <version>7.13.1-alpha-110-SNAPSHOT</version>
+    <version>7.15.0</version>
     <name>${project.groupId}:${project.artifactId}</name>
     <description>Library for all the shared uid2 operations</description>
     <url>https://github.com/IABTechLab/uid2docs</url>

diff --git a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java
@@ -0,0 +1,89 @@
+package com.uid2.shared.store;
+
+import com.uid2.shared.cloud.DownloadCloudStorage;
+import com.uid2.shared.model.S3Key;
+import com.uid2.shared.store.parser.Parser;
+import com.uid2.shared.store.parser.ParsingResult;
+import com.uid2.shared.store.scope.EncryptedScope;
+import com.uid2.shared.store.scope.StoreScope;
+import com.uid2.shared.store.reader.RotatingS3KeyProvider;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+
+import com.uid2.shared.encryption.AesGcm;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+
+public class EncryptedScopedStoreReader<T> extends ScopedStoreReader<T> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class);
+
+    private final int siteId;
+    private final RotatingS3KeyProvider s3KeyProvider;
+
+    public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, Parser<T> parser, String dataTypeName, RotatingS3KeyProvider s3KeyProvider) {
+        super(fileStreamProvider, scope, parser, dataTypeName);
+        this.siteId = scope.getId();
+        this.s3KeyProvider = s3KeyProvider;
+    }
+
+    @Override
+    protected long loadContent(String path) throws Exception {
+        try (InputStream inputStream = this.contentStreamProvider.download(path)) {
+            String encryptedContent = inputStreamToString(inputStream);
+            String decryptedContent = getDecryptedContent(encryptedContent);
+            ParsingResult<T> parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8)));
+            latestSnapshot.set(parsed.getData());
+
+            final int count = parsed.getCount();
+            latestEntryCount.set(count);
+            LOGGER.info(String.format("Loaded %d %s", count, dataTypeName));
+            return count;
+        } catch (Exception e) {
+            LOGGER.error(String.format("Unable to load %s", dataTypeName));
+            throw e;
+        }
+    }
+
+    protected String getDecryptedContent(String encryptedContent) throws Exception {
+        JsonObject json = new JsonObject(encryptedContent);
+        int keyId = json.getInteger("key_id");
+        String encryptedPayload = json.getString("encrypted_payload");
+
+        Map<Integer, S3Key> s3Keys = s3KeyProvider.getAll();
+        S3Key decryptionKey = null;
+
+        for (S3Key key : s3Keys.values()) {
+            if (key.getSiteId() == siteId && key.getId() == keyId) {
+                decryptionKey = key;
+                break;
+            }
+        }
+
+        if (decryptionKey == null) {
+            throw new IllegalStateException("No matching S3 key found for decryption for site ID: " + siteId + " and key ID: " + keyId);
+        }
+
+        byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
+        byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload);
+        byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
+
+        return new String(decryptedBytes, StandardCharsets.UTF_8);
+    }
+
+    public static String inputStreamToString(InputStream inputStream) throws IOException {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
+            StringBuilder stringBuilder = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                stringBuilder.append(line);
+            }
+            return stringBuilder.toString();
+        }
+    }
+}
diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java
@@ -21,11 +21,11 @@ public class ScopedStoreReader<T> {
 
     private final DownloadCloudStorage metadataStreamProvider;
     private final StoreScope scope;
-    private final Parser<T> parser;
-    private final String dataTypeName;
-    private final DownloadCloudStorage contentStreamProvider;
-    private final AtomicReference<T> latestSnapshot;
-    private final AtomicLong latestEntryCount = new AtomicLong(-1L);
+    protected final Parser<T> parser;
+    protected final String dataTypeName;
+    protected final DownloadCloudStorage contentStreamProvider;
+    protected final AtomicReference<T> latestSnapshot;
+    protected final AtomicLong latestEntryCount = new AtomicLong(-1L);
 
     public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser<T> parser, String dataTypeName) {
         this.metadataStreamProvider = fileStreamProvider;
@@ -60,7 +60,7 @@ public JsonObject getMetadata() throws Exception {
         }
     }
 
-    private long loadContent(String path) throws Exception {
+    protected long loadContent(String path) throws Exception {
         try (InputStream inputStream = this.contentStreamProvider.download(path)) {
             ParsingResult<T> parsed = parser.deserialize(inputStream);
             latestSnapshot.set(parsed.getData());

diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingKeyAclProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingKeyAclProvider.java
@@ -5,9 +5,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.cloud.ICloudStorage;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IKeyAclProvider;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeyAclParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -21,6 +23,10 @@ public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls");
     }
 
+    public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader =  new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() { return reader.getMetadataPath(); }
 

diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingKeyStore.java b/src/main/java/com/uid2/shared/store/reader/RotatingKeyStore.java
@@ -3,9 +3,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.EncryptionKey;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IKeyStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeyParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -50,6 +52,10 @@ public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, StoreScope scop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys");
     }
 
+    public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() {
         return reader.getMetadataPath();

diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingKeysetKeyStore.java b/src/main/java/com/uid2/shared/store/reader/RotatingKeysetKeyStore.java
@@ -7,7 +7,9 @@
 import com.uid2.shared.store.KeysetKeyStoreSnapshot;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeysetKeyParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import io.vertx.core.json.JsonObject;
 
 import java.time.Instant;
@@ -20,6 +22,10 @@ public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys");
     }
 
+    public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys", s3KeyProvider);
+    }
+
     @Override
     public long getVersion(JsonObject metadata) {
         return metadata.getLong("version");

diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingKeysetProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingKeysetProvider.java
@@ -4,8 +4,10 @@
 import com.uid2.shared.auth.KeysetSnapshot;
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeysetParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -19,6 +21,10 @@ public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetParser(), "keysets");
     }
 
+    public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider,scope,new KeysetParser(),"keysers",s3KeyProvider);
+    }
+
     public KeysetSnapshot getSnapshot(Instant asOf) {
         return reader.getSnapshot();
     }

diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingS3KeyProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingS3KeyProvider.java
@@ -9,12 +9,14 @@
 import io.vertx.core.json.JsonObject;
 
 import java.time.Instant;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import java.util.stream.Collectors;
 
 import com.uid2.shared.model.S3Key;
 
@@ -77,4 +79,29 @@ public Set<Integer> getAllSiteIds() {
     public int getTotalSites() {
         return siteToKeysMap.size();
     }
+
+    public Collection<S3Key> getKeysForSite(Integer siteId) {
+        Map<Integer, S3Key> allKeys = getAll();
+        return allKeys.values().stream()
+                .filter(key -> key.getSiteId()==(siteId))
+                .collect(Collectors.toList());
+    }
+
+    public S3Key getEncryptionKeyForSite(Integer siteId) {
+        Collection<S3Key> keys = getKeysForSite(siteId);
+        if (keys.isEmpty()) {
+            throw new IllegalStateException("No S3 keys available for encryption for site ID: " + siteId);
+        } else {
+            Map<Integer, S3Key> allKeys = getAll();
+            S3Key largestKey = null;
+            for (S3Key key : allKeys.values()) {
+                if (key.getSiteId() == siteId) {
+                    if (largestKey == null || key.getId() > largestKey.getId()) {
+                        largestKey = key;
+                    }
+                }
+            }
+            return largestKey;
+        }
+    }
 }
diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingSiteStore.java b/src/main/java/com/uid2/shared/store/reader/RotatingSiteStore.java
@@ -3,9 +3,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.Site;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.ISiteStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.SiteParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -21,6 +23,10 @@ public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, StoreScope sco
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites");
     }
 
+    public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() { return reader.getMetadataPath(); }
 

diff --git a/src/main/java/com/uid2/shared/store/scope/EncryptedScope.java b/src/main/java/com/uid2/shared/store/scope/EncryptedScope.java
@@ -0,0 +1,44 @@
+package com.uid2.shared.store.scope;
+
+import com.uid2.shared.store.CloudPath;
+
+public class EncryptedScope implements StoreScope {
+    private final Integer siteId;
+    private final CloudPath rootMetadataPath;
+    private final boolean isPublic;
+
+    public EncryptedScope(CloudPath rootMetadataPath, Integer siteId, boolean isPublic){
+        this.rootMetadataPath = rootMetadataPath;
+        this.siteId = siteId;
+        this.isPublic = isPublic;
+    }
+
+    @Override
+    public CloudPath getMetadataPath() {
+        return resolve(rootMetadataPath.getFileName());
+    }
+
+    @Override
+    public CloudPath resolve(CloudPath cloudPath) {
+        CloudPath directory = rootMetadataPath.getParent();
+        String siteType = isPublic ? "public" : "private";
+        return directory.resolve("encrypted").resolve(siteId + "_" + siteType).resolve(cloudPath);
+    }
+
+    @Override
+    public Integer getId() {
+        return siteId;
+    }
+
+    public boolean getPublic() {
+        return isPublic;
+    }
+
+    /* paths when we no longer need to distinguish private & public sites
+    @Override
+    public CloudPath resolve(CloudPath cloudPath) {
+        CloudPath directory = rootMetadataPath.getParent();
+        return directory.resolve("encryption").resolve("site").resolve(getId().toString()).resolve(cloudPath);
+    }
+    */
+}
diff --git a/src/test/java/com/uid2/shared/encryption/AesGcmTest.java b/src/test/java/com/uid2/shared/encryption/AesGcmTest.java
@@ -31,9 +31,9 @@ public void testEncryptionDecryptionString() {
     @Test
     public void testDecryption() {
         byte[] keyBytes = new byte[]{19, -105, -79, 100, 11, 38, -93, 100, 123, 111, 68, -57, 67, -12, -33, 14, -53, -120, -66, -116, 30, 9, -123, -50, -36, 73, 79, 85, 78, -18, 42, -108};
-        byte[] encypted = new byte[]{-2, 35, -109, -1, -123, 75, 111, 93, 83, 0, -11, -97, -69, -10, 88, 82, -62, 83, 76, -75, -51, 45, 87, 102, 38, 117, 98, 84, -78, -100, -2, -99, -3, -121, 94, 23, 75, 84, 20};
+        byte[] encrypted = new byte[]{-2, 35, -109, -1, -123, 75, 111, 93, 83, 0, -11, -97, -69, -10, 88, 82, -62, 83, 76, -75, -51, 45, 87, 102, 38, 117, 98, 84, -78, -100, -2, -99, -3, -121, 94, 23, 75, 84, 20};
         final KeysetKey key = new KeysetKey(1, keyBytes, Instant.now(), Instant.now(), Instant.now(), 123);
-        byte[] payload = AesGcm.decrypt(encypted,0, key);
+        byte[] payload = AesGcm.decrypt(encrypted,0, key);
         String expected = "hello world";
         String results = new String(payload);
         assertEquals(results, expected);