Skip to content

Commit

Permalink
fix:[NEXT-455] Vulnerability for multiple asset types for a plugin (#…
Browse files Browse the repository at this point in the history 
…2357)
  • Loading branch information
@arunpaladin
arunpaladin authored Feb 4, 2025
1 parent 369e660 commit e47803c
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 28 deletions.
2 changes: 1 addition & 1 deletion jobs/pacman-data-shipper/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.261</version>
<version>1.12.263</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
Expand Down
73 changes: 46 additions & 27 deletions .../main/java/com/tmobile/cso/pacman/datashipper/entity/VulnerabilityAssociationManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.util.StringUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -31,6 +30,8 @@ public class VulnerabilityAssociationManager {
private static final String BUCKET_NAME = System.getProperty("s3");
private static final String DATA_PATH = System.getProperty("s3.data");
private static final String DATE_FORMAT_SEC = "yyyy-MM-dd HH:mm:00Z";
private static final String VUL_FILE_SUFFIX = "-vulnerabilities.data";
private static final String DETECTION_FILE_SUFFIX = "-detections.data";
private static final Map<String, String> sourceFileToIndexMapping = new HashMap<>(2);

static {
Expand All @@ -50,31 +51,33 @@ public List<Map<String, String>> uploadVulnerabilityInfo(String dataSource) {
String indexName = String.format(entry.getValue(), dataSource);
String filePrefix = String.format(entry.getKey(), dataSource);
List<Map<String, Object>> entities;
S3Object entitiesData = s3Client.getObject(new GetObjectRequest(BUCKET_NAME, DATA_PATH + "/" + filePrefix + ".data"));
try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) {
entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")), new TypeReference<List<Map<String, Object>>>() {
});
} catch (Exception e) {
LOGGER.info("{} data is empty", filePrefix);
continue;
}
if (Objects.isNull(entities)) {
LOGGER.info("{} object is empty for dataSource - {}", filePrefix, dataSource);
continue;
}
String url = ESUtils.getEsUrl();
if (!ESUtils.isValidIndex(url, indexName)) {
ESUtils.createIndex(url, indexName);
}
String loaddate = new SimpleDateFormat(DATE_FORMAT_SEC).format(new java.util.Date());
entities.parallelStream().filter(obj -> obj.get("closedDate") == null || StringUtils.isNullOrEmpty(obj.get("closedDate").toString()))
.forEach((obj) -> {
obj.remove("closedDate");
obj.put("_loaddate", loaddate);
});
LOGGER.info("Collected vulnerabilities: {}", entities.size());
ESManager.uploadVulnerabilityData(indexName, entities);
ESManager.deleteOldDocuments(indexName, null, "_loaddate.keyword", loaddate);
ListObjectsV2Request listReq = new ListObjectsV2Request().
withBucketName(BUCKET_NAME)
.withPrefix(DATA_PATH); // List only files inside this folder
ListObjectsV2Result result = s3Client.listObjectsV2(listReq);
if(result != null && result.getKeyCount() > 0) {
String loadDate = new SimpleDateFormat(DATE_FORMAT_SEC).format(new java.util.Date());
for (S3ObjectSummary object : result.getObjectSummaries()) {
String fileName = object.getKey();
S3Object entitiesData = s3Client.getObject(BUCKET_NAME, fileName);
if (fileName.endsWith(VUL_FILE_SUFFIX) && entry.getKey().contains("vulnerabilities") ||
fileName.endsWith(DETECTION_FILE_SUFFIX) && entry.getKey().contains("-detections")) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) {
entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")), new TypeReference<List<Map<String, Object>>>() {
});
if (Objects.isNull(entities)) {
LOGGER.info("{} object is empty for dataSource - {}", filePrefix, dataSource);
continue;
}
uploadEntity(entities, indexName, loadDate);
} catch (Exception e) {
LOGGER.info("{} data is empty", filePrefix);
}
}
}

}

} catch (Exception e) {
LOGGER.error("Error in shipping vulnerability data for dataSource - {}", dataSource);
Map<String, String> errorMap = new HashMap<>();
Expand All @@ -87,4 +90,20 @@ public List<Map<String, String>> uploadVulnerabilityInfo(String dataSource) {
LOGGER.info("Completed Vulnerability collection for {}", dataSource);
return errorList;
}


public static void uploadEntity(List<Map<String, Object>> entities, String indexName, String loadDate) throws Exception{
String url = ESUtils.getEsUrl();
if (!ESUtils.isValidIndex(url, indexName)) {
ESUtils.createIndex(url, indexName);
}
entities.parallelStream().filter(obj -> obj.get("closedDate") == null || StringUtils.isNullOrEmpty(obj.get("closedDate").toString()))
.forEach((obj) -> {
obj.remove("closedDate");
obj.put("_loaddate", loadDate);
});
LOGGER.info("Collected vulnerabilities: {}", entities.size());
ESManager.uploadVulnerabilityData(indexName, entities);
ESManager.deleteOldDocuments(indexName, null, "_loaddate.keyword", loadDate);
}
}

0 comments on commit e47803c

Please sign in to comment.