From 92fdc6a889481300ba690ee0cae67ebc6b66b229 Mon Sep 17 00:00:00 2001 From: Nithin R <103981517+rnithinpaladin@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:38:26 +0530 Subject: [PATCH] Revert "fix:[NEXT-455] Vulnerability for multiple asset types for a plugin" (#2358) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "fix:[NEXT-455] Vulnerability for multiple asset types for a plugin (#…" This reverts commit e47803cbf763b74db8b43052e5a230e613e4a8f0. --- jobs/pacman-data-shipper/pom.xml | 2 +- .../VulnerabilityAssociationManager.java | 73 +++++++------------ 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/jobs/pacman-data-shipper/pom.xml b/jobs/pacman-data-shipper/pom.xml index 7783681e40..b8b9acb60f 100644 --- a/jobs/pacman-data-shipper/pom.xml +++ b/jobs/pacman-data-shipper/pom.xml @@ -12,7 +12,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.263 + 1.12.261 com.amazonaws diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/VulnerabilityAssociationManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/VulnerabilityAssociationManager.java index 2a113d814c..fe997ba36b 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/VulnerabilityAssociationManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/VulnerabilityAssociationManager.java @@ -3,7 +3,8 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.*; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.util.StringUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,8 +31,6 @@ public class VulnerabilityAssociationManager { private static final String BUCKET_NAME = System.getProperty("s3"); private static final String DATA_PATH = System.getProperty("s3.data"); private static final String DATE_FORMAT_SEC = "yyyy-MM-dd HH:mm:00Z"; - private static final String VUL_FILE_SUFFIX = "-vulnerabilities.data"; - private static final String DETECTION_FILE_SUFFIX = "-detections.data"; private static final Map sourceFileToIndexMapping = new HashMap<>(2); static { @@ -51,33 +50,31 @@ public List> uploadVulnerabilityInfo(String dataSource) { String indexName = String.format(entry.getValue(), dataSource); String filePrefix = String.format(entry.getKey(), dataSource); List> entities; - ListObjectsV2Request listReq = new ListObjectsV2Request(). - withBucketName(BUCKET_NAME) - .withPrefix(DATA_PATH); // List only files inside this folder - ListObjectsV2Result result = s3Client.listObjectsV2(listReq); - if(result != null && result.getKeyCount() > 0) { - String loadDate = new SimpleDateFormat(DATE_FORMAT_SEC).format(new java.util.Date()); - for (S3ObjectSummary object : result.getObjectSummaries()) { - String fileName = object.getKey(); - S3Object entitiesData = s3Client.getObject(BUCKET_NAME, fileName); - if (fileName.endsWith(VUL_FILE_SUFFIX) && entry.getKey().contains("vulnerabilities") || - fileName.endsWith(DETECTION_FILE_SUFFIX) && entry.getKey().contains("-detections")) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) { - entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")), new TypeReference>>() { - }); - if (Objects.isNull(entities)) { - LOGGER.info("{} object is empty for dataSource - {}", filePrefix, dataSource); - continue; - } - uploadEntity(entities, indexName, loadDate); - } catch (Exception e) { - LOGGER.info("{} data is empty", filePrefix); - } - } - } - - } - + S3Object entitiesData = s3Client.getObject(new GetObjectRequest(BUCKET_NAME, DATA_PATH + "/" + filePrefix + ".data")); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) { + entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")), new TypeReference>>() { + }); + } catch (Exception e) { + LOGGER.info("{} data is empty", filePrefix); + continue; + } + if (Objects.isNull(entities)) { + LOGGER.info("{} object is empty for dataSource - {}", filePrefix, dataSource); + continue; + } + String url = ESUtils.getEsUrl(); + if (!ESUtils.isValidIndex(url, indexName)) { + ESUtils.createIndex(url, indexName); + } + String loaddate = new SimpleDateFormat(DATE_FORMAT_SEC).format(new java.util.Date()); + entities.parallelStream().filter(obj -> obj.get("closedDate") == null || StringUtils.isNullOrEmpty(obj.get("closedDate").toString())) + .forEach((obj) -> { + obj.remove("closedDate"); + obj.put("_loaddate", loaddate); + }); + LOGGER.info("Collected vulnerabilities: {}", entities.size()); + ESManager.uploadVulnerabilityData(indexName, entities); + ESManager.deleteOldDocuments(indexName, null, "_loaddate.keyword", loaddate); } catch (Exception e) { LOGGER.error("Error in shipping vulnerability data for dataSource - {}", dataSource); Map errorMap = new HashMap<>(); @@ -90,20 +87,4 @@ public List> uploadVulnerabilityInfo(String dataSource) { LOGGER.info("Completed Vulnerability collection for {}", dataSource); return errorList; } - - - public static void uploadEntity(List> entities, String indexName, String loadDate) throws Exception{ - String url = ESUtils.getEsUrl(); - if (!ESUtils.isValidIndex(url, indexName)) { - ESUtils.createIndex(url, indexName); - } - entities.parallelStream().filter(obj -> obj.get("closedDate") == null || StringUtils.isNullOrEmpty(obj.get("closedDate").toString())) - .forEach((obj) -> { - obj.remove("closedDate"); - obj.put("_loaddate", loadDate); - }); - LOGGER.info("Collected vulnerabilities: {}", entities.size()); - ESManager.uploadVulnerabilityData(indexName, entities); - ESManager.deleteOldDocuments(indexName, null, "_loaddate.keyword", loadDate); - } }