Skip to content

Commit 435fe3f

Browse files
mneethirajmneethiraj
committed
ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
1 parent 0abf84c commit 435fe3f

File tree

385 files changed

+6559
-50561
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

385 files changed

+6559
-50561
lines changed

‎addons/falcon-bridge/pom.xml‎

Lines changed: 2 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,6 @@
4646
<artifactId>slf4j-log4j12</artifactId>
4747
</dependency>
4848

49-
<dependency>
50-
<groupId>org.apache.atlas</groupId>
51-
<artifactId>atlas-typesystem</artifactId>
52-
</dependency>
53-
5449
<dependency>
5550
<groupId>org.apache.atlas</groupId>
5651
<artifactId>atlas-client-v1</artifactId>
@@ -100,13 +95,6 @@
10095
<type>pom</type>
10196
<scope>test</scope>
10297
</dependency>
103-
104-
<dependency>
105-
<groupId>org.apache.atlas</groupId>
106-
<artifactId>atlas-typesystem</artifactId>
107-
<classifier>tests</classifier>
108-
<scope>test</scope>
109-
</dependency>
11098
</dependencies>
11199

112100
<profiles>
@@ -140,21 +128,6 @@
140128
<artifactId>hive-bridge</artifactId>
141129
<version>${project.version}</version>
142130
</artifactItem>
143-
<artifactItem>
144-
<groupId>org.json4s</groupId>
145-
<artifactId>json4s-native_${scala.binary.version}</artifactId>
146-
<version>${json.version}</version>
147-
</artifactItem>
148-
<artifactItem>
149-
<groupId>org.json4s</groupId>
150-
<artifactId>json4s-core_${scala.binary.version}</artifactId>
151-
<version>${json.version}</version>
152-
</artifactItem>
153-
<artifactItem>
154-
<groupId>org.json4s</groupId>
155-
<artifactId>json4s-ast_${scala.binary.version}</artifactId>
156-
<version>${json.version}</version>
157-
</artifactItem>
158131
<artifactItem>
159132
<groupId>${project.groupId}</groupId>
160133
<artifactId>atlas-client-common</artifactId>
@@ -170,11 +143,6 @@
170143
<artifactId>atlas-client-v2</artifactId>
171144
<version>${project.version}</version>
172145
</artifactItem>
173-
<artifactItem>
174-
<groupId>${project.groupId}</groupId>
175-
<artifactId>atlas-typesystem</artifactId>
176-
<version>${project.version}</version>
177-
</artifactItem>
178146
<artifactItem>
179147
<groupId>${project.groupId}</groupId>
180148
<artifactId>atlas-intg</artifactId>
@@ -195,41 +163,16 @@
195163
<artifactId>hdfs-model</artifactId>
196164
<version>${project.version}</version>
197165
</artifactItem>
198-
<artifactItem>
199-
<groupId>org.scala-lang</groupId>
200-
<artifactId>scala-compiler</artifactId>
201-
<version>${scala.version}</version>
202-
</artifactItem>
203-
<artifactItem>
204-
<groupId>org.scala-lang</groupId>
205-
<artifactId>scala-reflect</artifactId>
206-
<version>${scala.version}</version>
207-
</artifactItem>
208-
<artifactItem>
209-
<groupId>org.scala-lang</groupId>
210-
<artifactId>scala-library</artifactId>
211-
<version>${scala.version}</version>
212-
</artifactItem>
213-
<artifactItem>
214-
<groupId>org.scala-lang</groupId>
215-
<artifactId>scalap</artifactId>
216-
<version>${scala.version}</version>
217-
</artifactItem>
218166
<artifactItem>
219167
<groupId>org.apache.kafka</groupId>
220-
<artifactId>kafka_${scala.binary.version}</artifactId>
168+
<artifactId>kafka_${kafka.scala.binary.version}</artifactId>
221169
<version>${kafka.version}</version>
222170
</artifactItem>
223171
<artifactItem>
224172
<groupId>org.apache.kafka</groupId>
225173
<artifactId>kafka-clients</artifactId>
226174
<version>${kafka.version}</version>
227175
</artifactItem>
228-
<artifactItem>
229-
<groupId>com.google.code.gson</groupId>
230-
<artifactId>gson</artifactId>
231-
<version>${gson.version}</version>
232-
</artifactItem>
233176
</artifactItems>
234177
</configuration>
235178
</execution>
@@ -302,7 +245,7 @@
302245
</systemProperty>
303246
<systemProperty>
304247
<key>atlas.conf</key>
305-
<value>${project.build.directory}/../../../typesystem/target/test-classes</value>
248+
<value>${project.build.directory}/../../../intg/target/test-classes</value>
306249
</systemProperty>
307250
<systemProperty>
308251
<key>atlas.home</key>

‎addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import org.apache.atlas.falcon.model.FalconDataTypes;
2525
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
2626
import org.apache.atlas.hive.model.HiveDataTypes;
27-
import org.apache.atlas.typesystem.Referenceable;
27+
import org.apache.atlas.v1.model.instance.Referenceable;
2828
import org.apache.commons.collections.CollectionUtils;
2929
import org.apache.commons.lang3.StringUtils;
3030
import org.apache.falcon.FalconException;
@@ -204,8 +204,6 @@ public static List<Referenceable> createFeedCreationEntity(Feed feed, Configurat
204204
*
205205
* @param process process entity
206206
* @param falconStore config store
207-
* @param user falcon user
208-
* @param timestamp timestamp of entity
209207
* @return process instance reference
210208
*
211209
* @throws FalconException if retrieving from the configuration store fail

‎addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@
2525
import org.apache.atlas.falcon.publisher.FalconEventPublisher;
2626
import org.apache.atlas.hook.AtlasHook;
2727
import org.apache.atlas.kafka.NotificationProvider;
28-
import org.apache.atlas.notification.hook.HookNotification;
29-
import org.apache.atlas.typesystem.Referenceable;
28+
import org.apache.atlas.model.notification.HookNotification;
29+
import org.apache.atlas.v1.model.instance.Referenceable;
30+
import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest;
3031
import org.apache.falcon.FalconException;
3132
import org.apache.falcon.entity.store.ConfigurationStore;
3233
import org.apache.falcon.entity.v0.feed.Feed;
@@ -151,14 +152,14 @@ protected String getNumberOfRetriesPropertyKey() {
151152

152153
private void fireAndForget(FalconEvent event) throws FalconException, URISyntaxException {
153154
LOG.info("Entered Atlas hook for Falcon hook operation {}", event.getOperation());
154-
List<HookNotification.HookNotificationMessage> messages = new ArrayList<>();
155+
List<HookNotification> messages = new ArrayList<>();
155156

156157
Operation op = getOperation(event.getOperation());
157158
String user = getUser(event.getUser());
158159
LOG.info("fireAndForget user:{}", user);
159160
switch (op) {
160161
case ADD:
161-
messages.add(new HookNotification.EntityCreateRequest(user, createEntities(event, user)));
162+
messages.add(new EntityCreateRequest(user, createEntities(event, user)));
162163
break;
163164

164165
}

‎addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@
2626
import org.apache.atlas.falcon.model.FalconDataTypes;
2727
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
2828
import org.apache.atlas.hive.model.HiveDataTypes;
29-
import org.apache.atlas.typesystem.Referenceable;
30-
import org.apache.atlas.typesystem.persistence.Id;
31-
import org.apache.atlas.typesystem.types.TypeUtils;
29+
import org.apache.atlas.v1.model.instance.Id;
30+
import org.apache.atlas.v1.model.instance.Referenceable;
31+
import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
3232
import org.apache.atlas.utils.AuthenticationUtil;
3333
import org.apache.atlas.utils.ParamChecker;
3434
import org.apache.commons.configuration.Configuration;
@@ -162,7 +162,7 @@ private String assertClusterIsRegistered(Cluster cluster) throws Exception {
162162
AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
163163
}
164164

165-
private TypeUtils.Pair<String, Feed> getHDFSFeed(String feedResource, String clusterName) throws Exception {
165+
private TypesUtil.Pair<String, Feed> getHDFSFeed(String feedResource, String clusterName) throws Exception {
166166
Feed feed = loadEntity(EntityType.FEED, feedResource, "feed" + random());
167167
org.apache.falcon.entity.v0.feed.Cluster feedCluster = feed.getClusters().getClusters().get(0);
168168
feedCluster.setName(clusterName);
@@ -174,9 +174,9 @@ private TypeUtils.Pair<String, Feed> getHDFSFeed(String feedResource, String clu
174174
AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
175175
FalconBridge.getFeedQualifiedName(feed.getName(), clusterName));
176176
Referenceable processEntity = atlasClient.getEntity(processId);
177-
assertEquals(((List<Id>)processEntity.get("outputs")).get(0).getId()._getId(), feedId);
177+
assertEquals(((List<Id>)processEntity.get("outputs")).get(0).getId(), feedId);
178178

179-
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
179+
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId();
180180
Referenceable pathEntity = atlasClient.getEntity(inputId);
181181
assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH);
182182

@@ -185,7 +185,7 @@ private TypeUtils.Pair<String, Feed> getHDFSFeed(String feedResource, String clu
185185
assertEquals(pathEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME),
186186
FalconBridge.normalize(dataLocation.getPath()));
187187

188-
return TypeUtils.Pair.of(feedId, feed);
188+
return TypesUtil.Pair.of(feedId, feed);
189189
}
190190

191191
private Feed getTableFeed(String feedResource, String clusterName) throws Exception {
@@ -236,9 +236,9 @@ private void verifyFeedLineage(String feedName, String clusterName, String feedI
236236
AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
237237
FalconBridge.getFeedQualifiedName(feedName, clusterName));
238238
Referenceable processEntity = atlasClient.getEntity(processId);
239-
assertEquals(((List<Id>)processEntity.get("outputs")).get(0).getId()._getId(), feedId);
239+
assertEquals(((List<Id>)processEntity.get("outputs")).get(0).getId(), feedId);
240240

241-
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
241+
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId();
242242
Referenceable tableEntity = atlasClient.getEntity(inputId);
243243
assertEquals(tableEntity.getTypeName(), HiveDataTypes.HIVE_TABLE.getName());
244244
assertEquals(tableEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME),
@@ -280,7 +280,7 @@ public void testCreateProcessWithHDFSFeed() throws Exception {
280280
Cluster cluster = loadEntity(EntityType.CLUSTER, CLUSTER_RESOURCE, "cluster" + random());
281281
STORE.publish(EntityType.CLUSTER, cluster);
282282

283-
TypeUtils.Pair<String, Feed> result = getHDFSFeed(FEED_HDFS_RESOURCE, cluster.getName());
283+
TypesUtil.Pair<String, Feed> result = getHDFSFeed(FEED_HDFS_RESOURCE, cluster.getName());
284284
Feed infeed = result.right;
285285
String infeedId = result.left;
286286

0 commit comments

Comments
 (0)